libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {1, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {1, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {1, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t cl_curl_enabled = {1, "cl_curl_enabled","0", "whether client's download support is enabled"};
   9
  10 /*
  11 =================================================================
  12
  13   Minimal set of definitions from libcurl
  14
  15   WARNING: for a matter of simplicity, several pointer types are
  16   casted to "void*", and most enumerated values are not included
  17
  18 =================================================================
  19 */
  20
  21 typedef struct CURL_s CURL;
  22 typedef struct CURLM_s CURLM;
  23 typedef enum
  24 {
  25         CURLE_OK = 0
  26 }
  27 CURLcode;
  28 typedef enum
  29 {
  30         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  31         CURLM_OK = 0
  32 }
  33 CURLMcode;
  34 #define CURL_GLOBAL_NOTHING 0
  35 #define CURL_GLOBAL_SSL 1
  36 #define CURL_GLOBAL_WIN32 2
  37 #define CURLOPTTYPE_LONG          0
  38 #define CURLOPTTYPE_OBJECTPOINT   10000
  39 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  40 #define CURLOPTTYPE_OFF_T         30000
  41 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  42 typedef enum
  43 {
  44         CINIT(WRITEDATA, OBJECTPOINT, 1),
  45         CINIT(URL,  OBJECTPOINT, 2),
  46         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  47         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  48         CINIT(REFERER, OBJECTPOINT, 16),
  49         CINIT(USERAGENT, OBJECTPOINT, 18),
  50         CINIT(RESUME_FROM, LONG, 21),
  51         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  52         CINIT(PRIVATE, OBJECTPOINT, 103),
  53 }
  54 CURLoption;
  55 typedef enum
  56 {
  57         CURLINFO_TEXT = 0,
  58         CURLINFO_HEADER_IN,    /* 1 */
  59         CURLINFO_HEADER_OUT,   /* 2 */
  60         CURLINFO_DATA_IN,      /* 3 */
  61         CURLINFO_DATA_OUT,     /* 4 */
  62         CURLINFO_SSL_DATA_IN,  /* 5 */
  63         CURLINFO_SSL_DATA_OUT, /* 6 */
  64         CURLINFO_END
  65 }
  66 curl_infotype;
  67 #define CURLINFO_STRING   0x100000
  68 #define CURLINFO_LONG     0x200000
  69 #define CURLINFO_DOUBLE   0x300000
  70 #define CURLINFO_SLIST    0x400000
  71 #define CURLINFO_MASK     0x0fffff
  72 #define CURLINFO_TYPEMASK 0xf00000
  73 typedef enum
  74 {
  75         CURLINFO_NONE, /* first, never use this */
  76         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  77         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  78         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  79         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  80         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  81         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  82         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  83         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  84         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  85         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  86         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  87         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  88         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  89         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  90         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  91         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  92         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  93         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  94         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  95         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  96         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
  97         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
  98         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
  99         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 100         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 101         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 102         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 103 }
 104 CURLINFO;
 105
 106 typedef enum
 107 {
 108         CURLMSG_NONE, /* first, not used */
 109         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 110                                          the CURLcode of the transfer */
 111         CURLMSG_LAST
 112 }
 113 CURLMSG;
 114 typedef struct
 115 {
 116         CURLMSG msg;       /* what this message means */
 117         CURL *easy_handle; /* the handle it concerns */
 118         union
 119         {
 120                 void *whatever;    /* message-specific data */
 121                 CURLcode result;   /* return code for transfer */
 122         }
 123         data;
 124 }
 125 CURLMsg;
 126
 127 static void (*qcurl_global_init) (long flags);
 128 static void (*qcurl_global_cleanup) ();
 129
 130 static CURL * (*qcurl_easy_init) ();
 131 static void (*qcurl_easy_cleanup) (CURL *handle);
 132 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 133 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 134 static const char * (*qcurl_easy_strerror) (CURLcode);
 135
 136 static CURLM * (*qcurl_multi_init) ();
 137 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 138 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 139 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 140 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 141 static void (*qcurl_multi_cleanup) (CURLM *);
 142 static const char * (*qcurl_multi_strerror) (CURLcode);
 143
 144 static dllfunction_t curlfuncs[] =
 145 {
 146         {"curl_global_init",            (void **) &qcurl_global_init},
 147         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 148         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 149         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 150         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 151         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 152         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 153         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 154         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 155         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 156         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 157         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 158         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 159         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 160         {NULL, NULL}
 161 };
 162
 163 // Handle for CURL DLL
 164 static dllhandle_t curl_dll = NULL;
 165 // will be checked at many places to find out if qcurl calls are allowed
 166
 167 typedef struct downloadinfo_s
 168 {
 169         char filename[MAX_QPATH];
 170         char url[256];
 171         char referer[256];
 172         qfile_t *stream;
 173         fs_offset_t startpos;
 174         CURL *curle;
 175         qboolean started;
 176         qboolean ispak;
 177         unsigned long bytes_received;
 178         struct downloadinfo_s *next, *prev;
 179         qboolean forthismap;
 180 }
 181 downloadinfo;
 182 static downloadinfo *downloads = NULL;
 183 static int numdownloads = 0;
 184
 185 /*
 186 ====================
 187 CURL_CloseLibrary
 188
 189 Load the cURL DLL
 190 ====================
 191 */
 192 static qboolean CURL_OpenLibrary (void)
 193 {
 194         const char* dllnames [] =
 195         {
 196 #if defined(WIN64)
 197                 "libcurl64.dll",
 198 #elif defined(WIN32)
 199                 "libcurl-3.dll",
 200 #elif defined(MACOSX)
 201                 "libcurl.3.dylib",
 202 #else
 203                 "libcurl.so.3",
 204 #endif
 205                 NULL
 206         };
 207
 208         // Already loaded?
 209         if (curl_dll)
 210                 return true;
 211
 212         // Load the DLL
 213         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 214         {
 215                 Con_Printf ("cURL support disabled\n");
 216                 return false;
 217         }
 218
 219         Con_Printf ("cURL support enabled\n");
 220         return true;
 221 }
 222
 223
 224 /*
 225 ====================
 226 CURL_CloseLibrary
 227
 228 Unload the cURL DLL
 229 ====================
 230 */
 231 static void CURL_CloseLibrary (void)
 232 {
 233         Sys_UnloadLibrary (&curl_dll);
 234 }
 235
 236
 237 static CURLM *curlm = NULL;
 238 static unsigned long bytes_received = 0; // used for bandwidth throttling
 239 static double curltime = 0;
 240
 241 /*
 242 ====================
 243 CURL_fwrite
 244
 245 fwrite-compatible function that writes the data to a file. libcurl can call
 246 this.
 247 ====================
 248 */
 249 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 250 {
 251         fs_offset_t ret;
 252         size_t bytes = size * nmemb;
 253         downloadinfo *di = (downloadinfo *) vdi;
 254
 255         bytes_received += bytes;
 256         di->bytes_received += bytes;
 257
 258         ret = FS_Write(di->stream, data, bytes);
 259
 260         return ret; // why not ret / nmemb?
 261 }
 262
 263 typedef enum
 264 {
 265         CURL_DOWNLOAD_SUCCESS = 0,
 266         CURL_DOWNLOAD_FAILED,
 267         CURL_DOWNLOAD_ABORTED,
 268         CURL_DOWNLOAD_SERVERERROR
 269 }
 270 CurlStatus;
 271
 272 /*
 273 ====================
 274 Curl_Clear_forthismap
 275
 276 Clears the "will disconnect on failure" flags.
 277 ====================
 278 */
 279 void Curl_Clear_forthismap()
 280 {
 281         downloadinfo *di;
 282         for(di = downloads; di; di = di->next)
 283                 di->forthismap = false;
 284 }
 285
 286 static qboolean Curl_Have_forthismap()
 287 {
 288         downloadinfo *di;
 289         for(di = downloads; di; di = di->next)
 290                 if(di->forthismap)
 291                         return true;
 292         return false;
 293 }
 294
 295 /*
 296 ====================
 297 Curl_EndDownload
 298
 299 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 300 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 301 code from libcurl, or 0, if another error has occurred.
 302 ====================
 303 */
 304 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 305 {
 306         qboolean ok = false;
 307         if(!curl_dll)
 308                 return;
 309         switch(status)
 310         {
 311                 case CURL_DOWNLOAD_SUCCESS:
 312                         Con_Printf("Download of %s: OK\n", di->filename);
 313                         ok = true;
 314                         break;
 315                 case CURL_DOWNLOAD_FAILED:
 316                         Con_Printf("Download of %s: FAILED\n", di->filename);
 317                         if(error)
 318                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 319                         break;
 320                 case CURL_DOWNLOAD_ABORTED:
 321                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 322                         break;
 323                 case CURL_DOWNLOAD_SERVERERROR:
 324                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 325
 326                         // reopen to enforce it to have zero bytes again
 327                         FS_Close(di->stream);
 328                         di->stream = FS_Open(di->filename, "w", false, false);
 329
 330                         break;
 331         }
 332
 333         if(di->curle)
 334         {
 335                 qcurl_multi_remove_handle(curlm, di->curle);
 336                 qcurl_easy_cleanup(di->curle);
 337         }
 338
 339         if(ok && !di->bytes_received)
 340         {
 341                 Con_Printf("ERROR: empty file\n");
 342                 ok = false;
 343         }
 344
 345         if(di->stream)
 346                 FS_Close(di->stream);
 347
 348         if(ok && di->ispak)
 349         {
 350                 ok = FS_AddPack(di->filename, NULL, true);
 351                 if(ok && di->forthismap)
 352                         Mod_Reload();
 353         }
 354
 355         if(!ok && di->forthismap)
 356         {
 357                 // BAD. Something went totally wrong.
 358                 // The best we can do is clean up the forthismap flags...
 359                 Curl_Clear_forthismap();
 360                 // and disconnect.
 361                 CL_Disconnect_f();
 362         }
 363
 364         if(di->prev)
 365                 di->prev->next = di->next;
 366         else
 367                 downloads = di->next;
 368         if(di->next)
 369                 di->next->prev = di->prev;
 370         Z_Free(di);
 371
 372         --numdownloads;
 373 }
 374
 375 /*
 376 ====================
 377 CheckPendingDownloads
 378
 379 checks if there are free download slots to start new downloads in.
 380 To not start too many downloads at once, only one download is added at a time,
 381 up to a maximum number of cl_curl_maxdownloads are running.
 382 ====================
 383 */
 384 static void CheckPendingDownloads()
 385 {
 386         if(!curl_dll)
 387                 return;
 388         if(numdownloads < cl_curl_maxdownloads.integer)
 389         {
 390                 downloadinfo *di;
 391                 for(di = downloads; di; di = di->next)
 392                 {
 393                         if(!di->started)
 394                         {
 395                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 396
 397                                 di->stream = FS_Open(di->filename, "ab", false, false);
 398                                 if(!di->stream)
 399                                 {
 400                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 401                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 402                                         return;
 403                                 }
 404
 405                                 FS_Seek(di->stream, 0, SEEK_END);
 406                                 di->startpos = FS_Tell(di->stream);
 407                                 if(di->startpos > 0)
 408                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 409                                 Con_Print("...\n");
 410
 411                                 di->curle = qcurl_easy_init();
 412                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 413                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 414                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 415                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 416                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 417                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 418                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 419                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 420                                 qcurl_multi_add_handle(curlm, di->curle);
 421                                 di->started = true;
 422                                 ++numdownloads;
 423                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 424                                         break;
 425                         }
 426                 }
 427         }
 428 }
 429
 430 /*
 431 ====================
 432 Curl_Init
 433
 434 this function MUST be called before using anything else in this file.
 435 On Win32, this must be called AFTER WSAStartup has been done!
 436 ====================
 437 */
 438 void Curl_Init()
 439 {
 440         CURL_OpenLibrary();
 441         if(!curl_dll)
 442                 return;
 443         qcurl_global_init(CURL_GLOBAL_NOTHING);
 444         curlm = qcurl_multi_init();
 445 }
 446
 447 /*
 448 ====================
 449 Curl_Shutdown
 450
 451 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 452 ====================
 453 */
 454 void Curl_ClearRequirements();
 455 void Curl_Shutdown()
 456 {
 457         if(!curl_dll)
 458                 return;
 459         Curl_ClearRequirements();
 460         Curl_CancelAll();
 461         CURL_CloseLibrary();
 462         curl_dll = NULL;
 463 }
 464
 465 /*
 466 ====================
 467 Curl_Find
 468
 469 Finds the internal information block for a download given by file name.
 470 ====================
 471 */
 472 static downloadinfo *Curl_Find(const char *filename)
 473 {
 474         downloadinfo *di;
 475         if(!curl_dll)
 476                 return NULL;
 477         for(di = downloads; di; di = di->next)
 478                 if(!strcasecmp(di->filename, filename))
 479                         return di;
 480         return NULL;
 481 }
 482
 483 /*
 484 ====================
 485 Curl_Begin
 486
 487 Starts a download of a given URL to the file name portion of this URL (or name
 488 if given) in the "dlcache/" folder.
 489 ====================
 490 */
 491 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 492 {
 493         if(!curl_dll)
 494                 return;
 495         else
 496         {
 497                 char fn[MAX_QPATH];
 498                 const char *p, *q;
 499                 size_t length;
 500                 downloadinfo *di;
 501
 502                 // Note: This extraction of the file name portion is NOT entirely correct.
 503                 //
 504                 // It does the following:
 505                 //
 506                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 507                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 508                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 509                 //
 510                 // However, I'd like to keep this "buggy" behavior so that PHP script
 511                 // authors can write download scripts without having to enable
 512                 // AcceptPathInfo on Apache. They just have to ensure that their script
 513                 // can be called with such a "fake" path name like
 514                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 515                 //
 516                 // By the way, such PHP scripts should either send the file or a
 517                 // "Location:" redirect; PHP code example:
 518                 //
 519                 //   header("Location: http://www.example.com/");
 520                 //
 521                 // By the way, this will set User-Agent to something like
 522                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 523                 // dp://serverhost:serverport/ so you can filter on this; an example
 524                 // httpd log file line might be:
 525                 //
 526                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 527
 528                 if(!name)
 529                         name = URL;
 530                 p = strrchr(name, '/');
 531                 p = p ? (p+1) : name;
 532                 q = strchr(p, '?');
 533                 length = q ? (size_t)(q - p) : strlen(p);
 534                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 535
 536
 537                 // already downloading the file?
 538                 {
 539                         downloadinfo *di = Curl_Find(fn);
 540                         if(di)
 541                         {
 542                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 543
 544                                 // however, if it was not for this map yet...
 545                                 if(forthismap)
 546                                         di->forthismap = true;
 547
 548                                 return;
 549                         }
 550                 }
 551
 552                 if(ispak && FS_FileExists(fn))
 553                 {
 554                         qboolean already_loaded;
 555                         if(FS_AddPack(fn, &already_loaded, true))
 556                         {
 557                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 558                                 if(already_loaded)
 559                                         Con_DPrintf("(pak was already loaded)\n");
 560                                 else
 561                                         if(forthismap)
 562                                                 Mod_Reload();
 563                                 return;
 564                         }
 565                         else
 566                         {
 567                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 568                                 if(f)
 569                                 {
 570                                         char buf[4] = {0};
 571                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 572
 573                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 574                                         {
 575                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 576                                                 FS_Close(f);
 577                                                 f = FS_Open(fn, "w", false, false);
 578                                                 if(f)
 579                                                         FS_Close(f);
 580                                         }
 581                                         else
 582                                         {
 583                                                 // OK
 584                                                 FS_Close(f);
 585                                         }
 586                                 }
 587                         }
 588                 }
 589
 590                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 591                 strlcpy(di->filename, fn, sizeof(di->filename));
 592                 strlcpy(di->url, URL, sizeof(di->url));
 593                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 594                 di->forthismap = forthismap;
 595                 di->stream = NULL;
 596                 di->startpos = 0;
 597                 di->curle = NULL;
 598                 di->started = false;
 599                 di->ispak = ispak;
 600                 di->bytes_received = 0;
 601                 di->next = downloads;
 602                 di->prev = NULL;
 603                 if(di->next)
 604                         di->next->prev = di;
 605                 downloads = di;
 606         }
 607 }
 608
 609
 610 /*
 611 ====================
 612 Curl_Run
 613
 614 call this regularily as this will always download as much as possible without
 615 blocking.
 616 ====================
 617 */
 618 void Curl_Run()
 619 {
 620         if(!cl_curl_enabled.integer)
 621                 return;
 622
 623         if(!curl_dll)
 624                 return;
 625
 626         if(!downloads)
 627                 return;
 628
 629         if(realtime < curltime) // throttle
 630                 return;
 631
 632         {
 633                 int remaining;
 634                 CURLMcode mc;
 635
 636                 do
 637                 {
 638                         mc = qcurl_multi_perform(curlm, &remaining);
 639                 }
 640                 while(mc == CURLM_CALL_MULTI_PERFORM);
 641
 642                 for(;;)
 643                 {
 644                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 645                         if(!msg)
 646                                 break;
 647                         if(msg->msg == CURLMSG_DONE)
 648                         {
 649                                 downloadinfo *di;
 650                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 651                                 CURLcode result;
 652
 653                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 654                                 result = msg->data.result;
 655                                 if(result)
 656                                 {
 657                                         failed = CURL_DOWNLOAD_FAILED;
 658                                 }
 659                                 else
 660                                 {
 661                                         long code;
 662                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 663                                         switch(code / 100)
 664                                         {
 665                                                 case 4: // e.g. 404?
 666                                                 case 5: // e.g. 500?
 667                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 668                                                         result = code;
 669                                                         break;
 670                                         }
 671                                 }
 672
 673                                 Curl_EndDownload(di, failed, result);
 674                         }
 675                 }
 676         }
 677
 678         CheckPendingDownloads();
 679
 680         // when will we curl the next time?
 681         // we will wait a bit to ensure our download rate is kept.
 682         // we now know that realtime >= curltime... so set up a new curltime
 683         if(cl_curl_maxspeed.value > 0)
 684         {
 685                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 686                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 687                 bytes_received -= bytes;
 688         }
 689         else
 690                 curltime = realtime;
 691 }
 692
 693 /*
 694 ====================
 695 Curl_CancelAll
 696
 697 Stops ALL downloads.
 698 ====================
 699 */
 700 void Curl_CancelAll()
 701 {
 702         if(!curl_dll)
 703                 return;
 704
 705         while(downloads)
 706         {
 707                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 708                 // INVARIANT: downloads will point to the next download after that!
 709         }
 710 }
 711
 712 /*
 713 ====================
 714 Curl_Running
 715
 716 returns true iff there is a download running.
 717 ====================
 718 */
 719 qboolean Curl_Running()
 720 {
 721         if(!curl_dll)
 722                 return false;
 723
 724         return downloads != NULL;
 725 }
 726
 727 /*
 728 ====================
 729 Curl_GetDownloadAmount
 730
 731 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 732 for the given download.
 733 ====================
 734 */
 735 static double Curl_GetDownloadAmount(downloadinfo *di)
 736 {
 737         if(!curl_dll)
 738                 return -2;
 739         if(di->curle)
 740         {
 741                 double length;
 742                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 743                 if(length > 0)
 744                         return di->bytes_received / length;
 745                 else
 746                         return 0;
 747         }
 748         else
 749                 return -1;
 750 }
 751
 752 /*
 753 ====================
 754 Curl_GetDownloadSpeed
 755
 756 returns the speed of the given download in bytes per second
 757 ====================
 758 */
 759 static double Curl_GetDownloadSpeed(downloadinfo *di)
 760 {
 761         if(!curl_dll)
 762                 return -2;
 763         if(di->curle)
 764         {
 765                 double speed;
 766                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 767                 return speed;
 768         }
 769         else
 770                 return -1;
 771 }
 772
 773 /*
 774 ====================
 775 Curl_Info_f
 776
 777 prints the download list
 778 ====================
 779 */
 780 // TODO rewrite using Curl_GetDownloadInfo?
 781 static void Curl_Info_f()
 782 {
 783         downloadinfo *di;
 784         if(!curl_dll)
 785                 return;
 786         if(Curl_Running())
 787         {
 788                 Con_Print("Currently running downloads:\n");
 789                 for(di = downloads; di; di = di->next)
 790                 {
 791                         double speed, percent;
 792                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 793                         percent = 100.0 * Curl_GetDownloadAmount(di);
 794                         speed = Curl_GetDownloadSpeed(di);
 795                         if(percent >= 0)
 796                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 797                         else
 798                                 Con_Print("(queued)\n");
 799                 }
 800         }
 801         else
 802         {
 803                 Con_Print("No downloads running.\n");
 804         }
 805 }
 806
 807 /*
 808 ====================
 809 Curl_Curl_f
 810
 811 implements the "curl" console command
 812
 813 curl --info
 814 curl --cancel
 815 curl --cancel filename
 816 curl url
 817
 818 For internal use:
 819
 820 curl [--pak] [--forthismap] [--for filename filename...] url
 821         --pak: after downloading, load the package into the virtual file system
 822         --for filename...: only download of at least one of the named files is missing
 823         --forthismap: disconnect on failure
 824 ====================
 825 */
 826 void Curl_Curl_f(void)
 827 {
 828         int i;
 829         int end;
 830         qboolean pak = false;
 831         qboolean forthismap = false;
 832         const char *url;
 833         const char *name = 0;
 834
 835         if(!curl_dll)
 836         {
 837                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 838                 return;
 839         }
 840
 841         if(!cl_curl_enabled.integer)
 842         {
 843                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 844                 return;
 845         }
 846
 847         for(i = 0; i != Cmd_Argc(); ++i)
 848                 Con_DPrintf("%s ", Cmd_Argv(i));
 849         Con_DPrint("\n");
 850
 851         if(Cmd_Argc() < 2)
 852         {
 853                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 854                 return;
 855         }
 856
 857         url = Cmd_Argv(Cmd_Argc() - 1);
 858         end = Cmd_Argc();
 859
 860         for(i = 1; i != end; ++i)
 861         {
 862                 const char *a = Cmd_Argv(i);
 863                 if(!strcmp(a, "--info"))
 864                 {
 865                         Curl_Info_f();
 866                         return;
 867                 }
 868                 else if(!strcmp(a, "--cancel"))
 869                 {
 870                         if(i == end - 1) // last argument
 871                                 Curl_CancelAll();
 872                         else
 873                         {
 874                                 downloadinfo *di = Curl_Find(url);
 875                                 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 876                         }
 877                         return;
 878                 }
 879                 else if(!strcmp(a, "--pak"))
 880                 {
 881                         pak = true;
 882                 }
 883                 else if(!strcmp(a, "--for"))
 884                 {
 885                         for(i = i + 1; i != end - 1; ++i)
 886                         {
 887                                 if(!FS_FileExists(Cmd_Argv(i)))
 888                                         goto needthefile; // why can't I have a "double break"?
 889                         }
 890                         // if we get here, we have all the files...
 891                         return;
 892                 }
 893                 else if(!strcmp(a, "--forthismap"))
 894                 {
 895                         forthismap = true;
 896                 }
 897                 else if(!strcmp(a, "--as"))
 898                 {
 899                         if(i < end - 1)
 900                         {
 901                                 ++i;
 902                                 name = Cmd_Argv(i);
 903                         }
 904                 }
 905                 else if(!strcmp(a, "--clear_autodownload"))
 906                 {
 907                         // mark all running downloads as "not for this map", so if they
 908                         // fail, it does not matter
 909                         Curl_Clear_forthismap();
 910                         return;
 911                 }
 912                 else if(!strcmp(a, "--finish_autodownload"))
 913                 {
 914                         // nothing
 915                         return;
 916                 }
 917                 else if(*a == '-')
 918                 {
 919                         Con_Printf("invalid option %s\n", a);
 920                         return;
 921                 }
 922         }
 923
 924 needthefile:
 925         Curl_Begin(url, name, pak, forthismap);
 926 }
 927
 928 /*
 929 ====================
 930 Curl_Init_Commands
 931
 932 loads the commands and cvars this library uses
 933 ====================
 934 */
 935 void Curl_Init_Commands(void)
 936 {
 937         Cvar_RegisterVariable (&cl_curl_enabled);
 938         Cvar_RegisterVariable (&cl_curl_maxdownloads);
 939         Cvar_RegisterVariable (&cl_curl_maxspeed);
 940         Cvar_RegisterVariable (&sv_curl_defaulturl);
 941         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
 942 }
 943
 944 /*
 945 ====================
 946 Curl_GetDownloadInfo
 947
 948 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
 949 The number of elements in the array is returned in int *nDownloads.
 950 const char **additional_info may be set to a string of additional user
 951 information, or to NULL if no such display shall occur. The returned
 952 array must be freed later using Z_Free.
 953 ====================
 954 */
 955 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
 956 {
 957         int n, i;
 958         downloadinfo *di;
 959         Curl_downloadinfo_t *downinfo;
 960         static char addinfo[128];
 961
 962         if(!curl_dll)
 963         {
 964                 *nDownloads = 0;
 965                 if(additional_info)
 966                         *additional_info = NULL;
 967                 return NULL;
 968         }
 969
 970         n = 0;
 971         for(di = downloads; di; di = di->next)
 972                 ++n;
 973
 974         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
 975         i = 0;
 976         for(di = downloads; di; di = di->next)
 977         {
 978                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
 979                 if(di->curle)
 980                 {
 981                         downinfo[i].progress = Curl_GetDownloadAmount(di);
 982                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
 983                         downinfo[i].queued = false;
 984                 }
 985                 else
 986                 {
 987                         downinfo[i].queued = true;
 988                 }
 989                 ++i;
 990         }
 991
 992         if(additional_info)
 993         {
 994                 // TODO put something better here?
 995                 // maybe... check if the file is actually needed for the current map?
 996                 if(Curl_Have_forthismap())
 997                 {
 998                         dpsnprintf(addinfo, sizeof(addinfo), "please wait for the download to complete");
 999                         *additional_info = addinfo;
1000                 }
1001                 else
1002                         *additional_info = NULL;
1003         }
1004
1005         *nDownloads = n;
1006         return downinfo;
1007 }
1008
1009
1010 /*
1011 ====================
1012 Curl_FindPackURL
1013
1014 finds the URL where to find a given package.
1015
1016 For this, it reads a file "curl_urls.txt" of the following format:
1017
1018         data*.pk3       -
1019         revdm*.pk3      http://revdm/downloads/are/here/
1020         *                       http://any/other/stuff/is/here/
1021
1022 The URLs should end in /. If not, downloads will still work, but the cached files
1023 can't be just put into the data directory with the same download configuration
1024 (you might want to do this if you want to tag downloaded files from your
1025 server, but you should not). "-" means "don't download".
1026
1027 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1028 location instead.
1029
1030 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1031 this file for obvious reasons.
1032 ====================
1033 */
1034 static const char *Curl_FindPackURL(const char *filename)
1035 {
1036         static char foundurl[256];
1037         fs_offset_t filesize;
1038         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1039         if(buf && filesize)
1040         {
1041                 // read lines of format "pattern url"
1042                 char *p = buf;
1043                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1044                 qboolean eof = false;
1045
1046                 pattern = p;
1047                 while(!eof)
1048                 {
1049                         switch(*p)
1050                         {
1051                                 case 0:
1052                                         eof = true;
1053                                         // fallthrough
1054                                 case '\n':
1055                                 case '\r':
1056                                         if(pattern && url && patternend)
1057                                         {
1058                                                 if(!urlend)
1059                                                         urlend = p;
1060                                                 *patternend = 0;
1061                                                 *urlend = 0;
1062                                                 if(matchpattern(filename, pattern, true))
1063                                                 {
1064                                                         strlcpy(foundurl, url, sizeof(foundurl));
1065                                                         Z_Free(buf);
1066                                                         return foundurl;
1067                                                 }
1068                                         }
1069                                         pattern = NULL;
1070                                         patternend = NULL;
1071                                         url = NULL;
1072                                         urlend = NULL;
1073                                         break;
1074                                 case ' ':
1075                                 case '\t':
1076                                         if(pattern && !patternend)
1077                                                 patternend = p;
1078                                         else if(url && !urlend)
1079                                                 urlend = p;
1080                                         break;
1081                                 default:
1082                                         if(!pattern)
1083                                                 pattern = p;
1084                                         else if(pattern && patternend && !url)
1085                                                 url = p;
1086                                         break;
1087                         }
1088                         ++p;
1089                 }
1090         }
1091         if(buf)
1092                 Z_Free(buf);
1093         return sv_curl_defaulturl.string;
1094 }
1095
1096 typedef struct requirement_s
1097 {
1098         struct requirement_s *next;
1099         char filename[MAX_QPATH];
1100 }
1101 requirement;
1102 static requirement *requirements = NULL;
1103
1104
1105 /*
1106 ====================
1107 Curl_ClearRequirements
1108
1109 Clears the list of required files for playing on the current map.
1110 This should be called at every map change.
1111 ====================
1112 */
1113 void Curl_ClearRequirements()
1114 {
1115         while(requirements)
1116         {
1117                 requirement *req = requirements;
1118                 requirements = requirements->next;
1119                 Z_Free(req);
1120         }
1121 }
1122
1123 /*
1124 ====================
1125 Curl_RequireFile
1126
1127 Adds the given file to the list of requirements.
1128 ====================
1129 */
1130 void Curl_RequireFile(const char *filename)
1131 {
1132         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1133         req->next = requirements;
1134         strlcpy(req->filename, filename, sizeof(req->filename));
1135         requirements = req;
1136 }
1137
1138 /*
1139 ====================
1140 Curl_SendRequirements
1141
1142 Makes the current host_clients download all files he needs.
1143 This is done by sending him the following console commands:
1144
1145         curl --start_autodownload
1146         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1147         curl --finish_autodownload
1148 ====================
1149 */
1150 void Curl_SendRequirements()
1151 {
1152         // for each requirement, find the pack name
1153         char sendbuffer[4096] = "";
1154         requirement *req;
1155
1156         strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1157
1158         for(req = requirements; req; req = req->next)
1159         {
1160                 const char *p;
1161                 const char *thispack = FS_WhichPack(req->filename);
1162                 const char *packurl;
1163
1164                 if(!thispack)
1165                         continue;
1166
1167                 p = strrchr(thispack, '/');
1168                 if(p)
1169                         thispack = p + 1;
1170
1171                 packurl = Curl_FindPackURL(thispack);
1172
1173                 if(packurl && *packurl && strcmp(packurl, "-"))
1174                 {
1175                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1176                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1177                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1178                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1179                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1180                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1181                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1182                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1183                 }
1184         }
1185
1186         strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1187
1188         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1189                 Host_ClientCommands("%s", sendbuffer);
1190         else
1191                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1192 }