libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {1, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {1, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {1, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t cl_curl_enabled = {1, "cl_curl_enabled","0", "whether client's download support is enabled"};
   9
  10 /*
  11 =================================================================
  12
  13   Minimal set of definitions from libcurl
  14
  15   WARNING: for a matter of simplicity, several pointer types are
  16   casted to "void*", and most enumerated values are not included
  17
  18 =================================================================
  19 */
  20
  21 typedef struct CURL_s CURL;
  22 typedef struct CURLM_s CURLM;
  23 typedef enum
  24 {
  25         CURLE_OK = 0
  26 }
  27 CURLcode;
  28 typedef enum
  29 {
  30         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  31         CURLM_OK = 0
  32 }
  33 CURLMcode;
  34 #define CURL_GLOBAL_NOTHING 0
  35 #define CURL_GLOBAL_SSL 1
  36 #define CURL_GLOBAL_WIN32 2
  37 #define CURLOPTTYPE_LONG          0
  38 #define CURLOPTTYPE_OBJECTPOINT   10000
  39 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  40 #define CURLOPTTYPE_OFF_T         30000
  41 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  42 typedef enum
  43 {
  44         CINIT(WRITEDATA, OBJECTPOINT, 1),
  45         CINIT(URL,  OBJECTPOINT, 2),
  46         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  47         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  48         CINIT(REFERER, OBJECTPOINT, 16),
  49         CINIT(USERAGENT, OBJECTPOINT, 18),
  50         CINIT(RESUME_FROM, LONG, 21),
  51         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  52         CINIT(PRIVATE, OBJECTPOINT, 103),
  53 }
  54 CURLoption;
  55 typedef enum
  56 {
  57         CURLINFO_TEXT = 0,
  58         CURLINFO_HEADER_IN,    /* 1 */
  59         CURLINFO_HEADER_OUT,   /* 2 */
  60         CURLINFO_DATA_IN,      /* 3 */
  61         CURLINFO_DATA_OUT,     /* 4 */
  62         CURLINFO_SSL_DATA_IN,  /* 5 */
  63         CURLINFO_SSL_DATA_OUT, /* 6 */
  64         CURLINFO_END
  65 }
  66 curl_infotype;
  67 #define CURLINFO_STRING   0x100000
  68 #define CURLINFO_LONG     0x200000
  69 #define CURLINFO_DOUBLE   0x300000
  70 #define CURLINFO_SLIST    0x400000
  71 #define CURLINFO_MASK     0x0fffff
  72 #define CURLINFO_TYPEMASK 0xf00000
  73 typedef enum
  74 {
  75         CURLINFO_NONE, /* first, never use this */
  76         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  77         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  78         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  79         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  80         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  81         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  82         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  83         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  84         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  85         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  86         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  87         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  88         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  89         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  90         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  91         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  92         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  93         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  94         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  95         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  96         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
  97         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
  98         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
  99         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 100         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 101         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 102         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 103 }
 104 CURLINFO;
 105
 106 typedef enum
 107 {
 108         CURLMSG_NONE, /* first, not used */
 109         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 110                                          the CURLcode of the transfer */
 111         CURLMSG_LAST
 112 }
 113 CURLMSG;
 114 typedef struct
 115 {
 116         CURLMSG msg;       /* what this message means */
 117         CURL *easy_handle; /* the handle it concerns */
 118         union
 119         {
 120                 void *whatever;    /* message-specific data */
 121                 CURLcode result;   /* return code for transfer */
 122         }
 123         data;
 124 }
 125 CURLMsg;
 126
 127 static void (*qcurl_global_init) (long flags);
 128 static void (*qcurl_global_cleanup) ();
 129
 130 static CURL * (*qcurl_easy_init) ();
 131 static void (*qcurl_easy_cleanup) (CURL *handle);
 132 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 133 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 134 static const char * (*qcurl_easy_strerror) (CURLcode);
 135
 136 static CURLM * (*qcurl_multi_init) ();
 137 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 138 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 139 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 140 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 141 static void (*qcurl_multi_cleanup) (CURLM *);
 142 static const char * (*qcurl_multi_strerror) (CURLcode);
 143
 144 static dllfunction_t curlfuncs[] =
 145 {
 146         {"curl_global_init",            (void **) &qcurl_global_init},
 147         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 148         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 149         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 150         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 151         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 152         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 153         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 154         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 155         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 156         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 157         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 158         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 159         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 160         {NULL, NULL}
 161 };
 162
 163 // Handle for CURL DLL
 164 static dllhandle_t curl_dll = NULL;
 165 // will be checked at many places to find out if qcurl calls are allowed
 166
 167 typedef struct downloadinfo_s
 168 {
 169         char filename[MAX_QPATH];
 170         char url[256];
 171         char referer[256];
 172         qfile_t *stream;
 173         fs_offset_t startpos;
 174         CURL *curle;
 175         qboolean started;
 176         qboolean ispak;
 177         unsigned long bytes_received;
 178         struct downloadinfo_s *next, *prev;
 179         qboolean forthismap;
 180 }
 181 downloadinfo;
 182 static downloadinfo *downloads = NULL;
 183 static int numdownloads = 0;
 184
 185 /*
 186 ====================
 187 CURL_CloseLibrary
 188
 189 Load the cURL DLL
 190 ====================
 191 */
 192 static qboolean CURL_OpenLibrary (void)
 193 {
 194         const char* dllnames [] =
 195         {
 196 #if defined(WIN64)
 197                 "libcurl64.dll",
 198 #elif defined(WIN32)
 199                 "libcurl-3.dll",
 200 #elif defined(MACOSX)
 201                 "libcurl.3.dylib", // Mac OS X Tiger
 202                 "libcurl.2.dylib", // Mac OS X Panther
 203 #else
 204                 "libcurl.so.3",
 205 #endif
 206                 NULL
 207         };
 208
 209         // Already loaded?
 210         if (curl_dll)
 211                 return true;
 212
 213         // Load the DLL
 214         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 215         {
 216                 Con_Printf ("cURL support disabled\n");
 217                 return false;
 218         }
 219
 220         Con_Printf ("cURL support enabled\n");
 221         return true;
 222 }
 223
 224
 225 /*
 226 ====================
 227 CURL_CloseLibrary
 228
 229 Unload the cURL DLL
 230 ====================
 231 */
 232 static void CURL_CloseLibrary (void)
 233 {
 234         Sys_UnloadLibrary (&curl_dll);
 235 }
 236
 237
 238 static CURLM *curlm = NULL;
 239 static unsigned long bytes_received = 0; // used for bandwidth throttling
 240 static double curltime = 0;
 241
 242 /*
 243 ====================
 244 CURL_fwrite
 245
 246 fwrite-compatible function that writes the data to a file. libcurl can call
 247 this.
 248 ====================
 249 */
 250 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 251 {
 252         fs_offset_t ret;
 253         size_t bytes = size * nmemb;
 254         downloadinfo *di = (downloadinfo *) vdi;
 255
 256         bytes_received += bytes;
 257         di->bytes_received += bytes;
 258
 259         ret = FS_Write(di->stream, data, bytes);
 260
 261         return ret; // why not ret / nmemb?
 262 }
 263
 264 typedef enum
 265 {
 266         CURL_DOWNLOAD_SUCCESS = 0,
 267         CURL_DOWNLOAD_FAILED,
 268         CURL_DOWNLOAD_ABORTED,
 269         CURL_DOWNLOAD_SERVERERROR
 270 }
 271 CurlStatus;
 272
 273 /*
 274 ====================
 275 Curl_Clear_forthismap
 276
 277 Clears the "will disconnect on failure" flags.
 278 ====================
 279 */
 280 void Curl_Clear_forthismap()
 281 {
 282         downloadinfo *di;
 283         for(di = downloads; di; di = di->next)
 284                 di->forthismap = false;
 285 }
 286
 287 static qboolean Curl_Have_forthismap()
 288 {
 289         downloadinfo *di;
 290         for(di = downloads; di; di = di->next)
 291                 if(di->forthismap)
 292                         return true;
 293         return false;
 294 }
 295
 296 /*
 297 ====================
 298 Curl_EndDownload
 299
 300 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 301 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 302 code from libcurl, or 0, if another error has occurred.
 303 ====================
 304 */
 305 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 306 {
 307         qboolean ok = false;
 308         if(!curl_dll)
 309                 return;
 310         switch(status)
 311         {
 312                 case CURL_DOWNLOAD_SUCCESS:
 313                         Con_Printf("Download of %s: OK\n", di->filename);
 314                         ok = true;
 315                         break;
 316                 case CURL_DOWNLOAD_FAILED:
 317                         Con_Printf("Download of %s: FAILED\n", di->filename);
 318                         if(error)
 319                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 320                         break;
 321                 case CURL_DOWNLOAD_ABORTED:
 322                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 323                         break;
 324                 case CURL_DOWNLOAD_SERVERERROR:
 325                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 326
 327                         // reopen to enforce it to have zero bytes again
 328                         FS_Close(di->stream);
 329                         di->stream = FS_Open(di->filename, "w", false, false);
 330
 331                         break;
 332         }
 333
 334         if(di->curle)
 335         {
 336                 qcurl_multi_remove_handle(curlm, di->curle);
 337                 qcurl_easy_cleanup(di->curle);
 338         }
 339
 340         if(ok && !di->bytes_received)
 341         {
 342                 Con_Printf("ERROR: empty file\n");
 343                 ok = false;
 344         }
 345
 346         if(di->stream)
 347                 FS_Close(di->stream);
 348
 349         if(ok && di->ispak)
 350         {
 351                 ok = FS_AddPack(di->filename, NULL, true);
 352                 if(ok && di->forthismap)
 353                         Mod_Reload();
 354         }
 355
 356         if(!ok && di->forthismap)
 357         {
 358                 // BAD. Something went totally wrong.
 359                 // The best we can do is clean up the forthismap flags...
 360                 Curl_Clear_forthismap();
 361                 // and disconnect.
 362                 CL_Disconnect_f();
 363         }
 364
 365         if(di->prev)
 366                 di->prev->next = di->next;
 367         else
 368                 downloads = di->next;
 369         if(di->next)
 370                 di->next->prev = di->prev;
 371         Z_Free(di);
 372
 373         --numdownloads;
 374 }
 375
 376 /*
 377 ====================
 378 CheckPendingDownloads
 379
 380 checks if there are free download slots to start new downloads in.
 381 To not start too many downloads at once, only one download is added at a time,
 382 up to a maximum number of cl_curl_maxdownloads are running.
 383 ====================
 384 */
 385 static void CheckPendingDownloads()
 386 {
 387         if(!curl_dll)
 388                 return;
 389         if(numdownloads < cl_curl_maxdownloads.integer)
 390         {
 391                 downloadinfo *di;
 392                 for(di = downloads; di; di = di->next)
 393                 {
 394                         if(!di->started)
 395                         {
 396                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 397
 398                                 di->stream = FS_Open(di->filename, "ab", false, false);
 399                                 if(!di->stream)
 400                                 {
 401                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 402                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 403                                         return;
 404                                 }
 405
 406                                 FS_Seek(di->stream, 0, SEEK_END);
 407                                 di->startpos = FS_Tell(di->stream);
 408                                 if(di->startpos > 0)
 409                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 410                                 Con_Print("...\n");
 411
 412                                 di->curle = qcurl_easy_init();
 413                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 414                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 415                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 416                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 417                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 418                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 419                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 420                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 421                                 qcurl_multi_add_handle(curlm, di->curle);
 422                                 di->started = true;
 423                                 ++numdownloads;
 424                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 425                                         break;
 426                         }
 427                 }
 428         }
 429 }
 430
 431 /*
 432 ====================
 433 Curl_Init
 434
 435 this function MUST be called before using anything else in this file.
 436 On Win32, this must be called AFTER WSAStartup has been done!
 437 ====================
 438 */
 439 void Curl_Init()
 440 {
 441         CURL_OpenLibrary();
 442         if(!curl_dll)
 443                 return;
 444         qcurl_global_init(CURL_GLOBAL_NOTHING);
 445         curlm = qcurl_multi_init();
 446 }
 447
 448 /*
 449 ====================
 450 Curl_Shutdown
 451
 452 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 453 ====================
 454 */
 455 void Curl_ClearRequirements();
 456 void Curl_Shutdown()
 457 {
 458         if(!curl_dll)
 459                 return;
 460         Curl_ClearRequirements();
 461         Curl_CancelAll();
 462         CURL_CloseLibrary();
 463         curl_dll = NULL;
 464 }
 465
 466 /*
 467 ====================
 468 Curl_Find
 469
 470 Finds the internal information block for a download given by file name.
 471 ====================
 472 */
 473 static downloadinfo *Curl_Find(const char *filename)
 474 {
 475         downloadinfo *di;
 476         if(!curl_dll)
 477                 return NULL;
 478         for(di = downloads; di; di = di->next)
 479                 if(!strcasecmp(di->filename, filename))
 480                         return di;
 481         return NULL;
 482 }
 483
 484 /*
 485 ====================
 486 Curl_Begin
 487
 488 Starts a download of a given URL to the file name portion of this URL (or name
 489 if given) in the "dlcache/" folder.
 490 ====================
 491 */
 492 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 493 {
 494         if(!curl_dll)
 495                 return;
 496         else
 497         {
 498                 char fn[MAX_QPATH];
 499                 const char *p, *q;
 500                 size_t length;
 501                 downloadinfo *di;
 502
 503                 // Note: This extraction of the file name portion is NOT entirely correct.
 504                 //
 505                 // It does the following:
 506                 //
 507                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 508                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 509                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 510                 //
 511                 // However, I'd like to keep this "buggy" behavior so that PHP script
 512                 // authors can write download scripts without having to enable
 513                 // AcceptPathInfo on Apache. They just have to ensure that their script
 514                 // can be called with such a "fake" path name like
 515                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 516                 //
 517                 // By the way, such PHP scripts should either send the file or a
 518                 // "Location:" redirect; PHP code example:
 519                 //
 520                 //   header("Location: http://www.example.com/");
 521                 //
 522                 // By the way, this will set User-Agent to something like
 523                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 524                 // dp://serverhost:serverport/ so you can filter on this; an example
 525                 // httpd log file line might be:
 526                 //
 527                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 528
 529                 if(!name)
 530                         name = URL;
 531                 p = strrchr(name, '/');
 532                 p = p ? (p+1) : name;
 533                 q = strchr(p, '?');
 534                 length = q ? (size_t)(q - p) : strlen(p);
 535                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 536
 537
 538                 // already downloading the file?
 539                 {
 540                         downloadinfo *di = Curl_Find(fn);
 541                         if(di)
 542                         {
 543                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 544
 545                                 // however, if it was not for this map yet...
 546                                 if(forthismap)
 547                                         di->forthismap = true;
 548
 549                                 return;
 550                         }
 551                 }
 552
 553                 if(ispak && FS_FileExists(fn))
 554                 {
 555                         qboolean already_loaded;
 556                         if(FS_AddPack(fn, &already_loaded, true))
 557                         {
 558                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 559                                 if(already_loaded)
 560                                         Con_DPrintf("(pak was already loaded)\n");
 561                                 else
 562                                         if(forthismap)
 563                                                 Mod_Reload();
 564                                 return;
 565                         }
 566                         else
 567                         {
 568                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 569                                 if(f)
 570                                 {
 571                                         char buf[4] = {0};
 572                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 573
 574                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 575                                         {
 576                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 577                                                 FS_Close(f);
 578                                                 f = FS_Open(fn, "w", false, false);
 579                                                 if(f)
 580                                                         FS_Close(f);
 581                                         }
 582                                         else
 583                                         {
 584                                                 // OK
 585                                                 FS_Close(f);
 586                                         }
 587                                 }
 588                         }
 589                 }
 590
 591                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 592                 strlcpy(di->filename, fn, sizeof(di->filename));
 593                 strlcpy(di->url, URL, sizeof(di->url));
 594                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 595                 di->forthismap = forthismap;
 596                 di->stream = NULL;
 597                 di->startpos = 0;
 598                 di->curle = NULL;
 599                 di->started = false;
 600                 di->ispak = ispak;
 601                 di->bytes_received = 0;
 602                 di->next = downloads;
 603                 di->prev = NULL;
 604                 if(di->next)
 605                         di->next->prev = di;
 606                 downloads = di;
 607         }
 608 }
 609
 610
 611 /*
 612 ====================
 613 Curl_Run
 614
 615 call this regularily as this will always download as much as possible without
 616 blocking.
 617 ====================
 618 */
 619 void Curl_Run()
 620 {
 621         if(!cl_curl_enabled.integer)
 622                 return;
 623
 624         if(!curl_dll)
 625                 return;
 626
 627         if(!downloads)
 628                 return;
 629
 630         if(realtime < curltime) // throttle
 631                 return;
 632
 633         {
 634                 int remaining;
 635                 CURLMcode mc;
 636
 637                 do
 638                 {
 639                         mc = qcurl_multi_perform(curlm, &remaining);
 640                 }
 641                 while(mc == CURLM_CALL_MULTI_PERFORM);
 642
 643                 for(;;)
 644                 {
 645                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 646                         if(!msg)
 647                                 break;
 648                         if(msg->msg == CURLMSG_DONE)
 649                         {
 650                                 downloadinfo *di;
 651                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 652                                 CURLcode result;
 653
 654                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 655                                 result = msg->data.result;
 656                                 if(result)
 657                                 {
 658                                         failed = CURL_DOWNLOAD_FAILED;
 659                                 }
 660                                 else
 661                                 {
 662                                         long code;
 663                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 664                                         switch(code / 100)
 665                                         {
 666                                                 case 4: // e.g. 404?
 667                                                 case 5: // e.g. 500?
 668                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 669                                                         result = code;
 670                                                         break;
 671                                         }
 672                                 }
 673
 674                                 Curl_EndDownload(di, failed, result);
 675                         }
 676                 }
 677         }
 678
 679         CheckPendingDownloads();
 680
 681         // when will we curl the next time?
 682         // we will wait a bit to ensure our download rate is kept.
 683         // we now know that realtime >= curltime... so set up a new curltime
 684         if(cl_curl_maxspeed.value > 0)
 685         {
 686                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 687                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 688                 bytes_received -= bytes;
 689         }
 690         else
 691                 curltime = realtime;
 692 }
 693
 694 /*
 695 ====================
 696 Curl_CancelAll
 697
 698 Stops ALL downloads.
 699 ====================
 700 */
 701 void Curl_CancelAll()
 702 {
 703         if(!curl_dll)
 704                 return;
 705
 706         while(downloads)
 707         {
 708                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 709                 // INVARIANT: downloads will point to the next download after that!
 710         }
 711 }
 712
 713 /*
 714 ====================
 715 Curl_Running
 716
 717 returns true iff there is a download running.
 718 ====================
 719 */
 720 qboolean Curl_Running()
 721 {
 722         if(!curl_dll)
 723                 return false;
 724
 725         return downloads != NULL;
 726 }
 727
 728 /*
 729 ====================
 730 Curl_GetDownloadAmount
 731
 732 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 733 for the given download.
 734 ====================
 735 */
 736 static double Curl_GetDownloadAmount(downloadinfo *di)
 737 {
 738         if(!curl_dll)
 739                 return -2;
 740         if(di->curle)
 741         {
 742                 double length;
 743                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 744                 if(length > 0)
 745                         return di->bytes_received / length;
 746                 else
 747                         return 0;
 748         }
 749         else
 750                 return -1;
 751 }
 752
 753 /*
 754 ====================
 755 Curl_GetDownloadSpeed
 756
 757 returns the speed of the given download in bytes per second
 758 ====================
 759 */
 760 static double Curl_GetDownloadSpeed(downloadinfo *di)
 761 {
 762         if(!curl_dll)
 763                 return -2;
 764         if(di->curle)
 765         {
 766                 double speed;
 767                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 768                 return speed;
 769         }
 770         else
 771                 return -1;
 772 }
 773
 774 /*
 775 ====================
 776 Curl_Info_f
 777
 778 prints the download list
 779 ====================
 780 */
 781 // TODO rewrite using Curl_GetDownloadInfo?
 782 static void Curl_Info_f()
 783 {
 784         downloadinfo *di;
 785         if(!curl_dll)
 786                 return;
 787         if(Curl_Running())
 788         {
 789                 Con_Print("Currently running downloads:\n");
 790                 for(di = downloads; di; di = di->next)
 791                 {
 792                         double speed, percent;
 793                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 794                         percent = 100.0 * Curl_GetDownloadAmount(di);
 795                         speed = Curl_GetDownloadSpeed(di);
 796                         if(percent >= 0)
 797                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 798                         else
 799                                 Con_Print("(queued)\n");
 800                 }
 801         }
 802         else
 803         {
 804                 Con_Print("No downloads running.\n");
 805         }
 806 }
 807
 808 /*
 809 ====================
 810 Curl_Curl_f
 811
 812 implements the "curl" console command
 813
 814 curl --info
 815 curl --cancel
 816 curl --cancel filename
 817 curl url
 818
 819 For internal use:
 820
 821 curl [--pak] [--forthismap] [--for filename filename...] url
 822         --pak: after downloading, load the package into the virtual file system
 823         --for filename...: only download of at least one of the named files is missing
 824         --forthismap: disconnect on failure
 825 ====================
 826 */
 827 void Curl_Curl_f(void)
 828 {
 829         int i;
 830         int end;
 831         qboolean pak = false;
 832         qboolean forthismap = false;
 833         const char *url;
 834         const char *name = 0;
 835
 836         if(!curl_dll)
 837         {
 838                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 839                 return;
 840         }
 841
 842         if(!cl_curl_enabled.integer)
 843         {
 844                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 845                 return;
 846         }
 847
 848         for(i = 0; i != Cmd_Argc(); ++i)
 849                 Con_DPrintf("%s ", Cmd_Argv(i));
 850         Con_DPrint("\n");
 851
 852         if(Cmd_Argc() < 2)
 853         {
 854                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 855                 return;
 856         }
 857
 858         url = Cmd_Argv(Cmd_Argc() - 1);
 859         end = Cmd_Argc();
 860
 861         for(i = 1; i != end; ++i)
 862         {
 863                 const char *a = Cmd_Argv(i);
 864                 if(!strcmp(a, "--info"))
 865                 {
 866                         Curl_Info_f();
 867                         return;
 868                 }
 869                 else if(!strcmp(a, "--cancel"))
 870                 {
 871                         if(i == end - 1) // last argument
 872                                 Curl_CancelAll();
 873                         else
 874                         {
 875                                 downloadinfo *di = Curl_Find(url);
 876                                 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 877                         }
 878                         return;
 879                 }
 880                 else if(!strcmp(a, "--pak"))
 881                 {
 882                         pak = true;
 883                 }
 884                 else if(!strcmp(a, "--for"))
 885                 {
 886                         for(i = i + 1; i != end - 1; ++i)
 887                         {
 888                                 if(!FS_FileExists(Cmd_Argv(i)))
 889                                         goto needthefile; // why can't I have a "double break"?
 890                         }
 891                         // if we get here, we have all the files...
 892                         return;
 893                 }
 894                 else if(!strcmp(a, "--forthismap"))
 895                 {
 896                         forthismap = true;
 897                 }
 898                 else if(!strcmp(a, "--as"))
 899                 {
 900                         if(i < end - 1)
 901                         {
 902                                 ++i;
 903                                 name = Cmd_Argv(i);
 904                         }
 905                 }
 906                 else if(!strcmp(a, "--clear_autodownload"))
 907                 {
 908                         // mark all running downloads as "not for this map", so if they
 909                         // fail, it does not matter
 910                         Curl_Clear_forthismap();
 911                         return;
 912                 }
 913                 else if(!strcmp(a, "--finish_autodownload"))
 914                 {
 915                         // nothing
 916                         return;
 917                 }
 918                 else if(*a == '-')
 919                 {
 920                         Con_Printf("invalid option %s\n", a);
 921                         return;
 922                 }
 923         }
 924
 925 needthefile:
 926         Curl_Begin(url, name, pak, forthismap);
 927 }
 928
 929 /*
 930 ====================
 931 Curl_Init_Commands
 932
 933 loads the commands and cvars this library uses
 934 ====================
 935 */
 936 void Curl_Init_Commands(void)
 937 {
 938         Cvar_RegisterVariable (&cl_curl_enabled);
 939         Cvar_RegisterVariable (&cl_curl_maxdownloads);
 940         Cvar_RegisterVariable (&cl_curl_maxspeed);
 941         Cvar_RegisterVariable (&sv_curl_defaulturl);
 942         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
 943 }
 944
 945 /*
 946 ====================
 947 Curl_GetDownloadInfo
 948
 949 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
 950 The number of elements in the array is returned in int *nDownloads.
 951 const char **additional_info may be set to a string of additional user
 952 information, or to NULL if no such display shall occur. The returned
 953 array must be freed later using Z_Free.
 954 ====================
 955 */
 956 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
 957 {
 958         int n, i;
 959         downloadinfo *di;
 960         Curl_downloadinfo_t *downinfo;
 961         static char addinfo[128];
 962
 963         if(!curl_dll)
 964         {
 965                 *nDownloads = 0;
 966                 if(additional_info)
 967                         *additional_info = NULL;
 968                 return NULL;
 969         }
 970
 971         n = 0;
 972         for(di = downloads; di; di = di->next)
 973                 ++n;
 974
 975         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
 976         i = 0;
 977         for(di = downloads; di; di = di->next)
 978         {
 979                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
 980                 if(di->curle)
 981                 {
 982                         downinfo[i].progress = Curl_GetDownloadAmount(di);
 983                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
 984                         downinfo[i].queued = false;
 985                 }
 986                 else
 987                 {
 988                         downinfo[i].queued = true;
 989                 }
 990                 ++i;
 991         }
 992
 993         if(additional_info)
 994         {
 995                 // TODO put something better here?
 996                 // maybe... check if the file is actually needed for the current map?
 997                 if(Curl_Have_forthismap())
 998                 {
 999                         dpsnprintf(addinfo, sizeof(addinfo), "please wait for the download to complete");
1000                         *additional_info = addinfo;
1001                 }
1002                 else
1003                         *additional_info = NULL;
1004         }
1005
1006         *nDownloads = n;
1007         return downinfo;
1008 }
1009
1010
1011 /*
1012 ====================
1013 Curl_FindPackURL
1014
1015 finds the URL where to find a given package.
1016
1017 For this, it reads a file "curl_urls.txt" of the following format:
1018
1019         data*.pk3       -
1020         revdm*.pk3      http://revdm/downloads/are/here/
1021         *                       http://any/other/stuff/is/here/
1022
1023 The URLs should end in /. If not, downloads will still work, but the cached files
1024 can't be just put into the data directory with the same download configuration
1025 (you might want to do this if you want to tag downloaded files from your
1026 server, but you should not). "-" means "don't download".
1027
1028 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1029 location instead.
1030
1031 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1032 this file for obvious reasons.
1033 ====================
1034 */
1035 static const char *Curl_FindPackURL(const char *filename)
1036 {
1037         static char foundurl[256];
1038         fs_offset_t filesize;
1039         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1040         if(buf && filesize)
1041         {
1042                 // read lines of format "pattern url"
1043                 char *p = buf;
1044                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1045                 qboolean eof = false;
1046
1047                 pattern = p;
1048                 while(!eof)
1049                 {
1050                         switch(*p)
1051                         {
1052                                 case 0:
1053                                         eof = true;
1054                                         // fallthrough
1055                                 case '\n':
1056                                 case '\r':
1057                                         if(pattern && url && patternend)
1058                                         {
1059                                                 if(!urlend)
1060                                                         urlend = p;
1061                                                 *patternend = 0;
1062                                                 *urlend = 0;
1063                                                 if(matchpattern(filename, pattern, true))
1064                                                 {
1065                                                         strlcpy(foundurl, url, sizeof(foundurl));
1066                                                         Z_Free(buf);
1067                                                         return foundurl;
1068                                                 }
1069                                         }
1070                                         pattern = NULL;
1071                                         patternend = NULL;
1072                                         url = NULL;
1073                                         urlend = NULL;
1074                                         break;
1075                                 case ' ':
1076                                 case '\t':
1077                                         if(pattern && !patternend)
1078                                                 patternend = p;
1079                                         else if(url && !urlend)
1080                                                 urlend = p;
1081                                         break;
1082                                 default:
1083                                         if(!pattern)
1084                                                 pattern = p;
1085                                         else if(pattern && patternend && !url)
1086                                                 url = p;
1087                                         break;
1088                         }
1089                         ++p;
1090                 }
1091         }
1092         if(buf)
1093                 Z_Free(buf);
1094         return sv_curl_defaulturl.string;
1095 }
1096
1097 typedef struct requirement_s
1098 {
1099         struct requirement_s *next;
1100         char filename[MAX_QPATH];
1101 }
1102 requirement;
1103 static requirement *requirements = NULL;
1104
1105
1106 /*
1107 ====================
1108 Curl_ClearRequirements
1109
1110 Clears the list of required files for playing on the current map.
1111 This should be called at every map change.
1112 ====================
1113 */
1114 void Curl_ClearRequirements()
1115 {
1116         while(requirements)
1117         {
1118                 requirement *req = requirements;
1119                 requirements = requirements->next;
1120                 Z_Free(req);
1121         }
1122 }
1123
1124 /*
1125 ====================
1126 Curl_RequireFile
1127
1128 Adds the given file to the list of requirements.
1129 ====================
1130 */
1131 void Curl_RequireFile(const char *filename)
1132 {
1133         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1134         req->next = requirements;
1135         strlcpy(req->filename, filename, sizeof(req->filename));
1136         requirements = req;
1137 }
1138
1139 /*
1140 ====================
1141 Curl_SendRequirements
1142
1143 Makes the current host_clients download all files he needs.
1144 This is done by sending him the following console commands:
1145
1146         curl --start_autodownload
1147         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1148         curl --finish_autodownload
1149 ====================
1150 */
1151 void Curl_SendRequirements()
1152 {
1153         // for each requirement, find the pack name
1154         char sendbuffer[4096] = "";
1155         requirement *req;
1156         qboolean foundone = false;
1157
1158         for(req = requirements; req; req = req->next)
1159         {
1160                 const char *p;
1161                 const char *thispack = FS_WhichPack(req->filename);
1162                 const char *packurl;
1163
1164                 if(!thispack)
1165                         continue;
1166
1167                 p = strrchr(thispack, '/');
1168                 if(p)
1169                         thispack = p + 1;
1170
1171                 packurl = Curl_FindPackURL(thispack);
1172
1173                 if(packurl && *packurl && strcmp(packurl, "-"))
1174                 {
1175                         if(!foundone)
1176                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1177
1178                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1179                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1180                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1181                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1182                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1183                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1184                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1185                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1186
1187                         foundone = true;
1188                 }
1189         }
1190
1191         if(foundone)
1192                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1193
1194         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1195                 Host_ClientCommands("%s", sendbuffer);
1196         else
1197                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1198 }