libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
  10
  11 /*
  12 =================================================================
  13
  14   Minimal set of definitions from libcurl
  15
  16   WARNING: for a matter of simplicity, several pointer types are
  17   casted to "void*", and most enumerated values are not included
  18
  19 =================================================================
  20 */
  21
  22 typedef struct CURL_s CURL;
  23 typedef struct CURLM_s CURLM;
  24 typedef enum
  25 {
  26         CURLE_OK = 0
  27 }
  28 CURLcode;
  29 typedef enum
  30 {
  31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  32         CURLM_OK = 0
  33 }
  34 CURLMcode;
  35 #define CURL_GLOBAL_NOTHING 0
  36 #define CURL_GLOBAL_SSL 1
  37 #define CURL_GLOBAL_WIN32 2
  38 #define CURLOPTTYPE_LONG          0
  39 #define CURLOPTTYPE_OBJECTPOINT   10000
  40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  41 #define CURLOPTTYPE_OFF_T         30000
  42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  43 typedef enum
  44 {
  45         CINIT(WRITEDATA, OBJECTPOINT, 1),
  46         CINIT(URL,  OBJECTPOINT, 2),
  47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  49         CINIT(REFERER, OBJECTPOINT, 16),
  50         CINIT(USERAGENT, OBJECTPOINT, 18),
  51         CINIT(RESUME_FROM, LONG, 21),
  52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  53         CINIT(PRIVATE, OBJECTPOINT, 103),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56 }
  57 CURLoption;
  58 typedef enum
  59 {
  60         CURLINFO_TEXT = 0,
  61         CURLINFO_HEADER_IN,    /* 1 */
  62         CURLINFO_HEADER_OUT,   /* 2 */
  63         CURLINFO_DATA_IN,      /* 3 */
  64         CURLINFO_DATA_OUT,     /* 4 */
  65         CURLINFO_SSL_DATA_IN,  /* 5 */
  66         CURLINFO_SSL_DATA_OUT, /* 6 */
  67         CURLINFO_END
  68 }
  69 curl_infotype;
  70 #define CURLINFO_STRING   0x100000
  71 #define CURLINFO_LONG     0x200000
  72 #define CURLINFO_DOUBLE   0x300000
  73 #define CURLINFO_SLIST    0x400000
  74 #define CURLINFO_MASK     0x0fffff
  75 #define CURLINFO_TYPEMASK 0xf00000
  76 typedef enum
  77 {
  78         CURLINFO_NONE, /* first, never use this */
  79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 106 }
 107 CURLINFO;
 108
 109 typedef enum
 110 {
 111         CURLMSG_NONE, /* first, not used */
 112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 113                                          the CURLcode of the transfer */
 114         CURLMSG_LAST
 115 }
 116 CURLMSG;
 117 typedef struct
 118 {
 119         CURLMSG msg;       /* what this message means */
 120         CURL *easy_handle; /* the handle it concerns */
 121         union
 122         {
 123                 void *whatever;    /* message-specific data */
 124                 CURLcode result;   /* return code for transfer */
 125         }
 126         data;
 127 }
 128 CURLMsg;
 129
 130 static void (*qcurl_global_init) (long flags);
 131 static void (*qcurl_global_cleanup) ();
 132
 133 static CURL * (*qcurl_easy_init) ();
 134 static void (*qcurl_easy_cleanup) (CURL *handle);
 135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 137 static const char * (*qcurl_easy_strerror) (CURLcode);
 138
 139 static CURLM * (*qcurl_multi_init) ();
 140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 144 static void (*qcurl_multi_cleanup) (CURLM *);
 145 static const char * (*qcurl_multi_strerror) (CURLcode);
 146
 147 static dllfunction_t curlfuncs[] =
 148 {
 149         {"curl_global_init",            (void **) &qcurl_global_init},
 150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 163         {NULL, NULL}
 164 };
 165
 166 // Handle for CURL DLL
 167 static dllhandle_t curl_dll = NULL;
 168 // will be checked at many places to find out if qcurl calls are allowed
 169
 170 typedef struct downloadinfo_s
 171 {
 172         char filename[MAX_QPATH];
 173         char url[256];
 174         char referer[256];
 175         qfile_t *stream;
 176         fs_offset_t startpos;
 177         CURL *curle;
 178         qboolean started;
 179         qboolean ispak;
 180         unsigned long bytes_received;
 181         struct downloadinfo_s *next, *prev;
 182         qboolean forthismap;
 183 }
 184 downloadinfo;
 185 static downloadinfo *downloads = NULL;
 186 static int numdownloads = 0;
 187
 188 static qboolean noclear = FALSE;
 189
 190 static int numdownloads_fail = 0;
 191 static int numdownloads_success = 0;
 192 static int numdownloads_added = 0;
 193 static char command_when_done[256] = "";
 194 static char command_when_error[256] = "";
 195
 196 /*
 197 ====================
 198 Curl_CommandWhenDone
 199
 200 Sets the command which is to be executed when the last download completes AND
 201 all downloads since last server connect ended with a successful status.
 202 Setting the command to NULL clears it.
 203 ====================
 204 */
 205 void Curl_CommandWhenDone(const char *cmd)
 206 {
 207         if(!curl_dll)
 208                 return;
 209         if(cmd)
 210                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 211         else
 212                 *command_when_done = 0;
 213 }
 214
 215 /*
 216 FIXME
 217 Do not use yet. Not complete.
 218 Problem: what counts as an error?
 219 */
 220
 221 void Curl_CommandWhenError(const char *cmd)
 222 {
 223         if(!curl_dll)
 224                 return;
 225         if(cmd)
 226                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 227         else
 228                 *command_when_error = 0;
 229 }
 230
 231 /*
 232 ====================
 233 Curl_Clear_forthismap
 234
 235 Clears the "will disconnect on failure" flags.
 236 ====================
 237 */
 238 void Curl_Clear_forthismap()
 239 {
 240         downloadinfo *di;
 241         if(noclear)
 242                 return;
 243         for(di = downloads; di; di = di->next)
 244                 di->forthismap = false;
 245         Curl_CommandWhenError(NULL);
 246         Curl_CommandWhenDone(NULL);
 247         numdownloads_fail = 0;
 248         numdownloads_success = 0;
 249         numdownloads_added = 0;
 250 }
 251
 252 /*
 253 ====================
 254 Curl_Have_forthismap
 255
 256 Returns true if a download needed for the current game is running.
 257 ====================
 258 */
 259 qboolean Curl_Have_forthismap()
 260 {
 261         return numdownloads_added;
 262 }
 263
 264 void Curl_Register_predownload()
 265 {
 266         Curl_CommandWhenDone("cl_begindownloads");
 267         Curl_CommandWhenError("cl_begindownloads");
 268 }
 269
 270 /*
 271 ====================
 272 Curl_CheckCommandWhenDone
 273
 274 Checks if a "done command" is to be executed.
 275 All downloads finished, at least one success since connect, no single failure
 276 -> execute the command.
 277 */
 278 static void Curl_CheckCommandWhenDone()
 279 {
 280         if(!curl_dll)
 281                 return;
 282         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 283         {
 284                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 285                 Cbuf_AddText("\n");
 286                 Cbuf_AddText(command_when_done);
 287                 Cbuf_AddText("\n");
 288                 Curl_Clear_forthismap();
 289         }
 290         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 291         {
 292                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 293                 Cbuf_AddText("\n");
 294                 Cbuf_AddText(command_when_error);
 295                 Cbuf_AddText("\n");
 296                 Curl_Clear_forthismap();
 297         }
 298 }
 299
 300 /*
 301 ====================
 302 CURL_CloseLibrary
 303
 304 Load the cURL DLL
 305 ====================
 306 */
 307 static qboolean CURL_OpenLibrary (void)
 308 {
 309         const char* dllnames [] =
 310         {
 311 #if defined(WIN64)
 312                 "libcurl64.dll",
 313 #elif defined(WIN32)
 314                 "libcurl-4.dll",
 315                 "libcurl-3.dll",
 316 #elif defined(MACOSX)
 317                 "libcurl.4.dylib", // Mac OS X Notyetreleased
 318                 "libcurl.3.dylib", // Mac OS X Tiger
 319                 "libcurl.2.dylib", // Mac OS X Panther
 320 #else
 321                 "libcurl.so.4",
 322                 "libcurl.so.3",
 323                 "libcurl.so", // FreeBSD
 324 #endif
 325                 NULL
 326         };
 327
 328         // Already loaded?
 329         if (curl_dll)
 330                 return true;
 331
 332         // Load the DLL
 333         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 334         {
 335                 Con_Printf ("cURL support disabled\n");
 336                 return false;
 337         }
 338
 339         Con_Printf ("cURL support enabled\n");
 340         return true;
 341 }
 342
 343
 344 /*
 345 ====================
 346 CURL_CloseLibrary
 347
 348 Unload the cURL DLL
 349 ====================
 350 */
 351 static void CURL_CloseLibrary (void)
 352 {
 353         Sys_UnloadLibrary (&curl_dll);
 354 }
 355
 356
 357 static CURLM *curlm = NULL;
 358 static unsigned long bytes_received = 0; // used for bandwidth throttling
 359 static double curltime = 0;
 360
 361 /*
 362 ====================
 363 CURL_fwrite
 364
 365 fwrite-compatible function that writes the data to a file. libcurl can call
 366 this.
 367 ====================
 368 */
 369 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 370 {
 371         fs_offset_t ret;
 372         size_t bytes = size * nmemb;
 373         downloadinfo *di = (downloadinfo *) vdi;
 374
 375         bytes_received += bytes;
 376         di->bytes_received += bytes;
 377
 378         ret = FS_Write(di->stream, data, bytes);
 379
 380         return ret; // why not ret / nmemb?
 381 }
 382
 383 typedef enum
 384 {
 385         CURL_DOWNLOAD_SUCCESS = 0,
 386         CURL_DOWNLOAD_FAILED,
 387         CURL_DOWNLOAD_ABORTED,
 388         CURL_DOWNLOAD_SERVERERROR
 389 }
 390 CurlStatus;
 391
 392 /*
 393 ====================
 394 Curl_EndDownload
 395
 396 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 397 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 398 code from libcurl, or 0, if another error has occurred.
 399 ====================
 400 */
 401 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 402 {
 403         qboolean ok = false;
 404         if(!curl_dll)
 405                 return;
 406         switch(status)
 407         {
 408                 case CURL_DOWNLOAD_SUCCESS:
 409                         Con_Printf("Download of %s: OK\n", di->filename);
 410                         ok = true;
 411                         break;
 412                 case CURL_DOWNLOAD_FAILED:
 413                         Con_Printf("Download of %s: FAILED\n", di->filename);
 414                         if(error)
 415                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 416                         break;
 417                 case CURL_DOWNLOAD_ABORTED:
 418                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 419                         break;
 420                 case CURL_DOWNLOAD_SERVERERROR:
 421                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 422
 423                         // reopen to enforce it to have zero bytes again
 424                         FS_Close(di->stream);
 425                         di->stream = FS_Open(di->filename, "w", false, false);
 426
 427                         break;
 428         }
 429
 430         if(di->curle)
 431         {
 432                 qcurl_multi_remove_handle(curlm, di->curle);
 433                 qcurl_easy_cleanup(di->curle);
 434         }
 435
 436         if(ok && !di->bytes_received)
 437         {
 438                 Con_Printf("ERROR: empty file\n");
 439                 ok = false;
 440         }
 441
 442         if(di->stream)
 443                 FS_Close(di->stream);
 444
 445         if(ok && di->ispak)
 446                 ok = FS_AddPack(di->filename, NULL, true);
 447
 448         if(di->prev)
 449                 di->prev->next = di->next;
 450         else
 451                 downloads = di->next;
 452         if(di->next)
 453                 di->next->prev = di->prev;
 454
 455         --numdownloads;
 456         if(di->forthismap)
 457         {
 458                 if(ok)
 459                         ++numdownloads_success;
 460                 else
 461                         ++numdownloads_fail;
 462         }
 463         Z_Free(di);
 464 }
 465
 466 /*
 467 ====================
 468 CheckPendingDownloads
 469
 470 checks if there are free download slots to start new downloads in.
 471 To not start too many downloads at once, only one download is added at a time,
 472 up to a maximum number of cl_curl_maxdownloads are running.
 473 ====================
 474 */
 475 static void CheckPendingDownloads()
 476 {
 477         if(!curl_dll)
 478                 return;
 479         if(numdownloads < cl_curl_maxdownloads.integer)
 480         {
 481                 downloadinfo *di;
 482                 for(di = downloads; di; di = di->next)
 483                 {
 484                         if(!di->started)
 485                         {
 486                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 487
 488                                 di->stream = FS_Open(di->filename, "ab", false, false);
 489                                 if(!di->stream)
 490                                 {
 491                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 492                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 493                                         return;
 494                                 }
 495
 496                                 FS_Seek(di->stream, 0, SEEK_END);
 497                                 di->startpos = FS_Tell(di->stream);
 498                                 if(di->startpos > 0)
 499                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 500                                 Con_Print("...\n");
 501
 502                                 di->curle = qcurl_easy_init();
 503                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 504                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 505                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 506                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 507                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 508                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 509                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 510                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 511                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 512                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 513                                 qcurl_multi_add_handle(curlm, di->curle);
 514                                 di->started = true;
 515                                 ++numdownloads;
 516                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 517                                         break;
 518                         }
 519                 }
 520         }
 521 }
 522
 523 /*
 524 ====================
 525 Curl_Init
 526
 527 this function MUST be called before using anything else in this file.
 528 On Win32, this must be called AFTER WSAStartup has been done!
 529 ====================
 530 */
 531 void Curl_Init()
 532 {
 533         CURL_OpenLibrary();
 534         if(!curl_dll)
 535                 return;
 536         qcurl_global_init(CURL_GLOBAL_NOTHING);
 537         curlm = qcurl_multi_init();
 538 }
 539
 540 /*
 541 ====================
 542 Curl_Shutdown
 543
 544 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 545 ====================
 546 */
 547 void Curl_ClearRequirements();
 548 void Curl_Shutdown()
 549 {
 550         if(!curl_dll)
 551                 return;
 552         Curl_ClearRequirements();
 553         Curl_CancelAll();
 554         CURL_CloseLibrary();
 555         curl_dll = NULL;
 556 }
 557
 558 /*
 559 ====================
 560 Curl_Find
 561
 562 Finds the internal information block for a download given by file name.
 563 ====================
 564 */
 565 static downloadinfo *Curl_Find(const char *filename)
 566 {
 567         downloadinfo *di;
 568         if(!curl_dll)
 569                 return NULL;
 570         for(di = downloads; di; di = di->next)
 571                 if(!strcasecmp(di->filename, filename))
 572                         return di;
 573         return NULL;
 574 }
 575
 576 /*
 577 ====================
 578 Curl_Begin
 579
 580 Starts a download of a given URL to the file name portion of this URL (or name
 581 if given) in the "dlcache/" folder.
 582 ====================
 583 */
 584 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 585 {
 586         if(!curl_dll)
 587                 return;
 588         else
 589         {
 590                 char fn[MAX_QPATH];
 591                 const char *p, *q;
 592                 size_t length;
 593                 downloadinfo *di;
 594
 595                 // Note: This extraction of the file name portion is NOT entirely correct.
 596                 //
 597                 // It does the following:
 598                 //
 599                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 600                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 601                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 602                 //
 603                 // However, I'd like to keep this "buggy" behavior so that PHP script
 604                 // authors can write download scripts without having to enable
 605                 // AcceptPathInfo on Apache. They just have to ensure that their script
 606                 // can be called with such a "fake" path name like
 607                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 608                 //
 609                 // By the way, such PHP scripts should either send the file or a
 610                 // "Location:" redirect; PHP code example:
 611                 //
 612                 //   header("Location: http://www.example.com/");
 613                 //
 614                 // By the way, this will set User-Agent to something like
 615                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 616                 // dp://serverhost:serverport/ so you can filter on this; an example
 617                 // httpd log file line might be:
 618                 //
 619                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 620
 621                 if(!name)
 622                         name = URL;
 623                 p = strrchr(name, '/');
 624                 p = p ? (p+1) : name;
 625                 q = strchr(p, '?');
 626                 length = q ? (size_t)(q - p) : strlen(p);
 627                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 628
 629
 630                 // already downloading the file?
 631                 {
 632                         downloadinfo *di = Curl_Find(fn);
 633                         if(di)
 634                         {
 635                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 636
 637                                 // however, if it was not for this map yet...
 638                                 if(forthismap && !di->forthismap)
 639                                 {
 640                                         di->forthismap = true;
 641                                         // this "fakes" a download attempt so the client will wait for
 642                                         // the download to finish and then reconnect
 643                                         ++numdownloads_added;
 644                                 }
 645
 646                                 return;
 647                         }
 648                 }
 649
 650                 if(ispak && FS_FileExists(fn))
 651                 {
 652                         qboolean already_loaded;
 653                         if(FS_AddPack(fn, &already_loaded, true))
 654                         {
 655                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 656                                 if(already_loaded)
 657                                         Con_DPrintf("(pak was already loaded)\n");
 658                                 else
 659                                 {
 660                                         if(forthismap)
 661                                         {
 662                                                 ++numdownloads_added;
 663                                                 ++numdownloads_success;
 664                                         }
 665                                 }
 666                                 return;
 667                         }
 668                         else
 669                         {
 670                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 671                                 if(f)
 672                                 {
 673                                         char buf[4] = {0};
 674                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 675
 676                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 677                                         {
 678                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 679                                                 FS_Close(f);
 680                                                 f = FS_Open(fn, "w", false, false);
 681                                                 if(f)
 682                                                         FS_Close(f);
 683                                         }
 684                                         else
 685                                         {
 686                                                 // OK
 687                                                 FS_Close(f);
 688                                         }
 689                                 }
 690                         }
 691                 }
 692
 693                 if(forthismap)
 694                         ++numdownloads_added;
 695                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 696                 strlcpy(di->filename, fn, sizeof(di->filename));
 697                 strlcpy(di->url, URL, sizeof(di->url));
 698                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 699                 di->forthismap = forthismap;
 700                 di->stream = NULL;
 701                 di->startpos = 0;
 702                 di->curle = NULL;
 703                 di->started = false;
 704                 di->ispak = ispak;
 705                 di->bytes_received = 0;
 706                 di->next = downloads;
 707                 di->prev = NULL;
 708                 if(di->next)
 709                         di->next->prev = di;
 710                 downloads = di;
 711         }
 712 }
 713
 714
 715 /*
 716 ====================
 717 Curl_Run
 718
 719 call this regularily as this will always download as much as possible without
 720 blocking.
 721 ====================
 722 */
 723 void Curl_Run()
 724 {
 725         noclear = FALSE;
 726
 727         if(!cl_curl_enabled.integer)
 728                 return;
 729
 730         if(!curl_dll)
 731                 return;
 732
 733         Curl_CheckCommandWhenDone();
 734
 735         if(!downloads)
 736                 return;
 737
 738         if(realtime < curltime) // throttle
 739                 return;
 740
 741         {
 742                 int remaining;
 743                 CURLMcode mc;
 744
 745                 do
 746                 {
 747                         mc = qcurl_multi_perform(curlm, &remaining);
 748                 }
 749                 while(mc == CURLM_CALL_MULTI_PERFORM);
 750
 751                 for(;;)
 752                 {
 753                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 754                         if(!msg)
 755                                 break;
 756                         if(msg->msg == CURLMSG_DONE)
 757                         {
 758                                 downloadinfo *di;
 759                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 760                                 CURLcode result;
 761                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 762                                 result = msg->data.result;
 763                                 if(result)
 764                                 {
 765                                         failed = CURL_DOWNLOAD_FAILED;
 766                                 }
 767                                 else
 768                                 {
 769                                         long code;
 770                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 771                                         switch(code / 100)
 772                                         {
 773                                                 case 4: // e.g. 404?
 774                                                 case 5: // e.g. 500?
 775                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 776                                                         result = code;
 777                                                         break;
 778                                         }
 779                                 }
 780
 781                                 Curl_EndDownload(di, failed, result);
 782                         }
 783                 }
 784         }
 785
 786         CheckPendingDownloads();
 787
 788         // when will we curl the next time?
 789         // we will wait a bit to ensure our download rate is kept.
 790         // we now know that realtime >= curltime... so set up a new curltime
 791         if(cl_curl_maxspeed.value > 0)
 792         {
 793                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 794                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 795                 bytes_received -= bytes;
 796         }
 797         else
 798                 curltime = realtime;
 799 }
 800
 801 /*
 802 ====================
 803 Curl_CancelAll
 804
 805 Stops ALL downloads.
 806 ====================
 807 */
 808 void Curl_CancelAll()
 809 {
 810         if(!curl_dll)
 811                 return;
 812
 813         while(downloads)
 814         {
 815                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 816                 // INVARIANT: downloads will point to the next download after that!
 817         }
 818 }
 819
 820 /*
 821 ====================
 822 Curl_Running
 823
 824 returns true iff there is a download running.
 825 ====================
 826 */
 827 qboolean Curl_Running()
 828 {
 829         if(!curl_dll)
 830                 return false;
 831
 832         return downloads != NULL;
 833 }
 834
 835 /*
 836 ====================
 837 Curl_GetDownloadAmount
 838
 839 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 840 for the given download.
 841 ====================
 842 */
 843 static double Curl_GetDownloadAmount(downloadinfo *di)
 844 {
 845         if(!curl_dll)
 846                 return -2;
 847         if(di->curle)
 848         {
 849                 double length;
 850                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 851                 if(length > 0)
 852                         return di->bytes_received / length;
 853                 else
 854                         return 0;
 855         }
 856         else
 857                 return -1;
 858 }
 859
 860 /*
 861 ====================
 862 Curl_GetDownloadSpeed
 863
 864 returns the speed of the given download in bytes per second
 865 ====================
 866 */
 867 static double Curl_GetDownloadSpeed(downloadinfo *di)
 868 {
 869         if(!curl_dll)
 870                 return -2;
 871         if(di->curle)
 872         {
 873                 double speed;
 874                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 875                 return speed;
 876         }
 877         else
 878                 return -1;
 879 }
 880
 881 /*
 882 ====================
 883 Curl_Info_f
 884
 885 prints the download list
 886 ====================
 887 */
 888 // TODO rewrite using Curl_GetDownloadInfo?
 889 static void Curl_Info_f()
 890 {
 891         downloadinfo *di;
 892         if(!curl_dll)
 893                 return;
 894         if(Curl_Running())
 895         {
 896                 Con_Print("Currently running downloads:\n");
 897                 for(di = downloads; di; di = di->next)
 898                 {
 899                         double speed, percent;
 900                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 901                         percent = 100.0 * Curl_GetDownloadAmount(di);
 902                         speed = Curl_GetDownloadSpeed(di);
 903                         if(percent >= 0)
 904                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 905                         else
 906                                 Con_Print("(queued)\n");
 907                 }
 908         }
 909         else
 910         {
 911                 Con_Print("No downloads running.\n");
 912         }
 913 }
 914
 915 /*
 916 ====================
 917 Curl_Curl_f
 918
 919 implements the "curl" console command
 920
 921 curl --info
 922 curl --cancel
 923 curl --cancel filename
 924 curl url
 925
 926 For internal use:
 927
 928 curl [--pak] [--forthismap] [--for filename filename...] url
 929         --pak: after downloading, load the package into the virtual file system
 930         --for filename...: only download of at least one of the named files is missing
 931         --forthismap: don't reconnect on failure
 932
 933 curl --clear_autodownload
 934         clears the download success/failure counters
 935
 936 curl --finish_autodownload
 937         if at least one download has been started, disconnect and drop to the menu
 938         once the last download completes successfully, reconnect to the current server
 939 ====================
 940 */
 941 void Curl_Curl_f(void)
 942 {
 943         int i;
 944         int end;
 945         qboolean pak = false;
 946         qboolean forthismap = false;
 947         const char *url;
 948         const char *name = 0;
 949
 950         if(!curl_dll)
 951         {
 952                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 953                 return;
 954         }
 955
 956         if(!cl_curl_enabled.integer)
 957         {
 958                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 959                 return;
 960         }
 961
 962         for(i = 0; i != Cmd_Argc(); ++i)
 963                 Con_DPrintf("%s ", Cmd_Argv(i));
 964         Con_DPrint("\n");
 965
 966         if(Cmd_Argc() < 2)
 967         {
 968                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 969                 return;
 970         }
 971
 972         url = Cmd_Argv(Cmd_Argc() - 1);
 973         end = Cmd_Argc();
 974
 975         for(i = 1; i != end; ++i)
 976         {
 977                 const char *a = Cmd_Argv(i);
 978                 if(!strcmp(a, "--info"))
 979                 {
 980                         Curl_Info_f();
 981                         return;
 982                 }
 983                 else if(!strcmp(a, "--cancel"))
 984                 {
 985                         if(i == end - 1) // last argument
 986                                 Curl_CancelAll();
 987                         else
 988                         {
 989                                 downloadinfo *di = Curl_Find(url);
 990                                 if(di)
 991                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 992                                 else
 993                                         Con_Print("download not found\n");
 994                         }
 995                         return;
 996                 }
 997                 else if(!strcmp(a, "--pak"))
 998                 {
 999                         pak = true;
1000                 }
1001                 else if(!strcmp(a, "--for"))
1002                 {
1003                         for(i = i + 1; i != end - 1; ++i)
1004                         {
1005                                 if(!FS_FileExists(Cmd_Argv(i)))
1006                                         goto needthefile; // why can't I have a "double break"?
1007                         }
1008                         // if we get here, we have all the files...
1009                         return;
1010                 }
1011                 else if(!strcmp(a, "--forthismap"))
1012                 {
1013                         forthismap = true;
1014                 }
1015                 else if(!strcmp(a, "--as"))
1016                 {
1017                         if(i < end - 1)
1018                         {
1019                                 ++i;
1020                                 name = Cmd_Argv(i);
1021                         }
1022                 }
1023                 else if(!strcmp(a, "--clear_autodownload"))
1024                 {
1025                         // mark all running downloads as "not for this map", so if they
1026                         // fail, it does not matter
1027                         Curl_Clear_forthismap();
1028                         return;
1029                 }
1030                 else if(!strcmp(a, "--finish_autodownload"))
1031                 {
1032                         if(numdownloads_added)
1033                         {
1034                                 char donecommand[256];
1035                                 if(cls.netcon)
1036                                 {
1037                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1038                                         {
1039                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1040                                                 Curl_CommandWhenDone(donecommand);
1041                                                 noclear = TRUE;
1042                                                 CL_Disconnect();
1043                                                 noclear = FALSE;
1044                                                 Curl_CheckCommandWhenDone();
1045                                         }
1046                                         else
1047                                                 Curl_Register_predownload();
1048                                 }
1049                         }
1050                         return;
1051                 }
1052                 else if(*a == '-')
1053                 {
1054                         Con_Printf("invalid option %s\n", a);
1055                         return;
1056                 }
1057         }
1058
1059 needthefile:
1060         Curl_Begin(url, name, pak, forthismap);
1061 }
1062
1063 /*
1064 ====================
1065 Curl_Init_Commands
1066
1067 loads the commands and cvars this library uses
1068 ====================
1069 */
1070 void Curl_Init_Commands(void)
1071 {
1072         Cvar_RegisterVariable (&cl_curl_enabled);
1073         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1074         Cvar_RegisterVariable (&cl_curl_maxspeed);
1075         Cvar_RegisterVariable (&sv_curl_defaulturl);
1076         Cvar_RegisterVariable (&sv_curl_serverpackages);
1077         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1078 }
1079
1080 /*
1081 ====================
1082 Curl_GetDownloadInfo
1083
1084 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1085 The number of elements in the array is returned in int *nDownloads.
1086 const char **additional_info may be set to a string of additional user
1087 information, or to NULL if no such display shall occur. The returned
1088 array must be freed later using Z_Free.
1089 ====================
1090 */
1091 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1092 {
1093         int n, i;
1094         downloadinfo *di;
1095         Curl_downloadinfo_t *downinfo;
1096         static char addinfo[128];
1097
1098         if(!curl_dll)
1099         {
1100                 *nDownloads = 0;
1101                 if(additional_info)
1102                         *additional_info = NULL;
1103                 return NULL;
1104         }
1105
1106         n = 0;
1107         for(di = downloads; di; di = di->next)
1108                 ++n;
1109
1110         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1111         i = 0;
1112         for(di = downloads; di; di = di->next)
1113         {
1114                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1115                 if(di->curle)
1116                 {
1117                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1118                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1119                         downinfo[i].queued = false;
1120                 }
1121                 else
1122                 {
1123                         downinfo[i].queued = true;
1124                 }
1125                 ++i;
1126         }
1127
1128         if(additional_info)
1129         {
1130                 // TODO: can I clear command_when_done as soon as the first download fails?
1131                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1132                 {
1133                         if(!strncmp(command_when_done, "connect ", 8))
1134                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1135                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1136                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1137                         else
1138                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1139                         *additional_info = addinfo;
1140                 }
1141                 else
1142                         *additional_info = NULL;
1143         }
1144
1145         *nDownloads = n;
1146         return downinfo;
1147 }
1148
1149
1150 /*
1151 ====================
1152 Curl_FindPackURL
1153
1154 finds the URL where to find a given package.
1155
1156 For this, it reads a file "curl_urls.txt" of the following format:
1157
1158         data*.pk3       -
1159         revdm*.pk3      http://revdm/downloads/are/here/
1160         *                       http://any/other/stuff/is/here/
1161
1162 The URLs should end in /. If not, downloads will still work, but the cached files
1163 can't be just put into the data directory with the same download configuration
1164 (you might want to do this if you want to tag downloaded files from your
1165 server, but you should not). "-" means "don't download".
1166
1167 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1168 location instead.
1169
1170 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1171 this file for obvious reasons.
1172 ====================
1173 */
1174 static const char *Curl_FindPackURL(const char *filename)
1175 {
1176         static char foundurl[256];
1177         fs_offset_t filesize;
1178         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1179         if(buf && filesize)
1180         {
1181                 // read lines of format "pattern url"
1182                 char *p = buf;
1183                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1184                 qboolean eof = false;
1185
1186                 pattern = p;
1187                 while(!eof)
1188                 {
1189                         switch(*p)
1190                         {
1191                                 case 0:
1192                                         eof = true;
1193                                         // fallthrough
1194                                 case '\n':
1195                                 case '\r':
1196                                         if(pattern && url && patternend)
1197                                         {
1198                                                 if(!urlend)
1199                                                         urlend = p;
1200                                                 *patternend = 0;
1201                                                 *urlend = 0;
1202                                                 if(matchpattern(filename, pattern, true))
1203                                                 {
1204                                                         strlcpy(foundurl, url, sizeof(foundurl));
1205                                                         Z_Free(buf);
1206                                                         return foundurl;
1207                                                 }
1208                                         }
1209                                         pattern = NULL;
1210                                         patternend = NULL;
1211                                         url = NULL;
1212                                         urlend = NULL;
1213                                         break;
1214                                 case ' ':
1215                                 case '\t':
1216                                         if(pattern && !patternend)
1217                                                 patternend = p;
1218                                         else if(url && !urlend)
1219                                                 urlend = p;
1220                                         break;
1221                                 default:
1222                                         if(!pattern)
1223                                                 pattern = p;
1224                                         else if(pattern && patternend && !url)
1225                                                 url = p;
1226                                         break;
1227                         }
1228                         ++p;
1229                 }
1230         }
1231         if(buf)
1232                 Z_Free(buf);
1233         return sv_curl_defaulturl.string;
1234 }
1235
1236 typedef struct requirement_s
1237 {
1238         struct requirement_s *next;
1239         char filename[MAX_QPATH];
1240 }
1241 requirement;
1242 static requirement *requirements = NULL;
1243
1244
1245 /*
1246 ====================
1247 Curl_RequireFile
1248
1249 Adds the given file to the list of requirements.
1250 ====================
1251 */
1252 void Curl_RequireFile(const char *filename)
1253 {
1254         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1255         req->next = requirements;
1256         strlcpy(req->filename, filename, sizeof(req->filename));
1257         requirements = req;
1258 }
1259
1260 /*
1261 ====================
1262 Curl_ClearRequirements
1263
1264 Clears the list of required files for playing on the current map.
1265 This should be called at every map change.
1266 ====================
1267 */
1268 void Curl_ClearRequirements()
1269 {
1270         const char *p;
1271         while(requirements)
1272         {
1273                 requirement *req = requirements;
1274                 requirements = requirements->next;
1275                 Z_Free(req);
1276         }
1277         p = sv_curl_serverpackages.string;
1278         Con_DPrintf("Require all of: %s\n", p);
1279         while(COM_ParseToken_Simple(&p, false))
1280         {
1281                 Con_DPrintf("Require: %s\n", com_token);
1282                 Curl_RequireFile(com_token);
1283         }
1284 }
1285
1286 /*
1287 ====================
1288 Curl_SendRequirements
1289
1290 Makes the current host_clients download all files he needs.
1291 This is done by sending him the following console commands:
1292
1293         curl --clear_autodownload
1294         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1295         curl --finish_autodownload
1296 ====================
1297 */
1298 void Curl_SendRequirements()
1299 {
1300         // for each requirement, find the pack name
1301         char sendbuffer[4096] = "";
1302         requirement *req;
1303         qboolean foundone = false;
1304
1305         for(req = requirements; req; req = req->next)
1306         {
1307                 const char *p;
1308                 const char *thispack = FS_WhichPack(req->filename);
1309                 const char *packurl;
1310
1311                 if(!thispack)
1312                         continue;
1313
1314                 p = strrchr(thispack, '/');
1315                 if(p)
1316                         thispack = p + 1;
1317
1318                 packurl = Curl_FindPackURL(thispack);
1319
1320                 if(packurl && *packurl && strcmp(packurl, "-"))
1321                 {
1322                         if(!foundone)
1323                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1324
1325                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1326                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1327                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1328                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1329                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1330                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1331                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1332                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1333
1334                         foundone = true;
1335                 }
1336         }
1337
1338         if(foundone)
1339                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1340
1341         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1342                 Host_ClientCommands("%s", sendbuffer);
1343         else
1344                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1345 }