libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
   9 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","0", "whether client's download support is enabled"};
  10
  11 /*
  12 =================================================================
  13
  14   Minimal set of definitions from libcurl
  15
  16   WARNING: for a matter of simplicity, several pointer types are
  17   casted to "void*", and most enumerated values are not included
  18
  19 =================================================================
  20 */
  21
  22 typedef struct CURL_s CURL;
  23 typedef struct CURLM_s CURLM;
  24 typedef enum
  25 {
  26         CURLE_OK = 0
  27 }
  28 CURLcode;
  29 typedef enum
  30 {
  31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  32         CURLM_OK = 0
  33 }
  34 CURLMcode;
  35 #define CURL_GLOBAL_NOTHING 0
  36 #define CURL_GLOBAL_SSL 1
  37 #define CURL_GLOBAL_WIN32 2
  38 #define CURLOPTTYPE_LONG          0
  39 #define CURLOPTTYPE_OBJECTPOINT   10000
  40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  41 #define CURLOPTTYPE_OFF_T         30000
  42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  43 typedef enum
  44 {
  45         CINIT(WRITEDATA, OBJECTPOINT, 1),
  46         CINIT(URL,  OBJECTPOINT, 2),
  47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  49         CINIT(REFERER, OBJECTPOINT, 16),
  50         CINIT(USERAGENT, OBJECTPOINT, 18),
  51         CINIT(RESUME_FROM, LONG, 21),
  52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  53         CINIT(PRIVATE, OBJECTPOINT, 103),
  54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
  55         CINIT(LOW_SPEED_TIME, LONG, 20),
  56 }
  57 CURLoption;
  58 typedef enum
  59 {
  60         CURLINFO_TEXT = 0,
  61         CURLINFO_HEADER_IN,    /* 1 */
  62         CURLINFO_HEADER_OUT,   /* 2 */
  63         CURLINFO_DATA_IN,      /* 3 */
  64         CURLINFO_DATA_OUT,     /* 4 */
  65         CURLINFO_SSL_DATA_IN,  /* 5 */
  66         CURLINFO_SSL_DATA_OUT, /* 6 */
  67         CURLINFO_END
  68 }
  69 curl_infotype;
  70 #define CURLINFO_STRING   0x100000
  71 #define CURLINFO_LONG     0x200000
  72 #define CURLINFO_DOUBLE   0x300000
  73 #define CURLINFO_SLIST    0x400000
  74 #define CURLINFO_MASK     0x0fffff
  75 #define CURLINFO_TYPEMASK 0xf00000
  76 typedef enum
  77 {
  78         CURLINFO_NONE, /* first, never use this */
  79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
 100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
 101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
 102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 106 }
 107 CURLINFO;
 108
 109 typedef enum
 110 {
 111         CURLMSG_NONE, /* first, not used */
 112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 113                                          the CURLcode of the transfer */
 114         CURLMSG_LAST
 115 }
 116 CURLMSG;
 117 typedef struct
 118 {
 119         CURLMSG msg;       /* what this message means */
 120         CURL *easy_handle; /* the handle it concerns */
 121         union
 122         {
 123                 void *whatever;    /* message-specific data */
 124                 CURLcode result;   /* return code for transfer */
 125         }
 126         data;
 127 }
 128 CURLMsg;
 129
 130 static void (*qcurl_global_init) (long flags);
 131 static void (*qcurl_global_cleanup) ();
 132
 133 static CURL * (*qcurl_easy_init) ();
 134 static void (*qcurl_easy_cleanup) (CURL *handle);
 135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 137 static const char * (*qcurl_easy_strerror) (CURLcode);
 138
 139 static CURLM * (*qcurl_multi_init) ();
 140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 144 static void (*qcurl_multi_cleanup) (CURLM *);
 145 static const char * (*qcurl_multi_strerror) (CURLcode);
 146
 147 static dllfunction_t curlfuncs[] =
 148 {
 149         {"curl_global_init",            (void **) &qcurl_global_init},
 150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 163         {NULL, NULL}
 164 };
 165
 166 // Handle for CURL DLL
 167 static dllhandle_t curl_dll = NULL;
 168 // will be checked at many places to find out if qcurl calls are allowed
 169
 170 typedef struct downloadinfo_s
 171 {
 172         char filename[MAX_QPATH];
 173         char url[256];
 174         char referer[256];
 175         qfile_t *stream;
 176         fs_offset_t startpos;
 177         CURL *curle;
 178         qboolean started;
 179         qboolean ispak;
 180         unsigned long bytes_received;
 181         struct downloadinfo_s *next, *prev;
 182         qboolean forthismap;
 183 }
 184 downloadinfo;
 185 static downloadinfo *downloads = NULL;
 186 static int numdownloads = 0;
 187
 188 static qboolean noclear = FALSE;
 189
 190 static int numdownloads_fail = 0;
 191 static int numdownloads_success = 0;
 192 static int numdownloads_added = 0;
 193 static char command_when_done[256] = "";
 194 static char command_when_error[256] = "";
 195
 196 /*
 197 ====================
 198 Curl_CommandWhenDone
 199
 200 Sets the command which is to be executed when the last download completes AND
 201 all downloads since last server connect ended with a successful status.
 202 Setting the command to NULL clears it.
 203 ====================
 204 */
 205 void Curl_CommandWhenDone(const char *cmd)
 206 {
 207         if(!curl_dll)
 208                 return;
 209         if(cmd)
 210                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
 211         else
 212                 *command_when_done = 0;
 213 }
 214
 215 /*
 216 FIXME
 217 Do not use yet. Not complete.
 218 Problem: what counts as an error?
 219 */
 220
 221 void Curl_CommandWhenError(const char *cmd)
 222 {
 223         if(!curl_dll)
 224                 return;
 225         if(cmd)
 226                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
 227         else
 228                 *command_when_error = 0;
 229 }
 230
 231 /*
 232 ====================
 233 Curl_Clear_forthismap
 234
 235 Clears the "will disconnect on failure" flags.
 236 ====================
 237 */
 238 void Curl_Clear_forthismap()
 239 {
 240         downloadinfo *di;
 241         if(noclear)
 242                 return;
 243         for(di = downloads; di; di = di->next)
 244                 di->forthismap = false;
 245         Curl_CommandWhenError(NULL);
 246         Curl_CommandWhenDone(NULL);
 247         numdownloads_fail = 0;
 248         numdownloads_success = 0;
 249         numdownloads_added = 0;
 250 }
 251
 252 /*
 253 ====================
 254 Curl_Have_forthismap
 255
 256 Returns true if a download needed for the current game is running.
 257 ====================
 258 */
 259 qboolean Curl_Have_forthismap()
 260 {
 261         return numdownloads_added;
 262 }
 263
 264 void Curl_Register_predownload()
 265 {
 266         Curl_CommandWhenDone("cl_begindownloads");
 267         Curl_CommandWhenError("cl_begindownloads");
 268 }
 269
 270 /*
 271 ====================
 272 Curl_CheckCommandWhenDone
 273
 274 Checks if a "done command" is to be executed.
 275 All downloads finished, at least one success since connect, no single failure
 276 -> execute the command.
 277 */
 278 static void Curl_CheckCommandWhenDone()
 279 {
 280         if(!curl_dll)
 281                 return;
 282         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
 283         {
 284                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
 285                 Cbuf_AddText("\n");
 286                 Cbuf_AddText(command_when_done);
 287                 Cbuf_AddText("\n");
 288                 Curl_Clear_forthismap();
 289         }
 290         else if(numdownloads_added && numdownloads_fail && *command_when_error)
 291         {
 292                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
 293                 Cbuf_AddText("\n");
 294                 Cbuf_AddText(command_when_error);
 295                 Cbuf_AddText("\n");
 296                 Curl_Clear_forthismap();
 297         }
 298 }
 299
 300 /*
 301 ====================
 302 CURL_CloseLibrary
 303
 304 Load the cURL DLL
 305 ====================
 306 */
 307 static qboolean CURL_OpenLibrary (void)
 308 {
 309         const char* dllnames [] =
 310         {
 311 #if defined(WIN64)
 312                 "libcurl64.dll",
 313 #elif defined(WIN32)
 314                 "libcurl-3.dll",
 315 #elif defined(MACOSX)
 316                 "libcurl.3.dylib", // Mac OS X Tiger
 317                 "libcurl.2.dylib", // Mac OS X Panther
 318 #else
 319                 "libcurl.so.3",
 320 #endif
 321                 NULL
 322         };
 323
 324         // Already loaded?
 325         if (curl_dll)
 326                 return true;
 327
 328         // Load the DLL
 329         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 330         {
 331                 Con_Printf ("cURL support disabled\n");
 332                 return false;
 333         }
 334
 335         Con_Printf ("cURL support enabled\n");
 336         return true;
 337 }
 338
 339
 340 /*
 341 ====================
 342 CURL_CloseLibrary
 343
 344 Unload the cURL DLL
 345 ====================
 346 */
 347 static void CURL_CloseLibrary (void)
 348 {
 349         Sys_UnloadLibrary (&curl_dll);
 350 }
 351
 352
 353 static CURLM *curlm = NULL;
 354 static unsigned long bytes_received = 0; // used for bandwidth throttling
 355 static double curltime = 0;
 356
 357 /*
 358 ====================
 359 CURL_fwrite
 360
 361 fwrite-compatible function that writes the data to a file. libcurl can call
 362 this.
 363 ====================
 364 */
 365 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 366 {
 367         fs_offset_t ret;
 368         size_t bytes = size * nmemb;
 369         downloadinfo *di = (downloadinfo *) vdi;
 370
 371         bytes_received += bytes;
 372         di->bytes_received += bytes;
 373
 374         ret = FS_Write(di->stream, data, bytes);
 375
 376         return ret; // why not ret / nmemb?
 377 }
 378
 379 typedef enum
 380 {
 381         CURL_DOWNLOAD_SUCCESS = 0,
 382         CURL_DOWNLOAD_FAILED,
 383         CURL_DOWNLOAD_ABORTED,
 384         CURL_DOWNLOAD_SERVERERROR
 385 }
 386 CurlStatus;
 387
 388 /*
 389 ====================
 390 Curl_EndDownload
 391
 392 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 393 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 394 code from libcurl, or 0, if another error has occurred.
 395 ====================
 396 */
 397 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 398 {
 399         qboolean ok = false;
 400         if(!curl_dll)
 401                 return;
 402         switch(status)
 403         {
 404                 case CURL_DOWNLOAD_SUCCESS:
 405                         Con_Printf("Download of %s: OK\n", di->filename);
 406                         ok = true;
 407                         break;
 408                 case CURL_DOWNLOAD_FAILED:
 409                         Con_Printf("Download of %s: FAILED\n", di->filename);
 410                         if(error)
 411                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 412                         break;
 413                 case CURL_DOWNLOAD_ABORTED:
 414                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 415                         break;
 416                 case CURL_DOWNLOAD_SERVERERROR:
 417                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 418
 419                         // reopen to enforce it to have zero bytes again
 420                         FS_Close(di->stream);
 421                         di->stream = FS_Open(di->filename, "w", false, false);
 422
 423                         break;
 424         }
 425
 426         if(di->curle)
 427         {
 428                 qcurl_multi_remove_handle(curlm, di->curle);
 429                 qcurl_easy_cleanup(di->curle);
 430         }
 431
 432         if(ok && !di->bytes_received)
 433         {
 434                 Con_Printf("ERROR: empty file\n");
 435                 ok = false;
 436         }
 437
 438         if(di->stream)
 439                 FS_Close(di->stream);
 440
 441         if(ok && di->ispak)
 442                 ok = FS_AddPack(di->filename, NULL, true);
 443
 444         if(di->prev)
 445                 di->prev->next = di->next;
 446         else
 447                 downloads = di->next;
 448         if(di->next)
 449                 di->next->prev = di->prev;
 450
 451         --numdownloads;
 452         if(di->forthismap)
 453         {
 454                 if(ok)
 455                         ++numdownloads_success;
 456                 else
 457                         ++numdownloads_fail;
 458         }
 459         Z_Free(di);
 460 }
 461
 462 /*
 463 ====================
 464 CheckPendingDownloads
 465
 466 checks if there are free download slots to start new downloads in.
 467 To not start too many downloads at once, only one download is added at a time,
 468 up to a maximum number of cl_curl_maxdownloads are running.
 469 ====================
 470 */
 471 static void CheckPendingDownloads()
 472 {
 473         if(!curl_dll)
 474                 return;
 475         if(numdownloads < cl_curl_maxdownloads.integer)
 476         {
 477                 downloadinfo *di;
 478                 for(di = downloads; di; di = di->next)
 479                 {
 480                         if(!di->started)
 481                         {
 482                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 483
 484                                 di->stream = FS_Open(di->filename, "ab", false, false);
 485                                 if(!di->stream)
 486                                 {
 487                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 488                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 489                                         return;
 490                                 }
 491
 492                                 FS_Seek(di->stream, 0, SEEK_END);
 493                                 di->startpos = FS_Tell(di->stream);
 494                                 if(di->startpos > 0)
 495                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 496                                 Con_Print("...\n");
 497
 498                                 di->curle = qcurl_easy_init();
 499                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 500                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 501                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 502                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 503                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 504                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 505                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
 506                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
 507                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 508                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 509                                 qcurl_multi_add_handle(curlm, di->curle);
 510                                 di->started = true;
 511                                 ++numdownloads;
 512                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 513                                         break;
 514                         }
 515                 }
 516         }
 517 }
 518
 519 /*
 520 ====================
 521 Curl_Init
 522
 523 this function MUST be called before using anything else in this file.
 524 On Win32, this must be called AFTER WSAStartup has been done!
 525 ====================
 526 */
 527 void Curl_Init()
 528 {
 529         CURL_OpenLibrary();
 530         if(!curl_dll)
 531                 return;
 532         qcurl_global_init(CURL_GLOBAL_NOTHING);
 533         curlm = qcurl_multi_init();
 534 }
 535
 536 /*
 537 ====================
 538 Curl_Shutdown
 539
 540 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 541 ====================
 542 */
 543 void Curl_ClearRequirements();
 544 void Curl_Shutdown()
 545 {
 546         if(!curl_dll)
 547                 return;
 548         Curl_ClearRequirements();
 549         Curl_CancelAll();
 550         CURL_CloseLibrary();
 551         curl_dll = NULL;
 552 }
 553
 554 /*
 555 ====================
 556 Curl_Find
 557
 558 Finds the internal information block for a download given by file name.
 559 ====================
 560 */
 561 static downloadinfo *Curl_Find(const char *filename)
 562 {
 563         downloadinfo *di;
 564         if(!curl_dll)
 565                 return NULL;
 566         for(di = downloads; di; di = di->next)
 567                 if(!strcasecmp(di->filename, filename))
 568                         return di;
 569         return NULL;
 570 }
 571
 572 /*
 573 ====================
 574 Curl_Begin
 575
 576 Starts a download of a given URL to the file name portion of this URL (or name
 577 if given) in the "dlcache/" folder.
 578 ====================
 579 */
 580 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 581 {
 582         if(!curl_dll)
 583                 return;
 584         else
 585         {
 586                 char fn[MAX_QPATH];
 587                 const char *p, *q;
 588                 size_t length;
 589                 downloadinfo *di;
 590
 591                 // Note: This extraction of the file name portion is NOT entirely correct.
 592                 //
 593                 // It does the following:
 594                 //
 595                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 596                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 597                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 598                 //
 599                 // However, I'd like to keep this "buggy" behavior so that PHP script
 600                 // authors can write download scripts without having to enable
 601                 // AcceptPathInfo on Apache. They just have to ensure that their script
 602                 // can be called with such a "fake" path name like
 603                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 604                 //
 605                 // By the way, such PHP scripts should either send the file or a
 606                 // "Location:" redirect; PHP code example:
 607                 //
 608                 //   header("Location: http://www.example.com/");
 609                 //
 610                 // By the way, this will set User-Agent to something like
 611                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 612                 // dp://serverhost:serverport/ so you can filter on this; an example
 613                 // httpd log file line might be:
 614                 //
 615                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 616
 617                 if(!name)
 618                         name = URL;
 619                 p = strrchr(name, '/');
 620                 p = p ? (p+1) : name;
 621                 q = strchr(p, '?');
 622                 length = q ? (size_t)(q - p) : strlen(p);
 623                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 624
 625
 626                 // already downloading the file?
 627                 {
 628                         downloadinfo *di = Curl_Find(fn);
 629                         if(di)
 630                         {
 631                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 632
 633                                 // however, if it was not for this map yet...
 634                                 if(forthismap && !di->forthismap)
 635                                 {
 636                                         di->forthismap = true;
 637                                         // this "fakes" a download attempt so the client will wait for
 638                                         // the download to finish and then reconnect
 639                                         ++numdownloads_added;
 640                                 }
 641
 642                                 return;
 643                         }
 644                 }
 645
 646                 if(ispak && FS_FileExists(fn))
 647                 {
 648                         qboolean already_loaded;
 649                         if(FS_AddPack(fn, &already_loaded, true))
 650                         {
 651                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 652                                 if(already_loaded)
 653                                         Con_DPrintf("(pak was already loaded)\n");
 654                                 else
 655                                 {
 656                                         if(forthismap)
 657                                         {
 658                                                 ++numdownloads_added;
 659                                                 ++numdownloads_success;
 660                                         }
 661                                 }
 662                                 return;
 663                         }
 664                         else
 665                         {
 666                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 667                                 if(f)
 668                                 {
 669                                         char buf[4] = {0};
 670                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 671
 672                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 673                                         {
 674                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 675                                                 FS_Close(f);
 676                                                 f = FS_Open(fn, "w", false, false);
 677                                                 if(f)
 678                                                         FS_Close(f);
 679                                         }
 680                                         else
 681                                         {
 682                                                 // OK
 683                                                 FS_Close(f);
 684                                         }
 685                                 }
 686                         }
 687                 }
 688
 689                 if(forthismap)
 690                         ++numdownloads_added;
 691                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 692                 strlcpy(di->filename, fn, sizeof(di->filename));
 693                 strlcpy(di->url, URL, sizeof(di->url));
 694                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 695                 di->forthismap = forthismap;
 696                 di->stream = NULL;
 697                 di->startpos = 0;
 698                 di->curle = NULL;
 699                 di->started = false;
 700                 di->ispak = ispak;
 701                 di->bytes_received = 0;
 702                 di->next = downloads;
 703                 di->prev = NULL;
 704                 if(di->next)
 705                         di->next->prev = di;
 706                 downloads = di;
 707         }
 708 }
 709
 710
 711 /*
 712 ====================
 713 Curl_Run
 714
 715 call this regularily as this will always download as much as possible without
 716 blocking.
 717 ====================
 718 */
 719 void Curl_Run()
 720 {
 721         noclear = FALSE;
 722
 723         if(!cl_curl_enabled.integer)
 724                 return;
 725
 726         if(!curl_dll)
 727                 return;
 728
 729         Curl_CheckCommandWhenDone();
 730
 731         if(!downloads)
 732                 return;
 733
 734         if(realtime < curltime) // throttle
 735                 return;
 736
 737         {
 738                 int remaining;
 739                 CURLMcode mc;
 740
 741                 do
 742                 {
 743                         mc = qcurl_multi_perform(curlm, &remaining);
 744                 }
 745                 while(mc == CURLM_CALL_MULTI_PERFORM);
 746
 747                 for(;;)
 748                 {
 749                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 750                         if(!msg)
 751                                 break;
 752                         if(msg->msg == CURLMSG_DONE)
 753                         {
 754                                 downloadinfo *di;
 755                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 756                                 CURLcode result;
 757                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 758                                 result = msg->data.result;
 759                                 if(result)
 760                                 {
 761                                         failed = CURL_DOWNLOAD_FAILED;
 762                                 }
 763                                 else
 764                                 {
 765                                         long code;
 766                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 767                                         switch(code / 100)
 768                                         {
 769                                                 case 4: // e.g. 404?
 770                                                 case 5: // e.g. 500?
 771                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 772                                                         result = code;
 773                                                         break;
 774                                         }
 775                                 }
 776
 777                                 Curl_EndDownload(di, failed, result);
 778                         }
 779                 }
 780         }
 781
 782         CheckPendingDownloads();
 783
 784         // when will we curl the next time?
 785         // we will wait a bit to ensure our download rate is kept.
 786         // we now know that realtime >= curltime... so set up a new curltime
 787         if(cl_curl_maxspeed.value > 0)
 788         {
 789                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 790                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 791                 bytes_received -= bytes;
 792         }
 793         else
 794                 curltime = realtime;
 795 }
 796
 797 /*
 798 ====================
 799 Curl_CancelAll
 800
 801 Stops ALL downloads.
 802 ====================
 803 */
 804 void Curl_CancelAll()
 805 {
 806         if(!curl_dll)
 807                 return;
 808
 809         while(downloads)
 810         {
 811                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 812                 // INVARIANT: downloads will point to the next download after that!
 813         }
 814 }
 815
 816 /*
 817 ====================
 818 Curl_Running
 819
 820 returns true iff there is a download running.
 821 ====================
 822 */
 823 qboolean Curl_Running()
 824 {
 825         if(!curl_dll)
 826                 return false;
 827
 828         return downloads != NULL;
 829 }
 830
 831 /*
 832 ====================
 833 Curl_GetDownloadAmount
 834
 835 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 836 for the given download.
 837 ====================
 838 */
 839 static double Curl_GetDownloadAmount(downloadinfo *di)
 840 {
 841         if(!curl_dll)
 842                 return -2;
 843         if(di->curle)
 844         {
 845                 double length;
 846                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 847                 if(length > 0)
 848                         return di->bytes_received / length;
 849                 else
 850                         return 0;
 851         }
 852         else
 853                 return -1;
 854 }
 855
 856 /*
 857 ====================
 858 Curl_GetDownloadSpeed
 859
 860 returns the speed of the given download in bytes per second
 861 ====================
 862 */
 863 static double Curl_GetDownloadSpeed(downloadinfo *di)
 864 {
 865         if(!curl_dll)
 866                 return -2;
 867         if(di->curle)
 868         {
 869                 double speed;
 870                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 871                 return speed;
 872         }
 873         else
 874                 return -1;
 875 }
 876
 877 /*
 878 ====================
 879 Curl_Info_f
 880
 881 prints the download list
 882 ====================
 883 */
 884 // TODO rewrite using Curl_GetDownloadInfo?
 885 static void Curl_Info_f()
 886 {
 887         downloadinfo *di;
 888         if(!curl_dll)
 889                 return;
 890         if(Curl_Running())
 891         {
 892                 Con_Print("Currently running downloads:\n");
 893                 for(di = downloads; di; di = di->next)
 894                 {
 895                         double speed, percent;
 896                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 897                         percent = 100.0 * Curl_GetDownloadAmount(di);
 898                         speed = Curl_GetDownloadSpeed(di);
 899                         if(percent >= 0)
 900                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 901                         else
 902                                 Con_Print("(queued)\n");
 903                 }
 904         }
 905         else
 906         {
 907                 Con_Print("No downloads running.\n");
 908         }
 909 }
 910
 911 /*
 912 ====================
 913 Curl_Curl_f
 914
 915 implements the "curl" console command
 916
 917 curl --info
 918 curl --cancel
 919 curl --cancel filename
 920 curl url
 921
 922 For internal use:
 923
 924 curl [--pak] [--forthismap] [--for filename filename...] url
 925         --pak: after downloading, load the package into the virtual file system
 926         --for filename...: only download of at least one of the named files is missing
 927         --forthismap: don't reconnect on failure
 928
 929 curl --clear_autodownload
 930         clears the download success/failure counters
 931
 932 curl --finish_autodownload
 933         if at least one download has been started, disconnect and drop to the menu
 934         once the last download completes successfully, reconnect to the current server
 935 ====================
 936 */
 937 void Curl_Curl_f(void)
 938 {
 939         int i;
 940         int end;
 941         qboolean pak = false;
 942         qboolean forthismap = false;
 943         const char *url;
 944         const char *name = 0;
 945
 946         if(!curl_dll)
 947         {
 948                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 949                 return;
 950         }
 951
 952         if(!cl_curl_enabled.integer)
 953         {
 954                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 955                 return;
 956         }
 957
 958         for(i = 0; i != Cmd_Argc(); ++i)
 959                 Con_DPrintf("%s ", Cmd_Argv(i));
 960         Con_DPrint("\n");
 961
 962         if(Cmd_Argc() < 2)
 963         {
 964                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 965                 return;
 966         }
 967
 968         url = Cmd_Argv(Cmd_Argc() - 1);
 969         end = Cmd_Argc();
 970
 971         for(i = 1; i != end; ++i)
 972         {
 973                 const char *a = Cmd_Argv(i);
 974                 if(!strcmp(a, "--info"))
 975                 {
 976                         Curl_Info_f();
 977                         return;
 978                 }
 979                 else if(!strcmp(a, "--cancel"))
 980                 {
 981                         if(i == end - 1) // last argument
 982                                 Curl_CancelAll();
 983                         else
 984                         {
 985                                 downloadinfo *di = Curl_Find(url);
 986                                 if(di)
 987                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 988                                 else
 989                                         Con_Print("download not found\n");
 990                         }
 991                         return;
 992                 }
 993                 else if(!strcmp(a, "--pak"))
 994                 {
 995                         pak = true;
 996                 }
 997                 else if(!strcmp(a, "--for"))
 998                 {
 999                         for(i = i + 1; i != end - 1; ++i)
1000                         {
1001                                 if(!FS_FileExists(Cmd_Argv(i)))
1002                                         goto needthefile; // why can't I have a "double break"?
1003                         }
1004                         // if we get here, we have all the files...
1005                         return;
1006                 }
1007                 else if(!strcmp(a, "--forthismap"))
1008                 {
1009                         forthismap = true;
1010                 }
1011                 else if(!strcmp(a, "--as"))
1012                 {
1013                         if(i < end - 1)
1014                         {
1015                                 ++i;
1016                                 name = Cmd_Argv(i);
1017                         }
1018                 }
1019                 else if(!strcmp(a, "--clear_autodownload"))
1020                 {
1021                         // mark all running downloads as "not for this map", so if they
1022                         // fail, it does not matter
1023                         Curl_Clear_forthismap();
1024                         return;
1025                 }
1026                 else if(!strcmp(a, "--finish_autodownload"))
1027                 {
1028                         if(numdownloads_added)
1029                         {
1030                                 char donecommand[256];
1031                                 if(cls.netcon)
1032                                 {
1033                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1034                                         {
1035                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1036                                                 Curl_CommandWhenDone(donecommand);
1037                                                 noclear = TRUE;
1038                                                 CL_Disconnect();
1039                                                 noclear = FALSE;
1040                                                 Curl_CheckCommandWhenDone();
1041                                         }
1042                                         else
1043                                                 Curl_Register_predownload();
1044                                 }
1045                         }
1046                         return;
1047                 }
1048                 else if(*a == '-')
1049                 {
1050                         Con_Printf("invalid option %s\n", a);
1051                         return;
1052                 }
1053         }
1054
1055 needthefile:
1056         Curl_Begin(url, name, pak, forthismap);
1057 }
1058
1059 /*
1060 ====================
1061 Curl_Init_Commands
1062
1063 loads the commands and cvars this library uses
1064 ====================
1065 */
1066 void Curl_Init_Commands(void)
1067 {
1068         Cvar_RegisterVariable (&cl_curl_enabled);
1069         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1070         Cvar_RegisterVariable (&cl_curl_maxspeed);
1071         Cvar_RegisterVariable (&sv_curl_defaulturl);
1072         Cvar_RegisterVariable (&sv_curl_serverpackages);
1073         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1074 }
1075
1076 /*
1077 ====================
1078 Curl_GetDownloadInfo
1079
1080 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1081 The number of elements in the array is returned in int *nDownloads.
1082 const char **additional_info may be set to a string of additional user
1083 information, or to NULL if no such display shall occur. The returned
1084 array must be freed later using Z_Free.
1085 ====================
1086 */
1087 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1088 {
1089         int n, i;
1090         downloadinfo *di;
1091         Curl_downloadinfo_t *downinfo;
1092         static char addinfo[128];
1093
1094         if(!curl_dll)
1095         {
1096                 *nDownloads = 0;
1097                 if(additional_info)
1098                         *additional_info = NULL;
1099                 return NULL;
1100         }
1101
1102         n = 0;
1103         for(di = downloads; di; di = di->next)
1104                 ++n;
1105
1106         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1107         i = 0;
1108         for(di = downloads; di; di = di->next)
1109         {
1110                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1111                 if(di->curle)
1112                 {
1113                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1114                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1115                         downinfo[i].queued = false;
1116                 }
1117                 else
1118                 {
1119                         downinfo[i].queued = true;
1120                 }
1121                 ++i;
1122         }
1123
1124         if(additional_info)
1125         {
1126                 // TODO: can I clear command_when_done as soon as the first download fails?
1127                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1128                 {
1129                         if(!strncmp(command_when_done, "connect ", 8))
1130                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1131                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1132                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1133                         else
1134                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1135                         *additional_info = addinfo;
1136                 }
1137                 else
1138                         *additional_info = NULL;
1139         }
1140
1141         *nDownloads = n;
1142         return downinfo;
1143 }
1144
1145
1146 /*
1147 ====================
1148 Curl_FindPackURL
1149
1150 finds the URL where to find a given package.
1151
1152 For this, it reads a file "curl_urls.txt" of the following format:
1153
1154         data*.pk3       -
1155         revdm*.pk3      http://revdm/downloads/are/here/
1156         *                       http://any/other/stuff/is/here/
1157
1158 The URLs should end in /. If not, downloads will still work, but the cached files
1159 can't be just put into the data directory with the same download configuration
1160 (you might want to do this if you want to tag downloaded files from your
1161 server, but you should not). "-" means "don't download".
1162
1163 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1164 location instead.
1165
1166 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1167 this file for obvious reasons.
1168 ====================
1169 */
1170 static const char *Curl_FindPackURL(const char *filename)
1171 {
1172         static char foundurl[256];
1173         fs_offset_t filesize;
1174         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1175         if(buf && filesize)
1176         {
1177                 // read lines of format "pattern url"
1178                 char *p = buf;
1179                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1180                 qboolean eof = false;
1181
1182                 pattern = p;
1183                 while(!eof)
1184                 {
1185                         switch(*p)
1186                         {
1187                                 case 0:
1188                                         eof = true;
1189                                         // fallthrough
1190                                 case '\n':
1191                                 case '\r':
1192                                         if(pattern && url && patternend)
1193                                         {
1194                                                 if(!urlend)
1195                                                         urlend = p;
1196                                                 *patternend = 0;
1197                                                 *urlend = 0;
1198                                                 if(matchpattern(filename, pattern, true))
1199                                                 {
1200                                                         strlcpy(foundurl, url, sizeof(foundurl));
1201                                                         Z_Free(buf);
1202                                                         return foundurl;
1203                                                 }
1204                                         }
1205                                         pattern = NULL;
1206                                         patternend = NULL;
1207                                         url = NULL;
1208                                         urlend = NULL;
1209                                         break;
1210                                 case ' ':
1211                                 case '\t':
1212                                         if(pattern && !patternend)
1213                                                 patternend = p;
1214                                         else if(url && !urlend)
1215                                                 urlend = p;
1216                                         break;
1217                                 default:
1218                                         if(!pattern)
1219                                                 pattern = p;
1220                                         else if(pattern && patternend && !url)
1221                                                 url = p;
1222                                         break;
1223                         }
1224                         ++p;
1225                 }
1226         }
1227         if(buf)
1228                 Z_Free(buf);
1229         return sv_curl_defaulturl.string;
1230 }
1231
1232 typedef struct requirement_s
1233 {
1234         struct requirement_s *next;
1235         char filename[MAX_QPATH];
1236 }
1237 requirement;
1238 static requirement *requirements = NULL;
1239
1240
1241 /*
1242 ====================
1243 Curl_RequireFile
1244
1245 Adds the given file to the list of requirements.
1246 ====================
1247 */
1248 void Curl_RequireFile(const char *filename)
1249 {
1250         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1251         req->next = requirements;
1252         strlcpy(req->filename, filename, sizeof(req->filename));
1253         requirements = req;
1254 }
1255
1256 /*
1257 ====================
1258 Curl_ClearRequirements
1259
1260 Clears the list of required files for playing on the current map.
1261 This should be called at every map change.
1262 ====================
1263 */
1264 void Curl_ClearRequirements()
1265 {
1266         const char *p;
1267         while(requirements)
1268         {
1269                 requirement *req = requirements;
1270                 requirements = requirements->next;
1271                 Z_Free(req);
1272         }
1273         p = sv_curl_serverpackages.string;
1274         Con_DPrintf("Require all of: %s\n", p);
1275         while(COM_ParseTokenConsole(&p))
1276         {
1277                 Con_DPrintf("Require: %s\n", com_token);
1278                 Curl_RequireFile(com_token);
1279         }
1280 }
1281
1282 /*
1283 ====================
1284 Curl_SendRequirements
1285
1286 Makes the current host_clients download all files he needs.
1287 This is done by sending him the following console commands:
1288
1289         curl --clear_autodownload
1290         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1291         curl --finish_autodownload
1292 ====================
1293 */
1294 void Curl_SendRequirements()
1295 {
1296         // for each requirement, find the pack name
1297         char sendbuffer[4096] = "";
1298         requirement *req;
1299         qboolean foundone = false;
1300
1301         for(req = requirements; req; req = req->next)
1302         {
1303                 const char *p;
1304                 const char *thispack = FS_WhichPack(req->filename);
1305                 const char *packurl;
1306
1307                 if(!thispack)
1308                         continue;
1309
1310                 p = strrchr(thispack, '/');
1311                 if(p)
1312                         thispack = p + 1;
1313
1314                 packurl = Curl_FindPackURL(thispack);
1315
1316                 if(packurl && *packurl && strcmp(packurl, "-"))
1317                 {
1318                         if(!foundone)
1319                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1320
1321                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1322                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1323                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1324                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1325                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1326                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1327                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1328                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1329
1330                         foundone = true;
1331                 }
1332         }
1333
1334         if(foundone)
1335                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1336
1337         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1338                 Host_ClientCommands("%s", sendbuffer);
1339         else
1340                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1341 }