libcurl.c

   1 #include "quakedef.h"
   2 #include "fs.h"
   3 #include "libcurl.h"
   4
   5 static cvar_t cl_curl_maxdownloads = {1, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
   6 static cvar_t cl_curl_maxspeed = {1, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
   7 static cvar_t sv_curl_defaulturl = {1, "sv_curl_defaulturl","", "default autodownload source URL"};
   8 static cvar_t cl_curl_enabled = {1, "cl_curl_enabled","0", "whether client's download support is enabled"};
   9
  10 /*
  11 =================================================================
  12
  13   Minimal set of definitions from libcurl
  14
  15   WARNING: for a matter of simplicity, several pointer types are
  16   casted to "void*", and most enumerated values are not included
  17
  18 =================================================================
  19 */
  20
  21 typedef struct CURL_s CURL;
  22 typedef struct CURLM_s CURLM;
  23 typedef enum
  24 {
  25         CURLE_OK = 0
  26 }
  27 CURLcode;
  28 typedef enum
  29 {
  30         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
  31         CURLM_OK = 0
  32 }
  33 CURLMcode;
  34 #define CURL_GLOBAL_NOTHING 0
  35 #define CURL_GLOBAL_SSL 1
  36 #define CURL_GLOBAL_WIN32 2
  37 #define CURLOPTTYPE_LONG          0
  38 #define CURLOPTTYPE_OBJECTPOINT   10000
  39 #define CURLOPTTYPE_FUNCTIONPOINT 20000
  40 #define CURLOPTTYPE_OFF_T         30000
  41 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
  42 typedef enum
  43 {
  44         CINIT(WRITEDATA, OBJECTPOINT, 1),
  45         CINIT(URL,  OBJECTPOINT, 2),
  46         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
  47         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
  48         CINIT(REFERER, OBJECTPOINT, 16),
  49         CINIT(USERAGENT, OBJECTPOINT, 18),
  50         CINIT(RESUME_FROM, LONG, 21),
  51         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
  52         CINIT(PRIVATE, OBJECTPOINT, 103),
  53 }
  54 CURLoption;
  55 typedef enum
  56 {
  57         CURLINFO_TEXT = 0,
  58         CURLINFO_HEADER_IN,    /* 1 */
  59         CURLINFO_HEADER_OUT,   /* 2 */
  60         CURLINFO_DATA_IN,      /* 3 */
  61         CURLINFO_DATA_OUT,     /* 4 */
  62         CURLINFO_SSL_DATA_IN,  /* 5 */
  63         CURLINFO_SSL_DATA_OUT, /* 6 */
  64         CURLINFO_END
  65 }
  66 curl_infotype;
  67 #define CURLINFO_STRING   0x100000
  68 #define CURLINFO_LONG     0x200000
  69 #define CURLINFO_DOUBLE   0x300000
  70 #define CURLINFO_SLIST    0x400000
  71 #define CURLINFO_MASK     0x0fffff
  72 #define CURLINFO_TYPEMASK 0xf00000
  73 typedef enum
  74 {
  75         CURLINFO_NONE, /* first, never use this */
  76         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
  77         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
  78         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
  79         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
  80         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
  81         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
  82         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
  83         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
  84         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
  85         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
  86         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
  87         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
  88         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
  89         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
  90         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
  91         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
  92         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
  93         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
  94         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
  95         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
  96         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
  97         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
  98         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
  99         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
 100         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
 101         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
 102         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
 103 }
 104 CURLINFO;
 105
 106 typedef enum
 107 {
 108         CURLMSG_NONE, /* first, not used */
 109         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
 110                                          the CURLcode of the transfer */
 111         CURLMSG_LAST
 112 }
 113 CURLMSG;
 114 typedef struct
 115 {
 116         CURLMSG msg;       /* what this message means */
 117         CURL *easy_handle; /* the handle it concerns */
 118         union
 119         {
 120                 void *whatever;    /* message-specific data */
 121                 CURLcode result;   /* return code for transfer */
 122         }
 123         data;
 124 }
 125 CURLMsg;
 126
 127 static void (*qcurl_global_init) (long flags);
 128 static void (*qcurl_global_cleanup) ();
 129
 130 static CURL * (*qcurl_easy_init) ();
 131 static void (*qcurl_easy_cleanup) (CURL *handle);
 132 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
 133 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
 134 static const char * (*qcurl_easy_strerror) (CURLcode);
 135
 136 static CURLM * (*qcurl_multi_init) ();
 137 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
 138 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
 139 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
 140 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
 141 static void (*qcurl_multi_cleanup) (CURLM *);
 142 static const char * (*qcurl_multi_strerror) (CURLcode);
 143
 144 static dllfunction_t curlfuncs[] =
 145 {
 146         {"curl_global_init",            (void **) &qcurl_global_init},
 147         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
 148         {"curl_easy_init",                      (void **) &qcurl_easy_init},
 149         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
 150         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
 151         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
 152         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
 153         {"curl_multi_init",                     (void **) &qcurl_multi_init},
 154         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
 155         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
 156         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
 157         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
 158         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
 159         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
 160         {NULL, NULL}
 161 };
 162
 163 // Handle for CURL DLL
 164 static dllhandle_t curl_dll = NULL;
 165 // will be checked at many places to find out if qcurl calls are allowed
 166
 167 typedef struct downloadinfo_s
 168 {
 169         char filename[MAX_QPATH];
 170         char url[256];
 171         char referer[256];
 172         qfile_t *stream;
 173         fs_offset_t startpos;
 174         CURL *curle;
 175         qboolean started;
 176         qboolean ispak;
 177         unsigned long bytes_received;
 178         struct downloadinfo_s *next, *prev;
 179         qboolean forthismap;
 180 }
 181 downloadinfo;
 182 static downloadinfo *downloads = NULL;
 183 static int numdownloads = 0;
 184
 185 /*
 186 ====================
 187 CURL_CloseLibrary
 188
 189 Load the cURL DLL
 190 ====================
 191 */
 192 static qboolean CURL_OpenLibrary (void)
 193 {
 194         const char* dllnames [] =
 195         {
 196 #if defined(WIN64)
 197                 "libcurl64.dll",
 198 #elif defined(WIN32)
 199                 "libcurl-3.dll",
 200 #elif defined(MACOSX)
 201                 "libcurl.3.dylib", // Mac OS X Tiger
 202                 "libcurl.2.dylib", // Mac OS X Panther
 203 #else
 204                 "libcurl.so.3",
 205 #endif
 206                 NULL
 207         };
 208
 209         // Already loaded?
 210         if (curl_dll)
 211                 return true;
 212
 213         // Load the DLL
 214         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
 215         {
 216                 Con_Printf ("cURL support disabled\n");
 217                 return false;
 218         }
 219
 220         Con_Printf ("cURL support enabled\n");
 221         return true;
 222 }
 223
 224
 225 /*
 226 ====================
 227 CURL_CloseLibrary
 228
 229 Unload the cURL DLL
 230 ====================
 231 */
 232 static void CURL_CloseLibrary (void)
 233 {
 234         Sys_UnloadLibrary (&curl_dll);
 235 }
 236
 237
 238 static CURLM *curlm = NULL;
 239 static unsigned long bytes_received = 0; // used for bandwidth throttling
 240 static double curltime = 0;
 241
 242 /*
 243 ====================
 244 CURL_fwrite
 245
 246 fwrite-compatible function that writes the data to a file. libcurl can call
 247 this.
 248 ====================
 249 */
 250 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
 251 {
 252         fs_offset_t ret;
 253         size_t bytes = size * nmemb;
 254         downloadinfo *di = (downloadinfo *) vdi;
 255
 256         bytes_received += bytes;
 257         di->bytes_received += bytes;
 258
 259         ret = FS_Write(di->stream, data, bytes);
 260
 261         return ret; // why not ret / nmemb?
 262 }
 263
 264 typedef enum
 265 {
 266         CURL_DOWNLOAD_SUCCESS = 0,
 267         CURL_DOWNLOAD_FAILED,
 268         CURL_DOWNLOAD_ABORTED,
 269         CURL_DOWNLOAD_SERVERERROR
 270 }
 271 CurlStatus;
 272
 273 /*
 274 ====================
 275 Curl_Clear_forthismap
 276
 277 Clears the "will disconnect on failure" flags.
 278 ====================
 279 */
 280 void Curl_Clear_forthismap()
 281 {
 282         downloadinfo *di;
 283         for(di = downloads; di; di = di->next)
 284                 di->forthismap = false;
 285 }
 286
 287 static qboolean Curl_Have_forthismap()
 288 {
 289         downloadinfo *di;
 290         for(di = downloads; di; di = di->next)
 291                 if(di->forthismap)
 292                         return true;
 293         return false;
 294 }
 295
 296 /*
 297 ====================
 298 Curl_EndDownload
 299
 300 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
 301 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
 302 code from libcurl, or 0, if another error has occurred.
 303 ====================
 304 */
 305 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
 306 {
 307         qboolean ok = false;
 308         if(!curl_dll)
 309                 return;
 310         switch(status)
 311         {
 312                 case CURL_DOWNLOAD_SUCCESS:
 313                         Con_Printf("Download of %s: OK\n", di->filename);
 314                         ok = true;
 315                         break;
 316                 case CURL_DOWNLOAD_FAILED:
 317                         Con_Printf("Download of %s: FAILED\n", di->filename);
 318                         if(error)
 319                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
 320                         break;
 321                 case CURL_DOWNLOAD_ABORTED:
 322                         Con_Printf("Download of %s: ABORTED\n", di->filename);
 323                         break;
 324                 case CURL_DOWNLOAD_SERVERERROR:
 325                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
 326
 327                         // reopen to enforce it to have zero bytes again
 328                         FS_Close(di->stream);
 329                         di->stream = FS_Open(di->filename, "w", false, false);
 330
 331                         break;
 332         }
 333
 334         if(di->curle)
 335         {
 336                 qcurl_multi_remove_handle(curlm, di->curle);
 337                 qcurl_easy_cleanup(di->curle);
 338         }
 339
 340         if(ok && !di->bytes_received)
 341         {
 342                 Con_Printf("ERROR: empty file\n");
 343                 ok = false;
 344         }
 345
 346         if(di->stream)
 347                 FS_Close(di->stream);
 348
 349         if(ok && di->ispak)
 350         {
 351                 ok = FS_AddPack(di->filename, NULL, true);
 352                 if(ok && di->forthismap)
 353                 {
 354                         Mod_Reload();
 355                         R_Modules_NewMap();
 356                 }
 357         }
 358
 359         if(!ok && di->forthismap)
 360         {
 361                 // BAD. Something went totally wrong.
 362                 // The best we can do is clean up the forthismap flags...
 363                 Curl_Clear_forthismap();
 364                 // and disconnect.
 365                 CL_Disconnect_f();
 366         }
 367
 368         if(di->prev)
 369                 di->prev->next = di->next;
 370         else
 371                 downloads = di->next;
 372         if(di->next)
 373                 di->next->prev = di->prev;
 374         Z_Free(di);
 375
 376         --numdownloads;
 377 }
 378
 379 /*
 380 ====================
 381 CheckPendingDownloads
 382
 383 checks if there are free download slots to start new downloads in.
 384 To not start too many downloads at once, only one download is added at a time,
 385 up to a maximum number of cl_curl_maxdownloads are running.
 386 ====================
 387 */
 388 static void CheckPendingDownloads()
 389 {
 390         if(!curl_dll)
 391                 return;
 392         if(numdownloads < cl_curl_maxdownloads.integer)
 393         {
 394                 downloadinfo *di;
 395                 for(di = downloads; di; di = di->next)
 396                 {
 397                         if(!di->started)
 398                         {
 399                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
 400
 401                                 di->stream = FS_Open(di->filename, "ab", false, false);
 402                                 if(!di->stream)
 403                                 {
 404                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
 405                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
 406                                         return;
 407                                 }
 408
 409                                 FS_Seek(di->stream, 0, SEEK_END);
 410                                 di->startpos = FS_Tell(di->stream);
 411                                 if(di->startpos > 0)
 412                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
 413                                 Con_Print("...\n");
 414
 415                                 di->curle = qcurl_easy_init();
 416                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
 417                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
 418                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
 419                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
 420                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
 421                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
 422                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
 423                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
 424                                 qcurl_multi_add_handle(curlm, di->curle);
 425                                 di->started = true;
 426                                 ++numdownloads;
 427                                 if(numdownloads >= cl_curl_maxdownloads.integer)
 428                                         break;
 429                         }
 430                 }
 431         }
 432 }
 433
 434 /*
 435 ====================
 436 Curl_Init
 437
 438 this function MUST be called before using anything else in this file.
 439 On Win32, this must be called AFTER WSAStartup has been done!
 440 ====================
 441 */
 442 void Curl_Init()
 443 {
 444         CURL_OpenLibrary();
 445         if(!curl_dll)
 446                 return;
 447         qcurl_global_init(CURL_GLOBAL_NOTHING);
 448         curlm = qcurl_multi_init();
 449 }
 450
 451 /*
 452 ====================
 453 Curl_Shutdown
 454
 455 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
 456 ====================
 457 */
 458 void Curl_ClearRequirements();
 459 void Curl_Shutdown()
 460 {
 461         if(!curl_dll)
 462                 return;
 463         Curl_ClearRequirements();
 464         Curl_CancelAll();
 465         CURL_CloseLibrary();
 466         curl_dll = NULL;
 467 }
 468
 469 /*
 470 ====================
 471 Curl_Find
 472
 473 Finds the internal information block for a download given by file name.
 474 ====================
 475 */
 476 static downloadinfo *Curl_Find(const char *filename)
 477 {
 478         downloadinfo *di;
 479         if(!curl_dll)
 480                 return NULL;
 481         for(di = downloads; di; di = di->next)
 482                 if(!strcasecmp(di->filename, filename))
 483                         return di;
 484         return NULL;
 485 }
 486
 487 /*
 488 ====================
 489 Curl_Begin
 490
 491 Starts a download of a given URL to the file name portion of this URL (or name
 492 if given) in the "dlcache/" folder.
 493 ====================
 494 */
 495 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
 496 {
 497         if(!curl_dll)
 498                 return;
 499         else
 500         {
 501                 char fn[MAX_QPATH];
 502                 const char *p, *q;
 503                 size_t length;
 504                 downloadinfo *di;
 505
 506                 // Note: This extraction of the file name portion is NOT entirely correct.
 507                 //
 508                 // It does the following:
 509                 //
 510                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
 511                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
 512                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
 513                 //
 514                 // However, I'd like to keep this "buggy" behavior so that PHP script
 515                 // authors can write download scripts without having to enable
 516                 // AcceptPathInfo on Apache. They just have to ensure that their script
 517                 // can be called with such a "fake" path name like
 518                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
 519                 //
 520                 // By the way, such PHP scripts should either send the file or a
 521                 // "Location:" redirect; PHP code example:
 522                 //
 523                 //   header("Location: http://www.example.com/");
 524                 //
 525                 // By the way, this will set User-Agent to something like
 526                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
 527                 // dp://serverhost:serverport/ so you can filter on this; an example
 528                 // httpd log file line might be:
 529                 //
 530                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
 531
 532                 if(!name)
 533                         name = URL;
 534                 p = strrchr(name, '/');
 535                 p = p ? (p+1) : name;
 536                 q = strchr(p, '?');
 537                 length = q ? (size_t)(q - p) : strlen(p);
 538                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
 539
 540
 541                 // already downloading the file?
 542                 {
 543                         downloadinfo *di = Curl_Find(fn);
 544                         if(di)
 545                         {
 546                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
 547
 548                                 // however, if it was not for this map yet...
 549                                 if(forthismap)
 550                                         di->forthismap = true;
 551
 552                                 return;
 553                         }
 554                 }
 555
 556                 if(ispak && FS_FileExists(fn))
 557                 {
 558                         qboolean already_loaded;
 559                         if(FS_AddPack(fn, &already_loaded, true))
 560                         {
 561                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
 562                                 if(already_loaded)
 563                                         Con_DPrintf("(pak was already loaded)\n");
 564                                 else
 565                                         if(forthismap)
 566                                         {
 567                                                 Mod_Reload();
 568                                                 R_Modules_NewMap();
 569                                         }
 570                                 return;
 571                         }
 572                         else
 573                         {
 574                                 qfile_t *f = FS_Open(fn, "rb", false, false);
 575                                 if(f)
 576                                 {
 577                                         char buf[4] = {0};
 578                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
 579
 580                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
 581                                         {
 582                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
 583                                                 FS_Close(f);
 584                                                 f = FS_Open(fn, "w", false, false);
 585                                                 if(f)
 586                                                         FS_Close(f);
 587                                         }
 588                                         else
 589                                         {
 590                                                 // OK
 591                                                 FS_Close(f);
 592                                         }
 593                                 }
 594                         }
 595                 }
 596
 597                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
 598                 strlcpy(di->filename, fn, sizeof(di->filename));
 599                 strlcpy(di->url, URL, sizeof(di->url));
 600                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
 601                 di->forthismap = forthismap;
 602                 di->stream = NULL;
 603                 di->startpos = 0;
 604                 di->curle = NULL;
 605                 di->started = false;
 606                 di->ispak = ispak;
 607                 di->bytes_received = 0;
 608                 di->next = downloads;
 609                 di->prev = NULL;
 610                 if(di->next)
 611                         di->next->prev = di;
 612                 downloads = di;
 613         }
 614 }
 615
 616
 617 /*
 618 ====================
 619 Curl_Run
 620
 621 call this regularily as this will always download as much as possible without
 622 blocking.
 623 ====================
 624 */
 625 void Curl_Run()
 626 {
 627         if(!cl_curl_enabled.integer)
 628                 return;
 629
 630         if(!curl_dll)
 631                 return;
 632
 633         if(!downloads)
 634                 return;
 635
 636         if(realtime < curltime) // throttle
 637                 return;
 638
 639         {
 640                 int remaining;
 641                 CURLMcode mc;
 642
 643                 do
 644                 {
 645                         mc = qcurl_multi_perform(curlm, &remaining);
 646                 }
 647                 while(mc == CURLM_CALL_MULTI_PERFORM);
 648
 649                 for(;;)
 650                 {
 651                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
 652                         if(!msg)
 653                                 break;
 654                         if(msg->msg == CURLMSG_DONE)
 655                         {
 656                                 downloadinfo *di;
 657                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
 658                                 CURLcode result;
 659
 660                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
 661                                 result = msg->data.result;
 662                                 if(result)
 663                                 {
 664                                         failed = CURL_DOWNLOAD_FAILED;
 665                                 }
 666                                 else
 667                                 {
 668                                         long code;
 669                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 670                                         switch(code / 100)
 671                                         {
 672                                                 case 4: // e.g. 404?
 673                                                 case 5: // e.g. 500?
 674                                                         failed = CURL_DOWNLOAD_SERVERERROR;
 675                                                         result = code;
 676                                                         break;
 677                                         }
 678                                 }
 679
 680                                 Curl_EndDownload(di, failed, result);
 681                         }
 682                 }
 683         }
 684
 685         CheckPendingDownloads();
 686
 687         // when will we curl the next time?
 688         // we will wait a bit to ensure our download rate is kept.
 689         // we now know that realtime >= curltime... so set up a new curltime
 690         if(cl_curl_maxspeed.value > 0)
 691         {
 692                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
 693                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
 694                 bytes_received -= bytes;
 695         }
 696         else
 697                 curltime = realtime;
 698 }
 699
 700 /*
 701 ====================
 702 Curl_CancelAll
 703
 704 Stops ALL downloads.
 705 ====================
 706 */
 707 void Curl_CancelAll()
 708 {
 709         if(!curl_dll)
 710                 return;
 711
 712         while(downloads)
 713         {
 714                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 715                 // INVARIANT: downloads will point to the next download after that!
 716         }
 717 }
 718
 719 /*
 720 ====================
 721 Curl_Running
 722
 723 returns true iff there is a download running.
 724 ====================
 725 */
 726 qboolean Curl_Running()
 727 {
 728         if(!curl_dll)
 729                 return false;
 730
 731         return downloads != NULL;
 732 }
 733
 734 /*
 735 ====================
 736 Curl_GetDownloadAmount
 737
 738 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
 739 for the given download.
 740 ====================
 741 */
 742 static double Curl_GetDownloadAmount(downloadinfo *di)
 743 {
 744         if(!curl_dll)
 745                 return -2;
 746         if(di->curle)
 747         {
 748                 double length;
 749                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
 750                 if(length > 0)
 751                         return di->bytes_received / length;
 752                 else
 753                         return 0;
 754         }
 755         else
 756                 return -1;
 757 }
 758
 759 /*
 760 ====================
 761 Curl_GetDownloadSpeed
 762
 763 returns the speed of the given download in bytes per second
 764 ====================
 765 */
 766 static double Curl_GetDownloadSpeed(downloadinfo *di)
 767 {
 768         if(!curl_dll)
 769                 return -2;
 770         if(di->curle)
 771         {
 772                 double speed;
 773                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
 774                 return speed;
 775         }
 776         else
 777                 return -1;
 778 }
 779
 780 /*
 781 ====================
 782 Curl_Info_f
 783
 784 prints the download list
 785 ====================
 786 */
 787 // TODO rewrite using Curl_GetDownloadInfo?
 788 static void Curl_Info_f()
 789 {
 790         downloadinfo *di;
 791         if(!curl_dll)
 792                 return;
 793         if(Curl_Running())
 794         {
 795                 Con_Print("Currently running downloads:\n");
 796                 for(di = downloads; di; di = di->next)
 797                 {
 798                         double speed, percent;
 799                         Con_Printf("  %s -> %s ",  di->url, di->filename);
 800                         percent = 100.0 * Curl_GetDownloadAmount(di);
 801                         speed = Curl_GetDownloadSpeed(di);
 802                         if(percent >= 0)
 803                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
 804                         else
 805                                 Con_Print("(queued)\n");
 806                 }
 807         }
 808         else
 809         {
 810                 Con_Print("No downloads running.\n");
 811         }
 812 }
 813
 814 /*
 815 ====================
 816 Curl_Curl_f
 817
 818 implements the "curl" console command
 819
 820 curl --info
 821 curl --cancel
 822 curl --cancel filename
 823 curl url
 824
 825 For internal use:
 826
 827 curl [--pak] [--forthismap] [--for filename filename...] url
 828         --pak: after downloading, load the package into the virtual file system
 829         --for filename...: only download of at least one of the named files is missing
 830         --forthismap: disconnect on failure
 831 ====================
 832 */
 833 void Curl_Curl_f(void)
 834 {
 835         int i;
 836         int end;
 837         qboolean pak = false;
 838         qboolean forthismap = false;
 839         const char *url;
 840         const char *name = 0;
 841
 842         if(!curl_dll)
 843         {
 844                 Con_Print("libcurl DLL not found, this command is inactive.\n");
 845                 return;
 846         }
 847
 848         if(!cl_curl_enabled.integer)
 849         {
 850                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
 851                 return;
 852         }
 853
 854         for(i = 0; i != Cmd_Argc(); ++i)
 855                 Con_DPrintf("%s ", Cmd_Argv(i));
 856         Con_DPrint("\n");
 857
 858         if(Cmd_Argc() < 2)
 859         {
 860                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
 861                 return;
 862         }
 863
 864         url = Cmd_Argv(Cmd_Argc() - 1);
 865         end = Cmd_Argc();
 866
 867         for(i = 1; i != end; ++i)
 868         {
 869                 const char *a = Cmd_Argv(i);
 870                 if(!strcmp(a, "--info"))
 871                 {
 872                         Curl_Info_f();
 873                         return;
 874                 }
 875                 else if(!strcmp(a, "--cancel"))
 876                 {
 877                         if(i == end - 1) // last argument
 878                                 Curl_CancelAll();
 879                         else
 880                         {
 881                                 downloadinfo *di = Curl_Find(url);
 882                                 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
 883                         }
 884                         return;
 885                 }
 886                 else if(!strcmp(a, "--pak"))
 887                 {
 888                         pak = true;
 889                 }
 890                 else if(!strcmp(a, "--for"))
 891                 {
 892                         for(i = i + 1; i != end - 1; ++i)
 893                         {
 894                                 if(!FS_FileExists(Cmd_Argv(i)))
 895                                         goto needthefile; // why can't I have a "double break"?
 896                         }
 897                         // if we get here, we have all the files...
 898                         return;
 899                 }
 900                 else if(!strcmp(a, "--forthismap"))
 901                 {
 902                         forthismap = true;
 903                 }
 904                 else if(!strcmp(a, "--as"))
 905                 {
 906                         if(i < end - 1)
 907                         {
 908                                 ++i;
 909                                 name = Cmd_Argv(i);
 910                         }
 911                 }
 912                 else if(!strcmp(a, "--clear_autodownload"))
 913                 {
 914                         // mark all running downloads as "not for this map", so if they
 915                         // fail, it does not matter
 916                         Curl_Clear_forthismap();
 917                         return;
 918                 }
 919                 else if(!strcmp(a, "--finish_autodownload"))
 920                 {
 921                         // nothing
 922                         return;
 923                 }
 924                 else if(*a == '-')
 925                 {
 926                         Con_Printf("invalid option %s\n", a);
 927                         return;
 928                 }
 929         }
 930
 931 needthefile:
 932         Curl_Begin(url, name, pak, forthismap);
 933 }
 934
 935 /*
 936 ====================
 937 Curl_Init_Commands
 938
 939 loads the commands and cvars this library uses
 940 ====================
 941 */
 942 void Curl_Init_Commands(void)
 943 {
 944         Cvar_RegisterVariable (&cl_curl_enabled);
 945         Cvar_RegisterVariable (&cl_curl_maxdownloads);
 946         Cvar_RegisterVariable (&cl_curl_maxspeed);
 947         Cvar_RegisterVariable (&sv_curl_defaulturl);
 948         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
 949 }
 950
 951 /*
 952 ====================
 953 Curl_GetDownloadInfo
 954
 955 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
 956 The number of elements in the array is returned in int *nDownloads.
 957 const char **additional_info may be set to a string of additional user
 958 information, or to NULL if no such display shall occur. The returned
 959 array must be freed later using Z_Free.
 960 ====================
 961 */
 962 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
 963 {
 964         int n, i;
 965         downloadinfo *di;
 966         Curl_downloadinfo_t *downinfo;
 967         static char addinfo[128];
 968
 969         if(!curl_dll)
 970         {
 971                 *nDownloads = 0;
 972                 if(additional_info)
 973                         *additional_info = NULL;
 974                 return NULL;
 975         }
 976
 977         n = 0;
 978         for(di = downloads; di; di = di->next)
 979                 ++n;
 980
 981         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
 982         i = 0;
 983         for(di = downloads; di; di = di->next)
 984         {
 985                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
 986                 if(di->curle)
 987                 {
 988                         downinfo[i].progress = Curl_GetDownloadAmount(di);
 989                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
 990                         downinfo[i].queued = false;
 991                 }
 992                 else
 993                 {
 994                         downinfo[i].queued = true;
 995                 }
 996                 ++i;
 997         }
 998
 999         if(additional_info)
1000         {
1001                 // TODO put something better here?
1002                 // maybe... check if the file is actually needed for the current map?
1003                 if(Curl_Have_forthismap())
1004                 {
1005                         dpsnprintf(addinfo, sizeof(addinfo), "please wait for the download to complete");
1006                         *additional_info = addinfo;
1007                 }
1008                 else
1009                         *additional_info = NULL;
1010         }
1011
1012         *nDownloads = n;
1013         return downinfo;
1014 }
1015
1016
1017 /*
1018 ====================
1019 Curl_FindPackURL
1020
1021 finds the URL where to find a given package.
1022
1023 For this, it reads a file "curl_urls.txt" of the following format:
1024
1025         data*.pk3       -
1026         revdm*.pk3      http://revdm/downloads/are/here/
1027         *                       http://any/other/stuff/is/here/
1028
1029 The URLs should end in /. If not, downloads will still work, but the cached files
1030 can't be just put into the data directory with the same download configuration
1031 (you might want to do this if you want to tag downloaded files from your
1032 server, but you should not). "-" means "don't download".
1033
1034 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1035 location instead.
1036
1037 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1038 this file for obvious reasons.
1039 ====================
1040 */
1041 static const char *Curl_FindPackURL(const char *filename)
1042 {
1043         static char foundurl[256];
1044         fs_offset_t filesize;
1045         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1046         if(buf && filesize)
1047         {
1048                 // read lines of format "pattern url"
1049                 char *p = buf;
1050                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1051                 qboolean eof = false;
1052
1053                 pattern = p;
1054                 while(!eof)
1055                 {
1056                         switch(*p)
1057                         {
1058                                 case 0:
1059                                         eof = true;
1060                                         // fallthrough
1061                                 case '\n':
1062                                 case '\r':
1063                                         if(pattern && url && patternend)
1064                                         {
1065                                                 if(!urlend)
1066                                                         urlend = p;
1067                                                 *patternend = 0;
1068                                                 *urlend = 0;
1069                                                 if(matchpattern(filename, pattern, true))
1070                                                 {
1071                                                         strlcpy(foundurl, url, sizeof(foundurl));
1072                                                         Z_Free(buf);
1073                                                         return foundurl;
1074                                                 }
1075                                         }
1076                                         pattern = NULL;
1077                                         patternend = NULL;
1078                                         url = NULL;
1079                                         urlend = NULL;
1080                                         break;
1081                                 case ' ':
1082                                 case '\t':
1083                                         if(pattern && !patternend)
1084                                                 patternend = p;
1085                                         else if(url && !urlend)
1086                                                 urlend = p;
1087                                         break;
1088                                 default:
1089                                         if(!pattern)
1090                                                 pattern = p;
1091                                         else if(pattern && patternend && !url)
1092                                                 url = p;
1093                                         break;
1094                         }
1095                         ++p;
1096                 }
1097         }
1098         if(buf)
1099                 Z_Free(buf);
1100         return sv_curl_defaulturl.string;
1101 }
1102
1103 typedef struct requirement_s
1104 {
1105         struct requirement_s *next;
1106         char filename[MAX_QPATH];
1107 }
1108 requirement;
1109 static requirement *requirements = NULL;
1110
1111
1112 /*
1113 ====================
1114 Curl_ClearRequirements
1115
1116 Clears the list of required files for playing on the current map.
1117 This should be called at every map change.
1118 ====================
1119 */
1120 void Curl_ClearRequirements()
1121 {
1122         while(requirements)
1123         {
1124                 requirement *req = requirements;
1125                 requirements = requirements->next;
1126                 Z_Free(req);
1127         }
1128 }
1129
1130 /*
1131 ====================
1132 Curl_RequireFile
1133
1134 Adds the given file to the list of requirements.
1135 ====================
1136 */
1137 void Curl_RequireFile(const char *filename)
1138 {
1139         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1140         req->next = requirements;
1141         strlcpy(req->filename, filename, sizeof(req->filename));
1142         requirements = req;
1143 }
1144
1145 /*
1146 ====================
1147 Curl_SendRequirements
1148
1149 Makes the current host_clients download all files he needs.
1150 This is done by sending him the following console commands:
1151
1152         curl --start_autodownload
1153         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1154         curl --finish_autodownload
1155 ====================
1156 */
1157 void Curl_SendRequirements()
1158 {
1159         // for each requirement, find the pack name
1160         char sendbuffer[4096] = "";
1161         requirement *req;
1162         qboolean foundone = false;
1163
1164         for(req = requirements; req; req = req->next)
1165         {
1166                 const char *p;
1167                 const char *thispack = FS_WhichPack(req->filename);
1168                 const char *packurl;
1169
1170                 if(!thispack)
1171                         continue;
1172
1173                 p = strrchr(thispack, '/');
1174                 if(p)
1175                         thispack = p + 1;
1176
1177                 packurl = Curl_FindPackURL(thispack);
1178
1179                 if(packurl && *packurl && strcmp(packurl, "-"))
1180                 {
1181                         if(!foundone)
1182                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1183
1184                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1185                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1186                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1187                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1188                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1189                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1190                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1191                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1192
1193                         foundone = true;
1194                 }
1195         }
1196
1197         if(foundone)
1198                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1199
1200         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1201                 Host_ClientCommands("%s", sendbuffer);
1202         else
1203                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1204 }