oops, forgot these
[divverent/darkplaces.git] / libcurl.c
1 #include "quakedef.h"
2 #include "fs.h"
3 #include "libcurl.h"
4
5 static cvar_t cl_curl_maxdownloads = {1, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {1, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {1, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t cl_curl_enabled = {1, "cl_curl_enabled","0", "whether client's download support is enabled"};
9
10 /*
11 =================================================================
12
13   Minimal set of definitions from libcurl
14
15   WARNING: for a matter of simplicity, several pointer types are
16   casted to "void*", and most enumerated values are not included
17
18 =================================================================
19 */
20
21 typedef struct CURL_s CURL;
22 typedef struct CURLM_s CURLM;
23 typedef enum
24 {
25         CURLE_OK = 0
26 }
27 CURLcode;
28 typedef enum
29 {
30         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
31         CURLM_OK = 0
32 }
33 CURLMcode;
34 #define CURL_GLOBAL_NOTHING 0
35 #define CURL_GLOBAL_SSL 1
36 #define CURL_GLOBAL_WIN32 2
37 #define CURLOPTTYPE_LONG          0
38 #define CURLOPTTYPE_OBJECTPOINT   10000
39 #define CURLOPTTYPE_FUNCTIONPOINT 20000
40 #define CURLOPTTYPE_OFF_T         30000
41 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
42 typedef enum
43 {
44         CINIT(WRITEDATA, OBJECTPOINT, 1),
45         CINIT(URL,  OBJECTPOINT, 2),
46         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
47         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
48         CINIT(REFERER, OBJECTPOINT, 16),
49         CINIT(USERAGENT, OBJECTPOINT, 18),
50         CINIT(RESUME_FROM, LONG, 21),
51         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
52         CINIT(PRIVATE, OBJECTPOINT, 103),
53 }
54 CURLoption;
55 typedef enum
56 {
57         CURLINFO_TEXT = 0,
58         CURLINFO_HEADER_IN,    /* 1 */
59         CURLINFO_HEADER_OUT,   /* 2 */
60         CURLINFO_DATA_IN,      /* 3 */
61         CURLINFO_DATA_OUT,     /* 4 */
62         CURLINFO_SSL_DATA_IN,  /* 5 */
63         CURLINFO_SSL_DATA_OUT, /* 6 */
64         CURLINFO_END
65 }
66 curl_infotype;
67 #define CURLINFO_STRING   0x100000
68 #define CURLINFO_LONG     0x200000
69 #define CURLINFO_DOUBLE   0x300000
70 #define CURLINFO_SLIST    0x400000
71 #define CURLINFO_MASK     0x0fffff
72 #define CURLINFO_TYPEMASK 0xf00000
73 typedef enum
74 {
75         CURLINFO_NONE, /* first, never use this */
76         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
77         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
78         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
79         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
80         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
81         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
82         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
83         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
84         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
85         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
86         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
87         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
88         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
89         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
90         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
91         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
92         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
93         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
94         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
95         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
96         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
97         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
98         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
99         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
100         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
101         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
102         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
103 }
104 CURLINFO;
105
106 typedef enum
107 {
108         CURLMSG_NONE, /* first, not used */
109         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
110                                          the CURLcode of the transfer */
111         CURLMSG_LAST
112 }
113 CURLMSG;
114 typedef struct
115 {
116         CURLMSG msg;       /* what this message means */
117         CURL *easy_handle; /* the handle it concerns */
118         union
119         {
120                 void *whatever;    /* message-specific data */
121                 CURLcode result;   /* return code for transfer */
122         }
123         data;
124 }
125 CURLMsg;
126
127 static void (*qcurl_global_init) (long flags);
128 static void (*qcurl_global_cleanup) ();
129
130 static CURL * (*qcurl_easy_init) ();
131 static void (*qcurl_easy_cleanup) (CURL *handle);
132 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
133 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
134 static const char * (*qcurl_easy_strerror) (CURLcode);
135
136 static CURLM * (*qcurl_multi_init) ();
137 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
138 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
139 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
140 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
141 static void (*qcurl_multi_cleanup) (CURLM *);
142 static const char * (*qcurl_multi_strerror) (CURLcode);
143
144 static dllfunction_t curlfuncs[] =
145 {
146         {"curl_global_init",            (void **) &qcurl_global_init},
147         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
148         {"curl_easy_init",                      (void **) &qcurl_easy_init},
149         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
150         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
151         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
152         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
153         {"curl_multi_init",                     (void **) &qcurl_multi_init},
154         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
155         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
156         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
157         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
158         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
159         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
160         {NULL, NULL}
161 };
162
163 // Handle for CURL DLL
164 static dllhandle_t curl_dll = NULL;
165 // will be checked at many places to find out if qcurl calls are allowed
166
167 typedef struct downloadinfo_s
168 {
169         char filename[MAX_QPATH];
170         char url[256];
171         char referer[256];
172         qfile_t *stream;
173         fs_offset_t startpos;
174         CURL *curle;
175         qboolean started;
176         qboolean ispak;
177         unsigned long bytes_received;
178         struct downloadinfo_s *next, *prev;
179         qboolean forthismap;
180 }
181 downloadinfo;
182 static downloadinfo *downloads = NULL;
183 static int numdownloads = 0;
184
185 /*
186 ====================
187 CURL_CloseLibrary
188
189 Load the cURL DLL
190 ====================
191 */
192 static qboolean CURL_OpenLibrary (void)
193 {
194         const char* dllnames [] =
195         {
196 #if defined(WIN64)
197                 "libcurl64.dll",
198 #elif defined(WIN32)
199                 "libcurl-3.dll",
200 #elif defined(MACOSX)
201                 "libcurl.3.dylib",
202 #else
203                 "libcurl.so.3",
204 #endif
205                 NULL
206         };
207
208         // Already loaded?
209         if (curl_dll)
210                 return true;
211
212         // Load the DLL
213         if (! Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs))
214         {
215                 Con_Printf ("cURL support disabled\n");
216                 return false;
217         }
218
219         Con_Printf ("cURL support enabled\n");
220         return true;
221 }
222
223
224 /*
225 ====================
226 CURL_CloseLibrary
227
228 Unload the cURL DLL
229 ====================
230 */
231 static void CURL_CloseLibrary (void)
232 {
233         Sys_UnloadLibrary (&curl_dll);
234 }
235
236
237 static CURLM *curlm = NULL;
238 static unsigned long bytes_received = 0; // used for bandwidth throttling
239 static double curltime = 0;
240
241 /*
242 ====================
243 CURL_fwrite
244
245 fwrite-compatible function that writes the data to a file. libcurl can call
246 this.
247 ====================
248 */
249 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
250 {
251         fs_offset_t ret;
252         size_t bytes = size * nmemb;
253         downloadinfo *di = (downloadinfo *) vdi;
254
255         bytes_received += bytes;
256         di->bytes_received += bytes;
257
258         ret = FS_Write(di->stream, data, bytes);
259
260         return ret; // why not ret / nmemb?
261 }
262
263 typedef enum
264 {
265         CURL_DOWNLOAD_SUCCESS = 0,
266         CURL_DOWNLOAD_FAILED,
267         CURL_DOWNLOAD_ABORTED,
268         CURL_DOWNLOAD_SERVERERROR
269 }
270 CurlStatus;
271
272 /*
273 ====================
274 Curl_Clear_forthismap
275
276 Clears the "will disconnect on failure" flags.
277 ====================
278 */
279 void Curl_Clear_forthismap()
280 {
281         downloadinfo *di;
282         for(di = downloads; di; di = di->next)
283                 di->forthismap = false;
284 }
285
286 static qboolean Curl_Have_forthismap()
287 {
288         downloadinfo *di;
289         for(di = downloads; di; di = di->next)
290                 if(di->forthismap)
291                         return true;
292         return false;
293 }
294
295 /*
296 ====================
297 Curl_EndDownload
298
299 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
300 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
301 code from libcurl, or 0, if another error has occurred.
302 ====================
303 */
304 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
305 {
306         qboolean ok = false;
307         if(!curl_dll)
308                 return;
309         switch(status)
310         {
311                 case CURL_DOWNLOAD_SUCCESS:
312                         Con_Printf("Download of %s: OK\n", di->filename);
313                         ok = true;
314                         break;
315                 case CURL_DOWNLOAD_FAILED:
316                         Con_Printf("Download of %s: FAILED\n", di->filename);
317                         if(error)
318                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
319                         break;
320                 case CURL_DOWNLOAD_ABORTED:
321                         Con_Printf("Download of %s: ABORTED\n", di->filename);
322                         break;
323                 case CURL_DOWNLOAD_SERVERERROR:
324                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
325
326                         // reopen to enforce it to have zero bytes again
327                         FS_Close(di->stream);
328                         di->stream = FS_Open(di->filename, "w", false, false);
329
330                         break;
331         }
332
333         if(di->curle)
334         {
335                 qcurl_multi_remove_handle(curlm, di->curle);
336                 qcurl_easy_cleanup(di->curle);
337         }
338
339         if(ok && !di->bytes_received)
340         {
341                 Con_Printf("ERROR: empty file\n");
342                 ok = false;
343         }
344
345         if(di->stream)
346                 FS_Close(di->stream);
347
348         if(ok && di->ispak)
349         {
350                 ok = FS_AddPack(di->filename, NULL, true);
351                 if(ok && di->forthismap)
352                         Mod_Reload();
353         }
354
355         if(!ok && di->forthismap)
356         {
357                 // BAD. Something went totally wrong.
358                 // The best we can do is clean up the forthismap flags...
359                 Curl_Clear_forthismap();
360                 // and disconnect.
361                 CL_Disconnect_f();
362         }
363
364         if(di->prev)
365                 di->prev->next = di->next;
366         else
367                 downloads = di->next;
368         if(di->next)
369                 di->next->prev = di->prev;
370         Z_Free(di);
371
372         --numdownloads;
373 }
374
375 /*
376 ====================
377 CheckPendingDownloads
378
379 checks if there are free download slots to start new downloads in.
380 To not start too many downloads at once, only one download is added at a time,
381 up to a maximum number of cl_curl_maxdownloads are running.
382 ====================
383 */
384 static void CheckPendingDownloads()
385 {
386         if(!curl_dll)
387                 return;
388         if(numdownloads < cl_curl_maxdownloads.integer)
389         {
390                 downloadinfo *di;
391                 for(di = downloads; di; di = di->next)
392                 {
393                         if(!di->started)
394                         {
395                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
396
397                                 di->stream = FS_Open(di->filename, "ab", false, false);
398                                 if(!di->stream)
399                                 {
400                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
401                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
402                                         return;
403                                 }
404
405                                 FS_Seek(di->stream, 0, SEEK_END);
406                                 di->startpos = FS_Tell(di->stream);
407                                 if(di->startpos > 0)
408                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
409                                 Con_Print("...\n");
410
411                                 di->curle = qcurl_easy_init();
412                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
413                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
414                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
415                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
416                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
417                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
418                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
419                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
420                                 qcurl_multi_add_handle(curlm, di->curle);
421                                 di->started = true;
422                                 ++numdownloads;
423                                 if(numdownloads >= cl_curl_maxdownloads.integer)
424                                         break;
425                         }
426                 }
427         }
428 }
429
430 /*
431 ====================
432 Curl_Init
433
434 this function MUST be called before using anything else in this file.
435 On Win32, this must be called AFTER WSAStartup has been done!
436 ====================
437 */
438 void Curl_Init()
439 {
440         CURL_OpenLibrary();
441         if(!curl_dll)
442                 return;
443         qcurl_global_init(CURL_GLOBAL_NOTHING);
444         curlm = qcurl_multi_init();
445 }
446
447 /*
448 ====================
449 Curl_Shutdown
450
451 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
452 ====================
453 */
454 void Curl_ClearRequirements();
455 void Curl_Shutdown()
456 {
457         if(!curl_dll)
458                 return;
459         Curl_ClearRequirements();
460         Curl_CancelAll();
461         CURL_CloseLibrary();
462         curl_dll = NULL;
463 }
464
465 /*
466 ====================
467 Curl_Find
468
469 Finds the internal information block for a download given by file name.
470 ====================
471 */
472 static downloadinfo *Curl_Find(const char *filename)
473 {
474         downloadinfo *di;
475         if(!curl_dll)
476                 return NULL;
477         for(di = downloads; di; di = di->next)
478                 if(!strcasecmp(di->filename, filename))
479                         return di;
480         return NULL;
481 }
482
483 /*
484 ====================
485 Curl_Begin
486
487 Starts a download of a given URL to the file name portion of this URL (or name
488 if given) in the "dlcache/" folder.
489 ====================
490 */
491 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
492 {
493         if(!curl_dll)
494                 return;
495         else
496         {
497                 char fn[MAX_QPATH];
498                 const char *p, *q;
499                 size_t length;
500                 downloadinfo *di;
501
502                 // Note: This extraction of the file name portion is NOT entirely correct.
503                 //
504                 // It does the following:
505                 //
506                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
507                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
508                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
509                 //
510                 // However, I'd like to keep this "buggy" behavior so that PHP script
511                 // authors can write download scripts without having to enable
512                 // AcceptPathInfo on Apache. They just have to ensure that their script
513                 // can be called with such a "fake" path name like
514                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
515                 //
516                 // By the way, such PHP scripts should either send the file or a
517                 // "Location:" redirect; PHP code example:
518                 //
519                 //   header("Location: http://www.example.com/");
520                 //
521                 // By the way, this will set User-Agent to something like
522                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
523                 // dp://serverhost:serverport/ so you can filter on this; an example
524                 // httpd log file line might be:
525                 //
526                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
527
528                 if(!name)
529                         name = URL;
530                 p = strrchr(name, '/');
531                 p = p ? (p+1) : name;
532                 q = strchr(p, '?');
533                 length = q ? (size_t)(q - p) : strlen(p);
534                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
535
536
537                 // already downloading the file?
538                 {
539                         downloadinfo *di = Curl_Find(fn);
540                         if(di)
541                         {
542                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
543
544                                 // however, if it was not for this map yet...
545                                 if(forthismap)
546                                         di->forthismap = true;
547
548                                 return;
549                         }
550                 }
551                 
552                 if(ispak && FS_FileExists(fn))
553                 {
554                         qboolean already_loaded;
555                         if(FS_AddPack(fn, &already_loaded, true))
556                         {
557                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
558                                 if(already_loaded)
559                                         Con_DPrintf("(pak was already loaded)\n");
560                                 else
561                                         if(forthismap)
562                                                 Mod_Reload();
563                                 return;
564                         }
565                         else
566                         {
567                                 qfile_t *f = FS_Open(fn, "rb", false, false);
568                                 if(f)
569                                 {
570                                         char buf[4] = {0};
571                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
572
573                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
574                                         {
575                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
576                                                 FS_Close(f);
577                                                 f = FS_Open(fn, "w", false, false);
578                                                 if(f)
579                                                         FS_Close(f);
580                                         }
581                                         else
582                                         {
583                                                 // OK
584                                                 FS_Close(f);
585                                         }
586                                 }
587                         }
588                 }
589
590                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
591                 strlcpy(di->filename, fn, sizeof(di->filename));
592                 strlcpy(di->url, URL, sizeof(di->url));
593                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
594                 di->forthismap = forthismap;
595                 di->stream = NULL;
596                 di->startpos = 0;
597                 di->curle = NULL;
598                 di->started = false;
599                 di->ispak = ispak;
600                 di->bytes_received = 0;
601                 di->next = downloads;
602                 di->prev = NULL;
603                 if(di->next)
604                         di->next->prev = di;
605                 downloads = di;
606         }
607 }
608
609
610 /*
611 ====================
612 Curl_Run
613
614 call this regularily as this will always download as much as possible without
615 blocking.
616 ====================
617 */
618 void Curl_Run()
619 {
620         if(!cl_curl_enabled.integer)
621         {
622                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
623                 return;
624         }
625
626         if(!curl_dll)
627                 return;
628
629         if(!downloads)
630                 return;
631
632         if(realtime < curltime) // throttle
633                 return;
634
635         {
636                 int remaining;
637                 CURLMcode mc;
638                 
639                 do
640                 {
641                         mc = qcurl_multi_perform(curlm, &remaining);
642                 }
643                 while(mc == CURLM_CALL_MULTI_PERFORM);
644
645                 for(;;)
646                 {
647                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
648                         if(!msg)
649                                 break;
650                         if(msg->msg == CURLMSG_DONE)
651                         {
652                                 downloadinfo *di;
653                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
654                                 CURLcode result;
655
656                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
657                                 result = msg->data.result;
658                                 if(result)
659                                 {
660                                         failed = CURL_DOWNLOAD_FAILED;
661                                 }
662                                 else
663                                 {
664                                         long code;
665                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
666                                         switch(code / 100)
667                                         {
668                                                 case 4: // e.g. 404?
669                                                 case 5: // e.g. 500?
670                                                         failed = CURL_DOWNLOAD_SERVERERROR;
671                                                         result = code;
672                                                         break;
673                                         }
674                                 }
675                                 
676                                 Curl_EndDownload(di, failed, result);
677                         }
678                 }
679         }
680
681         CheckPendingDownloads();
682
683         // when will we curl the next time?
684         // we will wait a bit to ensure our download rate is kept.
685         // we now know that realtime >= curltime... so set up a new curltime
686         if(cl_curl_maxspeed.value > 0)
687         {
688                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
689                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
690                 bytes_received -= bytes;
691         }
692         else
693                 curltime = realtime;
694 }
695
696 /*
697 ====================
698 Curl_CancelAll
699
700 Stops ALL downloads.
701 ====================
702 */
703 void Curl_CancelAll()
704 {
705         if(!curl_dll)
706                 return;
707
708         while(downloads)
709         {
710                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
711                 // INVARIANT: downloads will point to the next download after that!
712         }
713 }
714
715 /*
716 ====================
717 Curl_Running
718
719 returns true iff there is a download running.
720 ====================
721 */
722 qboolean Curl_Running()
723 {
724         if(!curl_dll)
725                 return false;
726
727         return downloads != NULL;
728 }
729
730 /*
731 ====================
732 Curl_GetDownloadAmount
733
734 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
735 for the given download.
736 ====================
737 */
738 static double Curl_GetDownloadAmount(downloadinfo *di)
739 {
740         if(!curl_dll)
741                 return -2;
742         if(di->curle)
743         {
744                 double length;
745                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
746                 if(length > 0)
747                         return di->bytes_received / length;
748                 else
749                         return 0;
750         }
751         else
752                 return -1;
753 }
754
755 /*
756 ====================
757 Curl_GetDownloadSpeed
758
759 returns the speed of the given download in bytes per second
760 ====================
761 */
762 static double Curl_GetDownloadSpeed(downloadinfo *di)
763 {
764         if(!curl_dll)
765                 return -2;
766         if(di->curle)
767         {
768                 double speed;
769                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
770                 return speed;
771         }
772         else
773                 return -1;
774 }
775
776 /*
777 ====================
778 Curl_Info_f
779
780 prints the download list
781 ====================
782 */
783 // TODO rewrite using Curl_GetDownloadInfo?
784 static void Curl_Info_f()
785 {
786         downloadinfo *di;
787         if(!curl_dll)
788                 return;
789         if(Curl_Running())
790         {
791                 Con_Print("Currently running downloads:\n");
792                 for(di = downloads; di; di = di->next)
793                 {
794                         double speed, percent;
795                         Con_Printf("  %s -> %s ",  di->url, di->filename);
796                         percent = 100.0 * Curl_GetDownloadAmount(di);
797                         speed = Curl_GetDownloadSpeed(di);
798                         if(percent >= 0)
799                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
800                         else
801                                 Con_Print("(queued)\n");
802                 }
803         }
804         else
805         {
806                 Con_Print("No downloads running.\n");
807         }
808 }
809
810 /*
811 ====================
812 Curl_Curl_f
813
814 implements the "curl" console command
815
816 curl --info
817 curl --cancel
818 curl --cancel filename
819 curl url
820
821 For internal use:
822
823 curl [--pak] [--forthismap] [--for filename filename...] url
824         --pak: after downloading, load the package into the virtual file system
825         --for filename...: only download of at least one of the named files is missing
826         --forthismap: disconnect on failure
827 ====================
828 */
829 void Curl_Curl_f(void)
830 {
831         int i;
832         int end;
833         qboolean pak = false;
834         qboolean forthismap = false;
835         const char *url;
836         const char *name = 0;
837
838         if(!cl_curl_enabled.integer)
839                 return;
840
841         if(!curl_dll)
842                 return;
843
844         for(i = 0; i != Cmd_Argc(); ++i)
845                 Con_DPrintf("%s ", Cmd_Argv(i));
846         Con_DPrint("\n");
847
848         if(Cmd_Argc() < 2)
849         {
850                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
851                 return;
852         }
853
854         url = Cmd_Argv(Cmd_Argc() - 1);
855         end = Cmd_Argc();
856
857         for(i = 1; i != end; ++i)
858         {
859                 const char *a = Cmd_Argv(i);
860                 if(!strcmp(a, "--info"))
861                 {
862                         Curl_Info_f();
863                         return;
864                 }
865                 else if(!strcmp(a, "--cancel"))
866                 {
867                         if(i == end - 1) // last argument
868                                 Curl_CancelAll();
869                         else
870                         {
871                                 downloadinfo *di = Curl_Find(url);
872                                 Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
873                         }
874                         return;
875                 }
876                 else if(!strcmp(a, "--pak"))
877                 {
878                         pak = true;
879                 }
880                 else if(!strcmp(a, "--for"))
881                 {
882                         for(i = i + 1; i != end - 1; ++i)
883                         {
884                                 if(!FS_FileExists(Cmd_Argv(i)))
885                                         goto needthefile; // why can't I have a "double break"?
886                         }
887                         // if we get here, we have all the files...
888                         return;
889                 }
890                 else if(!strcmp(a, "--forthismap"))
891                 {
892                         forthismap = true;
893                 }
894                 else if(!strcmp(a, "--as"))
895                 {
896                         if(i < end - 1)
897                         {
898                                 ++i;
899                                 name = Cmd_Argv(i);
900                         }
901                 }
902                 else if(!strcmp(a, "--clear_autodownload"))
903                 {
904                         // mark all running downloads as "not for this map", so if they
905                         // fail, it does not matter
906                         Curl_Clear_forthismap();
907                         return;
908                 }
909                 else if(!strcmp(a, "--finish_autodownload"))
910                 {
911                         // nothing
912                         return;
913                 }
914                 else if(*a == '-')
915                 {
916                         Con_Printf("invalid option %s\n", a);
917                         return;
918                 }
919         }
920
921 needthefile:
922         Curl_Begin(url, name, pak, forthismap);
923 }
924
925 /*
926 ====================
927 Curl_Init_Commands
928
929 loads the commands and cvars this library uses
930 ====================
931 */
932 void Curl_Init_Commands(void)
933 {
934         Cvar_RegisterVariable (&cl_curl_enabled);
935         Cvar_RegisterVariable (&cl_curl_maxdownloads);
936         Cvar_RegisterVariable (&cl_curl_maxspeed);
937         Cvar_RegisterVariable (&sv_curl_defaulturl);
938         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
939 }
940
941 /*
942 ====================
943 Curl_GetDownloadInfo
944
945 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
946 The number of elements in the array is returned in int *nDownloads.
947 const char **additional_info may be set to a string of additional user
948 information, or to NULL if no such display shall occur. The returned
949 array must be freed later using Z_Free.
950 ====================
951 */
952 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
953 {
954         int n, i;
955         downloadinfo *di;
956         Curl_downloadinfo_t *downinfo;
957         static char addinfo[128];
958
959         if(!curl_dll)
960         {
961                 *nDownloads = 0;
962                 if(additional_info)
963                         *additional_info = NULL;
964                 return NULL;
965         }
966
967         n = 0;
968         for(di = downloads; di; di = di->next)
969                 ++n;
970
971         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
972         i = 0;
973         for(di = downloads; di; di = di->next)
974         {
975                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
976                 if(di->curle)
977                 {
978                         downinfo[i].progress = Curl_GetDownloadAmount(di);
979                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
980                         downinfo[i].queued = false;
981                 }
982                 else
983                 {
984                         downinfo[i].queued = true;
985                 }
986                 ++i;
987         }
988         
989         if(additional_info)
990         {
991                 // TODO put something better here?
992                 // maybe... check if the file is actually needed for the current map?
993                 if(Curl_Have_forthismap())
994                 {
995                         dpsnprintf(addinfo, sizeof(addinfo), "please wait for the download to complete");
996                         *additional_info = addinfo;
997                 }
998                 else
999                         *additional_info = NULL;
1000         }
1001
1002         *nDownloads = n;
1003         return downinfo;
1004 }
1005
1006
1007 /*
1008 ====================
1009 Curl_FindPackURL
1010
1011 finds the URL where to find a given package.
1012
1013 For this, it reads a file "curl_urls.txt" of the following format:
1014
1015         data*.pk3       -
1016         revdm*.pk3      http://revdm/downloads/are/here/
1017         *                       http://any/other/stuff/is/here/
1018
1019 The URLs should end in /. If not, downloads will still work, but the cached files
1020 can't be just put into the data directory with the same download configuration
1021 (you might want to do this if you want to tag downloaded files from your
1022 server, but you should not). "-" means "don't download".
1023
1024 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1025 location instead.
1026
1027 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1028 this file for obvious reasons.
1029 ====================
1030 */
1031 static const char *Curl_FindPackURL(const char *filename)
1032 {
1033         static char foundurl[256];
1034         fs_offset_t filesize;
1035         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1036         if(buf && filesize)
1037         {
1038                 // read lines of format "pattern url"
1039                 char *p = buf;
1040                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1041                 qboolean eof = false;
1042                 
1043                 pattern = p;
1044                 while(!eof)
1045                 {
1046                         switch(*p)
1047                         {
1048                                 case 0:
1049                                         eof = true;
1050                                         // fallthrough
1051                                 case '\n':
1052                                 case '\r':
1053                                         if(pattern && url && patternend)
1054                                         {
1055                                                 if(!urlend)
1056                                                         urlend = p;
1057                                                 *patternend = 0;
1058                                                 *urlend = 0;
1059                                                 if(matchpattern(filename, pattern, true))
1060                                                 {
1061                                                         strlcpy(foundurl, url, sizeof(foundurl));
1062                                                         Z_Free(buf);
1063                                                         return foundurl;
1064                                                 }
1065                                         }
1066                                         pattern = NULL;
1067                                         patternend = NULL;
1068                                         url = NULL;
1069                                         urlend = NULL;
1070                                         break;
1071                                 case ' ':
1072                                 case '\t':
1073                                         if(pattern && !patternend)
1074                                                 patternend = p;
1075                                         else if(url && !urlend)
1076                                                 urlend = p;
1077                                         break;
1078                                 default:
1079                                         if(!pattern)
1080                                                 pattern = p;
1081                                         else if(pattern && patternend && !url)
1082                                                 url = p;
1083                                         break;
1084                         }
1085                         ++p;
1086                 }
1087         }
1088         if(buf)
1089                 Z_Free(buf);
1090         return sv_curl_defaulturl.string;
1091 }
1092
1093 typedef struct requirement_s
1094 {
1095         struct requirement_s *next;
1096         char filename[MAX_QPATH];
1097 }
1098 requirement;
1099 static requirement *requirements = NULL;
1100
1101
1102 /*
1103 ====================
1104 Curl_ClearRequirements
1105
1106 Clears the list of required files for playing on the current map.
1107 This should be called at every map change.
1108 ====================
1109 */
1110 void Curl_ClearRequirements()
1111 {
1112         while(requirements)
1113         {
1114                 requirement *req = requirements;
1115                 requirements = requirements->next;
1116                 Z_Free(req);
1117         }
1118 }
1119
1120 /*
1121 ====================
1122 Curl_RequireFile
1123
1124 Adds the given file to the list of requirements.
1125 ====================
1126 */
1127 void Curl_RequireFile(const char *filename)
1128 {
1129         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1130         req->next = requirements;
1131         strlcpy(req->filename, filename, sizeof(req->filename));
1132         requirements = req;
1133 }
1134
1135 /*
1136 ====================
1137 Curl_SendRequirements
1138
1139 Makes the current host_clients download all files he needs.
1140 This is done by sending him the following console commands:
1141
1142         curl --start_autodownload
1143         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1144         curl --finish_autodownload
1145 ====================
1146 */
1147 void Curl_SendRequirements()
1148 {
1149         // for each requirement, find the pack name
1150         char sendbuffer[4096] = "";
1151         requirement *req;
1152
1153         strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1154
1155         for(req = requirements; req; req = req->next)
1156         {
1157                 const char *p;
1158                 const char *thispack = FS_WhichPack(req->filename);
1159                 const char *packurl;
1160
1161                 if(!thispack)
1162                         continue;
1163
1164                 p = strrchr(thispack, '/');
1165                 if(p)
1166                         thispack = p + 1;
1167
1168                 packurl = Curl_FindPackURL(thispack);
1169
1170                 if(packurl && *packurl && strcmp(packurl, "-"))
1171                 {
1172                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1173                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1174                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1175                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1176                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1177                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1178                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1179                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1180                 }
1181         }
1182
1183         strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1184
1185         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1186                 Host_ClientCommands("%s", sendbuffer);
1187         else
1188                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1189 }