quick hack to fix missing PK3s when loading demos with curl downloads (works for...
[divverent/darkplaces.git] / libcurl.c
1 #include "quakedef.h"
2 #include "fs.h"
3 #include "libcurl.h"
4
5 static cvar_t cl_curl_maxdownloads = {CVAR_SAVE, "cl_curl_maxdownloads","1", "maximum number of concurrent HTTP/FTP downloads"};
6 static cvar_t cl_curl_maxspeed = {CVAR_SAVE, "cl_curl_maxspeed","100", "maximum download speed (KiB/s)"};
7 static cvar_t sv_curl_defaulturl = {CVAR_SAVE, "sv_curl_defaulturl","", "default autodownload source URL"};
8 static cvar_t sv_curl_serverpackages = {CVAR_SAVE, "sv_curl_serverpackages","", "list of required files for the clients, separated by spaces"};
9 static cvar_t cl_curl_enabled = {CVAR_SAVE, "cl_curl_enabled","1", "whether client's download support is enabled"};
10
11 /*
12 =================================================================
13
14   Minimal set of definitions from libcurl
15
16   WARNING: for a matter of simplicity, several pointer types are
17   casted to "void*", and most enumerated values are not included
18
19 =================================================================
20 */
21
22 typedef struct CURL_s CURL;
23 typedef struct CURLM_s CURLM;
24 typedef enum
25 {
26         CURLE_OK = 0
27 }
28 CURLcode;
29 typedef enum
30 {
31         CURLM_CALL_MULTI_PERFORM=-1, /* please call curl_multi_perform() soon */
32         CURLM_OK = 0
33 }
34 CURLMcode;
35 #define CURL_GLOBAL_NOTHING 0
36 #define CURL_GLOBAL_SSL 1
37 #define CURL_GLOBAL_WIN32 2
38 #define CURLOPTTYPE_LONG          0
39 #define CURLOPTTYPE_OBJECTPOINT   10000
40 #define CURLOPTTYPE_FUNCTIONPOINT 20000
41 #define CURLOPTTYPE_OFF_T         30000
42 #define CINIT(name,type,number) CURLOPT_ ## name = CURLOPTTYPE_ ## type + number
43 typedef enum
44 {
45         CINIT(WRITEDATA, OBJECTPOINT, 1),
46         CINIT(URL,  OBJECTPOINT, 2),
47         CINIT(ERRORBUFFER, OBJECTPOINT, 10),
48         CINIT(WRITEFUNCTION, FUNCTIONPOINT, 11),
49         CINIT(REFERER, OBJECTPOINT, 16),
50         CINIT(USERAGENT, OBJECTPOINT, 18),
51         CINIT(RESUME_FROM, LONG, 21),
52         CINIT(FOLLOWLOCATION, LONG, 52),  /* use Location: Luke! */
53         CINIT(PRIVATE, OBJECTPOINT, 103),
54         CINIT(LOW_SPEED_LIMIT, LONG , 19),
55         CINIT(LOW_SPEED_TIME, LONG, 20),
56 }
57 CURLoption;
58 typedef enum
59 {
60         CURLINFO_TEXT = 0,
61         CURLINFO_HEADER_IN,    /* 1 */
62         CURLINFO_HEADER_OUT,   /* 2 */
63         CURLINFO_DATA_IN,      /* 3 */
64         CURLINFO_DATA_OUT,     /* 4 */
65         CURLINFO_SSL_DATA_IN,  /* 5 */
66         CURLINFO_SSL_DATA_OUT, /* 6 */
67         CURLINFO_END
68 }
69 curl_infotype;
70 #define CURLINFO_STRING   0x100000
71 #define CURLINFO_LONG     0x200000
72 #define CURLINFO_DOUBLE   0x300000
73 #define CURLINFO_SLIST    0x400000
74 #define CURLINFO_MASK     0x0fffff
75 #define CURLINFO_TYPEMASK 0xf00000
76 typedef enum
77 {
78         CURLINFO_NONE, /* first, never use this */
79         CURLINFO_EFFECTIVE_URL    = CURLINFO_STRING + 1,
80         CURLINFO_RESPONSE_CODE    = CURLINFO_LONG   + 2,
81         CURLINFO_TOTAL_TIME       = CURLINFO_DOUBLE + 3,
82         CURLINFO_NAMELOOKUP_TIME  = CURLINFO_DOUBLE + 4,
83         CURLINFO_CONNECT_TIME     = CURLINFO_DOUBLE + 5,
84         CURLINFO_PRETRANSFER_TIME = CURLINFO_DOUBLE + 6,
85         CURLINFO_SIZE_UPLOAD      = CURLINFO_DOUBLE + 7,
86         CURLINFO_SIZE_DOWNLOAD    = CURLINFO_DOUBLE + 8,
87         CURLINFO_SPEED_DOWNLOAD   = CURLINFO_DOUBLE + 9,
88         CURLINFO_SPEED_UPLOAD     = CURLINFO_DOUBLE + 10,
89         CURLINFO_HEADER_SIZE      = CURLINFO_LONG   + 11,
90         CURLINFO_REQUEST_SIZE     = CURLINFO_LONG   + 12,
91         CURLINFO_SSL_VERIFYRESULT = CURLINFO_LONG   + 13,
92         CURLINFO_FILETIME         = CURLINFO_LONG   + 14,
93         CURLINFO_CONTENT_LENGTH_DOWNLOAD   = CURLINFO_DOUBLE + 15,
94         CURLINFO_CONTENT_LENGTH_UPLOAD     = CURLINFO_DOUBLE + 16,
95         CURLINFO_STARTTRANSFER_TIME = CURLINFO_DOUBLE + 17,
96         CURLINFO_CONTENT_TYPE     = CURLINFO_STRING + 18,
97         CURLINFO_REDIRECT_TIME    = CURLINFO_DOUBLE + 19,
98         CURLINFO_REDIRECT_COUNT   = CURLINFO_LONG   + 20,
99         CURLINFO_PRIVATE          = CURLINFO_STRING + 21,
100         CURLINFO_HTTP_CONNECTCODE = CURLINFO_LONG   + 22,
101         CURLINFO_HTTPAUTH_AVAIL   = CURLINFO_LONG   + 23,
102         CURLINFO_PROXYAUTH_AVAIL  = CURLINFO_LONG   + 24,
103         CURLINFO_OS_ERRNO         = CURLINFO_LONG   + 25,
104         CURLINFO_NUM_CONNECTS     = CURLINFO_LONG   + 26,
105         CURLINFO_SSL_ENGINES      = CURLINFO_SLIST  + 27,
106 }
107 CURLINFO;
108
109 typedef enum
110 {
111         CURLMSG_NONE, /* first, not used */
112         CURLMSG_DONE, /* This easy handle has completed. 'result' contains
113                                          the CURLcode of the transfer */
114         CURLMSG_LAST
115 }
116 CURLMSG;
117 typedef struct
118 {
119         CURLMSG msg;       /* what this message means */
120         CURL *easy_handle; /* the handle it concerns */
121         union
122         {
123                 void *whatever;    /* message-specific data */
124                 CURLcode result;   /* return code for transfer */
125         }
126         data;
127 }
128 CURLMsg;
129
130 static void (*qcurl_global_init) (long flags);
131 static void (*qcurl_global_cleanup) ();
132
133 static CURL * (*qcurl_easy_init) ();
134 static void (*qcurl_easy_cleanup) (CURL *handle);
135 static CURLcode (*qcurl_easy_setopt) (CURL *handle, CURLoption option, ...);
136 static CURLcode (*qcurl_easy_getinfo) (CURL *handle, CURLINFO info, ...);
137 static const char * (*qcurl_easy_strerror) (CURLcode);
138
139 static CURLM * (*qcurl_multi_init) ();
140 static CURLMcode (*qcurl_multi_perform) (CURLM *multi_handle, int *running_handles);
141 static CURLMcode (*qcurl_multi_add_handle) (CURLM *multi_handle, CURL *easy_handle);
142 static CURLMcode (*qcurl_multi_remove_handle) (CURLM *multi_handle, CURL *easy_handle);
143 static CURLMsg * (*qcurl_multi_info_read) (CURLM *multi_handle, int *msgs_in_queue);
144 static void (*qcurl_multi_cleanup) (CURLM *);
145 static const char * (*qcurl_multi_strerror) (CURLcode);
146
147 static dllfunction_t curlfuncs[] =
148 {
149         {"curl_global_init",            (void **) &qcurl_global_init},
150         {"curl_global_cleanup",         (void **) &qcurl_global_cleanup},
151         {"curl_easy_init",                      (void **) &qcurl_easy_init},
152         {"curl_easy_cleanup",           (void **) &qcurl_easy_cleanup},
153         {"curl_easy_setopt",            (void **) &qcurl_easy_setopt},
154         {"curl_easy_strerror",          (void **) &qcurl_easy_strerror},
155         {"curl_easy_getinfo",           (void **) &qcurl_easy_getinfo},
156         {"curl_multi_init",                     (void **) &qcurl_multi_init},
157         {"curl_multi_perform",          (void **) &qcurl_multi_perform},
158         {"curl_multi_add_handle",       (void **) &qcurl_multi_add_handle},
159         {"curl_multi_remove_handle",(void **) &qcurl_multi_remove_handle},
160         {"curl_multi_info_read",        (void **) &qcurl_multi_info_read},
161         {"curl_multi_cleanup",          (void **) &qcurl_multi_cleanup},
162         {"curl_multi_strerror",         (void **) &qcurl_multi_strerror},
163         {NULL, NULL}
164 };
165
166 // Handle for CURL DLL
167 static dllhandle_t curl_dll = NULL;
168 // will be checked at many places to find out if qcurl calls are allowed
169
170 typedef struct downloadinfo_s
171 {
172         char filename[MAX_QPATH];
173         char url[256];
174         char referer[256];
175         qfile_t *stream;
176         fs_offset_t startpos;
177         CURL *curle;
178         qboolean started;
179         qboolean ispak;
180         unsigned long bytes_received;
181         struct downloadinfo_s *next, *prev;
182         qboolean forthismap;
183 }
184 downloadinfo;
185 static downloadinfo *downloads = NULL;
186 static int numdownloads = 0;
187
188 static qboolean noclear = FALSE;
189
190 static int numdownloads_fail = 0;
191 static int numdownloads_success = 0;
192 static int numdownloads_added = 0;
193 static char command_when_done[256] = "";
194 static char command_when_error[256] = "";
195
196 /*
197 ====================
198 Curl_CommandWhenDone
199
200 Sets the command which is to be executed when the last download completes AND
201 all downloads since last server connect ended with a successful status.
202 Setting the command to NULL clears it.
203 ====================
204 */
205 void Curl_CommandWhenDone(const char *cmd)
206 {
207         if(!curl_dll)
208                 return;
209         if(cmd)
210                 strlcpy(command_when_done, cmd, sizeof(command_when_done));
211         else
212                 *command_when_done = 0;
213 }
214
215 /*
216 FIXME
217 Do not use yet. Not complete.
218 Problem: what counts as an error?
219 */
220
221 void Curl_CommandWhenError(const char *cmd)
222 {
223         if(!curl_dll)
224                 return;
225         if(cmd)
226                 strlcpy(command_when_error, cmd, sizeof(command_when_error));
227         else
228                 *command_when_error = 0;
229 }
230
231 /*
232 ====================
233 Curl_Clear_forthismap
234
235 Clears the "will disconnect on failure" flags.
236 ====================
237 */
238 void Curl_Clear_forthismap()
239 {
240         downloadinfo *di;
241         if(noclear)
242                 return;
243         for(di = downloads; di; di = di->next)
244                 di->forthismap = false;
245         Curl_CommandWhenError(NULL);
246         Curl_CommandWhenDone(NULL);
247         numdownloads_fail = 0;
248         numdownloads_success = 0;
249         numdownloads_added = 0;
250 }
251
252 /*
253 ====================
254 Curl_Have_forthismap
255
256 Returns true if a download needed for the current game is running.
257 ====================
258 */
259 qboolean Curl_Have_forthismap()
260 {
261         return numdownloads_added;
262 }
263
264 void Curl_Register_predownload()
265 {
266         Curl_CommandWhenDone("cl_begindownloads");
267         Curl_CommandWhenError("cl_begindownloads");
268 }
269
270 /*
271 ====================
272 Curl_CheckCommandWhenDone
273
274 Checks if a "done command" is to be executed.
275 All downloads finished, at least one success since connect, no single failure
276 -> execute the command.
277 */
278 static void Curl_CheckCommandWhenDone()
279 {
280         if(!curl_dll)
281                 return;
282         if(numdownloads_added && (numdownloads_success == numdownloads_added) && *command_when_done)
283         {
284                 Con_DPrintf("cURL downloads occurred, executing %s\n", command_when_done);
285                 Cbuf_AddText("\n");
286                 Cbuf_AddText(command_when_done);
287                 Cbuf_AddText("\n");
288                 Curl_Clear_forthismap();
289         }
290         else if(numdownloads_added && numdownloads_fail && *command_when_error)
291         {
292                 Con_DPrintf("cURL downloads FAILED, executing %s\n", command_when_error);
293                 Cbuf_AddText("\n");
294                 Cbuf_AddText(command_when_error);
295                 Cbuf_AddText("\n");
296                 Curl_Clear_forthismap();
297         }
298 }
299
300 /*
301 ====================
302 CURL_CloseLibrary
303
304 Load the cURL DLL
305 ====================
306 */
307 static qboolean CURL_OpenLibrary (void)
308 {
309         const char* dllnames [] =
310         {
311 #if defined(WIN64)
312                 "libcurl64.dll",
313 #elif defined(WIN32)
314                 "libcurl-4.dll",
315                 "libcurl-3.dll",
316 #elif defined(MACOSX)
317                 "libcurl.4.dylib", // Mac OS X Notyetreleased
318                 "libcurl.3.dylib", // Mac OS X Tiger
319                 "libcurl.2.dylib", // Mac OS X Panther
320 #else
321                 "libcurl.so.4",
322                 "libcurl.so.3",
323                 "libcurl.so", // FreeBSD
324 #endif
325                 NULL
326         };
327
328         // Already loaded?
329         if (curl_dll)
330                 return true;
331
332         // Load the DLL
333         return Sys_LoadLibrary (dllnames, &curl_dll, curlfuncs);
334 }
335
336
337 /*
338 ====================
339 CURL_CloseLibrary
340
341 Unload the cURL DLL
342 ====================
343 */
344 static void CURL_CloseLibrary (void)
345 {
346         Sys_UnloadLibrary (&curl_dll);
347 }
348
349
350 static CURLM *curlm = NULL;
351 static unsigned long bytes_received = 0; // used for bandwidth throttling
352 static double curltime = 0;
353
354 /*
355 ====================
356 CURL_fwrite
357
358 fwrite-compatible function that writes the data to a file. libcurl can call
359 this.
360 ====================
361 */
362 static size_t CURL_fwrite(void *data, size_t size, size_t nmemb, void *vdi)
363 {
364         fs_offset_t ret;
365         size_t bytes = size * nmemb;
366         downloadinfo *di = (downloadinfo *) vdi;
367
368         bytes_received += bytes;
369         di->bytes_received += bytes;
370
371         ret = FS_Write(di->stream, data, bytes);
372
373         return ret; // why not ret / nmemb?
374 }
375
376 typedef enum
377 {
378         CURL_DOWNLOAD_SUCCESS = 0,
379         CURL_DOWNLOAD_FAILED,
380         CURL_DOWNLOAD_ABORTED,
381         CURL_DOWNLOAD_SERVERERROR
382 }
383 CurlStatus;
384
385 /*
386 ====================
387 Curl_EndDownload
388
389 stops a download. It receives a status (CURL_DOWNLOAD_SUCCESS,
390 CURL_DOWNLOAD_FAILED or CURL_DOWNLOAD_ABORTED) and in the second case the error
391 code from libcurl, or 0, if another error has occurred.
392 ====================
393 */
394 static void Curl_EndDownload(downloadinfo *di, CurlStatus status, CURLcode error)
395 {
396         qboolean ok = false;
397         if(!curl_dll)
398                 return;
399         switch(status)
400         {
401                 case CURL_DOWNLOAD_SUCCESS:
402                         Con_Printf("Download of %s: OK\n", di->filename);
403                         ok = true;
404                         break;
405                 case CURL_DOWNLOAD_FAILED:
406                         Con_Printf("Download of %s: FAILED\n", di->filename);
407                         if(error)
408                                 Con_Printf("Reason given by libcurl: %s\n", qcurl_easy_strerror(error));
409                         break;
410                 case CURL_DOWNLOAD_ABORTED:
411                         Con_Printf("Download of %s: ABORTED\n", di->filename);
412                         break;
413                 case CURL_DOWNLOAD_SERVERERROR:
414                         Con_Printf("Download of %s: %d\n", di->filename, (int) error);
415
416                         // reopen to enforce it to have zero bytes again
417                         FS_Close(di->stream);
418                         di->stream = FS_Open(di->filename, "w", false, false);
419
420                         break;
421         }
422
423         if(di->curle)
424         {
425                 qcurl_multi_remove_handle(curlm, di->curle);
426                 qcurl_easy_cleanup(di->curle);
427         }
428
429         if(ok && !di->bytes_received)
430         {
431                 Con_Printf("ERROR: empty file\n");
432                 ok = false;
433         }
434
435         if(di->stream)
436                 FS_Close(di->stream);
437
438         if(ok && di->ispak)
439                 ok = FS_AddPack(di->filename, NULL, true);
440
441         if(di->prev)
442                 di->prev->next = di->next;
443         else
444                 downloads = di->next;
445         if(di->next)
446                 di->next->prev = di->prev;
447
448         --numdownloads;
449         if(di->forthismap)
450         {
451                 if(ok)
452                         ++numdownloads_success;
453                 else
454                         ++numdownloads_fail;
455         }
456         Z_Free(di);
457 }
458
459 /*
460 ====================
461 CheckPendingDownloads
462
463 checks if there are free download slots to start new downloads in.
464 To not start too many downloads at once, only one download is added at a time,
465 up to a maximum number of cl_curl_maxdownloads are running.
466 ====================
467 */
468 static void CheckPendingDownloads()
469 {
470         if(!curl_dll)
471                 return;
472         if(numdownloads < cl_curl_maxdownloads.integer)
473         {
474                 downloadinfo *di;
475                 for(di = downloads; di; di = di->next)
476                 {
477                         if(!di->started)
478                         {
479                                 Con_Printf("Downloading %s -> %s", di->url, di->filename);
480
481                                 di->stream = FS_Open(di->filename, "ab", false, false);
482                                 if(!di->stream)
483                                 {
484                                         Con_Printf("\nFAILED: Could not open output file %s\n", di->filename);
485                                         Curl_EndDownload(di, CURL_DOWNLOAD_FAILED, CURLE_OK);
486                                         return;
487                                 }
488
489                                 FS_Seek(di->stream, 0, SEEK_END);
490                                 di->startpos = FS_Tell(di->stream);
491                                 if(di->startpos > 0)
492                                         Con_Printf(", resuming from position %ld", (long) di->startpos);
493                                 Con_Print("...\n");
494
495                                 di->curle = qcurl_easy_init();
496                                 qcurl_easy_setopt(di->curle, CURLOPT_URL, di->url);
497                                 qcurl_easy_setopt(di->curle, CURLOPT_USERAGENT, engineversion);
498                                 qcurl_easy_setopt(di->curle, CURLOPT_REFERER, di->referer);
499                                 qcurl_easy_setopt(di->curle, CURLOPT_RESUME_FROM, (long) di->startpos);
500                                 qcurl_easy_setopt(di->curle, CURLOPT_FOLLOWLOCATION, 1);
501                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEFUNCTION, CURL_fwrite);
502                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_LIMIT, (long) 256);
503                                 qcurl_easy_setopt(di->curle, CURLOPT_LOW_SPEED_TIME, (long) 45);
504                                 qcurl_easy_setopt(di->curle, CURLOPT_WRITEDATA, (void *) di);
505                                 qcurl_easy_setopt(di->curle, CURLOPT_PRIVATE, (void *) di);
506                                 qcurl_multi_add_handle(curlm, di->curle);
507                                 di->started = true;
508                                 ++numdownloads;
509                                 if(numdownloads >= cl_curl_maxdownloads.integer)
510                                         break;
511                         }
512                 }
513         }
514 }
515
516 /*
517 ====================
518 Curl_Init
519
520 this function MUST be called before using anything else in this file.
521 On Win32, this must be called AFTER WSAStartup has been done!
522 ====================
523 */
524 void Curl_Init()
525 {
526         CURL_OpenLibrary();
527         if(!curl_dll)
528                 return;
529         qcurl_global_init(CURL_GLOBAL_NOTHING);
530         curlm = qcurl_multi_init();
531 }
532
533 /*
534 ====================
535 Curl_Shutdown
536
537 Surprise... closes all the stuff. Please do this BEFORE shutting down LHNET.
538 ====================
539 */
540 void Curl_ClearRequirements();
541 void Curl_Shutdown()
542 {
543         if(!curl_dll)
544                 return;
545         Curl_ClearRequirements();
546         Curl_CancelAll();
547         CURL_CloseLibrary();
548         curl_dll = NULL;
549 }
550
551 /*
552 ====================
553 Curl_Find
554
555 Finds the internal information block for a download given by file name.
556 ====================
557 */
558 static downloadinfo *Curl_Find(const char *filename)
559 {
560         downloadinfo *di;
561         if(!curl_dll)
562                 return NULL;
563         for(di = downloads; di; di = di->next)
564                 if(!strcasecmp(di->filename, filename))
565                         return di;
566         return NULL;
567 }
568
569 /*
570 ====================
571 Curl_Begin
572
573 Starts a download of a given URL to the file name portion of this URL (or name
574 if given) in the "dlcache/" folder.
575 ====================
576 */
577 void Curl_Begin(const char *URL, const char *name, qboolean ispak, qboolean forthismap)
578 {
579         if(!curl_dll)
580                 return;
581         else
582         {
583                 char fn[MAX_QPATH];
584                 const char *p, *q;
585                 size_t length;
586                 downloadinfo *di;
587
588                 // Note: This extraction of the file name portion is NOT entirely correct.
589                 //
590                 // It does the following:
591                 //
592                 //   http://host/some/script.cgi/SomeFile.pk3?uid=ABCDE -> SomeFile.pk3
593                 //   http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3 -> SomeFile.pk3
594                 //   http://host/some/script.php?uid=ABCDE&file=SomeFile.pk3 -> script.php
595                 //
596                 // However, I'd like to keep this "buggy" behavior so that PHP script
597                 // authors can write download scripts without having to enable
598                 // AcceptPathInfo on Apache. They just have to ensure that their script
599                 // can be called with such a "fake" path name like
600                 // http://host/some/script.php?uid=ABCDE&file=/SomeFile.pk3
601                 //
602                 // By the way, such PHP scripts should either send the file or a
603                 // "Location:" redirect; PHP code example:
604                 //
605                 //   header("Location: http://www.example.com/");
606                 //
607                 // By the way, this will set User-Agent to something like
608                 // "Nexuiz build 22:27:55 Mar 17 2006" (engineversion) and Referer to
609                 // dp://serverhost:serverport/ so you can filter on this; an example
610                 // httpd log file line might be:
611                 //
612                 //   141.2.16.3 - - [17/Mar/2006:22:32:43 +0100] "GET /maps/tznex07.pk3 HTTP/1.1" 200 1077455 "dp://141.2.16.7:26000/" "Nexuiz Linux 22:07:43 Mar 17 2006"
613
614                 if(!name)
615                         name = URL;
616                 p = strrchr(name, '/');
617                 p = p ? (p+1) : name;
618                 q = strchr(p, '?');
619                 length = q ? (size_t)(q - p) : strlen(p);
620                 dpsnprintf(fn, sizeof(fn), "dlcache/%.*s", (int)length, p);
621
622                 if(cls.demorecording)
623                 {
624                         void *demobuf; fs_offset_t demofilesize;
625                         char msg[MAX_QPATH + 16];
626                         sizebuf_t sb;
627
628                         sb.data = (void *) msg;
629                         sb.maxsize = sizeof(msg);
630                         SZ_Clear(&sb);
631                         MSG_WriteByte(&sb, svc_stufftext);
632                         MSG_WriteString(&sb, va("\ncurl --pak \"%.*s\"\n", (int)length, p));
633
634                         CL_CutDemo(&demobuf, &demofilesize);
635                         CL_WriteDemoMessage(&sb);
636                         CL_PasteDemo(&demobuf, &demofilesize);
637                 }
638
639                 // already downloading the file?
640                 {
641                         downloadinfo *di = Curl_Find(fn);
642                         if(di)
643                         {
644                                 Con_Printf("Can't download %s, already getting it from %s!\n", fn, di->url);
645
646                                 // however, if it was not for this map yet...
647                                 if(forthismap && !di->forthismap)
648                                 {
649                                         di->forthismap = true;
650                                         // this "fakes" a download attempt so the client will wait for
651                                         // the download to finish and then reconnect
652                                         ++numdownloads_added;
653                                 }
654
655                                 return;
656                         }
657                 }
658
659                 if(ispak && FS_FileExists(fn))
660                 {
661                         qboolean already_loaded;
662                         if(FS_AddPack(fn, &already_loaded, true))
663                         {
664                                 Con_DPrintf("%s already exists, not downloading!\n", fn);
665                                 if(already_loaded)
666                                         Con_DPrintf("(pak was already loaded)\n");
667                                 else
668                                 {
669                                         if(forthismap)
670                                         {
671                                                 ++numdownloads_added;
672                                                 ++numdownloads_success;
673                                         }
674                                 }
675                                 return;
676                         }
677                         else
678                         {
679                                 qfile_t *f = FS_Open(fn, "rb", false, false);
680                                 if(f)
681                                 {
682                                         char buf[4] = {0};
683                                         FS_Read(f, buf, sizeof(buf)); // no "-1", I will use memcmp
684
685                                         if(memcmp(buf, "PK\x03\x04", 4) && memcmp(buf, "PACK", 4))
686                                         {
687                                                 Con_DPrintf("Detected non-PAK %s, clearing and NOT resuming.\n", fn);
688                                                 FS_Close(f);
689                                                 f = FS_Open(fn, "w", false, false);
690                                                 if(f)
691                                                         FS_Close(f);
692                                         }
693                                         else
694                                         {
695                                                 // OK
696                                                 FS_Close(f);
697                                         }
698                                 }
699                         }
700                 }
701
702                 if(forthismap)
703                         ++numdownloads_added;
704                 di = (downloadinfo *) Z_Malloc(sizeof(*di));
705                 strlcpy(di->filename, fn, sizeof(di->filename));
706                 strlcpy(di->url, URL, sizeof(di->url));
707                 dpsnprintf(di->referer, sizeof(di->referer), "dp://%s/", cls.netcon ? cls.netcon->address : "notconnected.invalid");
708                 di->forthismap = forthismap;
709                 di->stream = NULL;
710                 di->startpos = 0;
711                 di->curle = NULL;
712                 di->started = false;
713                 di->ispak = ispak;
714                 di->bytes_received = 0;
715                 di->next = downloads;
716                 di->prev = NULL;
717                 if(di->next)
718                         di->next->prev = di;
719                 downloads = di;
720         }
721 }
722
723
724 /*
725 ====================
726 Curl_Run
727
728 call this regularily as this will always download as much as possible without
729 blocking.
730 ====================
731 */
732 void Curl_Run()
733 {
734         noclear = FALSE;
735
736         if(!cl_curl_enabled.integer)
737                 return;
738
739         if(!curl_dll)
740                 return;
741
742         Curl_CheckCommandWhenDone();
743
744         if(!downloads)
745                 return;
746
747         if(realtime < curltime) // throttle
748                 return;
749
750         {
751                 int remaining;
752                 CURLMcode mc;
753
754                 do
755                 {
756                         mc = qcurl_multi_perform(curlm, &remaining);
757                 }
758                 while(mc == CURLM_CALL_MULTI_PERFORM);
759
760                 for(;;)
761                 {
762                         CURLMsg *msg = qcurl_multi_info_read(curlm, &remaining);
763                         if(!msg)
764                                 break;
765                         if(msg->msg == CURLMSG_DONE)
766                         {
767                                 downloadinfo *di;
768                                 CurlStatus failed = CURL_DOWNLOAD_SUCCESS;
769                                 CURLcode result;
770                                 qcurl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &di);
771                                 result = msg->data.result;
772                                 if(result)
773                                 {
774                                         failed = CURL_DOWNLOAD_FAILED;
775                                 }
776                                 else
777                                 {
778                                         long code;
779                                         qcurl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
780                                         switch(code / 100)
781                                         {
782                                                 case 4: // e.g. 404?
783                                                 case 5: // e.g. 500?
784                                                         failed = CURL_DOWNLOAD_SERVERERROR;
785                                                         result = code;
786                                                         break;
787                                         }
788                                 }
789
790                                 Curl_EndDownload(di, failed, result);
791                         }
792                 }
793         }
794
795         CheckPendingDownloads();
796
797         // when will we curl the next time?
798         // we will wait a bit to ensure our download rate is kept.
799         // we now know that realtime >= curltime... so set up a new curltime
800         if(cl_curl_maxspeed.value > 0)
801         {
802                 unsigned long bytes = bytes_received; // maybe smoothen a bit?
803                 curltime = realtime + bytes / (cl_curl_maxspeed.value * 1024.0);
804                 bytes_received -= bytes;
805         }
806         else
807                 curltime = realtime;
808 }
809
810 /*
811 ====================
812 Curl_CancelAll
813
814 Stops ALL downloads.
815 ====================
816 */
817 void Curl_CancelAll()
818 {
819         if(!curl_dll)
820                 return;
821
822         while(downloads)
823         {
824                 Curl_EndDownload(downloads, CURL_DOWNLOAD_ABORTED, CURLE_OK);
825                 // INVARIANT: downloads will point to the next download after that!
826         }
827 }
828
829 /*
830 ====================
831 Curl_Running
832
833 returns true iff there is a download running.
834 ====================
835 */
836 qboolean Curl_Running()
837 {
838         if(!curl_dll)
839                 return false;
840
841         return downloads != NULL;
842 }
843
844 /*
845 ====================
846 Curl_GetDownloadAmount
847
848 returns a value from 0.0 to 1.0 which represents the downloaded amount of data
849 for the given download.
850 ====================
851 */
852 static double Curl_GetDownloadAmount(downloadinfo *di)
853 {
854         if(!curl_dll)
855                 return -2;
856         if(di->curle)
857         {
858                 double length;
859                 qcurl_easy_getinfo(di->curle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &length);
860                 if(length > 0)
861                         return di->bytes_received / length;
862                 else
863                         return 0;
864         }
865         else
866                 return -1;
867 }
868
869 /*
870 ====================
871 Curl_GetDownloadSpeed
872
873 returns the speed of the given download in bytes per second
874 ====================
875 */
876 static double Curl_GetDownloadSpeed(downloadinfo *di)
877 {
878         if(!curl_dll)
879                 return -2;
880         if(di->curle)
881         {
882                 double speed;
883                 qcurl_easy_getinfo(di->curle, CURLINFO_SPEED_DOWNLOAD, &speed);
884                 return speed;
885         }
886         else
887                 return -1;
888 }
889
890 /*
891 ====================
892 Curl_Info_f
893
894 prints the download list
895 ====================
896 */
897 // TODO rewrite using Curl_GetDownloadInfo?
898 static void Curl_Info_f()
899 {
900         downloadinfo *di;
901         if(!curl_dll)
902                 return;
903         if(Curl_Running())
904         {
905                 Con_Print("Currently running downloads:\n");
906                 for(di = downloads; di; di = di->next)
907                 {
908                         double speed, percent;
909                         Con_Printf("  %s -> %s ",  di->url, di->filename);
910                         percent = 100.0 * Curl_GetDownloadAmount(di);
911                         speed = Curl_GetDownloadSpeed(di);
912                         if(percent >= 0)
913                                 Con_Printf("(%.1f%% @ %.1f KiB/s)\n", percent, speed / 1024.0);
914                         else
915                                 Con_Print("(queued)\n");
916                 }
917         }
918         else
919         {
920                 Con_Print("No downloads running.\n");
921         }
922 }
923
924 /*
925 ====================
926 Curl_Curl_f
927
928 implements the "curl" console command
929
930 curl --info
931 curl --cancel
932 curl --cancel filename
933 curl url
934
935 For internal use:
936
937 curl [--pak] [--forthismap] [--for filename filename...] url
938         --pak: after downloading, load the package into the virtual file system
939         --for filename...: only download of at least one of the named files is missing
940         --forthismap: don't reconnect on failure
941
942 curl --clear_autodownload
943         clears the download success/failure counters
944
945 curl --finish_autodownload
946         if at least one download has been started, disconnect and drop to the menu
947         once the last download completes successfully, reconnect to the current server
948 ====================
949 */
950 void Curl_Curl_f(void)
951 {
952         int i;
953         int end;
954         qboolean pak = false;
955         qboolean forthismap = false;
956         const char *url;
957         const char *name = 0;
958
959         if(!curl_dll)
960         {
961                 Con_Print("libcurl DLL not found, this command is inactive.\n");
962                 return;
963         }
964
965         if(!cl_curl_enabled.integer)
966         {
967                 Con_Print("curl support not enabled. Set cl_curl_enabled to 1 to enable.\n");
968                 return;
969         }
970
971         for(i = 0; i != Cmd_Argc(); ++i)
972                 Con_DPrintf("%s ", Cmd_Argv(i));
973         Con_DPrint("\n");
974
975         if(Cmd_Argc() < 2)
976         {
977                 Con_Print("usage:\ncurl --info, curl --cancel [filename], curl url\n");
978                 return;
979         }
980
981         url = Cmd_Argv(Cmd_Argc() - 1);
982         end = Cmd_Argc();
983
984         for(i = 1; i != end; ++i)
985         {
986                 const char *a = Cmd_Argv(i);
987                 if(!strcmp(a, "--info"))
988                 {
989                         Curl_Info_f();
990                         return;
991                 }
992                 else if(!strcmp(a, "--cancel"))
993                 {
994                         if(i == end - 1) // last argument
995                                 Curl_CancelAll();
996                         else
997                         {
998                                 downloadinfo *di = Curl_Find(url);
999                                 if(di)
1000                                         Curl_EndDownload(di, CURL_DOWNLOAD_ABORTED, CURLE_OK);
1001                                 else
1002                                         Con_Print("download not found\n");
1003                         }
1004                         return;
1005                 }
1006                 else if(!strcmp(a, "--pak"))
1007                 {
1008                         pak = true;
1009                 }
1010                 else if(!strcmp(a, "--for"))
1011                 {
1012                         for(i = i + 1; i != end - 1; ++i)
1013                         {
1014                                 if(!FS_FileExists(Cmd_Argv(i)))
1015                                         goto needthefile; // why can't I have a "double break"?
1016                         }
1017                         // if we get here, we have all the files...
1018                         return;
1019                 }
1020                 else if(!strcmp(a, "--forthismap"))
1021                 {
1022                         forthismap = true;
1023                 }
1024                 else if(!strcmp(a, "--as"))
1025                 {
1026                         if(i < end - 1)
1027                         {
1028                                 ++i;
1029                                 name = Cmd_Argv(i);
1030                         }
1031                 }
1032                 else if(!strcmp(a, "--clear_autodownload"))
1033                 {
1034                         // mark all running downloads as "not for this map", so if they
1035                         // fail, it does not matter
1036                         Curl_Clear_forthismap();
1037                         return;
1038                 }
1039                 else if(!strcmp(a, "--finish_autodownload"))
1040                 {
1041                         if(numdownloads_added)
1042                         {
1043                                 char donecommand[256];
1044                                 if(cls.netcon)
1045                                 {
1046                                         if(cl.loadbegun) // curling won't inhibit loading the map any more when at this stage, so bail out and force a reconnect
1047                                         {
1048                                                 dpsnprintf(donecommand, sizeof(donecommand), "connect %s", cls.netcon->address);
1049                                                 Curl_CommandWhenDone(donecommand);
1050                                                 noclear = TRUE;
1051                                                 CL_Disconnect();
1052                                                 noclear = FALSE;
1053                                                 Curl_CheckCommandWhenDone();
1054                                         }
1055                                         else
1056                                                 Curl_Register_predownload();
1057                                 }
1058                         }
1059                         return;
1060                 }
1061                 else if(*a == '-')
1062                 {
1063                         Con_Printf("invalid option %s\n", a);
1064                         return;
1065                 }
1066         }
1067
1068 needthefile:
1069         Curl_Begin(url, name, pak, forthismap);
1070 }
1071
1072 /*
1073 ====================
1074 Curl_Init_Commands
1075
1076 loads the commands and cvars this library uses
1077 ====================
1078 */
1079 void Curl_Init_Commands(void)
1080 {
1081         Cvar_RegisterVariable (&cl_curl_enabled);
1082         Cvar_RegisterVariable (&cl_curl_maxdownloads);
1083         Cvar_RegisterVariable (&cl_curl_maxspeed);
1084         Cvar_RegisterVariable (&sv_curl_defaulturl);
1085         Cvar_RegisterVariable (&sv_curl_serverpackages);
1086         Cmd_AddCommand ("curl", Curl_Curl_f, "download data from an URL and add to search path");
1087 }
1088
1089 /*
1090 ====================
1091 Curl_GetDownloadInfo
1092
1093 returns an array of Curl_downloadinfo_t structs for usage by GUIs.
1094 The number of elements in the array is returned in int *nDownloads.
1095 const char **additional_info may be set to a string of additional user
1096 information, or to NULL if no such display shall occur. The returned
1097 array must be freed later using Z_Free.
1098 ====================
1099 */
1100 Curl_downloadinfo_t *Curl_GetDownloadInfo(int *nDownloads, const char **additional_info)
1101 {
1102         int n, i;
1103         downloadinfo *di;
1104         Curl_downloadinfo_t *downinfo;
1105         static char addinfo[128];
1106
1107         if(!curl_dll)
1108         {
1109                 *nDownloads = 0;
1110                 if(additional_info)
1111                         *additional_info = NULL;
1112                 return NULL;
1113         }
1114
1115         n = 0;
1116         for(di = downloads; di; di = di->next)
1117                 ++n;
1118
1119         downinfo = (Curl_downloadinfo_t *) Z_Malloc(sizeof(*downinfo) * n);
1120         i = 0;
1121         for(di = downloads; di; di = di->next)
1122         {
1123                 strlcpy(downinfo[i].filename, di->filename, sizeof(downinfo[i].filename));
1124                 if(di->curle)
1125                 {
1126                         downinfo[i].progress = Curl_GetDownloadAmount(di);
1127                         downinfo[i].speed = Curl_GetDownloadSpeed(di);
1128                         downinfo[i].queued = false;
1129                 }
1130                 else
1131                 {
1132                         downinfo[i].queued = true;
1133                 }
1134                 ++i;
1135         }
1136
1137         if(additional_info)
1138         {
1139                 // TODO: can I clear command_when_done as soon as the first download fails?
1140                 if(*command_when_done && !numdownloads_fail && numdownloads_added)
1141                 {
1142                         if(!strncmp(command_when_done, "connect ", 8))
1143                                 dpsnprintf(addinfo, sizeof(addinfo), "(will join %s when done)", command_when_done + 8);
1144                         else if(!strcmp(command_when_done, "cl_begindownloads"))
1145                                 dpsnprintf(addinfo, sizeof(addinfo), "(will enter the game when done)");
1146                         else
1147                                 dpsnprintf(addinfo, sizeof(addinfo), "(will do '%s' when done)", command_when_done);
1148                         *additional_info = addinfo;
1149                 }
1150                 else
1151                         *additional_info = NULL;
1152         }
1153
1154         *nDownloads = n;
1155         return downinfo;
1156 }
1157
1158
1159 /*
1160 ====================
1161 Curl_FindPackURL
1162
1163 finds the URL where to find a given package.
1164
1165 For this, it reads a file "curl_urls.txt" of the following format:
1166
1167         data*.pk3       -
1168         revdm*.pk3      http://revdm/downloads/are/here/
1169         *                       http://any/other/stuff/is/here/
1170
1171 The URLs should end in /. If not, downloads will still work, but the cached files
1172 can't be just put into the data directory with the same download configuration
1173 (you might want to do this if you want to tag downloaded files from your
1174 server, but you should not). "-" means "don't download".
1175
1176 If no single pattern matched, the cvar sv_curl_defaulturl is used as download
1177 location instead.
1178
1179 Note: pak1.pak and data*.pk3 are excluded from autodownload at another point in
1180 this file for obvious reasons.
1181 ====================
1182 */
1183 static const char *Curl_FindPackURL(const char *filename)
1184 {
1185         static char foundurl[256];
1186         fs_offset_t filesize;
1187         char *buf = (char *) FS_LoadFile("curl_urls.txt", tempmempool, true, &filesize);
1188         if(buf && filesize)
1189         {
1190                 // read lines of format "pattern url"
1191                 char *p = buf;
1192                 char *pattern = NULL, *patternend = NULL, *url = NULL, *urlend = NULL;
1193                 qboolean eof = false;
1194
1195                 pattern = p;
1196                 while(!eof)
1197                 {
1198                         switch(*p)
1199                         {
1200                                 case 0:
1201                                         eof = true;
1202                                         // fallthrough
1203                                 case '\n':
1204                                 case '\r':
1205                                         if(pattern && url && patternend)
1206                                         {
1207                                                 if(!urlend)
1208                                                         urlend = p;
1209                                                 *patternend = 0;
1210                                                 *urlend = 0;
1211                                                 if(matchpattern(filename, pattern, true))
1212                                                 {
1213                                                         strlcpy(foundurl, url, sizeof(foundurl));
1214                                                         Z_Free(buf);
1215                                                         return foundurl;
1216                                                 }
1217                                         }
1218                                         pattern = NULL;
1219                                         patternend = NULL;
1220                                         url = NULL;
1221                                         urlend = NULL;
1222                                         break;
1223                                 case ' ':
1224                                 case '\t':
1225                                         if(pattern && !patternend)
1226                                                 patternend = p;
1227                                         else if(url && !urlend)
1228                                                 urlend = p;
1229                                         break;
1230                                 default:
1231                                         if(!pattern)
1232                                                 pattern = p;
1233                                         else if(pattern && patternend && !url)
1234                                                 url = p;
1235                                         break;
1236                         }
1237                         ++p;
1238                 }
1239         }
1240         if(buf)
1241                 Z_Free(buf);
1242         return sv_curl_defaulturl.string;
1243 }
1244
1245 typedef struct requirement_s
1246 {
1247         struct requirement_s *next;
1248         char filename[MAX_QPATH];
1249 }
1250 requirement;
1251 static requirement *requirements = NULL;
1252
1253
1254 /*
1255 ====================
1256 Curl_RequireFile
1257
1258 Adds the given file to the list of requirements.
1259 ====================
1260 */
1261 void Curl_RequireFile(const char *filename)
1262 {
1263         requirement *req = (requirement *) Z_Malloc(sizeof(*requirements));
1264         req->next = requirements;
1265         strlcpy(req->filename, filename, sizeof(req->filename));
1266         requirements = req;
1267 }
1268
1269 /*
1270 ====================
1271 Curl_ClearRequirements
1272
1273 Clears the list of required files for playing on the current map.
1274 This should be called at every map change.
1275 ====================
1276 */
1277 void Curl_ClearRequirements()
1278 {
1279         const char *p;
1280         while(requirements)
1281         {
1282                 requirement *req = requirements;
1283                 requirements = requirements->next;
1284                 Z_Free(req);
1285         }
1286         p = sv_curl_serverpackages.string;
1287         Con_DPrintf("Require all of: %s\n", p);
1288         while(COM_ParseToken_Simple(&p, false, false))
1289         {
1290                 Con_DPrintf("Require: %s\n", com_token);
1291                 Curl_RequireFile(com_token);
1292         }
1293 }
1294
1295 /*
1296 ====================
1297 Curl_SendRequirements
1298
1299 Makes the current host_clients download all files he needs.
1300 This is done by sending him the following console commands:
1301
1302         curl --clear_autodownload
1303         curl --pak --for maps/pushmoddm1.bsp --forthismap http://where/this/darn/map/is/pushmoddm1.pk3
1304         curl --finish_autodownload
1305 ====================
1306 */
1307 void Curl_SendRequirements()
1308 {
1309         // for each requirement, find the pack name
1310         char sendbuffer[4096] = "";
1311         requirement *req;
1312         qboolean foundone = false;
1313
1314         for(req = requirements; req; req = req->next)
1315         {
1316                 const char *p;
1317                 const char *thispack = FS_WhichPack(req->filename);
1318                 const char *packurl;
1319
1320                 if(!thispack)
1321                         continue;
1322
1323                 p = strrchr(thispack, '/');
1324                 if(p)
1325                         thispack = p + 1;
1326
1327                 packurl = Curl_FindPackURL(thispack);
1328
1329                 if(packurl && *packurl && strcmp(packurl, "-"))
1330                 {
1331                         if(!foundone)
1332                                 strlcat(sendbuffer, "curl --clear_autodownload\n", sizeof(sendbuffer));
1333
1334                         strlcat(sendbuffer, "curl --pak --forthismap --as ", sizeof(sendbuffer));
1335                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1336                         strlcat(sendbuffer, " --for ", sizeof(sendbuffer));
1337                         strlcat(sendbuffer, req->filename, sizeof(sendbuffer));
1338                         strlcat(sendbuffer, " ", sizeof(sendbuffer));
1339                         strlcat(sendbuffer, packurl, sizeof(sendbuffer));
1340                         strlcat(sendbuffer, thispack, sizeof(sendbuffer));
1341                         strlcat(sendbuffer, "\n", sizeof(sendbuffer));
1342
1343                         foundone = true;
1344                 }
1345         }
1346
1347         if(foundone)
1348                 strlcat(sendbuffer, "curl --finish_autodownload\n", sizeof(sendbuffer));
1349
1350         if(strlen(sendbuffer) + 1 < sizeof(sendbuffer))
1351                 Host_ClientCommands("%s", sendbuffer);
1352         else
1353                 Con_Printf("Could not initiate autodownload due to URL buffer overflow\n");
1354 }