DEADSOFTWARE

utils: more useless APIs
[d2df-sdl.git] / src / game / g_res_downloader.pas
1 (* Copyright (C) Doom 2D: Forever Developers
2 *
3 * This program is free software: you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation, version 3 of the License ONLY.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 *)
15 {$INCLUDE ../shared/a_modes.inc}
16 unit g_res_downloader;
18 interface
20 uses sysutils, Classes, md5, g_net, g_netmsg, g_console, g_main, e_log;
23 // download map wad from server (if necessary)
24 // download all required map resource wads too
25 // registers all required replacement wads
26 // returns name of the map wad (relative to mapdir), or empty string on error
27 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
29 // returns original name, or replacement name
30 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
32 // call this somewhere in startup sequence
33 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
36 implementation
38 uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb, e_res;
40 var
41 // cvars
42 g_res_ignore_names: AnsiString = 'standart;shrshade';
43 g_res_ignore_enabled: Boolean = true;
44 g_res_save_databases: Boolean = true;
45 // other vars
46 replacements: THashStrStr = nil;
47 knownMaps: TFileHashDB = nil;
48 knownRes: TFileHashDB = nil;
49 saveDBsToDiskEnabled: Boolean = false; // this will be set to `true` if initial database saving succeed
52 //==========================================================================
53 //
54 // saveDatabases
55 //
56 //==========================================================================
57 procedure saveDatabases (saveMap, saveRes: Boolean);
58 var
59 err: Boolean;
60 st: TStream;
61 ccdir: AnsiString = '';
62 begin
63 if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit;
64 ccdir := e_GetWriteableDir(CacheDirs, false);
65 if (length(ccdir) = 0) then exit;
66 // rescan dirs
67 // save map database
68 if (saveMap) then
69 begin
70 err := true;
71 st := nil;
72 try
73 st := createDiskFile(ccdir+'/maphash.db');
74 knownMaps.saveTo(st);
75 err := false;
76 except
77 end;
78 st.Free;
79 if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write map database, disk refresh disabled'); exit; end;
80 end;
81 // save resource database
82 if (saveRes) then
83 begin
84 err := true;
85 st := nil;
86 try
87 st := createDiskFile(ccdir+'/reshash.db');
88 knownRes.saveTo(st);
89 err := false;
90 except
91 end;
92 st.Free;
93 if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write resource database, disk refresh disabled'); exit; end;
94 end;
95 end;
98 //==========================================================================
99 //
100 // g_Res_CreateDatabases
101 //
102 //==========================================================================
103 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
104 var
105 st: TStream;
106 upmap: Boolean;
107 upres: Boolean;
108 forcesave: Boolean;
109 ccdir: AnsiString = '';
110 begin
111 if not assigned(knownMaps) then
112 begin
113 // create and load a know map database, if necessary
114 knownMaps := TFileHashDB.Create({GameDir}'', MapDirs);
115 knownMaps.appendMoreDirs(MapDownloadDirs);
116 knownRes := TFileHashDB.Create({GameDir}'', WadDirs);
117 knownRes.appendMoreDirs(WadDownloadDirs);
118 saveDBsToDiskEnabled := true;
119 // load map database
120 st := nil;
121 try
122 ccdir := e_GetWriteableDir(CacheDirs, false);
123 if (length(ccdir) > 0) then
124 begin
125 st := openDiskFileRO(ccdir+'/maphash.db');
126 knownMaps.loadFrom(st);
127 e_LogWriteln('loaded map database');
128 end;
129 except
130 end;
131 st.Free;
132 // load resource database
133 st := nil;
134 try
135 if (length(ccdir) > 0) then
136 begin
137 st := openDiskFileRO(ccdir+'/reshash.db');
138 knownRes.loadFrom(st);
139 e_LogWriteln('loaded resource database');
140 end;
141 except
142 end;
143 st.Free;
144 forcesave := true;
145 end
146 else
147 begin
148 if (not allowRescan) then exit;
149 forcesave := false;
150 end;
151 // rescan dirs
152 e_LogWriteln('refreshing map database');
153 upmap := knownMaps.scanFiles();
154 e_LogWriteln('refreshing resource database');
155 upres := knownRes.scanFiles();
156 // save databases
157 if (forcesave) then begin upmap := true; upres := true; end;
158 if upmap or upres then saveDatabases(upmap, upres);
159 end;
162 //==========================================================================
163 //
164 // getWord
165 //
166 // get next word from a string
167 // words are delimited with ';'
168 // ignores leading and trailing spaces
169 // returns empty string if there are no more words
170 //
171 //==========================================================================
172 function getWord (var list: AnsiString): AnsiString;
173 var
174 pos: Integer;
175 begin
176 result := '';
177 while (length(list) > 0) do
178 begin
179 if (ord(list[1]) <= 32) or (list[1] = ';') or (list[1] = ':') then begin Delete(list, 1, 1); continue; end;
180 pos := 1;
181 while (pos <= length(list)) and (list[pos] <> ';') and (list[pos] <> ':') do Inc(pos);
182 result := Copy(list, 1, pos-1);
183 Delete(list, 1, pos);
184 while (length(result) > 0) and (ord(result[length(result)]) <= 32) do Delete(result, length(result), 1);
185 if (length(result) > 0) then exit;
186 end;
187 end;
190 //==========================================================================
191 //
192 // isIgnoredResWad
193 //
194 // checks if the given resource wad can be ignored
195 //
196 // FIXME: preparse name list?
197 //
198 //==========================================================================
199 function isIgnoredResWad (fname: AnsiString): Boolean;
200 var
201 list: AnsiString;
202 name: AnsiString;
203 begin
204 result := false;
205 if (not g_res_ignore_enabled) then exit;
206 fname := forceFilenameExt(ExtractFileName(fname), '');
207 list := g_res_ignore_names;
208 name := getWord(list);
209 while (length(name) > 0) do
210 begin
211 name := forceFilenameExt(name, '');
212 //writeln('*** name=[', name, ']; fname=[', fname, ']');
213 if (StrEquCI1251(name, fname)) then begin result := true; exit; end;
214 name := getWord(list);
215 end;
216 end;
219 //==========================================================================
220 //
221 // clearReplacementWads
222 //
223 // call this before downloading a new map from a server
224 //
225 //==========================================================================
226 procedure clearReplacementWads ();
227 begin
228 if assigned(replacements) then replacements.clear();
229 e_LogWriteln('cleared replacement wads');
230 end;
233 //==========================================================================
234 //
235 // addReplacementWad
236 //
237 // register new replacement wad
238 //
239 //==========================================================================
240 procedure addReplacementWad (oldname: AnsiString; newDiskName: AnsiString);
241 begin
242 e_LogWritefln('adding replacement wad: oldname=%s; newname=%s', [oldname, newDiskName]);
243 if not assigned(replacements) then replacements := THashStrStr.Create();
244 replacements.put(toLowerCase1251(oldname), newDiskName);
245 end;
248 //==========================================================================
249 //
250 // g_Res_FindReplacementWad
251 //
252 // returns original name, or replacement name
253 //
254 //==========================================================================
255 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
256 var
257 fn: AnsiString;
258 begin
259 //e_LogWritefln('LOOKING for replacement wad for [%s]...', [oldname], TMsgType.Notify);
260 result := oldname;
261 if not assigned(replacements) then exit;
262 if (replacements.get(toLowerCase1251(ExtractFileName(oldname)), fn)) then
263 begin
264 //e_LogWritefln('found replacement wad for [%s] -> [%s]', [oldname, fn], TMsgType.Notify);
265 result := fn;
266 end;
267 end;
270 //==========================================================================
271 //
272 // findExistingMapWadWithHash
273 //
274 // find map or resource wad using its base name and hash
275 //
276 // returns found wad disk name, or empty string
277 //
278 //==========================================================================
279 function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
280 begin
281 result := knownMaps.findByHash(resMd5);
282 if (length(result) > 0) then
283 begin
284 //result := GameDir+'/maps/'+result;
285 if not FileExists(result) then
286 begin
287 if (knownMaps.scanFiles()) then saveDatabases(true, false);
288 result := '';
289 end;
290 end;
291 end;
294 //==========================================================================
295 //
296 // findExistingResWadWithHash
297 //
298 // find map or resource wad using its base name and hash
299 //
300 // returns found wad disk name, or empty string
301 //
302 //==========================================================================
303 function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
304 begin
305 result := knownRes.findByHash(resMd5);
306 if (length(result) > 0) then
307 begin
308 //result := GameDir+'/wads/'+result;
309 if not FileExists(result) then
310 begin
311 if (knownRes.scanFiles()) then saveDatabases(false, true);
312 result := '';
313 end;
314 end;
315 end;
318 //==========================================================================
319 //
320 // generateFileName
321 //
322 // generate new file name based on the given one and the hash
323 // you can pass files with pathes here too
324 //
325 //==========================================================================
326 function generateFileName (fname: AnsiString; const hash: TMD5Digest): AnsiString;
327 var
328 mds: AnsiString;
329 path: AnsiString;
330 base: AnsiString;
331 ext: AnsiString;
332 begin
333 mds := MD5Print(hash);
334 if (length(mds) > 16) then mds := Copy(mds, 1, 16);
335 mds := '_'+mds;
336 if (length(fname) = 0) then begin result := mds; exit; end;
337 path := ExtractFilePath(fname);
338 base := ExtractFileName(fname);
339 ext := getFilenameExt(base);
340 base := forceFilenameExt(base, '');
341 if (length(path) > 0) then result := IncludeTrailingPathDelimiter(path) else result := '';
342 result := result+base+mds+ext;
343 end;
346 //==========================================================================
347 //
348 // g_Res_DownloadMapWAD
349 //
350 // download map wad from server (if necessary)
351 // download all required map resource wads too
352 // registers all required replacement wads
353 //
354 // returns name of the map wad (relative to mapdir), or empty string on error
355 //
356 //==========================================================================
357 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
358 var
359 tf: TNetFileTransfer;
360 resList: array of TNetMapResourceInfo = nil;
361 f, res: Integer;
362 strm: TStream;
363 fname: AnsiString;
364 wadname: AnsiString;
365 md5: TMD5Digest;
366 mapdbUpdated: Boolean = false;
367 resdbUpdated: Boolean = false;
368 transStarted: Boolean;
369 destMapDir: AnsiString = '';
370 destResDir: AnsiString = '';
371 begin
372 result := '';
373 clearReplacementWads();
374 sfsGCCollect(); // why not?
375 g_Res_CreateDatabases();
377 try
378 g_Res_received_map_start := 1;
379 g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
380 e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
381 g_Game_SetLoadingText(FileName + '...', 0, False);
383 FileName := ExtractFileName(FileName);
384 if (length(FileName) = 0) then FileName := 'fucked_map_wad.wad';
386 // this also sends map request
387 res := g_Net_Wait_MapInfo(tf, resList);
388 if (res <> 0) then exit;
390 // find or download a map
391 result := findExistingMapWadWithHash(tf.diskName, mapHash);
392 if (length(result) = 0) then
393 begin
394 // download map
395 res := g_Net_RequestResFileInfo(-1{map}, tf);
396 if (res <> 0) then
397 begin
398 e_LogWriteln('error requesting map wad');
399 result := '';
400 exit;
401 end;
402 try
403 destMapDir := e_GetWriteableDir(MapDownloadDirs, false); // not required
404 except
405 end;
406 if (length(destMapDir) = 0) then
407 begin
408 e_LogWriteln('cannot create map download directory', TMsgType.Fatal);
409 result := '';
410 exit;
411 end;
412 fname := destMapDir+'/'+generateFileName(FileName, mapHash);
413 tf.diskName := fname;
414 e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal);
415 try
416 strm := openDiskFileRW(fname);
417 except
418 e_WriteLog('cannot create map file `'+fname+'`', TMsgType.Fatal);
419 result := '';
420 exit;
421 end;
422 try
423 res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
424 except
425 e_WriteLog('error downloading map file (exception) `'+FileName+'`', TMsgType.Fatal);
426 strm.Free;
427 result := '';
428 exit;
429 end;
430 strm.Free;
431 if (res <> 0) then
432 begin
433 e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
434 result := '';
435 exit;
436 end;
437 // if it was resumed, check md5 and initiate full download if necessary
438 if tf.resumed then
439 begin
440 md5 := MD5File(fname);
441 // sorry for pasta, i am asshole
442 if not MD5Match(md5, tf.hash) then
443 begin
444 e_LogWritefln('resuming failed; downloading map `%s` from scratch...', [fname]);
445 try
446 DeleteFile(fname);
447 strm := createDiskFile(fname);
448 except
449 e_WriteLog('cannot create map file `'+fname+'` (exception)', TMsgType.Fatal);
450 result := '';
451 exit;
452 end;
453 try
454 res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
455 except
456 e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
457 strm.Free;
458 result := '';
459 exit;
460 end;
461 strm.Free;
462 if (res <> 0) then
463 begin
464 e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
465 result := '';
466 exit;
467 end;
468 end;
469 end;
470 if (knownMaps.addWithHash(fname, mapHash)) then mapdbUpdated := true;
471 result := fname;
472 end;
474 // download resources
475 for f := 0 to High(resList) do
476 begin
477 // if we got a new-style reslist packet, use received data to check for resource files
478 if (resList[f].size < 0) then
479 begin
480 // old-style packet
481 transStarted := true;
482 res := g_Net_RequestResFileInfo(f, tf);
483 if (res <> 0) then begin result := ''; exit; end;
484 end
485 else
486 begin
487 // new-style packet
488 transStarted := false;
489 tf.diskName := resList[f].wadName;
490 tf.hash := resList[f].hash;
491 tf.size := resList[f].size;
492 end;
493 if (isIgnoredResWad(tf.diskName)) then
494 begin
495 // ignored file, abort download
496 if (transStarted) then g_Net_AbortResTransfer(tf);
497 e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]);
498 continue;
499 end;
500 wadname := findExistingResWadWithHash(tf.diskName, tf.hash);
501 if (length(wadname) <> 0) then
502 begin
503 // already here
504 if (transStarted) then g_Net_AbortResTransfer(tf);
505 addReplacementWad(tf.diskName, wadname);
506 end
507 else
508 begin
509 if (not transStarted) then
510 begin
511 res := g_Net_RequestResFileInfo(f, tf);
512 if (res <> 0) then begin result := ''; exit; end;
513 end;
514 try
515 destResDir := e_GetWriteableDir(WadDownloadDirs, false); // not required
516 except
517 end;
518 if (length(destResDir) = 0) then
519 begin
520 e_LogWriteln('cannot create wad download directory', TMsgType.Fatal);
521 result := '';
522 exit;
523 end;
524 fname := destResDir+'/'+generateFileName(tf.diskName, tf.hash);
525 e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]);
526 try
527 strm := openDiskFileRW(fname);
528 except
529 e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
530 result := '';
531 exit;
532 end;
533 try
534 res := g_Net_ReceiveResourceFile(f, tf, strm);
535 except
536 e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
537 strm.Free;
538 result := '';
539 exit;
540 end;
541 strm.Free;
542 if (res <> 0) then
543 begin
544 e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
545 result := '';
546 exit;
547 end;
548 // if it was resumed, check md5 and initiate full download if necessary
549 if tf.resumed then
550 begin
551 md5 := MD5File(fname);
552 // sorry for pasta, i am asshole
553 if not MD5Match(md5, tf.hash) then
554 begin
555 e_LogWritefln('resuming failed; downloading resource `%s` to `%s` from scratch...', [tf.diskName, fname]);
556 try
557 DeleteFile(fname);
558 strm := createDiskFile(fname);
559 except
560 e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
561 result := '';
562 exit;
563 end;
564 try
565 res := g_Net_ReceiveResourceFile(f, tf, strm);
566 except
567 e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
568 strm.Free;
569 result := '';
570 exit;
571 end;
572 strm.Free;
573 if (res <> 0) then
574 begin
575 e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
576 result := '';
577 exit;
578 end;
579 end;
580 end;
581 addReplacementWad(tf.diskName, fname);
582 if (knownRes.addWithHash(fname, tf.hash)) then resdbUpdated := true;
583 end;
584 end;
585 finally
586 SetLength(resList, 0);
587 g_Res_received_map_start := 0;
588 end;
590 if saveDBsToDiskEnabled and (mapdbUpdated or resdbUpdated) then saveDatabases(mapdbUpdated, resdbUpdated);
591 end;
594 initialization
595 conRegVar('rdl_ignore_names', @g_res_ignore_names, 'list of resource wad names (without extensions) to ignore in dl hash checks', 'dl ignore wads');
596 conRegVar('rdl_ignore_enabled', @g_res_ignore_enabled, 'enable dl hash check ignore list', 'dl hash check ignore list active');
597 conRegVar('rdl_hashdb_save_enabled', @g_res_save_databases, 'enable saving map/resource hash databases to disk', 'controls storing hash databases to disk');
598 end.