src/game/g_res_downloader.pas

   1 (* Copyright (C)  Doom 2D: Forever Developers
   2  *
   3  * This program is free software: you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation, version 3 of the License ONLY.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  14  *)
  15 {$INCLUDE ../shared/a_modes.inc}
  16 unit g_res_downloader;
  17
  18 interface
  19
  20 uses sysutils, Classes, md5, g_net, g_netmsg, g_console, g_main, e_log;
  21
  22
  23 // download map wad from server (if necessary)
  24 // download all required map resource wads too
  25 // registers all required replacement wads
  26 // returns name of the map wad (relative to mapdir), or empty string on error
  27 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
  28
  29 // returns original name, or replacement name
  30 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
  31
  32 // call this somewhere in startup sequence
  33 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
  34
  35
  36 implementation
  37
  38 uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb, e_res;
  39
  40 var
  41   // cvars
  42   g_res_ignore_names: AnsiString = 'standart;shrshade';
  43   g_res_ignore_enabled: Boolean = true;
  44   g_res_save_databases: Boolean = true;
  45   // other vars
  46   replacements: THashStrStr = nil;
  47   knownMaps: TFileHashDB = nil;
  48   knownRes: TFileHashDB = nil;
  49   saveDBsToDiskEnabled: Boolean = false; // this will be set to `true` if initial database saving succeed
  50
  51
  52 //==========================================================================
  53 //
  54 //  saveDatabases
  55 //
  56 //==========================================================================
  57 procedure saveDatabases (saveMap, saveRes: Boolean);
  58 var
  59   err: Boolean;
  60   st: TStream;
  61   ccdir: AnsiString = '';
  62 begin
  63   if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit;
  64   ccdir := e_GetWriteableDir(CacheDirs, false);
  65   if (length(ccdir) = 0) then exit;
  66   // rescan dirs
  67   // save map database
  68   if (saveMap) then
  69   begin
  70     err := true;
  71     st := nil;
  72     try
  73       st := createDiskFile(ccdir+'/maphash.db');
  74       knownMaps.saveTo(st);
  75       err := false;
  76     except
  77     end;
  78     st.Free;
  79     if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write map database, disk refresh disabled'); exit; end;
  80   end;
  81   // save resource database
  82   if (saveRes) then
  83   begin
  84     err := true;
  85     st := nil;
  86     try
  87       st := createDiskFile(ccdir+'/reshash.db');
  88       knownRes.saveTo(st);
  89       err := false;
  90     except
  91     end;
  92     st.Free;
  93     if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write resource database, disk refresh disabled'); exit; end;
  94   end;
  95 end;
  96
  97
  98 //==========================================================================
  99 //
 100 //  g_Res_CreateDatabases
 101 //
 102 //==========================================================================
 103 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
 104 var
 105   st: TStream;
 106   upmap: Boolean;
 107   upres: Boolean;
 108   forcesave: Boolean;
 109   ccdir: AnsiString = '';
 110 begin
 111   if not assigned(knownMaps) then
 112   begin
 113     // create and load a know map database, if necessary
 114     knownMaps := TFileHashDB.Create({GameDir}'', MapDirs);
 115     knownMaps.appendMoreDirs(MapDownloadDirs);
 116     knownRes := TFileHashDB.Create({GameDir}'', WadDirs);
 117     knownRes.appendMoreDirs(WadDownloadDirs);
 118     saveDBsToDiskEnabled := true;
 119     // load map database
 120     st := nil;
 121     try
 122       ccdir := e_GetWriteableDir(CacheDirs, false);
 123       if (length(ccdir) > 0) then
 124       begin
 125         st := openDiskFileRO(ccdir+'/maphash.db');
 126         knownMaps.loadFrom(st);
 127         e_LogWriteln('loaded map database');
 128       end;
 129     except
 130     end;
 131     st.Free;
 132     // load resource database
 133     st := nil;
 134     try
 135       if (length(ccdir) > 0) then
 136       begin
 137         st := openDiskFileRO(ccdir+'/reshash.db');
 138         knownRes.loadFrom(st);
 139         e_LogWriteln('loaded resource database');
 140       end;
 141     except
 142     end;
 143     st.Free;
 144     forcesave := true;
 145   end
 146   else
 147   begin
 148     if (not allowRescan) then exit;
 149     forcesave := false;
 150   end;
 151   // rescan dirs
 152   e_LogWriteln('refreshing map database');
 153   upmap := knownMaps.scanFiles();
 154   e_LogWriteln('refreshing resource database');
 155   upres := knownRes.scanFiles();
 156   // save databases
 157   if (forcesave) then begin upmap := true; upres := true; end;
 158   if upmap or upres then saveDatabases(upmap, upres);
 159 end;
 160
 161
 162 //==========================================================================
 163 //
 164 //  getWord
 165 //
 166 //  get next word from a string
 167 //  words are delimited with ';'
 168 //  ignores leading and trailing spaces
 169 //  returns empty string if there are no more words
 170 //
 171 //==========================================================================
 172 function getWord (var list: AnsiString): AnsiString;
 173 var
 174   pos: Integer;
 175 begin
 176   result := '';
 177   while (length(list) > 0) do
 178   begin
 179     if (ord(list[1]) <= 32) or (list[1] = ';') or (list[1] = ':') then begin Delete(list, 1, 1); continue; end;
 180     pos := 1;
 181     while (pos <= length(list)) and (list[pos] <> ';') and (list[pos] <> ':') do Inc(pos);
 182     result := Copy(list, 1, pos-1);
 183     Delete(list, 1, pos);
 184     while (length(result) > 0) and (ord(result[length(result)]) <= 32) do Delete(result, length(result), 1);
 185     if (length(result) > 0) then exit;
 186   end;
 187 end;
 188
 189
 190 //==========================================================================
 191 //
 192 //  isIgnoredResWad
 193 //
 194 //  checks if the given resource wad can be ignored
 195 //
 196 //  FIXME: preparse name list?
 197 //
 198 //==========================================================================
 199 function isIgnoredResWad (fname: AnsiString): Boolean;
 200 var
 201   list: AnsiString;
 202   name: AnsiString;
 203 begin
 204   result := false;
 205   if (not g_res_ignore_enabled) then exit;
 206   fname := forceFilenameExt(ExtractFileName(fname), '');
 207   list := g_res_ignore_names;
 208   name := getWord(list);
 209   while (length(name) > 0) do
 210   begin
 211     name := forceFilenameExt(name, '');
 212     //writeln('*** name=[', name, ']; fname=[', fname, ']');
 213     if (StrEquCI1251(name, fname)) then begin result := true; exit; end;
 214     name := getWord(list);
 215   end;
 216 end;
 217
 218
 219 //==========================================================================
 220 //
 221 //  clearReplacementWads
 222 //
 223 //  call this before downloading a new map from a server
 224 //
 225 //==========================================================================
 226 procedure clearReplacementWads ();
 227 begin
 228   if assigned(replacements) then replacements.clear();
 229   e_LogWriteln('cleared replacement wads');
 230 end;
 231
 232
 233 //==========================================================================
 234 //
 235 //  addReplacementWad
 236 //
 237 //  register new replacement wad
 238 //
 239 //==========================================================================
 240 procedure addReplacementWad (oldname: AnsiString; newDiskName: AnsiString);
 241 begin
 242   e_LogWritefln('adding replacement wad: oldname=%s; newname=%s', [oldname, newDiskName]);
 243   if not assigned(replacements) then replacements := THashStrStr.Create();
 244   replacements.put(toLowerCase1251(oldname), newDiskName);
 245 end;
 246
 247
 248 //==========================================================================
 249 //
 250 //  g_Res_FindReplacementWad
 251 //
 252 //  returns original name, or replacement name
 253 //
 254 //==========================================================================
 255 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
 256 var
 257   fn: AnsiString;
 258 begin
 259   //e_LogWritefln('LOOKING for replacement wad for [%s]...', [oldname], TMsgType.Notify);
 260   result := oldname;
 261   if not assigned(replacements) then exit;
 262   if (replacements.get(toLowerCase1251(ExtractFileName(oldname)), fn)) then
 263   begin
 264     //e_LogWritefln('found replacement wad for [%s] -> [%s]', [oldname, fn], TMsgType.Notify);
 265     result := fn;
 266   end;
 267 end;
 268
 269
 270 //==========================================================================
 271 //
 272 //  findExistingMapWadWithHash
 273 //
 274 //  find map or resource wad using its base name and hash
 275 //
 276 //  returns found wad disk name, or empty string
 277 //
 278 //==========================================================================
 279 function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
 280 begin
 281   result := knownMaps.findByHash(resMd5);
 282   if (length(result) > 0) then
 283   begin
 284     //result := GameDir+'/maps/'+result;
 285     if not FileExists(result) then
 286     begin
 287       if (knownMaps.scanFiles()) then saveDatabases(true, false);
 288       result := '';
 289     end;
 290   end;
 291 end;
 292
 293
 294 //==========================================================================
 295 //
 296 //  findExistingResWadWithHash
 297 //
 298 //  find map or resource wad using its base name and hash
 299 //
 300 //  returns found wad disk name, or empty string
 301 //
 302 //==========================================================================
 303 function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
 304 begin
 305   result := knownRes.findByHash(resMd5);
 306   if (length(result) > 0) then
 307   begin
 308     //result := GameDir+'/wads/'+result;
 309     if not FileExists(result) then
 310     begin
 311       if (knownRes.scanFiles()) then saveDatabases(false, true);
 312       result := '';
 313     end;
 314   end;
 315 end;
 316
 317
 318 //==========================================================================
 319 //
 320 //  generateFileName
 321 //
 322 //  generate new file name based on the given one and the hash
 323 //  you can pass files with pathes here too
 324 //
 325 //==========================================================================
 326 function generateFileName (fname: AnsiString; const hash: TMD5Digest): AnsiString;
 327 var
 328   mds: AnsiString;
 329   path: AnsiString;
 330   base: AnsiString;
 331   ext: AnsiString;
 332 begin
 333   mds := MD5Print(hash);
 334   if (length(mds) > 16) then mds := Copy(mds, 1, 16);
 335   mds := '_'+mds;
 336   if (length(fname) = 0) then begin result := mds; exit; end;
 337   path := ExtractFilePath(fname);
 338   base := ExtractFileName(fname);
 339   ext := getFilenameExt(base);
 340   base := forceFilenameExt(base, '');
 341   if (length(path) > 0) then result := IncludeTrailingPathDelimiter(path) else result := '';
 342   result := result+base+mds+ext;
 343 end;
 344
 345
 346 //==========================================================================
 347 //
 348 //  g_Res_DownloadMapWAD
 349 //
 350 //  download map wad from server (if necessary)
 351 //  download all required map resource wads too
 352 //  registers all required replacement wads
 353 //
 354 //  returns name of the map wad (relative to mapdir), or empty string on error
 355 //
 356 //==========================================================================
 357 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
 358 var
 359   tf: TNetFileTransfer;
 360   resList: array of TNetMapResourceInfo = nil;
 361   f, res: Integer;
 362   strm: TStream;
 363   fname: AnsiString;
 364   wadname: AnsiString;
 365   md5: TMD5Digest;
 366   mapdbUpdated: Boolean = false;
 367   resdbUpdated: Boolean = false;
 368   transStarted: Boolean;
 369   destMapDir: AnsiString = '';
 370   destResDir: AnsiString = '';
 371 begin
 372   result := '';
 373   clearReplacementWads();
 374   sfsGCCollect(); // why not?
 375   g_Res_CreateDatabases();
 376
 377   try
 378     g_Res_received_map_start := 1;
 379     g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
 380     e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
 381     g_Game_SetLoadingText(FileName + '...', 0, False);
 382
 383     FileName := ExtractFileName(FileName);
 384     if (length(FileName) = 0) then FileName := 'fucked_map_wad.wad';
 385
 386     // this also sends map request
 387     res := g_Net_Wait_MapInfo(tf, resList);
 388     if (res <> 0) then exit;
 389
 390     // find or download a map
 391     result := findExistingMapWadWithHash(tf.diskName, mapHash);
 392     if (length(result) = 0) then
 393     begin
 394       // download map
 395       res := g_Net_RequestResFileInfo(-1{map}, tf);
 396       if (res <> 0) then
 397       begin
 398         e_LogWriteln('error requesting map wad');
 399         result := '';
 400         exit;
 401       end;
 402       try
 403         destMapDir := e_GetWriteableDir(MapDownloadDirs, false); // not required
 404       except
 405       end;
 406       if (length(destMapDir) = 0) then
 407       begin
 408         e_LogWriteln('cannot create map download directory', TMsgType.Fatal);
 409         result := '';
 410         exit;
 411       end;
 412       fname := destMapDir+'/'+generateFileName(FileName, mapHash);
 413       tf.diskName := fname;
 414       e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal);
 415       try
 416         strm := openDiskFileRW(fname);
 417       except
 418         e_WriteLog('cannot create map file `'+fname+'`', TMsgType.Fatal);
 419         result := '';
 420         exit;
 421       end;
 422       try
 423         res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
 424       except
 425         e_WriteLog('error downloading map file (exception) `'+FileName+'`', TMsgType.Fatal);
 426         strm.Free;
 427         result := '';
 428         exit;
 429       end;
 430       strm.Free;
 431       if (res <> 0) then
 432       begin
 433         e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
 434         result := '';
 435         exit;
 436       end;
 437       // if it was resumed, check md5 and initiate full download if necessary
 438       if tf.resumed then
 439       begin
 440         md5 := MD5File(fname);
 441         // sorry for pasta, i am asshole
 442         if not MD5Match(md5, tf.hash) then
 443         begin
 444           e_LogWritefln('resuming failed; downloading map `%s` from scratch...', [fname]);
 445           try
 446             DeleteFile(fname);
 447             strm := createDiskFile(fname);
 448           except
 449             e_WriteLog('cannot create map file `'+fname+'` (exception)', TMsgType.Fatal);
 450             result := '';
 451             exit;
 452           end;
 453           try
 454             res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
 455           except
 456             e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 457             strm.Free;
 458             result := '';
 459             exit;
 460           end;
 461           strm.Free;
 462           if (res <> 0) then
 463           begin
 464             e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
 465             result := '';
 466             exit;
 467           end;
 468         end;
 469       end;
 470       if (knownMaps.addWithHash(fname, mapHash)) then mapdbUpdated := true;
 471       result := fname;
 472     end;
 473
 474     // download resources
 475     for f := 0 to High(resList) do
 476     begin
 477       // if we got a new-style reslist packet, use received data to check for resource files
 478       if (resList[f].size < 0) then
 479       begin
 480         // old-style packet
 481         transStarted := true;
 482         res := g_Net_RequestResFileInfo(f, tf);
 483         if (res <> 0) then begin result := ''; exit; end;
 484       end
 485       else
 486       begin
 487         // new-style packet
 488         transStarted := false;
 489         tf.diskName := resList[f].wadName;
 490         tf.hash := resList[f].hash;
 491         tf.size := resList[f].size;
 492       end;
 493       if (isIgnoredResWad(tf.diskName)) then
 494       begin
 495         // ignored file, abort download
 496         if (transStarted) then g_Net_AbortResTransfer(tf);
 497         e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]);
 498         continue;
 499       end;
 500       wadname := findExistingResWadWithHash(tf.diskName, tf.hash);
 501       if (length(wadname) <> 0) then
 502       begin
 503         // already here
 504         if (transStarted) then g_Net_AbortResTransfer(tf);
 505         addReplacementWad(tf.diskName, wadname);
 506       end
 507       else
 508       begin
 509         if (not transStarted) then
 510         begin
 511           res := g_Net_RequestResFileInfo(f, tf);
 512           if (res <> 0) then begin result := ''; exit; end;
 513         end;
 514         try
 515           destResDir := e_GetWriteableDir(WadDownloadDirs, false); // not required
 516         except
 517         end;
 518         if (length(destResDir) = 0) then
 519         begin
 520           e_LogWriteln('cannot create wad download directory', TMsgType.Fatal);
 521           result := '';
 522           exit;
 523         end;
 524         fname := destResDir+'/'+generateFileName(tf.diskName, tf.hash);
 525         e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]);
 526         try
 527           strm := openDiskFileRW(fname);
 528         except
 529           e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
 530           result := '';
 531           exit;
 532         end;
 533         try
 534           res := g_Net_ReceiveResourceFile(f, tf, strm);
 535         except
 536           e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 537           strm.Free;
 538           result := '';
 539           exit;
 540         end;
 541         strm.Free;
 542         if (res <> 0) then
 543         begin
 544           e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 545           result := '';
 546           exit;
 547         end;
 548         // if it was resumed, check md5 and initiate full download if necessary
 549         if tf.resumed then
 550         begin
 551           md5 := MD5File(fname);
 552           // sorry for pasta, i am asshole
 553           if not MD5Match(md5, tf.hash) then
 554           begin
 555             e_LogWritefln('resuming failed; downloading resource `%s` to `%s` from scratch...', [tf.diskName, fname]);
 556             try
 557               DeleteFile(fname);
 558               strm := createDiskFile(fname);
 559             except
 560               e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
 561               result := '';
 562               exit;
 563             end;
 564             try
 565               res := g_Net_ReceiveResourceFile(f, tf, strm);
 566             except
 567               e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 568               strm.Free;
 569               result := '';
 570               exit;
 571             end;
 572             strm.Free;
 573             if (res <> 0) then
 574             begin
 575               e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 576               result := '';
 577               exit;
 578             end;
 579           end;
 580         end;
 581         addReplacementWad(tf.diskName, fname);
 582         if (knownRes.addWithHash(fname, tf.hash)) then resdbUpdated := true;
 583       end;
 584     end;
 585   finally
 586     SetLength(resList, 0);
 587     g_Res_received_map_start := 0;
 588   end;
 589
 590   if saveDBsToDiskEnabled and (mapdbUpdated or resdbUpdated) then saveDatabases(mapdbUpdated, resdbUpdated);
 591 end;
 592
 593
 594 initialization
 595   conRegVar('rdl_ignore_names', @g_res_ignore_names, 'list of resource wad names (without extensions) to ignore in dl hash checks', 'dl ignore wads');
 596   conRegVar('rdl_ignore_enabled', @g_res_ignore_enabled, 'enable dl hash check ignore list', 'dl hash check ignore list active');
 597   conRegVar('rdl_hashdb_save_enabled', @g_res_save_databases, 'enable saving map/resource hash databases to disk', 'controls storing hash databases to disk');
 598 end.