src/game/g_res_downloader.pas

   1 (* Copyright (C)  Doom 2D: Forever Developers
   2  *
   3  * This program is free software: you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation, version 3 of the License ONLY.
   6  *
   7  * This program is distributed in the hope that it will be useful,
   8  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  * GNU General Public License for more details.
  11  *
  12  * You should have received a copy of the GNU General Public License
  13  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  14  *)
  15 {$INCLUDE ../shared/a_modes.inc}
  16 unit g_res_downloader;
  17
  18 interface
  19
  20 uses sysutils, Classes, md5, g_net, g_netmsg, g_console, g_main, e_log;
  21
  22
  23 // download map wad from server (if necessary)
  24 // download all required map resource wads too
  25 // registers all required replacement wads
  26 // returns name of the map wad (relative to mapdir), or empty string on error
  27 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
  28
  29 // returns original name, or replacement name
  30 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
  31
  32 // call this somewhere in startup sequence
  33 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
  34
  35
  36 implementation
  37
  38 uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb;
  39
  40 var
  41   // cvars
  42   g_res_ignore_names: AnsiString = 'standart;shrshade';
  43   g_res_ignore_enabled: Boolean = true;
  44   g_res_save_databases: Boolean = true;
  45   // other vars
  46   replacements: THashStrStr = nil;
  47   knownMaps: TFileHashDB = nil;
  48   knownRes: TFileHashDB = nil;
  49   saveDBsToDiskEnabled: Boolean = false; // this will be set to `true` if initial database saving succeed
  50
  51
  52 //==========================================================================
  53 //
  54 //  saveDatabases
  55 //
  56 //==========================================================================
  57 procedure saveDatabases (saveMap, saveRes: Boolean);
  58 var
  59   err: Boolean;
  60   st: TStream;
  61 begin
  62   if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit;
  63   // rescan dirs
  64   // save map database
  65   if (saveMap) then
  66   begin
  67     err := true;
  68     st := nil;
  69     try
  70       st := createDiskFile(GameDir+'/data/maphash.db');
  71       knownMaps.saveTo(st);
  72       err := false;
  73     except
  74     end;
  75     st.Free;
  76     if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write map database, disk refresh disabled'); exit; end;
  77   end;
  78   // save resource database
  79   if (saveRes) then
  80   begin
  81     err := true;
  82     st := nil;
  83     try
  84       st := createDiskFile(GameDir+'/data/reshash.db');
  85       knownRes.saveTo(st);
  86       err := false;
  87     except
  88     end;
  89     st.Free;
  90     if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write resource database, disk refresh disabled'); exit; end;
  91   end;
  92 end;
  93
  94
  95 //==========================================================================
  96 //
  97 //  g_Res_CreateDatabases
  98 //
  99 //==========================================================================
 100 procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
 101 var
 102   st: TStream;
 103   upmap: Boolean;
 104   upres: Boolean;
 105   forcesave: Boolean;
 106 begin
 107   if not assigned(knownMaps) then
 108   begin
 109     // create and load a know map database, if necessary
 110     knownMaps := TFileHashDB.Create(GameDir+'/maps/');
 111     knownRes := TFileHashDB.Create(GameDir+'/wads/');
 112     saveDBsToDiskEnabled := true;
 113     // load map database
 114     st := nil;
 115     try
 116       st := openDiskFileRO(GameDir+'/data/maphash.db');
 117       knownMaps.loadFrom(st);
 118       e_LogWriteln('loaded map database');
 119     except
 120     end;
 121     st.Free;
 122     // load resource database
 123     st := nil;
 124     try
 125       st := openDiskFileRO(GameDir+'/data/reshash.db');
 126       knownRes.loadFrom(st);
 127       e_LogWriteln('loaded resource database');
 128     except
 129     end;
 130     st.Free;
 131     forcesave := true;
 132   end
 133   else
 134   begin
 135     if (not allowRescan) then exit;
 136     forcesave := false;
 137   end;
 138   // rescan dirs
 139   e_LogWriteln('refreshing map database');
 140   upmap := knownMaps.scanFiles();
 141   e_LogWriteln('refreshing resource database');
 142   upres := knownRes.scanFiles();
 143   // save databases
 144   if (forcesave) then begin upmap := true; upres := true; end;
 145   if upmap or upres then saveDatabases(upmap, upres);
 146 end;
 147
 148
 149 //==========================================================================
 150 //
 151 //  getWord
 152 //
 153 //  get next word from a string
 154 //  words are delimited with ';'
 155 //  ignores leading and trailing spaces
 156 //  returns empty string if there are no more words
 157 //
 158 //==========================================================================
 159 function getWord (var list: AnsiString): AnsiString;
 160 var
 161   pos: Integer;
 162 begin
 163   result := '';
 164   while (length(list) > 0) do
 165   begin
 166     if (ord(list[1]) <= 32) or (list[1] = ';') or (list[1] = ':') then begin Delete(list, 1, 1); continue; end;
 167     pos := 1;
 168     while (pos <= length(list)) and (list[pos] <> ';') and (list[pos] <> ':') do Inc(pos);
 169     result := Copy(list, 1, pos-1);
 170     Delete(list, 1, pos);
 171     while (length(result) > 0) and (ord(result[length(result)]) <= 32) do Delete(result, length(result), 1);
 172     if (length(result) > 0) then exit;
 173   end;
 174 end;
 175
 176
 177 //==========================================================================
 178 //
 179 //  isIgnoredResWad
 180 //
 181 //  checks if the given resource wad can be ignored
 182 //
 183 //  FIXME: preparse name list?
 184 //
 185 //==========================================================================
 186 function isIgnoredResWad (fname: AnsiString): Boolean;
 187 var
 188   list: AnsiString;
 189   name: AnsiString;
 190 begin
 191   result := false;
 192   if (not g_res_ignore_enabled) then exit;
 193   fname := forceFilenameExt(ExtractFileName(fname), '');
 194   list := g_res_ignore_names;
 195   name := getWord(list);
 196   while (length(name) > 0) do
 197   begin
 198     name := forceFilenameExt(name, '');
 199     //writeln('*** name=[', name, ']; fname=[', fname, ']');
 200     if (StrEquCI1251(name, fname)) then begin result := true; exit; end;
 201     name := getWord(list);
 202   end;
 203 end;
 204
 205
 206 //==========================================================================
 207 //
 208 //  clearReplacementWads
 209 //
 210 //  call this before downloading a new map from a server
 211 //
 212 //==========================================================================
 213 procedure clearReplacementWads ();
 214 begin
 215   if assigned(replacements) then replacements.clear();
 216   e_LogWriteln('cleared replacement wads');
 217 end;
 218
 219
 220 //==========================================================================
 221 //
 222 //  addReplacementWad
 223 //
 224 //  register new replacement wad
 225 //
 226 //==========================================================================
 227 procedure addReplacementWad (oldname: AnsiString; newDiskName: AnsiString);
 228 begin
 229   e_LogWritefln('adding replacement wad: oldname=%s; newname=%s', [oldname, newDiskName]);
 230   if not assigned(replacements) then replacements := THashStrStr.Create();
 231   replacements.put(toLowerCase1251(oldname), newDiskName);
 232 end;
 233
 234
 235 //==========================================================================
 236 //
 237 //  g_Res_FindReplacementWad
 238 //
 239 //  returns original name, or replacement name
 240 //
 241 //==========================================================================
 242 function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
 243 var
 244   fn: AnsiString;
 245 begin
 246   //e_LogWritefln('LOOKING for replacement wad for [%s]...', [oldname], TMsgType.Notify);
 247   result := oldname;
 248   if not assigned(replacements) then exit;
 249   if (replacements.get(toLowerCase1251(ExtractFileName(oldname)), fn)) then
 250   begin
 251     //e_LogWritefln('found replacement wad for [%s] -> [%s]', [oldname, fn], TMsgType.Notify);
 252     result := fn;
 253   end;
 254 end;
 255
 256
 257 //==========================================================================
 258 //
 259 //  findExistingMapWadWithHash
 260 //
 261 //  find map or resource wad using its base name and hash
 262 //
 263 //  returns found wad disk name, or empty string
 264 //
 265 //==========================================================================
 266 function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
 267 begin
 268   //result := scanDir(GameDir+'/maps', ExtractFileName(fname), resMd5);
 269   result := knownMaps.findByHash(resMd5);
 270   if (length(result) > 0) then
 271   begin
 272     result := GameDir+'/maps/'+result;
 273     if not FileExists(result) then
 274     begin
 275       if (knownMaps.scanFiles()) then saveDatabases(true, false);
 276       result := '';
 277     end;
 278   end;
 279 end;
 280
 281
 282 //==========================================================================
 283 //
 284 //  findExistingResWadWithHash
 285 //
 286 //  find map or resource wad using its base name and hash
 287 //
 288 //  returns found wad disk name, or empty string
 289 //
 290 //==========================================================================
 291 function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
 292 begin
 293   //result := scanDir(GameDir+'/wads', ExtractFileName(fname), resMd5);
 294   result := knownRes.findByHash(resMd5);
 295   if (length(result) > 0) then
 296   begin
 297     result := GameDir+'/wads/'+result;
 298     if not FileExists(result) then
 299     begin
 300       if (knownRes.scanFiles()) then saveDatabases(false, true);
 301       result := '';
 302     end;
 303   end;
 304 end;
 305
 306
 307 //==========================================================================
 308 //
 309 //  generateFileName
 310 //
 311 //  generate new file name based on the given one and the hash
 312 //  you can pass files with pathes here too
 313 //
 314 //==========================================================================
 315 function generateFileName (fname: AnsiString; const hash: TMD5Digest): AnsiString;
 316 var
 317   mds: AnsiString;
 318   path: AnsiString;
 319   base: AnsiString;
 320   ext: AnsiString;
 321 begin
 322   mds := MD5Print(hash);
 323   if (length(mds) > 16) then mds := Copy(mds, 1, 16);
 324   mds := '_'+mds;
 325   if (length(fname) = 0) then begin result := mds; exit; end;
 326   path := ExtractFilePath(fname);
 327   base := ExtractFileName(fname);
 328   ext := getFilenameExt(base);
 329   base := forceFilenameExt(base, '');
 330   if (length(path) > 0) then result := IncludeTrailingPathDelimiter(path) else result := '';
 331   result := result+base+mds+ext;
 332 end;
 333
 334
 335 //==========================================================================
 336 //
 337 //  g_Res_DownloadMapWAD
 338 //
 339 //  download map wad from server (if necessary)
 340 //  download all required map resource wads too
 341 //  registers all required replacement wads
 342 //
 343 //  returns name of the map wad (relative to mapdir), or empty string on error
 344 //
 345 //==========================================================================
 346 function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
 347 var
 348   tf: TNetFileTransfer;
 349   resList: array of TNetMapResourceInfo = nil;
 350   f, res: Integer;
 351   strm: TStream;
 352   fname: AnsiString;
 353   wadname: AnsiString;
 354   md5: TMD5Digest;
 355   mapdbUpdated: Boolean = false;
 356   resdbUpdated: Boolean = false;
 357   transStarted: Boolean;
 358 begin
 359   result := '';
 360   clearReplacementWads();
 361   g_Res_CreateDatabases();
 362
 363   try
 364     g_Res_received_map_start := 1;
 365     g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
 366     e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
 367     g_Game_SetLoadingText(FileName + '...', 0, False);
 368
 369     FileName := ExtractFileName(FileName);
 370     if (length(FileName) = 0) then FileName := 'fucked_map_wad.wad';
 371
 372     // this also sends map request
 373     res := g_Net_Wait_MapInfo(tf, resList);
 374     if (res <> 0) then exit;
 375
 376     // find or download a map
 377     result := findExistingMapWadWithHash(tf.diskName, mapHash);
 378     if (length(result) = 0) then
 379     begin
 380       // download map
 381       res := g_Net_RequestResFileInfo(-1{map}, tf);
 382       if (res <> 0) then
 383       begin
 384         e_LogWriteln('error requesting map wad');
 385         result := '';
 386         exit;
 387       end;
 388       try
 389         CreateDir(GameDir+'/maps/downloads');
 390       except
 391       end;
 392       fname := GameDir+'/maps/downloads/'+generateFileName(FileName, mapHash);
 393       tf.diskName := fname;
 394       e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal);
 395       try
 396         strm := openDiskFileRW(fname);
 397       except
 398         e_WriteLog('cannot create map file `'+fname+'`', TMsgType.Fatal);
 399         result := '';
 400         exit;
 401       end;
 402       try
 403         res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
 404       except
 405         e_WriteLog('error downloading map file (exception) `'+FileName+'`', TMsgType.Fatal);
 406         strm.Free;
 407         result := '';
 408         exit;
 409       end;
 410       strm.Free;
 411       if (res <> 0) then
 412       begin
 413         e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
 414         result := '';
 415         exit;
 416       end;
 417       // if it was resumed, check md5 and initiate full download if necessary
 418       if tf.resumed then
 419       begin
 420         md5 := MD5File(fname);
 421         // sorry for pasta, i am asshole
 422         if not MD5Match(md5, tf.hash) then
 423         begin
 424           e_LogWritefln('resuming failed; downloading map `%s` from scratch...', [fname]);
 425           try
 426             DeleteFile(fname);
 427             strm := createDiskFile(fname);
 428           except
 429             e_WriteLog('cannot create map file `'+fname+'` (exception)', TMsgType.Fatal);
 430             result := '';
 431             exit;
 432           end;
 433           try
 434             res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
 435           except
 436             e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 437             strm.Free;
 438             result := '';
 439             exit;
 440           end;
 441           strm.Free;
 442           if (res <> 0) then
 443           begin
 444             e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
 445             result := '';
 446             exit;
 447           end;
 448         end;
 449       end;
 450       if (knownMaps.addWithHash(fname, mapHash)) then mapdbUpdated := true;
 451       result := fname;
 452     end;
 453
 454     // download resources
 455     for f := 0 to High(resList) do
 456     begin
 457       // if we got a new-style reslist packet, use received data to check for resource files
 458       if (resList[f].size < 0) then
 459       begin
 460         // old-style packet
 461         transStarted := true;
 462         res := g_Net_RequestResFileInfo(f, tf);
 463         if (res <> 0) then begin result := ''; exit; end;
 464       end
 465       else
 466       begin
 467         // new-style packet
 468         transStarted := false;
 469         tf.diskName := resList[f].wadName;
 470         tf.hash := resList[f].hash;
 471         tf.size := resList[f].size;
 472       end;
 473       if (isIgnoredResWad(tf.diskName)) then
 474       begin
 475         // ignored file, abort download
 476         if (transStarted) then g_Net_AbortResTransfer(tf);
 477         e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]);
 478         continue;
 479       end;
 480       wadname := findExistingResWadWithHash(tf.diskName, tf.hash);
 481       if (length(wadname) <> 0) then
 482       begin
 483         // already here
 484         if (transStarted) then g_Net_AbortResTransfer(tf);
 485         addReplacementWad(tf.diskName, wadname);
 486       end
 487       else
 488       begin
 489         if (not transStarted) then
 490         begin
 491           res := g_Net_RequestResFileInfo(f, tf);
 492           if (res <> 0) then begin result := ''; exit; end;
 493         end;
 494         try
 495           CreateDir(GameDir+'/wads/downloads');
 496         except
 497         end;
 498         fname := GameDir+'/wads/downloads/'+generateFileName(tf.diskName, tf.hash);
 499         e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]);
 500         try
 501           strm := openDiskFileRW(fname);
 502         except
 503           e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
 504           result := '';
 505           exit;
 506         end;
 507         try
 508           res := g_Net_ReceiveResourceFile(f, tf, strm);
 509         except
 510           e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 511           strm.Free;
 512           result := '';
 513           exit;
 514         end;
 515         strm.Free;
 516         if (res <> 0) then
 517         begin
 518           e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 519           result := '';
 520           exit;
 521         end;
 522         // if it was resumed, check md5 and initiate full download if necessary
 523         if tf.resumed then
 524         begin
 525           md5 := MD5File(fname);
 526           // sorry for pasta, i am asshole
 527           if not MD5Match(md5, tf.hash) then
 528           begin
 529             e_LogWritefln('resuming failed; downloading resource `%s` to `%s` from scratch...', [tf.diskName, fname]);
 530             try
 531               DeleteFile(fname);
 532               strm := createDiskFile(fname);
 533             except
 534               e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
 535               result := '';
 536               exit;
 537             end;
 538             try
 539               res := g_Net_ReceiveResourceFile(f, tf, strm);
 540             except
 541               e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 542               strm.Free;
 543               result := '';
 544               exit;
 545             end;
 546             strm.Free;
 547             if (res <> 0) then
 548             begin
 549               e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
 550               result := '';
 551               exit;
 552             end;
 553           end;
 554         end;
 555         addReplacementWad(tf.diskName, fname);
 556         if (knownRes.addWithHash(fname, tf.hash)) then resdbUpdated := true;
 557       end;
 558     end;
 559   finally
 560     SetLength(resList, 0);
 561     g_Res_received_map_start := 0;
 562   end;
 563
 564   if saveDBsToDiskEnabled and (mapdbUpdated or resdbUpdated) then saveDatabases(mapdbUpdated, resdbUpdated);
 565 end;
 566
 567
 568 initialization
 569   conRegVar('rdl_ignore_names', @g_res_ignore_names, 'list of resource wad names (without extensions) to ignore in dl hash checks', 'dl ignore wads');
 570   conRegVar('rdl_ignore_enabled', @g_res_ignore_enabled, 'enable dl hash check ignore list', 'dl hash check ignore list active');
 571   conRegVar('rdl_hashdb_save_enabled', @g_res_save_databases, 'enable saving map/resource hash databases to disk', 'controls storing hash databases to disk');
 572 end.