X-Git-Url: https://deadsoftware.ru/gitweb?a=blobdiff_plain;f=src%2Fgame%2Fg_res_downloader.pas;h=d67dc4c52db84c01abb20d58eb8f3ccae120908b;hb=4d7452d2a7340e4c245f5de3eec3c4d4a7e98fe2;hp=77db011c0c546aa9358fba72bf7b3cccfb0a4257;hpb=423556f23c02a18964bd2c1e125516c0c902ca46;p=d2df-sdl.git

diff --git a/src/game/g_res_downloader.pas b/src/game/g_res_downloader.pas
index 77db011..d67dc4c 100644
--- a/src/game/g_res_downloader.pas
+++ b/src/game/g_res_downloader.pas
@@ -1,155 +1,572 @@
+(* Copyright (C)  Doom 2D: Forever Developers
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3 of the License ONLY.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *)
+{$INCLUDE ../shared/a_modes.inc}
 unit g_res_downloader;
 
 interface
 
 uses sysutils, Classes, md5, g_net, g_netmsg, g_console, g_main, e_log;
 
-function g_Res_SearchSameWAD(const path, filename: string; const resMd5: TMD5Digest): string;
-function g_Res_DownloadWAD(const FileName: string): string;
+
+// download map wad from server (if necessary)
+// download all required map resource wads too
+// registers all required replacement wads
+// returns name of the map wad (relative to mapdir), or empty string on error
+function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
+
+// returns original name, or replacement name
+function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
+
+// call this somewhere in startup sequence
+procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
+
 
 implementation
 
-uses g_language, sfs, utils, wadreader;
+uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb;
 
-const DOWNLOAD_DIR = 'downloads';
+var
+  // cvars
+  g_res_ignore_names: AnsiString = 'standart;shrshade';
+  g_res_ignore_enabled: Boolean = true;
+  g_res_save_databases: Boolean = true;
+  // other vars
+  replacements: THashStrStr = nil;
+  knownMaps: TFileHashDB = nil;
+  knownRes: TFileHashDB = nil;
+  saveDBsToDiskEnabled: Boolean = false; // this will be set to `true` if initial database saving succeed
 
-procedure FindFiles(const dirName, filename: string; var files: TStringList);
+
+//==========================================================================
+//
+//  saveDatabases
+//
+//==========================================================================
+procedure saveDatabases (saveMap, saveRes: Boolean);
 var
-  searchResult: TSearchRec;
+  err: Boolean;
+  st: TStream;
 begin
-  if FindFirst(dirName+'/*', faAnyFile, searchResult) = 0 then
+  if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit;
+  // rescan dirs
+  // save map database
+  if (saveMap) then
   begin
+    err := true;
+    st := nil;
     try
-      repeat
-        if (searchResult.Attr and faDirectory) = 0 then
-        begin
-          if StrEquCI1251(searchResult.Name, filename) then
-          begin
-            files.Add(dirName+'/'+filename);
-            Exit;
-          end;
-        end
-        else if (searchResult.Name <> '.') and (searchResult.Name <> '..') then
-          FindFiles(IncludeTrailingPathDelimiter(dirName)+searchResult.Name, filename, files);
-      until FindNext(searchResult) <> 0;
-    finally
-      FindClose(searchResult);
+      st := createDiskFile(GameDir+'/data/maphash.db');
+      knownMaps.saveTo(st);
+      err := false;
+    except
     end;
+    st.Free;
+    if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write map database, disk refresh disabled'); exit; end;
+  end;
+  // save resource database
+  if (saveRes) then
+  begin
+    err := true;
+    st := nil;
+    try
+      st := createDiskFile(GameDir+'/data/reshash.db');
+      knownRes.saveTo(st);
+      err := false;
+    except
+    end;
+    st.Free;
+    if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write resource database, disk refresh disabled'); exit; end;
   end;
 end;
 
-function CompareFileHash(const filename: string; const resMd5: TMD5Digest): Boolean;
+
+//==========================================================================
+//
+//  g_Res_CreateDatabases
+//
+//==========================================================================
+procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
 var
-  gResHash: TMD5Digest;
-  fname: string;
+  st: TStream;
+  upmap: Boolean;
+  upres: Boolean;
+  forcesave: Boolean;
 begin
-  fname := findDiskWad(filename);
-  if length(fname) = 0 then begin result := false; exit; end;
-  gResHash := MD5File(fname);
-  Result := MD5Match(gResHash, resMd5);
+  if not assigned(knownMaps) then
+  begin
+    // create and load a know map database, if necessary
+    knownMaps := TFileHashDB.Create(GameDir+'/maps/');
+    knownRes := TFileHashDB.Create(GameDir+'/wads/');
+    saveDBsToDiskEnabled := true;
+    // load map database
+    st := nil;
+    try
+      st := openDiskFileRO(GameDir+'/data/maphash.db');
+      knownMaps.loadFrom(st);
+      e_LogWriteln('loaded map database');
+    except
+    end;
+    st.Free;
+    // load resource database
+    st := nil;
+    try
+      st := openDiskFileRO(GameDir+'/data/reshash.db');
+      knownRes.loadFrom(st);
+      e_LogWriteln('loaded resource database');
+    except
+    end;
+    st.Free;
+    forcesave := true;
+  end
+  else
+  begin
+    if (not allowRescan) then exit;
+    forcesave := false;
+  end;
+  // rescan dirs
+  e_LogWriteln('refreshing map database');
+  upmap := knownMaps.scanFiles();
+  e_LogWriteln('refreshing resource database');
+  upres := knownRes.scanFiles();
+  // save databases
+  if (forcesave) then begin upmap := true; upres := true; end;
+  if upmap or upres then saveDatabases(upmap, upres);
 end;
 
-function CheckFileHash(const path, filename: string; const resMd5: TMD5Digest): Boolean;
+
+//==========================================================================
+//
+//  getWord
+//
+//  get next word from a string
+//  words are delimited with ';'
+//  ignores leading and trailing spaces
+//  returns empty string if there are no more words
+//
+//==========================================================================
+function getWord (var list: AnsiString): AnsiString;
 var
-  fname: string;
+  pos: Integer;
 begin
-  fname := findDiskWad(path+filename);
-  if length(fname) = 0 then begin result := false; exit; end;
-  Result := FileExists(fname) and CompareFileHash(fname, resMd5);
+  result := '';
+  while (length(list) > 0) do
+  begin
+    if (ord(list[1]) <= 32) or (list[1] = ';') or (list[1] = ':') then begin Delete(list, 1, 1); continue; end;
+    pos := 1;
+    while (pos <= length(list)) and (list[pos] <> ';') and (list[pos] <> ':') do Inc(pos);
+    result := Copy(list, 1, pos-1);
+    Delete(list, 1, pos);
+    while (length(result) > 0) and (ord(result[length(result)]) <= 32) do Delete(result, length(result), 1);
+    if (length(result) > 0) then exit;
+  end;
 end;
 
-function g_Res_SearchSameWAD(const path, filename: string; const resMd5: TMD5Digest): string;
+
+//==========================================================================
+//
+//  isIgnoredResWad
+//
+//  checks if the given resource wad can be ignored
+//
+//  FIXME: preparse name list?
+//
+//==========================================================================
+function isIgnoredResWad (fname: AnsiString): Boolean;
 var
-  res: string;
-  files: TStringList;
-  i: Integer;
+  list: AnsiString;
+  name: AnsiString;
+begin
+  result := false;
+  if (not g_res_ignore_enabled) then exit;
+  fname := forceFilenameExt(ExtractFileName(fname), '');
+  list := g_res_ignore_names;
+  name := getWord(list);
+  while (length(name) > 0) do
+  begin
+    name := forceFilenameExt(name, '');
+    //writeln('*** name=[', name, ']; fname=[', fname, ']');
+    if (StrEquCI1251(name, fname)) then begin result := true; exit; end;
+    name := getWord(list);
+  end;
+end;
+
+
+//==========================================================================
+//
+//  clearReplacementWads
+//
+//  call this before downloading a new map from a server
+//
+//==========================================================================
+procedure clearReplacementWads ();
 begin
-  Result := '';
+  if assigned(replacements) then replacements.clear();
+  e_LogWriteln('cleared replacement wads');
+end;
+
+
+//==========================================================================
+//
+//  addReplacementWad
+//
+//  register new replacement wad
+//
+//==========================================================================
+procedure addReplacementWad (oldname: AnsiString; newDiskName: AnsiString);
+begin
+  e_LogWritefln('adding replacement wad: oldname=%s; newname=%s', [oldname, newDiskName]);
+  if not assigned(replacements) then replacements := THashStrStr.Create();
+  replacements.put(toLowerCase1251(oldname), newDiskName);
+end;
 
-  if CheckFileHash(path, filename, resMd5) then
+
+//==========================================================================
+//
+//  g_Res_FindReplacementWad
+//
+//  returns original name, or replacement name
+//
+//==========================================================================
+function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
+var
+  fn: AnsiString;
+begin
+  //e_LogWritefln('LOOKING for replacement wad for [%s]...', [oldname], TMsgType.Notify);
+  result := oldname;
+  if not assigned(replacements) then exit;
+  if (replacements.get(toLowerCase1251(ExtractFileName(oldname)), fn)) then
   begin
-    Result := path + filename;
-    Exit;
+    //e_LogWritefln('found replacement wad for [%s] -> [%s]', [oldname, fn], TMsgType.Notify);
+    result := fn;
   end;
+end;
 
-  files := TStringList.Create;
 
-  FindFiles(path, filename, files);
-  for i := 0 to files.Count - 1 do
+//==========================================================================
+//
+//  findExistingMapWadWithHash
+//
+//  find map or resource wad using its base name and hash
+//
+//  returns found wad disk name, or empty string
+//
+//==========================================================================
+function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
+begin
+  //result := scanDir(GameDir+'/maps', ExtractFileName(fname), resMd5);
+  result := knownMaps.findByHash(resMd5);
+  if (length(result) > 0) then
   begin
-    res := files.Strings[i];
-    if CompareFileHash(res, resMd5) then
+    result := GameDir+'/maps/'+result;
+    if not FileExists(result) then
     begin
-      Result := res;
-      Break;
+      if (knownMaps.scanFiles()) then saveDatabases(true, false);
+      result := '';
     end;
   end;
+end;
+
 
-  files.Free;
+//==========================================================================
+//
+//  findExistingResWadWithHash
+//
+//  find map or resource wad using its base name and hash
+//
+//  returns found wad disk name, or empty string
+//
+//==========================================================================
+function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
+begin
+  //result := scanDir(GameDir+'/wads', ExtractFileName(fname), resMd5);
+  result := knownRes.findByHash(resMd5);
+  if (length(result) > 0) then
+  begin
+    result := GameDir+'/wads/'+result;
+    if not FileExists(result) then
+    begin
+      if (knownRes.scanFiles()) then saveDatabases(false, true);
+      result := '';
+    end;
+  end;
 end;
 
-function SaveWAD(const path, filename: string; const data: array of Byte): string;
+
+//==========================================================================
+//
+//  generateFileName
+//
+//  generate new file name based on the given one and the hash
+//  you can pass files with pathes here too
+//
+//==========================================================================
+function generateFileName (fname: AnsiString; const hash: TMD5Digest): AnsiString;
 var
-  resFile: TStream;
-  dpt: string;
+  mds: AnsiString;
+  path: AnsiString;
+  base: AnsiString;
+  ext: AnsiString;
 begin
-  try
-    result := path+DOWNLOAD_DIR+'/'+filename;
-    dpt := path+DOWNLOAD_DIR;
-    if not findFileCI(dpt, true) then CreateDir(dpt);
-    resFile := createDiskFile(result);
-    resFile.WriteBuffer(data[0], Length(data));
-    resFile.Free
-  except
-    Result := '';
-  end;
+  mds := MD5Print(hash);
+  if (length(mds) > 16) then mds := Copy(mds, 1, 16);
+  mds := '_'+mds;
+  if (length(fname) = 0) then begin result := mds; exit; end;
+  path := ExtractFilePath(fname);
+  base := ExtractFileName(fname);
+  ext := getFilenameExt(base);
+  base := forceFilenameExt(base, '');
+  if (length(path) > 0) then result := IncludeTrailingPathDelimiter(path) else result := '';
+  result := result+base+mds+ext;
 end;
 
-function g_Res_DownloadWAD(const FileName: string): string;
+
+//==========================================================================
+//
+//  g_Res_DownloadMapWAD
+//
+//  download map wad from server (if necessary)
+//  download all required map resource wads too
+//  registers all required replacement wads
+//
+//  returns name of the map wad (relative to mapdir), or empty string on error
+//
+//==========================================================================
+function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
 var
-  msgStream: TMemoryStream;
-  resStream: TStream;
-  mapData: TMapDataMsg;
-  i: Integer;
-  resData: TResDataMsg;
+  tf: TNetFileTransfer;
+  resList: array of TNetMapResourceInfo = nil;
+  f, res: Integer;
+  strm: TStream;
+  fname: AnsiString;
+  wadname: AnsiString;
+  md5: TMD5Digest;
+  mapdbUpdated: Boolean = false;
+  resdbUpdated: Boolean = false;
+  transStarted: Boolean;
 begin
-  SetLength(mapData.ExternalResources, 0);
-  g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
-  e_WriteLog('Downloading map `' + FileName + '` from server', MSG_NOTIFY);
-  MC_SEND_MapRequest();
+  result := '';
+  clearReplacementWads();
+  g_Res_CreateDatabases();
 
-  msgStream := g_Net_Wait_Event(NET_MSG_MAP_RESPONSE);
-  if msgStream <> nil then
-  begin
-    mapData := MapDataFromMsgStream(msgStream);
-    msgStream.Free;
-  end;
+  try
+    g_Res_received_map_start := 1;
+    g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
+    e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
+    g_Game_SetLoadingText(FileName + '...', 0, False);
 
-  for i := 0 to High(mapData.ExternalResources) do
-  begin
-    if not CheckFileHash(GameDir + '/wads/',
-                         mapData.ExternalResources[i].Name,
-                         mapData.ExternalResources[i].md5) then
-    begin
-      g_Console_Add(Format(_lc[I_NET_WAD_DL],
-                           [mapData.ExternalResources[i].Name]));
-      e_WriteLog('Downloading Wad `' + mapData.ExternalResources[i].Name +
-                 '` from server', MSG_NOTIFY);
-      MC_SEND_ResRequest(mapData.ExternalResources[i].Name);
+    FileName := ExtractFileName(FileName);
+    if (length(FileName) = 0) then FileName := 'fucked_map_wad.wad';
 
-      msgStream := g_Net_Wait_Event(NET_MSG_RES_RESPONSE);
-      resData := ResDataFromMsgStream(msgStream);
+    // this also sends map request
+    res := g_Net_Wait_MapInfo(tf, resList);
+    if (res <> 0) then exit;
 
-      resStream := createDiskFile(GameDir+'/wads/'+mapData.ExternalResources[i].Name);
-      resStream.WriteBuffer(resData.FileData[0], resData.FileSize);
+    // find or download a map
+    result := findExistingMapWadWithHash(tf.diskName, mapHash);
+    if (length(result) = 0) then
+    begin
+      // download map
+      res := g_Net_RequestResFileInfo(-1{map}, tf);
+      if (res <> 0) then
+      begin
+        e_LogWriteln('error requesting map wad');
+        result := '';
+        exit;
+      end;
+      try
+        CreateDir(GameDir+'/maps/downloads');
+      except
+      end;
+      fname := GameDir+'/maps/downloads/'+generateFileName(FileName, mapHash);
+      tf.diskName := fname;
+      e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal);
+      try
+        strm := openDiskFileRW(fname);
+      except
+        e_WriteLog('cannot create map file `'+fname+'`', TMsgType.Fatal);
+        result := '';
+        exit;
+      end;
+      try
+        res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
+      except
+        e_WriteLog('error downloading map file (exception) `'+FileName+'`', TMsgType.Fatal);
+        strm.Free;
+        result := '';
+        exit;
+      end;
+      strm.Free;
+      if (res <> 0) then
+      begin
+        e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
+        result := '';
+        exit;
+      end;
+      // if it was resumed, check md5 and initiate full download if necessary
+      if tf.resumed then
+      begin
+        md5 := MD5File(fname);
+        // sorry for pasta, i am asshole
+        if not MD5Match(md5, tf.hash) then
+        begin
+          e_LogWritefln('resuming failed; downloading map `%s` from scratch...', [fname]);
+          try
+            DeleteFile(fname);
+            strm := createDiskFile(fname);
+          except
+            e_WriteLog('cannot create map file `'+fname+'` (exception)', TMsgType.Fatal);
+            result := '';
+            exit;
+          end;
+          try
+            res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
+          except
+            e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+            strm.Free;
+            result := '';
+            exit;
+          end;
+          strm.Free;
+          if (res <> 0) then
+          begin
+            e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
+            result := '';
+            exit;
+          end;
+        end;
+      end;
+      if (knownMaps.addWithHash(fname, mapHash)) then mapdbUpdated := true;
+      result := fname;
+    end;
 
-      resData.FileData := nil;
-      resStream.Free;
-      msgStream.Free;
+    // download resources
+    for f := 0 to High(resList) do
+    begin
+      // if we got a new-style reslist packet, use received data to check for resource files
+      if (resList[f].size < 0) then
+      begin
+        // old-style packet
+        transStarted := true;
+        res := g_Net_RequestResFileInfo(f, tf);
+        if (res <> 0) then begin result := ''; exit; end;
+      end
+      else
+      begin
+        // new-style packet
+        transStarted := false;
+        tf.diskName := resList[f].wadName;
+        tf.hash := resList[f].hash;
+        tf.size := resList[f].size;
+      end;
+      if (isIgnoredResWad(tf.diskName)) then
+      begin
+        // ignored file, abort download
+        if (transStarted) then g_Net_AbortResTransfer(tf);
+        e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]);
+        continue;
+      end;
+      wadname := findExistingResWadWithHash(tf.diskName, tf.hash);
+      if (length(wadname) <> 0) then
+      begin
+        // already here
+        if (transStarted) then g_Net_AbortResTransfer(tf);
+        addReplacementWad(tf.diskName, wadname);
+      end
+      else
+      begin
+        if (not transStarted) then
+        begin
+          res := g_Net_RequestResFileInfo(f, tf);
+          if (res <> 0) then begin result := ''; exit; end;
+        end;
+        try
+          CreateDir(GameDir+'/wads/downloads');
+        except
+        end;
+        fname := GameDir+'/wads/downloads/'+generateFileName(tf.diskName, tf.hash);
+        e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]);
+        try
+          strm := openDiskFileRW(fname);
+        except
+          e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
+          result := '';
+          exit;
+        end;
+        try
+          res := g_Net_ReceiveResourceFile(f, tf, strm);
+        except
+          e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+          strm.Free;
+          result := '';
+          exit;
+        end;
+        strm.Free;
+        if (res <> 0) then
+        begin
+          e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+          result := '';
+          exit;
+        end;
+        // if it was resumed, check md5 and initiate full download if necessary
+        if tf.resumed then
+        begin
+          md5 := MD5File(fname);
+          // sorry for pasta, i am asshole
+          if not MD5Match(md5, tf.hash) then
+          begin
+            e_LogWritefln('resuming failed; downloading resource `%s` to `%s` from scratch...', [tf.diskName, fname]);
+            try
+              DeleteFile(fname);
+              strm := createDiskFile(fname);
+            except
+              e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
+              result := '';
+              exit;
+            end;
+            try
+              res := g_Net_ReceiveResourceFile(f, tf, strm);
+            except
+              e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+              strm.Free;
+              result := '';
+              exit;
+            end;
+            strm.Free;
+            if (res <> 0) then
+            begin
+              e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+              result := '';
+              exit;
+            end;
+          end;
+        end;
+        addReplacementWad(tf.diskName, fname);
+        if (knownRes.addWithHash(fname, tf.hash)) then resdbUpdated := true;
+      end;
     end;
+  finally
+    SetLength(resList, 0);
+    g_Res_received_map_start := 0;
   end;
 
-  Result := SaveWAD(MapsDir, ExtractFileName(FileName), mapData.FileData);
+  if saveDBsToDiskEnabled and (mapdbUpdated or resdbUpdated) then saveDatabases(mapdbUpdated, resdbUpdated);
 end;
 
+
+initialization
+  conRegVar('rdl_ignore_names', @g_res_ignore_names, 'list of resource wad names (without extensions) to ignore in dl hash checks', 'dl ignore wads');
+  conRegVar('rdl_ignore_enabled', @g_res_ignore_enabled, 'enable dl hash check ignore list', 'dl hash check ignore list active');
+  conRegVar('rdl_hashdb_save_enabled', @g_res_save_databases, 'enable saving map/resource hash databases to disk', 'controls storing hash databases to disk');
 end.