index 78b0b2803ddccde223d34b5705f9def7191d5b63..f334e2f806d30ce5753aa533449888a99d15cfe8 100644 (file)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
+ * the Free Software Foundation, version 3 of the License ONLY.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
uses sysutils, Classes, md5, g_net, g_netmsg, g_console, g_main, e_log;
-function g_Res_SearchSameWAD(const path, filename: string; const resMd5: TMD5Digest): string;
-function g_Res_DownloadWAD(const FileName: string): string;
+
+// download map wad from server (if necessary)
+// download all required map resource wads too
+// registers all required replacement wads
+// returns name of the map wad (relative to mapdir), or empty string on error
+function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
+
+// returns original name, or replacement name
+function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
+
+// call this somewhere in startup sequence
+procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
+
implementation
-uses g_language, sfs, utils, wadreader, g_game;
+uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb, e_res;
+
+var
+ // cvars
+ g_res_ignore_names: AnsiString = 'standart;shrshade';
+ g_res_ignore_enabled: Boolean = true;
+ g_res_save_databases: Boolean = true;
+ // other vars
+ replacements: THashStrStr = nil;
+ knownMaps: TFileHashDB = nil;
+ knownRes: TFileHashDB = nil;
+ saveDBsToDiskEnabled: Boolean = false; // this will be set to `true` if initial database saving succeed
-const DOWNLOAD_DIR = 'downloads';
-procedure FindFiles(const dirName, filename: string; var files: TStringList);
+//==========================================================================
+//
+// saveDatabases
+//
+//==========================================================================
+procedure saveDatabases (saveMap, saveRes: Boolean);
var
- searchResult: TSearchRec;
+ err: Boolean;
+ st: TStream;
+ ccdir: AnsiString = '';
begin
- if FindFirst(dirName+'/*', faAnyFile, searchResult) = 0 then
+ if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit;
+ ccdir := e_GetWriteableDir(CacheDirs, false);
+ if (length(ccdir) = 0) then exit;
+ // rescan dirs
+ // save map database
+ if (saveMap) then
begin
+ err := true;
+ st := nil;
try
- repeat
- if (searchResult.Attr and faDirectory) = 0 then
- begin
- if StrEquCI1251(searchResult.Name, filename) then
- begin
- files.Add(dirName+'/'+filename);
- Exit;
- end;
- end
- else if (searchResult.Name <> '.') and (searchResult.Name <> '..') then
- FindFiles(IncludeTrailingPathDelimiter(dirName)+searchResult.Name, filename, files);
- until FindNext(searchResult) <> 0;
- finally
- FindClose(searchResult);
+ st := createDiskFile(ccdir+'/maphash.db');
+ knownMaps.saveTo(st);
+ err := false;
+ except
end;
+ st.Free;
+ if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write map database, disk refresh disabled'); exit; end;
+ end;
+ // save resource database
+ if (saveRes) then
+ begin
+ err := true;
+ st := nil;
+ try
+ st := createDiskFile(ccdir+'/reshash.db');
+ knownRes.saveTo(st);
+ err := false;
+ except
+ end;
+ st.Free;
+ if (err) then begin saveDBsToDiskEnabled := false; e_LogWriteln('cannot write resource database, disk refresh disabled'); exit; end;
end;
end;
-function CompareFileHash(const filename: string; const resMd5: TMD5Digest): Boolean;
+
+//==========================================================================
+//
+// g_Res_CreateDatabases
+//
+//==========================================================================
+procedure g_Res_CreateDatabases (allowRescan: Boolean=false);
var
- gResHash: TMD5Digest;
- fname: string;
+ st: TStream;
+ upmap: Boolean;
+ upres: Boolean;
+ forcesave: Boolean;
+ ccdir: AnsiString = '';
begin
- fname := findDiskWad(filename);
- if length(fname) = 0 then begin result := false; exit; end;
- gResHash := MD5File(fname);
- Result := MD5Match(gResHash, resMd5);
+ if not assigned(knownMaps) then
+ begin
+ // create and load a know map database, if necessary
+ knownMaps := TFileHashDB.Create({GameDir}'', MapDirs);
+ knownMaps.appendMoreDirs(MapDownloadDirs);
+ knownRes := TFileHashDB.Create({GameDir}'', WadDirs);
+ knownRes.appendMoreDirs(WadDownloadDirs);
+ saveDBsToDiskEnabled := true;
+ // load map database
+ st := nil;
+ try
+ ccdir := e_GetWriteableDir(CacheDirs, false);
+ if (length(ccdir) > 0) then
+ begin
+ st := openDiskFileRO(ccdir+'/maphash.db');
+ knownMaps.loadFrom(st);
+ e_LogWriteln('loaded map database');
+ end;
+ except
+ end;
+ st.Free;
+ // load resource database
+ st := nil;
+ try
+ if (length(ccdir) > 0) then
+ begin
+ st := openDiskFileRO(ccdir+'/reshash.db');
+ knownRes.loadFrom(st);
+ e_LogWriteln('loaded resource database');
+ end;
+ except
+ end;
+ st.Free;
+ forcesave := true;
+ end
+ else
+ begin
+ if (not allowRescan) then exit;
+ forcesave := false;
+ end;
+ // rescan dirs
+ e_LogWriteln('refreshing map database');
+ upmap := knownMaps.scanFiles();
+ e_LogWriteln('refreshing resource database');
+ upres := knownRes.scanFiles();
+ // save databases
+ if (forcesave) then begin upmap := true; upres := true; end;
+ if upmap or upres then saveDatabases(upmap, upres);
end;
-function CheckFileHash(const path, filename: string; const resMd5: TMD5Digest): Boolean;
+
+//==========================================================================
+//
+// getWord
+//
+// get next word from a string
+// words are delimited with ';'
+// ignores leading and trailing spaces
+// returns empty string if there are no more words
+//
+//==========================================================================
+function getWord (var list: AnsiString): AnsiString;
var
- fname: string;
+ pos: Integer;
begin
- fname := findDiskWad(path+filename);
- if length(fname) = 0 then begin result := false; exit; end;
- Result := FileExists(fname) and CompareFileHash(fname, resMd5);
+ result := '';
+ while (length(list) > 0) do
+ begin
+ if (ord(list[1]) <= 32) or (list[1] = ';') or (list[1] = ':') then begin Delete(list, 1, 1); continue; end;
+ pos := 1;
+ while (pos <= length(list)) and (list[pos] <> ';') and (list[pos] <> ':') do Inc(pos);
+ result := Copy(list, 1, pos-1);
+ Delete(list, 1, pos);
+ while (length(result) > 0) and (ord(result[length(result)]) <= 32) do Delete(result, length(result), 1);
+ if (length(result) > 0) then exit;
+ end;
end;
-function g_Res_SearchSameWAD(const path, filename: string; const resMd5: TMD5Digest): string;
+
+//==========================================================================
+//
+// isIgnoredResWad
+//
+// checks if the given resource wad can be ignored
+//
+// FIXME: preparse name list?
+//
+//==========================================================================
+function isIgnoredResWad (fname: AnsiString): Boolean;
var
- res: string;
- files: TStringList;
- i: Integer;
+ list: AnsiString;
+ name: AnsiString;
+begin
+ result := false;
+ if (not g_res_ignore_enabled) then exit;
+ fname := forceFilenameExt(ExtractFileName(fname), '');
+ list := g_res_ignore_names;
+ name := getWord(list);
+ while (length(name) > 0) do
+ begin
+ name := forceFilenameExt(name, '');
+ //writeln('*** name=[', name, ']; fname=[', fname, ']');
+ if (StrEquCI1251(name, fname)) then begin result := true; exit; end;
+ name := getWord(list);
+ end;
+end;
+
+
+//==========================================================================
+//
+// clearReplacementWads
+//
+// call this before downloading a new map from a server
+//
+//==========================================================================
+procedure clearReplacementWads ();
begin
- Result := '';
+ if assigned(replacements) then replacements.clear();
+ e_LogWriteln('cleared replacement wads');
+end;
+
+
+//==========================================================================
+//
+// addReplacementWad
+//
+// register new replacement wad
+//
+//==========================================================================
+procedure addReplacementWad (oldname: AnsiString; newDiskName: AnsiString);
+begin
+ e_LogWritefln('adding replacement wad: oldname=%s; newname=%s', [oldname, newDiskName]);
+ if not assigned(replacements) then replacements := THashStrStr.Create();
+ replacements.put(toLowerCase1251(oldname), newDiskName);
+end;
+
- if CheckFileHash(path, filename, resMd5) then
+//==========================================================================
+//
+// g_Res_FindReplacementWad
+//
+// returns original name, or replacement name
+//
+//==========================================================================
+function g_Res_FindReplacementWad (oldname: AnsiString): AnsiString;
+var
+ fn: AnsiString;
+begin
+ //e_LogWritefln('LOOKING for replacement wad for [%s]...', [oldname], TMsgType.Notify);
+ result := oldname;
+ if not assigned(replacements) then exit;
+ if (replacements.get(toLowerCase1251(ExtractFileName(oldname)), fn)) then
begin
- Result := path + filename;
- Exit;
+ //e_LogWritefln('found replacement wad for [%s] -> [%s]', [oldname, fn], TMsgType.Notify);
+ result := fn;
end;
+end;
- files := TStringList.Create;
- FindFiles(path, filename, files);
- for i := 0 to files.Count - 1 do
+//==========================================================================
+//
+// findExistingMapWadWithHash
+//
+// find map or resource wad using its base name and hash
+//
+// returns found wad disk name, or empty string
+//
+//==========================================================================
+function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
+begin
+ result := knownMaps.findByHash(resMd5);
+ if (length(result) > 0) then
begin
- res := files.Strings[i];
- if CompareFileHash(res, resMd5) then
+ //result := GameDir+'/maps/'+result;
+ if not FileExists(result) then
begin
- Result := res;
- Break;
+ if (knownMaps.scanFiles()) then saveDatabases(true, false);
+ result := '';
end;
end;
+end;
- files.Free;
+
+//==========================================================================
+//
+// findExistingResWadWithHash
+//
+// find map or resource wad using its base name and hash
+//
+// returns found wad disk name, or empty string
+//
+//==========================================================================
+function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString;
+begin
+ result := knownRes.findByHash(resMd5);
+ if (length(result) > 0) then
+ begin
+ //result := GameDir+'/wads/'+result;
+ if not FileExists(result) then
+ begin
+ if (knownRes.scanFiles()) then saveDatabases(false, true);
+ result := '';
+ end;
+ end;
end;
-function SaveWAD(const path, filename: string; const data: array of Byte): string;
+
+//==========================================================================
+//
+// generateFileName
+//
+// generate new file name based on the given one and the hash
+// you can pass files with pathes here too
+//
+//==========================================================================
+function generateFileName (fname: AnsiString; const hash: TMD5Digest): AnsiString;
var
- resFile: TStream;
- dpt: string;
+ mds: AnsiString;
+ path: AnsiString;
+ base: AnsiString;
+ ext: AnsiString;
begin
- try
- result := path+DOWNLOAD_DIR+'/'+filename;
- dpt := path+DOWNLOAD_DIR;
- if not findFileCI(dpt, true) then CreateDir(dpt);
- resFile := createDiskFile(result);
- resFile.WriteBuffer(data[0], Length(data));
- resFile.Free
- except
- Result := '';
- end;
+ mds := MD5Print(hash);
+ if (length(mds) > 16) then mds := Copy(mds, 1, 16);
+ mds := '_'+mds;
+ if (length(fname) = 0) then begin result := mds; exit; end;
+ path := ExtractFilePath(fname);
+ base := ExtractFileName(fname);
+ ext := getFilenameExt(base);
+ base := forceFilenameExt(base, '');
+ if (length(path) > 0) then result := IncludeTrailingPathDelimiter(path) else result := '';
+ result := result+base+mds+ext;
end;
-function g_Res_DownloadWAD(const FileName: string): string;
+
+//==========================================================================
+//
+// g_Res_DownloadMapWAD
+//
+// download map wad from server (if necessary)
+// download all required map resource wads too
+// registers all required replacement wads
+//
+// returns name of the map wad (relative to mapdir), or empty string on error
+//
+//==========================================================================
+function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString;
var
- msgStream: TMemoryStream;
- resStream: TStream;
- mapData: TMapDataMsg;
- i: Integer;
- resData: TResDataMsg;
+ tf: TNetFileTransfer;
+ resList: array of TNetMapResourceInfo = nil;
+ f, res: Integer;
+ strm: TStream;
+ fname: AnsiString;
+ wadname: AnsiString;
+ md5: TMD5Digest;
+ mapdbUpdated: Boolean = false;
+ resdbUpdated: Boolean = false;
+ transStarted: Boolean;
+ destMapDir: AnsiString = '';
+ destResDir: AnsiString = '';
begin
- SetLength(mapData.ExternalResources, 0);
- g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
- e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
- g_Game_SetLoadingText(FileName + '...', 0, False);
- MC_SEND_MapRequest();
-
- msgStream := g_Net_Wait_Event(NET_MSG_MAP_RESPONSE);
- if msgStream <> nil then
- begin
- mapData := MapDataFromMsgStream(msgStream);
- msgStream.Free;
- end else
- mapData.FileSize := 0;
+ result := '';
+ clearReplacementWads();
+ sfsGCCollect(); // why not?
+ g_Res_CreateDatabases();
- for i := 0 to High(mapData.ExternalResources) do
- begin
- if not CheckFileHash(GameDir + '/wads/',
- mapData.ExternalResources[i].Name,
- mapData.ExternalResources[i].md5) then
- begin
- g_Console_Add(Format(_lc[I_NET_WAD_DL],
- [mapData.ExternalResources[i].Name]));
- e_WriteLog('Downloading Wad `' + mapData.ExternalResources[i].Name +
- '` from server', TMsgType.Notify);
- g_Game_SetLoadingText(mapData.ExternalResources[i].Name + '...', 0, False);
- MC_SEND_ResRequest(mapData.ExternalResources[i].Name);
-
- msgStream := g_Net_Wait_Event(NET_MSG_RES_RESPONSE);
- if msgStream = nil then
- continue;
+ try
+ g_Res_received_map_start := 1;
+ g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName]));
+ e_WriteLog('Downloading map `' + FileName + '` from server', TMsgType.Notify);
+ g_Game_SetLoadingText(FileName + '...', 0, False);
+
+ FileName := ExtractFileName(FileName);
+ if (length(FileName) = 0) then FileName := 'fucked_map_wad.wad';
- resData := ResDataFromMsgStream(msgStream);
+ // this also sends map request
+ res := g_Net_Wait_MapInfo(tf, resList);
+ if (res <> 0) then exit;
- resStream := createDiskFile(GameDir+'/wads/'+mapData.ExternalResources[i].Name);
- resStream.WriteBuffer(resData.FileData[0], resData.FileSize);
+ // find or download a map
+ result := findExistingMapWadWithHash(tf.diskName, mapHash);
+ if (length(result) = 0) then
+ begin
+ // download map
+ res := g_Net_RequestResFileInfo(-1{map}, tf);
+ if (res <> 0) then
+ begin
+ e_LogWriteln('error requesting map wad');
+ result := '';
+ exit;
+ end;
+ try
+ destMapDir := e_GetWriteableDir(MapDownloadDirs, false); // not required
+ except
+ end;
+ if (length(destMapDir) = 0) then
+ begin
+ e_LogWriteln('cannot create map download directory', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ fname := destMapDir+'/'+generateFileName(FileName, mapHash);
+ tf.diskName := fname;
+ e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal);
+ try
+ strm := openDiskFileRW(fname);
+ except
+ e_WriteLog('cannot create map file `'+fname+'`', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ try
+ res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
+ except
+ e_WriteLog('error downloading map file (exception) `'+FileName+'`', TMsgType.Fatal);
+ strm.Free;
+ result := '';
+ exit;
+ end;
+ strm.Free;
+ if (res <> 0) then
+ begin
+ e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ // if it was resumed, check md5 and initiate full download if necessary
+ if tf.resumed then
+ begin
+ md5 := MD5File(fname);
+ // sorry for pasta, i am asshole
+ if not MD5Match(md5, tf.hash) then
+ begin
+ e_LogWritefln('resuming failed; downloading map `%s` from scratch...', [fname]);
+ try
+ DeleteFile(fname);
+ strm := createDiskFile(fname);
+ except
+ e_WriteLog('cannot create map file `'+fname+'` (exception)', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ try
+ res := g_Net_ReceiveResourceFile(-1{map}, tf, strm);
+ except
+ e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+ strm.Free;
+ result := '';
+ exit;
+ end;
+ strm.Free;
+ if (res <> 0) then
+ begin
+ e_LogWritefln('error downloading map `%s` (res=%d)', [FileName, res], TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ end;
+ end;
+ if (knownMaps.addWithHash(fname, mapHash)) then mapdbUpdated := true;
+ result := fname;
+ end;
- resData.FileData := nil;
- resStream.Free;
- msgStream.Free;
+ // download resources
+ for f := 0 to High(resList) do
+ begin
+ // if we got a new-style reslist packet, use received data to check for resource files
+ if (resList[f].size < 0) then
+ begin
+ // old-style packet
+ transStarted := true;
+ res := g_Net_RequestResFileInfo(f, tf);
+ if (res <> 0) then begin result := ''; exit; end;
+ end
+ else
+ begin
+ // new-style packet
+ transStarted := false;
+ tf.diskName := resList[f].wadName;
+ tf.hash := resList[f].hash;
+ tf.size := resList[f].size;
+ end;
+ if (isIgnoredResWad(tf.diskName)) then
+ begin
+ // ignored file, abort download
+ if (transStarted) then g_Net_AbortResTransfer(tf);
+ e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]);
+ continue;
+ end;
+ wadname := findExistingResWadWithHash(tf.diskName, tf.hash);
+ if (length(wadname) <> 0) then
+ begin
+ // already here
+ if (transStarted) then g_Net_AbortResTransfer(tf);
+ addReplacementWad(tf.diskName, wadname);
+ end
+ else
+ begin
+ if (not transStarted) then
+ begin
+ res := g_Net_RequestResFileInfo(f, tf);
+ if (res <> 0) then begin result := ''; exit; end;
+ end;
+ try
+ destResDir := e_GetWriteableDir(WadDownloadDirs, false); // not required
+ except
+ end;
+ if (length(destResDir) = 0) then
+ begin
+ e_LogWriteln('cannot create wad download directory', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ fname := destResDir+'/'+generateFileName(tf.diskName, tf.hash);
+ e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]);
+ try
+ strm := openDiskFileRW(fname);
+ except
+ e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ try
+ res := g_Net_ReceiveResourceFile(f, tf, strm);
+ except
+ e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+ strm.Free;
+ result := '';
+ exit;
+ end;
+ strm.Free;
+ if (res <> 0) then
+ begin
+ e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ // if it was resumed, check md5 and initiate full download if necessary
+ if tf.resumed then
+ begin
+ md5 := MD5File(fname);
+ // sorry for pasta, i am asshole
+ if not MD5Match(md5, tf.hash) then
+ begin
+ e_LogWritefln('resuming failed; downloading resource `%s` to `%s` from scratch...', [tf.diskName, fname]);
+ try
+ DeleteFile(fname);
+ strm := createDiskFile(fname);
+ except
+ e_WriteLog('cannot create resource file `'+fname+'`', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ try
+ res := g_Net_ReceiveResourceFile(f, tf, strm);
+ except
+ e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+ strm.Free;
+ result := '';
+ exit;
+ end;
+ strm.Free;
+ if (res <> 0) then
+ begin
+ e_WriteLog('error downloading map file `'+FileName+'`', TMsgType.Fatal);
+ result := '';
+ exit;
+ end;
+ end;
+ end;
+ addReplacementWad(tf.diskName, fname);
+ if (knownRes.addWithHash(fname, tf.hash)) then resdbUpdated := true;
+ end;
end;
+ finally
+ SetLength(resList, 0);
+ g_Res_received_map_start := 0;
end;
- Result := SaveWAD(MapsDir, ExtractFileName(FileName), mapData.FileData);
- if mapData.FileSize = 0 then
- DeleteFile(Result);
+ if saveDBsToDiskEnabled and (mapdbUpdated or resdbUpdated) then saveDatabases(mapdbUpdated, resdbUpdated);
end;
+
+initialization
+ conRegVar('rdl_ignore_names', @g_res_ignore_names, 'list of resource wad names (without extensions) to ignore in dl hash checks', 'dl ignore wads');
+ conRegVar('rdl_ignore_enabled', @g_res_ignore_enabled, 'enable dl hash check ignore list', 'dl hash check ignore list active');
+ conRegVar('rdl_hashdb_save_enabled', @g_res_save_databases, 'enable saving map/resource hash databases to disk', 'controls storing hash databases to disk');
end.