From: Ketmar Dark Date: Sun, 20 Oct 2019 23:15:06 +0000 (+0300) Subject: net: game: other: hash database and resource downloader converted to new dirsys X-Git-Url: https://deadsoftware.ru/gitweb?p=d2df-sdl.git;a=commitdiff_plain;h=93f620df25ad634372657a64d609dcaf44e718bd net: game: other: hash database and resource downloader converted to new dirsys --- diff --git a/src/game/g_game.pas b/src/game/g_game.pas index e371e0b..6564a23 100644 --- a/src/game/g_game.pas +++ b/src/game/g_game.pas @@ -6750,6 +6750,7 @@ begin if found then begin // no such map, found wad + pw := P[1]; SetLength(P, 3); P[1] := ExpandFileName(pw); P[2] := g_Game_GetFirstMap(P[1]); diff --git a/src/game/g_main.pas b/src/game/g_main.pas index b064804..808f7f6 100644 --- a/src/game/g_main.pas +++ b/src/game/g_main.pas @@ -29,7 +29,7 @@ procedure CharPress (C: AnsiChar); var {--- TO REMOVE ---} - GameDir: string; + //GameDir: string; {-----------------} {--- Read-only dirs ---} @@ -86,7 +86,7 @@ procedure InitPath; end; begin - GetDir(0, GameDir); + //GetDir(0, GameDir); i := 1; while i < ParamCount do @@ -138,7 +138,7 @@ begin AddDef(CacheDirs, 'data/cache'); AddDef(ConfigDirs, '.'); AddDef(MapDownloadDirs, 'maps/downloads'); - AddDef(WadDownloadDirs, 'wad/downloads'); + AddDef(WadDownloadDirs, 'wads/downloads'); AddDef(ScreenshotDirs, 'screenshots'); for i := 0 to High(MapDirs) do @@ -266,7 +266,7 @@ begin if assigned(oglInitCB) then oglInitCB; {$ENDIF} - //g_Res_CreateDatabases(); // it will be done before connecting to the server for the first time + //g_Res_CreateDatabases(true); // it will be done before connecting to the server for the first time e_WriteLog('Entering SDLMain', TMsgType.Notify); diff --git a/src/game/g_map.pas b/src/game/g_map.pas index 95bb1a0..e99d47b 100644 --- a/src/game/g_map.pas +++ b/src/game/g_map.pas @@ -900,21 +900,100 @@ begin end; +function extractWadName (resourceName: string): string; +var + posN: Integer; +begin + posN := Pos(':', resourceName); + if posN > 0 then + Result:= Copy(resourceName, 0, posN-1) + else + Result := ''; +end; + + +procedure addResToExternalResList (res: AnsiString); +var + uname: AnsiString; + f: Integer; + fi: TDiskFileInfo; +begin + if g_Game_IsClient or not g_Game_IsNet then exit; + if (length(res) = 0) then exit; // map wad + //res := extractWadName(res); + //if (length(res) = 0) then exit; // map wad + uname := toLowerCase1251(res); + // do not add duplicates + for f := 0 to High(gExternalResources) do + begin + if (gExternalResources[f].userName = uname) then exit; + end; + //writeln('***(000) addResToExternalResList: res=[', res, ']'); + // add new resource + fi.userName := uname; + if not findFileCI(res) then exit; + //writeln('***(001) addResToExternalResList: res=[', res, ']'); + fi.diskName := res; + if (not GetDiskFileInfo(res, fi)) then + begin + fi.tag := -1; + end + else + begin + //writeln('***(002) addResToExternalResList: res=[', res, ']'); + fi.tag := 0; // non-zero means "cannot caclucate hash" + try + fi.hash := MD5File(fi.diskName); + except + fi.tag := -1; + end; + end; + //e_LogWritefln('addext: res=[%s]; uname=[%s]; diskName=[%s]', [res, fi.userName, fi.diskName]); + SetLength(gExternalResources, length(gExternalResources)+1); + gExternalResources[High(gExternalResources)] := fi; +end; + + +procedure compactExtResList (); +var + src, dest: Integer; +begin + src := 0; + dest := 0; + for src := 0 to High(gExternalResources) do + begin + if (gExternalResources[src].tag = 0) then + begin + // copy it + if (dest <> src) then gExternalResources[dest] := gExternalResources[src]; + Inc(dest); + end; + end; + if (dest <> length(gExternalResources)) then SetLength(gExternalResources, dest); +end; + + function GetReplacementWad (WadName: AnsiString): AnsiString; begin result := ''; if WadName <> '' then begin result := WadName; - if g_Game_IsClient then - result := g_Res_FindReplacementWad(WadName); - if (result = WadName) then - result := e_FindWad(WadDirs, result) + if g_Game_IsClient then result := g_Res_FindReplacementWad(WadName); + if (result = WadName) then result := e_FindWad(WadDirs, result) end; end; -function CreateTexture(RecName: AnsiString; Map: string; log: Boolean): Integer; +procedure generateExternalResourcesList (map: TDynRecord); +begin + SetLength(gExternalResources, 0); + addResToExternalResList(GetReplacementWad(g_ExtractWadName(map.MusicName))); + addResToExternalResList(GetReplacementWad(g_ExtractWadName(map.SkyName))); +end; + + +function CreateTexture (RecName: AnsiString; Map: string; log: Boolean): Integer; var WAD: TWADFile; TextureData: Pointer; @@ -973,6 +1052,7 @@ begin // Çàãðóæàåì ðåñóðñ òåêñòóðû â ïàìÿòü èç WAD'à: WADName := GetReplacementWad(g_ExtractWadName(RecName)); + if (WADName <> '') then addResToExternalResList(WADName); if WADName = '' then WADName := Map; //WADName := GameDir+'/wads/'+WADName else WAD := TWADFile.Create(); @@ -1059,6 +1139,7 @@ begin // ×èòàåì WAD-ðåñóðñ àíèì.òåêñòóðû èç WAD'à â ïàìÿòü: WADName := GetReplacementWad(g_ExtractWadName(RecName)); + if (WADName <> '') then addResToExternalResList(WADName); if WADName = '' then WADName := Map; //WADName := GameDir+'/wads/'+WADName else WAD := TWADFile.Create(); @@ -1449,82 +1530,6 @@ begin g_Mons_ForEach(monsDieTrig); end; -function extractWadName(resourceName: string): string; -var - posN: Integer; -begin - posN := Pos(':', resourceName); - if posN > 0 then - Result:= Copy(resourceName, 0, posN-1) - else - Result := ''; -end; - - -procedure addResToExternalResList (res: AnsiString); -var - uname: AnsiString; - f: Integer; - fi: TDiskFileInfo; -begin - if g_Game_IsClient or not g_Game_IsNet then exit; - if (length(res) = 0) then exit; // map wad - res := extractWadName(res); - if (length(res) = 0) then exit; // map wad - uname := toLowerCase1251(res); - // do not add duplicates - for f := 0 to High(gExternalResources) do - begin - if (gExternalResources[f].userName = uname) then exit; - end; - // add new resource - fi.userName := uname; - if (not GetDiskFileInfo(GameDir+'/wads/'+res, fi)) then - begin - fi.tag := -1; - end - else - begin - fi.tag := 0; // non-zero means "cannot caclucate hash" - try - fi.hash := MD5File(fi.diskName); - except - fi.tag := -1; - end; - end; - //e_LogWritefln('addext: res=[%s]; uname=[%s]; diskName=[%s]', [res, fi.userName, fi.diskName]); - SetLength(gExternalResources, length(gExternalResources)+1); - gExternalResources[High(gExternalResources)] := fi; -end; - - -procedure compactExtResList (); -var - src, dest: Integer; -begin - src := 0; - dest := 0; - for src := 0 to High(gExternalResources) do - begin - if (gExternalResources[src].tag = 0) then - begin - // copy it - if (dest <> src) then gExternalResources[dest] := gExternalResources[src]; - Inc(dest); - end; - end; - if (dest <> length(gExternalResources)) then SetLength(gExternalResources, dest); -end; - - -procedure generateExternalResourcesList (map: TDynRecord); -begin - SetLength(gExternalResources, 0); - addResToExternalResList(map.MusicName); - addResToExternalResList(map.SkyName); -end; - - procedure mapCreateGrid (); var mapX0: Integer = $3fffffff; @@ -1864,14 +1869,7 @@ begin ntn := CreateTexture(rec.Resource, FileName, True); if (ntn < 0) then g_SimpleError(Format(_lc[I_GAME_ERROR_TEXTURE_SIMPLE], [rec.Resource])); end; - if (ntn < 0) then - begin - ntn := CreateNullTexture(rec.Resource); - end - else - begin - addResToExternalResList(rec.Resource); - end; + if (ntn < 0) then ntn := CreateNullTexture(rec.Resource); rec.tagInt := ntn; // remember texture number end; diff --git a/src/game/g_net.pas b/src/game/g_net.pas index 099ab76..2e00600 100644 --- a/src/game/g_net.pas +++ b/src/game/g_net.pas @@ -540,7 +540,6 @@ begin exit; end; tf.diskName := findDiskWad(fname); - //if (length(tf.diskName) = 0) then tf.diskName := findDiskWad(GameDir+'/wads/'+fname); if (length(tf.diskName) = 0) then begin e_LogWritefln('NETWORK: file "%s" not found!', [fname], TMsgType.Fatal); diff --git a/src/game/g_res_downloader.pas b/src/game/g_res_downloader.pas index 43a6087..f334e2f 100644 --- a/src/game/g_res_downloader.pas +++ b/src/game/g_res_downloader.pas @@ -35,7 +35,7 @@ procedure g_Res_CreateDatabases (allowRescan: Boolean=false); implementation -uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb; +uses g_language, sfs, utils, wadreader, g_game, hashtable, fhashdb, e_res; var // cvars @@ -58,8 +58,11 @@ procedure saveDatabases (saveMap, saveRes: Boolean); var err: Boolean; st: TStream; + ccdir: AnsiString = ''; begin if (not saveDBsToDiskEnabled) or (not g_res_save_databases) then exit; + ccdir := e_GetWriteableDir(CacheDirs, false); + if (length(ccdir) = 0) then exit; // rescan dirs // save map database if (saveMap) then @@ -67,7 +70,7 @@ begin err := true; st := nil; try - st := createDiskFile(GameDir+'/data/maphash.db'); + st := createDiskFile(ccdir+'/maphash.db'); knownMaps.saveTo(st); err := false; except @@ -81,7 +84,7 @@ begin err := true; st := nil; try - st := createDiskFile(GameDir+'/data/reshash.db'); + st := createDiskFile(ccdir+'/reshash.db'); knownRes.saveTo(st); err := false; except @@ -103,28 +106,38 @@ var upmap: Boolean; upres: Boolean; forcesave: Boolean; + ccdir: AnsiString = ''; begin if not assigned(knownMaps) then begin // create and load a know map database, if necessary - knownMaps := TFileHashDB.Create(GameDir+'/maps/'); - knownRes := TFileHashDB.Create(GameDir+'/wads/'); + knownMaps := TFileHashDB.Create({GameDir}'', MapDirs); + knownMaps.appendMoreDirs(MapDownloadDirs); + knownRes := TFileHashDB.Create({GameDir}'', WadDirs); + knownRes.appendMoreDirs(WadDownloadDirs); saveDBsToDiskEnabled := true; // load map database st := nil; try - st := openDiskFileRO(GameDir+'/data/maphash.db'); - knownMaps.loadFrom(st); - e_LogWriteln('loaded map database'); + ccdir := e_GetWriteableDir(CacheDirs, false); + if (length(ccdir) > 0) then + begin + st := openDiskFileRO(ccdir+'/maphash.db'); + knownMaps.loadFrom(st); + e_LogWriteln('loaded map database'); + end; except end; st.Free; // load resource database st := nil; try - st := openDiskFileRO(GameDir+'/data/reshash.db'); - knownRes.loadFrom(st); - e_LogWriteln('loaded resource database'); + if (length(ccdir) > 0) then + begin + st := openDiskFileRO(ccdir+'/reshash.db'); + knownRes.loadFrom(st); + e_LogWriteln('loaded resource database'); + end; except end; st.Free; @@ -265,11 +278,10 @@ end; //========================================================================== function findExistingMapWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString; begin - //result := scanDir(GameDir+'/maps', ExtractFileName(fname), resMd5); result := knownMaps.findByHash(resMd5); if (length(result) > 0) then begin - result := GameDir+'/maps/'+result; + //result := GameDir+'/maps/'+result; if not FileExists(result) then begin if (knownMaps.scanFiles()) then saveDatabases(true, false); @@ -290,11 +302,10 @@ end; //========================================================================== function findExistingResWadWithHash (fname: AnsiString; const resMd5: TMD5Digest): AnsiString; begin - //result := scanDir(GameDir+'/wads', ExtractFileName(fname), resMd5); result := knownRes.findByHash(resMd5); if (length(result) > 0) then begin - result := GameDir+'/wads/'+result; + //result := GameDir+'/wads/'+result; if not FileExists(result) then begin if (knownRes.scanFiles()) then saveDatabases(false, true); @@ -355,6 +366,8 @@ var mapdbUpdated: Boolean = false; resdbUpdated: Boolean = false; transStarted: Boolean; + destMapDir: AnsiString = ''; + destResDir: AnsiString = ''; begin result := ''; clearReplacementWads(); @@ -387,10 +400,16 @@ begin exit; end; try - CreateDir(GameDir+'/maps/downloads'); + destMapDir := e_GetWriteableDir(MapDownloadDirs, false); // not required except end; - fname := GameDir+'/maps/downloads/'+generateFileName(FileName, mapHash); + if (length(destMapDir) = 0) then + begin + e_LogWriteln('cannot create map download directory', TMsgType.Fatal); + result := ''; + exit; + end; + fname := destMapDir+'/'+generateFileName(FileName, mapHash); tf.diskName := fname; e_LogWritefln('map disk file for `%s` is `%s`', [FileName, fname], TMsgType.Fatal); try @@ -493,10 +512,16 @@ begin if (res <> 0) then begin result := ''; exit; end; end; try - CreateDir(GameDir+'/wads/downloads'); + destResDir := e_GetWriteableDir(WadDownloadDirs, false); // not required except end; - fname := GameDir+'/wads/downloads/'+generateFileName(tf.diskName, tf.hash); + if (length(destResDir) = 0) then + begin + e_LogWriteln('cannot create wad download directory', TMsgType.Fatal); + result := ''; + exit; + end; + fname := destResDir+'/'+generateFileName(tf.diskName, tf.hash); e_LogWritefln('downloading resource `%s` to `%s`...', [tf.diskName, fname]); try strm := openDiskFileRW(fname); diff --git a/src/shared/fhashdb.pas b/src/shared/fhashdb.pas index 7e45c10..09797e8 100644 --- a/src/shared/fhashdb.pas +++ b/src/shared/fhashdb.pas @@ -36,9 +36,11 @@ type TFileHashDB = class private + type TStrDynArray = array of AnsiString; + type TFileInfo = record - name: AnsiString; // names are relative to `mBasePath` + name: AnsiString; // name includes `mBasePath`, if necessary hash: TMD5Digest; size: LongWord; age: LongInt; @@ -48,7 +50,8 @@ type end; private - mBasePath: AnsiString; // ends with '/' + mBasePath: AnsiString; // ends with '/', or empty string + mPathList: TStrDynArray; mHash2List: THashMD5Int; // hash -> list index mFile2List: THashStrCIInt; // file name -> list index mFileList: array of TFileInfo; @@ -60,10 +63,18 @@ type procedure scanDir (path: AnsiString; var changed: Boolean); + procedure appendOneDir (dir: AnsiString); + + procedure setup (aBasePath: AnsiString; const aPathList: TStrDynArray); + public - constructor Create (aBasePath: AnsiString); + constructor Create (aBasePath: AnsiString; const aPathList: TStrDynArray); + constructor Create (aBasePath: AnsiString; const aPathList: SSArray); destructor Destroy (); override; + // doesn't automatically rescans + procedure appendMoreDirs (const aPathList: SSArray); + // doesn't clear base path procedure clear (); @@ -81,7 +92,7 @@ type function findByHash (const md5: TMD5Digest): AnsiString; // returns `true` if something was changed // name is relative to base - function addWithHash (relname: AnsiString; const md5: TMD5Digest): Boolean; + function addWithHash (fdiskname: AnsiString; const md5: TMD5Digest): Boolean; end; @@ -116,10 +127,42 @@ end; //========================================================================== // -// TFileHashDB.Create +// TFileHashDB.appendOneDir // //========================================================================== -constructor TFileHashDB.Create (aBasePath: AnsiString); +procedure TFileHashDB.appendOneDir (dir: AnsiString); +var + mps: AnsiString; + found: Boolean; +begin + if (length(dir) = 0) then exit; + if not findFileCI(dir, true) then exit; + dir := fixSlashes(dir, true); + if (mBasePath <> '') and (dir[1] <> '/') then + begin + dir := mBasePath+dir; + if not findFileCI(dir, true) then exit; + dir := fixSlashes(dir, true); + end; + if (dir = '/') then exit; + found := false; + for mps in mPathList do if (dir = mps) then begin found := true; break; end; + if not found then + begin + SetLength(mPathList, length(mPathList)+1); + mPathList[High(mPathList)] := dir; + end; +end; + + +//========================================================================== +// +// TFileHashDB.setup +// +//========================================================================== +procedure TFileHashDB.setup (aBasePath: AnsiString; const aPathList: TStrDynArray); +var + s: AnsiString; begin mBasePath := aBasePath; if (length(aBasePath) <> 0) then @@ -127,6 +170,8 @@ begin if not findFileCI(mBasePath, true) then mBasePath := aBasePath; end; mBasePath := fixSlashes(mBasePath, true); + SetLength(mPathList, 0); + for s in aPathList do appendOneDir(s); mHash2List := THashMD5Int.Create(); mFile2List := THashStrCIInt.Create(); SetLength(mFileList, 0); @@ -134,6 +179,46 @@ begin end; +//========================================================================== +// +// TFileHashDB.Create +// +//========================================================================== +constructor TFileHashDB.Create (aBasePath: AnsiString; const aPathList: TStrDynArray); +begin + setup(aBasePath, aPathList); +end; + + +//========================================================================== +// +// TFileHashDB.Create +// +//========================================================================== +constructor TFileHashDB.Create (aBasePath: AnsiString; const aPathList: SSArray); +var + f: Integer; + pl: TStrDynArray = nil; +begin + SetLength(pl, length(aPathList)); + for f := Low(pl) to High(pl) do pl[f] := aPathList[f-Low(pl)+Low(aPathList)]; + setup(aBasePath, pl); +end; + + +//========================================================================== +// +// TFileHashDB.appendMoreDirs +// +//========================================================================== +procedure TFileHashDB.appendMoreDirs (const aPathList: SSArray); +var + f: Integer; +begin + for f := Low(aPathList) to High(aPathList) do appendOneDir(aPathList[f]); +end; + + //========================================================================== // // TFileHashDB.Destroy @@ -145,6 +230,7 @@ begin mHash2List.Free; mFile2List.Free; SetLength(mFileList, 0); + SetLength(mPathList, 0); mFreeHead := -1; end; @@ -161,6 +247,7 @@ begin mHash2List.clear(); mFile2List.clear(); SetLength(mFileList, 0); + //SetLength(mPathList, 0); mFreeHead := -1; end; @@ -177,7 +264,7 @@ var begin sign := 'FHDB'; st.WriteBuffer(sign, 4); - st.WriteWord(0); // version + st.WriteWord(1); // version st.WriteDWord(LongWord(mFile2List.count)); for f := Low(mFileList) to High(mFileList) do begin @@ -207,7 +294,7 @@ begin st.ReadBuffer(sign, 4); if (sign <> 'FHDB') then raise Exception.Create('invalid database signature'); count := st.ReadWord(); - if (count <> 0) then raise Exception.Create('invalid database version'); + if (count <> 1) then raise Exception.Create('invalid database version'); count := Integer(st.ReadDWord()); if (count < 0) or (count > 1024*1024) then raise Exception.Create('invalid database file count'); while (count > 0) do @@ -285,7 +372,13 @@ var age: LongInt; needUpdate: Boolean; begin - if (FindFirst(path+'*', faAnyFile, sr) <> 0) then exit; + //writeln('TFileHashDB.scanDir(000): [', path, ']'); + if (FindFirst(path+'*', faAnyFile, sr) <> 0) then + begin + FindClose(sr); + exit; + end; + //writeln('TFileHashDB.scanDir(001): [', path, ']'); try repeat if ((sr.Attr and faDirectory) <> 0) then @@ -299,7 +392,7 @@ begin dfn := fixSlashes(path+sr.Name, false); // build internal file name hfn := dfn; - Delete(hfn, 1, length(mBasePath)); // remove prefix + //Delete(hfn, 1, length(mBasePath)); // remove prefix // find file in hash if not mFile2List.get(hfn, idx) then idx := -1; // check if we already have this file @@ -395,7 +488,9 @@ var begin result := false; for f := Low(mFileList) to High(mFileList) do mFileList[f].wasSeen := false; - scanDir(mBasePath, result); + //scanDir(mBasePath, result); + //writeln('TFileHashDB.scanFiles: dll=', length(mPathList)); + for f := Low(mPathList) to High(mPathList) do scanDir(mPathList[f], result); // remove all unseen files f := High(mFileList); while (f >= 0) do @@ -427,10 +522,10 @@ end; // TFileHashDB.addWithHash // // returns `true` if something was changed -// name is relative to base +// name is *NOT* relative to base // //========================================================================== -function TFileHashDB.addWithHash (relname: AnsiString; const md5: TMD5Digest): Boolean; +function TFileHashDB.addWithHash (fdiskname: AnsiString; const md5: TMD5Digest): Boolean; var age: LongInt; size: LongInt; @@ -439,9 +534,10 @@ var idx: Integer; begin result := false; - if (length(relname) > length(mBasePath)) and strEquCI1251(mBasePath, Copy(relname, 1, length(mBasePath))) then Delete(relname, 1, Length(mBasePath)); - if (length(relname) = 0) then exit; - fn := mBasePath+relname; + //if (length(fdiskname) > length(mBasePath)) and strEquCI1251(mBasePath, Copy(fdiskname, 1, length(mBasePath))) then Delete(fdiskname, 1, Length(mBasePath)); + if (length(fdiskname) = 0) then exit; + //fn := mBasePath+fdiskname; + fn := fdiskname; if not findFileCI(fn) then exit; // get age age := FileAge(fn); @@ -453,7 +549,7 @@ begin FileClose(handle); if (size = -1) then exit; // find old file, if any - Delete(fn, 1, length(mBasePath)); + //Delete(fn, 1, length(mBasePath)); if not mFile2List.get(fn, idx) then idx := -1; // check for changes if (idx >= 0) then