From: Ketmar Dark Date: Sun, 13 Oct 2019 00:21:57 +0000 (+0300) Subject: net: game: better hash management; calculate resource hashes only once, send all... X-Git-Url: https://deadsoftware.ru/gitweb?p=d2df-sdl.git;a=commitdiff_plain;h=986383de4f166773e41335f5b0fec5ee5c0128f0 net: game: better hash management; calculate resource hashes only once, send all resource hashes in "map info reply" --- diff --git a/src/game/g_game.pas b/src/game/g_game.pas index 317f0c4..4e23826 100644 --- a/src/game/g_game.pas +++ b/src/game/g_game.pas @@ -4825,7 +4825,7 @@ begin end else begin - gWADHash := MD5File(nws); + if (g_Game_IsNet) then gWADHash := MD5File(nws); //writeln('********: nws=', nws, ' : Map=', Map, ' : nw=', NewWAD, ' : resname=', ResName); g_Game_LoadWAD(NewWAD); end; diff --git a/src/game/g_map.pas b/src/game/g_map.pas index 392d45b..60aefbd 100644 --- a/src/game/g_map.pas +++ b/src/game/g_map.pas @@ -226,7 +226,7 @@ var gLiftMap: array of array of DWORD; gWADHash: TMD5Digest; BackID: DWORD = DWORD(-1); - gExternalResources: TStringList; + gExternalResources: array of TDiskFileInfo = nil; gMovingWallIds: array of Integer = nil; gdbg_map_use_accel_render: Boolean = true; @@ -1466,55 +1466,66 @@ begin Result := ''; end; -procedure addResToExternalResList(res: string); + +procedure addResToExternalResList (res: AnsiString); +var + uname: AnsiString; + f: Integer; + fi: TDiskFileInfo; begin - //e_LogWritefln('DBG: ***trying external resource %s', [res]); - res := toLowerCase1251(extractWadName(res)); - // ignore "standart.wad" - if (res <> '') {and (res <> 'standart.wad')} and (gExternalResources.IndexOf(res) = -1) then + if g_Game_IsClient or not g_Game_IsNet then exit; + if (length(res) = 0) then exit; // map wad + res := extractWadName(res); + if (length(res) = 0) then exit; // map wad + uname := toLowerCase1251(res); + // do not add duplicates + for f := 0 to High(gExternalResources) do begin - //e_LogWritefln('DBG: added external resource %s', [res]); - gExternalResources.Add(res); + if (gExternalResources[f].userName = uname) then exit; end; -end; - -procedure generateExternalResourcesList({mapReader: TMapReader_1}map: TDynRecord); -//var - //textures: TTexturesRec1Array; - //textures: TDynField; - //trec: TDynRecord; - //mapHeader: TMapHeaderRec_1; - //i: integer; - //resFile: String = ''; -begin - if gExternalResources = nil then - gExternalResources := TStringList.Create; - - gExternalResources.Clear; - - (* - { - textures := GetTextures(map); - for i := 0 to High(textures) do + // add new resource + fi.userName := uname; + if (not GetDiskFileInfo(GameDir+'/wads/'+res, fi)) then + begin + fi.tag := -1; + end + else begin - addResToExternalResList(resFile); + fi.tag := 0; // non-zero means "cannot caclucate hash" + try + fi.hash := MD5File(fi.diskName); + except + fi.tag := -1; + end; end; - } + //e_LogWritefln('addext: res=[%s]; uname=[%s]; diskName=[%s]', [res, fi.userName, fi.diskName]); + SetLength(gExternalResources, length(gExternalResources)+1); + gExternalResources[High(gExternalResources)] := fi; +end; + - textures := map['texture']; - if (textures <> nil) then +procedure compactExtResList (); +var + src, dest: Integer; +begin + src := 0; + dest := 0; + for src := 0 to High(gExternalResources) do begin - for trec in textures do + if (gExternalResources[src].tag = 0) then begin - addResToExternalResList(resFile); + // copy it + if (dest <> src) then gExternalResources[dest] := gExternalResources[src]; + Inc(dest); end; end; + if (dest <> length(gExternalResources)) then SetLength(gExternalResources, dest); +end; - textures := nil; - *) - - //mapHeader := GetMapHeader(map); +procedure generateExternalResourcesList (map: TDynRecord); +begin + SetLength(gExternalResources, 0); addResToExternalResList(map.MusicName); addResToExternalResList(map.SkyName); end; @@ -2300,6 +2311,7 @@ begin end; end; + compactExtResList(); e_WriteLog('Done loading map.', TMsgType.Notify); Result := True; end; diff --git a/src/game/g_net.pas b/src/game/g_net.pas index d3e67cf..391e09c 100644 --- a/src/game/g_net.pas +++ b/src/game/g_net.pas @@ -79,6 +79,14 @@ const {$ENDIF} type + TNetMapResourceInfo = record + wadName: AnsiString; // wad file name, without a path + size: Integer; // wad file size (-1: size and hash are not known) + hash: TMD5Digest; // wad hash + end; + + TNetMapResourceInfoArray = array of TNetMapResourceInfo; + TNetFileTransfer = record diskName: string; hash: TMD5Digest; @@ -230,7 +238,7 @@ procedure g_Net_UnforwardPorts(); function g_Net_UserRequestExit: Boolean; -function g_Net_Wait_MapInfo (var tf: TNetFileTransfer; resList: TStringList): Integer; +function g_Net_Wait_MapInfo (var tf: TNetFileTransfer; var resList: TNetMapResourceInfoArray): Integer; function g_Net_RequestResFileInfo (resIndex: LongInt; out tf: TNetFileTransfer): Integer; function g_Net_AbortResTransfer (var tf: TNetFileTransfer): Boolean; function g_Net_ReceiveResourceFile (resIndex: LongInt; var tf: TNetFileTransfer; strm: TStream): Integer; @@ -433,8 +441,9 @@ var ridx: Integer; dfn: AnsiString; md5: TMD5Digest; - st: TStream; + //st: TStream; size: LongInt; + fi: TDiskFileInfo; begin // find client index by peer for f := Low(NetClients) to High(NetClients) do @@ -483,13 +492,13 @@ begin end; // get resource index ridx := msg.ReadLongInt(); - if (ridx < -1) or (ridx >= gExternalResources.Count) then + if (ridx < -1) or (ridx >= length(gExternalResources)) then begin e_LogWritefln('Invalid resource index %d', [ridx], TMsgType.Warning); killClientByFT(nc^); exit; end; - if (ridx < 0) then fname := MapsDir+gGameSettings.WAD else fname := GameDir+'/wads/'+gExternalResources[ridx]; + if (ridx < 0) then fname := MapsDir+gGameSettings.WAD else fname := {GameDir+'/wads/'+}gExternalResources[ridx].diskName; if (length(fname) = 0) then begin e_WriteLog('Invalid filename: '+fname, TMsgType.Warning); @@ -505,8 +514,8 @@ begin exit; end; // calculate hash - //TODO: cache hashes - tf.hash := MD5File(tf.diskName); + //tf.hash := MD5File(tf.diskName); + if (ridx < 0) then tf.hash := gWADHash else tf.hash := gExternalResources[ridx].hash; // create file stream tf.diskName := findDiskWad(fname); try @@ -621,11 +630,21 @@ begin trans_omsg.Clear(); dfn := findDiskWad(MapsDir+gGameSettings.WAD); if (dfn = '') then dfn := '!wad_not_found!.wad'; //FIXME - md5 := MD5File(dfn); + //md5 := MD5File(dfn); + md5 := gWADHash; + if (not GetDiskFileInfo(dfn, fi)) then + begin + e_LogWritefln('client #%d requested map info, but i cannot get file info', [nc.ID]); + killClientByFT(nc^); + exit; + end; + size := fi.size; + { st := openDiskFileRO(dfn); if not assigned(st) then exit; //wtf?! size := st.size; st.Free; + } // packet type trans_omsg.Write(Byte(NTF_SERVER_MAP_INFO)); // map wad name @@ -635,11 +654,17 @@ begin // map wad size trans_omsg.Write(size); // number of external resources for map - trans_omsg.Write(LongInt(gExternalResources.Count)); + trans_omsg.Write(LongInt(length(gExternalResources))); // external resource names - for f := 0 to gExternalResources.Count-1 do + for f := 0 to High(gExternalResources) do begin - trans_omsg.Write(ExtractFileName(gExternalResources[f])); // GameDir+'/wads/'+ResList.Strings[i] + // old style packet + //trans_omsg.Write(ExtractFileName(gExternalResources[f])); // GameDir+'/wads/'+ResList.Strings[i] + // new style packet + trans_omsg.Write('!'); + trans_omsg.Write(LongInt(gExternalResources[f].size)); + trans_omsg.Write(gExternalResources[f].hash); + trans_omsg.Write(ExtractFileName(gExternalResources[f].diskName)); end; // send packet if not ftransSendServerMsg(nc^, trans_omsg) then exit; @@ -677,16 +702,18 @@ end; // // returns `false` on error or user abort // fills: -// hash -// size -// chunkSize +// diskName: map wad file name (without a path) +// hash: map wad hash +// size: map wad size +// chunkSize: set too +// resList: list of resource wads // returns: // <0 on error // 0 on success // 1 on user abort // 2 on server abort // for maps, first `tf.diskName` name will be map wad name, and `tf.hash`/`tf.size` will contain map info -function g_Net_Wait_MapInfo (var tf: TNetFileTransfer; resList: TStringList): Integer; +function g_Net_Wait_MapInfo (var tf: TNetFileTransfer; var resList: TNetMapResourceInfoArray): Integer; var ev: ENetEvent; rMsgId: Byte; @@ -697,7 +724,10 @@ var status: cint; s: AnsiString; rc, f: LongInt; + ri: ^TNetMapResourceInfo; begin + SetLength(resList, 0); + // send request trans_omsg.Clear(); trans_omsg.Write(Byte(NTF_CLIENT_MAP_REQUEST)); @@ -769,7 +799,7 @@ begin e_LogWritefln('g_Net_Wait_MapInfo: creating map info packet...', []); if not msg.Init(ev.packet^.data+1, ev.packet^.dataLength-1, True) then exit; e_LogWritefln('g_Net_Wait_MapInfo: parsing map info packet (rd=%d; max=%d)...', [msg.ReadCount, msg.MaxSize]); - resList.Clear(); + SetLength(resList, 0); // just in case // map wad name tf.diskName := msg.ReadString(); e_LogWritefln('g_Net_Wait_MapInfo: map wad is `%s`', [tf.diskName]); @@ -787,16 +817,28 @@ begin exit; end; e_LogWritefln('g_Net_Wait_MapInfo: map external resource count is %d', [rc]); + SetLength(resList, rc); // external resource names for f := 0 to rc-1 do begin - s := ExtractFileName(msg.ReadString()); - if (length(s) = 0) then + ri := @resList[f]; + s := msg.ReadString(); + if (length(s) = 0) then begin result := -1; exit; end; + if (s = '!') then + begin + // extended packet + ri.size := msg.ReadLongInt(); + ri.hash := msg.ReadMD5(); + ri.wadName := ExtractFileName(msg.ReadString()); + if (length(ri.wadName) = 0) or (ri.size < 0) then begin result := -1; exit; end; + end + else begin - Result := -1; - exit; + // old-style packet, only name + ri.wadName := ExtractFileName(s); + if (length(ri.wadName) = 0) then begin result := -1; exit; end; + ri.size := -1; // unknown end; - resList.append(s); end; e_LogWritefln('g_Net_Wait_MapInfo: got map info', []); Result := 0; // success diff --git a/src/game/g_res_downloader.pas b/src/game/g_res_downloader.pas index 68ceb0a..438cd58 100644 --- a/src/game/g_res_downloader.pas +++ b/src/game/g_res_downloader.pas @@ -242,80 +242,6 @@ begin end; -//========================================================================== -// -// scanDir -// -// look for a wad to match the hash -// scans subdirs, ignores known wad extensions -// -// returns found wad disk name, or empty string -// -//========================================================================== -(* -function scanDir (dirName: AnsiString; baseName: AnsiString; const resMd5: TMD5Digest): AnsiString; -var - searchResult: TSearchRec; - dfn: AnsiString; - md5: TMD5Digest; - dirs: array of AnsiString; - f: Integer; -begin - result := ''; - SetLength(dirs, 0); - if (length(baseName) = 0) then exit; - dirName := IncludeTrailingPathDelimiter(dirName); - e_LogWritefln('scanning dir `%s` for file `%s`...', [dirName, baseName]); - - // scan files - if (FindFirst(dirName+'*', faAnyFile, searchResult) <> 0) then exit; - try - repeat - if ((searchResult.Attr and faDirectory) = 0) then - begin - if (isWadNamesEqu(searchResult.Name, baseName)) then - begin - dfn := dirName+searchResult.Name; - if FileExists(dfn) then - begin - e_LogWritefln(' found `%s`...', [dfn]); - md5 := MD5File(dfn); - if MD5Match(md5, resMd5) then - begin - e_LogWritefln(' MATCH `%s`...', [dfn]); - SetLength(dirs, 0); - result := dfn; - exit; - end; - end; - end; - end - else - begin - if (searchResult.Name <> '.') and (searchResult.Name <> '..') then - begin - dfn := dirName+searchResult.Name; - SetLength(dirs, Length(dirs)+1); - dirs[length(dirs)-1] := dfn; - end; - end; - until (FindNext(searchResult) <> 0); - finally - FindClose(searchResult); - end; - - // scan subdirs - for f := 0 to High(dirs) do - begin - dfn := dirs[f]; - result := scanDir(dfn, baseName, resMd5); - if (length(result) <> 0) then begin SetLength(dirs, 0); exit; end; - end; - SetLength(dirs, 0); -end; -*) - - //========================================================================== // // findExistingMapWadWithHash @@ -408,7 +334,7 @@ end; function g_Res_DownloadMapWAD (FileName: AnsiString; const mapHash: TMD5Digest): AnsiString; var tf: TNetFileTransfer; - resList: TStringList; + resList: array of TNetMapResourceInfo = nil; f, res: Integer; strm: TStream; fname: AnsiString; @@ -416,12 +342,11 @@ var md5: TMD5Digest; mapdbUpdated: Boolean = false; resdbUpdated: Boolean = false; + transStarted: Boolean; begin result := ''; clearReplacementWads(); - resList := TStringList.Create(); - try g_Res_received_map_start := 1; g_Console_Add(Format(_lc[I_NET_MAP_DL], [FileName])); @@ -513,14 +438,28 @@ begin end; // download resources - for f := 0 to resList.Count-1 do + for f := 0 to High(resList) do begin - res := g_Net_RequestResFileInfo(f, tf); - if (res <> 0) then begin result := ''; exit; end; + // if we got a new-style reslist packet, use received data to check for resource files + if (resList[f].size < 0) then + begin + // old-style packet + transStarted := true; + res := g_Net_RequestResFileInfo(f, tf); + if (res <> 0) then begin result := ''; exit; end; + end + else + begin + // new-style packet + transStarted := false; + tf.diskName := resList[f].wadName; + tf.hash := resList[f].hash; + tf.size := resList[f].size; + end; if (isIgnoredResWad(tf.diskName)) then begin // ignored file, abort download - g_Net_AbortResTransfer(tf); + if (transStarted) then g_Net_AbortResTransfer(tf); e_LogWritefln('ignoring wad resource `%s` by user request', [tf.diskName]); continue; end; @@ -528,11 +467,16 @@ begin if (length(wadname) <> 0) then begin // already here - g_Net_AbortResTransfer(tf); + if (transStarted) then g_Net_AbortResTransfer(tf); addReplacementWad(tf.diskName, wadname); end else begin + if (not transStarted) then + begin + res := g_Net_RequestResFileInfo(f, tf); + if (res <> 0) then begin result := ''; exit; end; + end; try CreateDir(GameDir+'/wads/downloads'); except @@ -599,7 +543,7 @@ begin end; end; finally - resList.Free; + SetLength(resList, 0); g_Res_received_map_start := 0; end;