DEADSOFTWARE

ee1cc1c272b7e6c42e884a78f643632fe376f570
[d2df-sdl.git] / src / sfs / sfsZipFS.pas
1 // Streaming R/O Virtual File System v0.2.0
2 // Copyright (C) XL A.S. Ketmar. All rights reserved
3 // See the file aplicense.txt for conditions of use.
4 //
5 // grouping files with packing:
6 // zip, jar: PKZIP-compatible archives (store, deflate)
7 // fout2 : Fallout II .DAT
8 // vtdb : Asphyre's VTDb
9 // dfwad : D2D:F wad archives
10 //
11 {.$DEFINE SFS_DEBUG_ZIPFS}
12 {.$DEFINE SFS_ZIPFS_FULL}
13 {$MODE DELPHI}
14 {.$R-}
15 unit sfsZipFS;
17 interface
19 uses
20 SysUtils, Classes, Contnrs, sfs;
23 type
24 TSFSZipVolumeType = (
25 sfszvNone,
26 sfszvZIP,
27 {$IFDEF SFS_ZIPFS_FULL}
28 sfszvF2DAT,
29 sfszvVTDB,
30 {$ENDIF}
31 sfszvDFWAD
32 );
34 TSFSZipVolume = class(TSFSVolume)
35 protected
36 fType: TSFSZipVolumeType;
38 procedure ZIPReadDirectory ();
39 procedure DFWADReadDirectory ();
40 {$IFDEF SFS_ZIPFS_FULL}
41 procedure F2DATReadDirectory ();
42 procedure VTDBReadDirectory ();
43 {$ENDIF}
45 procedure ReadDirectory (); override;
46 procedure removeCommonPath (); override;
48 public
49 function OpenFileByIndex (const index: Integer): TStream; override;
50 end;
52 TSFSZipVolumeFactory = class (TSFSVolumeFactory)
53 public
54 function IsMyVolumePrefix (const prefix: TSFSString): Boolean; override;
55 function Produce (const prefix, fileName: TSFSString; st: TStream): TSFSVolume; override;
56 procedure Recycle (vol: TSFSVolume); override;
57 end;
61 implementation
63 uses
64 zstream, xstreams;
67 type
68 TZDecompressionStream = TDecompressionStream;
70 type
71 TSFSZipFileInfo = class (TSFSFileInfo)
72 public
73 fMethod: Byte; // 0: store; 8: deflate; 255: other
74 fPackSz: Int64;
75 end;
77 TZLocalFileHeader = packed record
78 version: Byte;
79 hostOS: Byte;
80 flags: Word;
81 method: Word;
82 time: LongWord;
83 crc: LongWord;
84 packSz: LongWord;
85 unpackSz: LongWord;
86 fnameSz: Word;
87 localExtraSz: Word;
88 end;
91 function ZIPCheckMagic (st: TStream): Boolean;
92 var
93 sign: packed array [0..3] of Char;
94 begin
95 result := false;
96 st.ReadBuffer(sign[0], 4);
97 st.Seek(-4, soCurrent);
98 if (sign <> 'PK'#3#4) and (sign <> 'PK'#5#6) then exit;
99 result := true;
100 end;
102 {$IFDEF SFS_ZIPFS_FULL}
103 function F2DATCheckMagic (st: TStream): Boolean;
104 var
105 dsize, fiSz: Integer;
106 begin
107 result := false;
108 st.Position := st.Size-8;
109 st.ReadBuffer(dsize, 4); st.ReadBuffer(fiSz, 4);
110 st.Position := 0;
111 if (fiSz <> st.Size) or (dsize < 5+13) or (dsize > fiSz-4) then exit;
112 result := true;
113 end;
115 function VTDBCheckMagic (st: TStream): Boolean;
116 var
117 sign: packed array [0..3] of Char;
118 fcnt, dofs: Integer;
119 begin
120 result := false;
121 if st.Size < 32 then exit;
122 st.ReadBuffer(sign[0], 4);
123 st.ReadBuffer(fcnt, 4); st.ReadBuffer(dofs, 4);
124 st.Seek(-12, soCurrent);
125 if sign <> 'vtdm' then exit;
126 if (fcnt < 0) or (dofs < 32) or (dofs+fcnt*8 > st.Size) then exit;
127 result := true;
128 end;
129 {$ENDIF}
131 function DFWADCheckMagic (st: TStream): Boolean;
132 var
133 sign: packed array [0..5] of Char;
134 fcnt: Word;
135 begin
136 result := false;
137 if st.Size < 10 then exit;
138 st.ReadBuffer(sign[0], 6);
139 st.ReadBuffer(fcnt, 2);
140 st.Seek(-8, soCurrent);
141 //writeln('trying DFWAD... [', sign, ']');
142 if (sign[0] <> 'D') and (sign[1] <> 'F') and (sign[2] <> 'W') and
143 (sign[3] <> 'A') and (sign[4] <> 'D') and (sign[5] <> #$01) then exit;
144 //writeln('DFWAD FOUND, with ', fcnt, ' files');
145 //if (fcnt < 0) then exit;
146 result := true;
147 end;
149 function maxPrefix (s0: string; s1: string): Integer;
150 var
151 f: Integer;
152 begin
153 for f := 1 to length(s0) do
154 begin
155 if f > length(s1) then begin result := f; exit; end;
156 if SFSUpCase(s0[f]) <> SFSUpCase(s1[f]) then begin result := f; exit; end;
157 end;
158 result := length(s0);
159 end;
161 procedure TSFSZipVolume.removeCommonPath ();
162 var
163 f, pl, maxsc, sc, c: integer;
164 cp, s: string;
165 fi: TSFSZipFileInfo;
166 begin
167 if fType <> sfszvZIP then exit;
168 maxsc := 0;
169 if fFiles.Count = 0 then exit;
170 cp := '';
171 for f := 0 to fFiles.Count-1 do
172 begin
173 fi := TSFSZipFileInfo(fFiles[f]);
174 s := fi.fPath;
175 if length(s) > 0 then begin cp := s; break; end;
176 end;
177 if length(cp) = 0 then exit;
178 for f := 0 to fFiles.Count-1 do
179 begin
180 fi := TSFSZipFileInfo(fFiles[f]);
181 s := fi.fPath;
182 if length(s) = 0 then continue;
183 pl := maxPrefix(cp, s);
184 //writeln('s=[', s, ']; cp=[', cp, ']; pl=', pl);
185 if pl = 0 then exit; // no common prefix at all
186 cp := Copy(cp, 1, pl);
187 sc := 0;
188 for c := 1 to length(s) do if s[c] = '/' then Inc(sc);
189 if sc > maxsc then maxsc := sc;
190 end;
191 if maxsc < 2 then exit; // alas
192 while (length(cp) > 0) and (cp[length(cp)] <> '/') do cp := Copy(cp, 1, length(cp)-1);
193 if length(cp) < 2 then exit; // nothing to do
194 for f := 0 to fFiles.Count-1 do
195 begin
196 fi := TSFSZipFileInfo(fFiles[f]);
197 if length(fi.fPath) >= length(cp) then
198 begin
199 s := fi.fPath;
200 fi.fPath := Copy(fi.fPath, length(cp)+1, length(fi.fPath));
201 //writeln('FIXED [', s, '] -> [', fi.fPath, ']');
202 end;
203 end;
204 end;
207 { TSFSZipVolume }
208 procedure TSFSZipVolume.ZIPReadDirectory ();
209 var
210 fi: TSFSZipFileInfo;
211 name: ShortString;
212 sign, dSign: packed array [0..3] of Char;
213 lhdr: TZLocalFileHeader;
214 ignoreFile, skipped: Boolean;
215 crc, psz, usz: LongWord;
216 buf: packed array of Byte;
217 bufPos, bufUsed: Integer;
218 begin
219 SetLength(buf, 0);
220 // read local directory
221 repeat
222 fFileStream.ReadBuffer(sign[0], Length(sign));
224 if sign <> 'PK'#3#4 then break;
226 ignoreFile := false;
227 skipped := false;
229 fi := TSFSZipFileInfo.Create(self);
230 fi.fPackSz := 0;
231 fi.fMethod := 0;
233 fFileStream.ReadBuffer(lhdr, SizeOf(lhdr));
234 if lhdr.fnameSz > 255 then name[0] := #255 else name[0] := chr(lhdr.fnameSz);
235 fFileStream.ReadBuffer(name[1], Length(name));
236 fFileStream.Seek(lhdr.fnameSz-Length(name), soCurrent); // rest of the name (if any)
237 fi.fName := name;
238 fFileStream.Seek(lhdr.localExtraSz, soCurrent);
240 if (lhdr.flags and 1) <> 0 then
241 begin
242 // encrypted file: skip it
243 ignoreFile := true;
244 end;
246 if (lhdr.method <> 0) and (lhdr.method <> 8) then
247 begin
248 // not stored. not deflated. skip.
249 ignoreFile := true;
250 end;
252 fi.fOfs := fFileStream.Position;
253 fi.fSize := lhdr.unpackSz;
254 fi.fPackSz := lhdr.packSz;
255 fi.fMethod := lhdr.method;
257 if (lhdr.flags and (1 shl 3)) <> 0 then
258 begin
259 // it has a descriptor. stupid thing at all...
260 {$IFDEF SFS_DEBUG_ZIPFS}
261 WriteLn(ErrOutput, 'descr: $', IntToHex(fFileStream.Position, 8));
262 WriteLn(ErrOutput, 'size: ', lhdr.unpackSz);
263 WriteLn(ErrOutput, 'psize: ', lhdr.packSz);
264 {$ENDIF}
265 skipped := true;
267 if lhdr.packSz <> 0 then
268 begin
269 // some kind of idiot already did our work (maybe paritally)
270 // trust him (her? %-)
271 fFileStream.Seek(lhdr.packSz, soCurrent);
272 end;
274 // scan for descriptor
275 if Length(buf) = 0 then SetLength(buf, 65536);
276 bufPos := 0; bufUsed := 0;
277 fFileStream.ReadBuffer(dSign[0], 4);
278 repeat
279 if dSign <> 'PK'#7#8 then
280 begin
281 // skip one byte
282 Move(dSign[1], dSign[0], 3);
283 if bufPos >= bufUsed then
284 begin
285 bufPos := 0;
286 // int64!
287 if fFileStream.Size-fFileStream.Position > Length(buf) then
288 bufUsed := Length(buf)
289 else bufUsed := fFileStream.Size-fFileStream.Position;
290 if bufUsed = 0 then raise ESFSError.Create('invalid ZIP file');
291 fFileStream.ReadBuffer(buf[0], bufUsed);
292 end;
293 dSign[3] := chr(buf[bufPos]); Inc(bufPos);
294 Inc(lhdr.packSz);
295 continue;
296 end;
297 // signature found: check if it is a real one
298 // ???: make stronger check (for the correct following signature)?
299 // sign, crc, packsize, unpacksize
300 fFileStream.Seek(-bufUsed+bufPos, soCurrent); bufPos := 0; bufUsed := 0;
301 fFileStream.ReadBuffer(crc, 4); // crc
302 fFileStream.ReadBuffer(psz, 4); // packed size
303 // is size correct?
304 if psz = lhdr.packSz then
305 begin
306 // this is a real description. fuck it off
307 fFileStream.ReadBuffer(usz, 4); // unpacked size
308 break;
309 end;
310 // this is just a sequence of bytes
311 fFileStream.Seek(-8, soCurrent);
312 fFileStream.ReadBuffer(dSign[0], 4);
313 Inc(lhdr.packSz, 4);
314 until false;
315 // store correct values
316 fi.fSize := usz;
317 fi.fPackSz := psz;
318 end;
320 // skip packed data
321 if not skipped then fFileStream.Seek(lhdr.packSz, soCurrent);
322 if ignoreFile then fi.Free();
323 until false;
325 if (sign <> 'PK'#1#2) and (sign <> 'PK'#5#6) then
326 begin
327 {$IFDEF SFS_DEBUG_ZIPFS}
328 WriteLn(ErrOutput, 'end: $', IntToHex(fFileStream.Position, 8));
329 WriteLn(ErrOutput, 'sign: $', sign[0], sign[1], '#', ord(sign[2]), '#', ord(sign[3]));
330 {$ENDIF}
331 raise ESFSError.Create('invalid .ZIP archive (no central dir)');
332 end;
333 end;
335 {$IFDEF SFS_ZIPFS_FULL}
336 procedure TSFSZipVolume.F2DATReadDirectory ();
337 var
338 dsize: Integer;
339 fi: TSFSZipFileInfo;
340 name: ShortString;
341 f: Integer;
342 b: Byte;
343 begin
344 fFileStream.Position := fFileStream.Size-8;
345 fFileStream.ReadBuffer(dsize, 4);
346 fFileStream.Seek(-dsize, soCurrent); Dec(dsize, 4);
347 while dsize > 0 do
348 begin
349 fi := TSFSZipFileInfo.Create(self);
350 fFileStream.ReadBuffer(f, 4);
351 if (f < 1) or (f > 255) then raise ESFSError.Create('invalid Fallout II .DAT file');
352 Dec(dsize, 4+f+13);
353 if dsize < 0 then raise ESFSError.Create('invalid Fallout II .DAT file');
354 name[0] := chr(f); if f > 0 then fFileStream.ReadBuffer(name[1], f);
355 f := 1; while (f <= ord(name[0])) and (name[f] <> #0) do Inc(f); name[0] := chr(f-1);
356 fi.fName := name;
357 fFileStream.ReadBuffer(b, 1); // packed?
358 if b = 0 then fi.fMethod := 0 else fi.fMethod := 255;
359 fFileStream.ReadBuffer(fi.fSize, 4);
360 fFileStream.ReadBuffer(fi.fPackSz, 4);
361 fFileStream.ReadBuffer(fi.fOfs, 4);
362 end;
363 end;
365 procedure TSFSZipVolume.VTDBReadDirectory ();
366 // idiotic format
367 var
368 fcnt, dofs: Integer;
369 keys: array of record name: string; ofs: Integer; end;
370 fi: TSFSZipFileInfo;
371 f, c: Integer;
372 rtype: Word;
373 begin
374 fFileStream.Seek(4, soCurrent); // skip signature
375 fFileStream.ReadBuffer(fcnt, 4);
376 fFileStream.ReadBuffer(dofs, 4);
377 fFileStream.Seek(dofs, soBeginning);
379 // read keys
380 SetLength(keys, fcnt);
381 for f := 0 to fcnt-1 do
382 begin
383 fFileStream.ReadBuffer(c, 4);
384 if (c < 0) or (c > 1023) then raise ESFSError.Create('invalid VTDB file');
385 SetLength(keys[f].name, c);
386 if c > 0 then
387 begin
388 fFileStream.ReadBuffer(keys[f].name[1], c);
389 keys[f].name := SFSReplacePathDelims(keys[f].name, '/');
390 if keys[f].name[1] = '/' then Delete(keys[f].name, 1, 1);
391 end;
392 fFileStream.ReadBuffer(keys[f].ofs, 4);
393 end;
395 // read records (record type will be converted to directory name)
396 for f := 0 to fcnt-1 do
397 begin
398 fFileStream.Position := keys[f].ofs;
399 fi := TSFSZipFileInfo.Create(self);
400 fFileStream.ReadBuffer(rtype, 2);
401 fFileStream.ReadBuffer(fi.fSize, 4);
402 fFileStream.ReadBuffer(fi.fPackSz, 4);
403 fi.fOfs := fFileStream.Position+12;
404 fi.fName := keys[f].name;
405 fi.fPath := IntToHex(rtype, 4)+'/';
406 fi.fMethod := 255;
407 end;
408 end;
409 {$ENDIF}
411 procedure TSFSZipVolume.DFWADReadDirectory ();
412 // idiotic format
413 var
414 fcnt: Word;
415 fi: TSFSZipFileInfo;
416 f, c, fofs, fpksize: Integer;
417 curpath, fname: string;
418 name: packed array [0..15] of Char;
419 begin
420 curpath := '';
421 fFileStream.Seek(6, soCurrent); // skip signature
422 fFileStream.ReadBuffer(fcnt, 2);
423 if fcnt = 0 then exit;
424 // read files
425 for f := 0 to fcnt-1 do
426 begin
427 fFileStream.ReadBuffer(name[0], 16);
428 fFileStream.ReadBuffer(fofs, 4);
429 fFileStream.ReadBuffer(fpksize, 4);
430 c := 0;
431 fname := '';
432 while (c < 16) and (name[c] <> #0) do
433 begin
434 if name[c] = '\' then name[c] := '/'
435 else if name[c] = '/' then name[c] := '_';
436 fname := fname+name[c];
437 Inc(c);
438 end;
439 // new directory?
440 if (fofs = 0) and (fpksize = 0) then
441 begin
442 if length(fname) <> 0 then fname := fname+'/';
443 curpath := fname;
444 continue;
445 end;
446 if length(fname) = 0 then continue; // just in case
447 //writeln('DFWAD: [', curpath, '] [', fname, '] at ', fofs, ', size ', fpksize);
448 // create file record
449 fi := TSFSZipFileInfo.Create(self);
450 fi.fOfs := fofs;
451 fi.fSize := -1;
452 fi.fPackSz := fpksize;
453 fi.fName := fname;
454 fi.fPath := curpath;
455 fi.fMethod := 255;
456 end;
457 end;
459 procedure TSFSZipVolume.ReadDirectory ();
460 begin
461 case fType of
462 sfszvZIP: ZIPReadDirectory();
463 {$IFDEF SFS_ZIPFS_FULL}
464 sfszvF2DAT: F2DATReadDirectory();
465 sfszvVTDB: VTDBReadDirectory();
466 {$ENDIF}
467 sfszvDFWAD: DFWADReadDirectory();
468 else raise ESFSError.Create('invalid zipped SFS');
469 end;
470 end;
472 function TSFSZipVolume.OpenFileByIndex (const index: Integer): TStream;
473 var
474 zs: TZDecompressionStream;
475 fs, rs: TStream;
476 gs: TSFSGuardStream;
477 kill: Boolean;
478 buf: packed array [0..1023] of Char;
479 rd: LongInt;
480 begin
481 result := nil;
482 zs := nil;
483 fs := nil;
484 gs := nil;
485 rs := nil;
486 if fFiles = nil then exit;
487 if (index < 0) or (index >= fFiles.Count) or (fFiles[index] = nil) then exit;
488 kill := false;
489 try
491 try
492 fs := TFileStream.Create(fFileName, fmOpenRead or fmShareDenyWrite);
493 kill := true;
494 except
495 fs := fFileStream;
496 end;
498 fs := fFileStream;
499 if TSFSZipFileInfo(fFiles[index]).fMethod = 0 then
500 begin
501 result := TSFSPartialStream.Create(fs,
502 TSFSZipFileInfo(fFiles[index]).fOfs,
503 TSFSZipFileInfo(fFiles[index]).fSize, kill);
504 end
505 else
506 begin
507 fs.Seek(TSFSZipFileInfo(fFiles[index]).fOfs, soBeginning);
508 if TSFSZipFileInfo(fFiles[index]).fMethod = 255 then
509 begin
510 // sorry, pals, DFWAD is completely broken, so users of it should SUFFER
511 if TSFSZipFileInfo(fFiles[index]).fSize = -1 then
512 begin
513 TSFSZipFileInfo(fFiles[index]).fSize := 0;
514 //writeln('trying to determine file size for [', TSFSZipFileInfo(fFiles[index]).fPath, TSFSZipFileInfo(fFiles[index]).fName, ']');
515 zs := TZDecompressionStream.Create(fs);
516 try
517 while true do
518 begin
519 rd := zs.read(buf, 1024);
520 //writeln(' got ', rd, ' bytes');
521 if rd > 0 then Inc(TSFSZipFileInfo(fFiles[index]).fSize, rd);
522 if rd < 1024 then break;
523 end;
524 //writeln(' resulting size: ', TSFSZipFileInfo(fFiles[index]).fSize, ' bytes');
525 // recreate stream
526 FreeAndNil(zs);
527 fs.Seek(TSFSZipFileInfo(fFiles[index]).fOfs, soBeginning);
528 except
529 //writeln('*** CAN''T determine file size for [', TSFSZipFileInfo(fFiles[index]).fPath, TSFSZipFileInfo(fFiles[index]).fName, ']');
530 FreeAndNil(zs);
531 if kill then FreeAndNil(fs);
532 result := nil;
533 exit;
534 end;
535 end;
536 rs := TSFSPartialStream.Create(fs, TSFSZipFileInfo(fFiles[index]).fOfs, TSFSZipFileInfo(fFiles[index]).fPackSz, true);
537 zs := TZDecompressionStream.Create(rs);
538 rs := nil;
539 end
540 else
541 begin
542 rs := TSFSPartialStream.Create(fs, TSFSZipFileInfo(fFiles[index]).fOfs, TSFSZipFileInfo(fFiles[index]).fPackSz, true);
543 zs := TZDecompressionStream.Create(rs, true {-15}{MAX_WBITS});
544 rs := nil;
545 end;
546 gs := TSFSGuardStream.Create(zs, fs, true, kill, false);
547 zs := nil;
548 fs := nil;
549 result := TSFSPartialStream.Create(gs, 0, TSFSZipFileInfo(fFiles[index]).fSize, true);
550 end;
551 except
552 FreeAndNil(rs);
553 FreeAndNil(gs);
554 FreeAndNil(zs);
555 if kill then FreeAndNil(fs);
556 result := nil;
557 exit;
558 end;
559 end;
562 { TSFSZipVolumeFactory }
563 function TSFSZipVolumeFactory.IsMyVolumePrefix (const prefix: TSFSString): Boolean;
564 begin
565 result :=
566 SFSStrEqu(prefix, 'zip') or
567 SFSStrEqu(prefix, 'dfwad')
568 {$IFDEF SFS_ZIPFS_FULL}
569 or SFSStrEqu(prefix, 'jar') or
570 SFSStrEqu(prefix, 'fout2') or
571 SFSStrEqu(prefix, 'vtdb') or
572 SFSStrEqu(prefix, 'wad')
573 {$ENDIF}
575 end;
577 procedure TSFSZipVolumeFactory.Recycle (vol: TSFSVolume);
578 begin
579 vol.Free();
580 end;
582 function TSFSZipVolumeFactory.Produce (const prefix, fileName: TSFSString; st: TStream): TSFSVolume;
583 var
584 vt: TSFSZipVolumeType;
585 begin
586 vt := sfszvNone;
587 if ZIPCheckMagic(st) then vt := sfszvZIP
588 else if DFWADCheckMagic(st) then vt := sfszvDFWAD
589 {$IFDEF SFS_ZIPFS_FULL}
590 else if F2DATCheckMagic(st) then vt := sfszvF2DAT
591 else if VTDBCheckMagic(st) then vt := sfszvVTDB
592 {$ENDIF}
595 if vt <> sfszvNone then
596 begin
597 result := TSFSZipVolume.Create(fileName, st);
598 TSFSZipVolume(result).fType := vt;
599 try
600 result.DoDirectoryRead();
601 except {$IFDEF SFS_DEBUG_ZIPFS} on e: Exception do begin
602 WriteLn(errOutput, 'ZIP ERROR: [', e.ClassName, ']: ', e.Message);
603 {$ENDIF}
604 FreeAndNil(result);
605 raise;
606 {$IFDEF SFS_DEBUG_ZIPFS}end;{$ENDIF}
607 end;
608 end
609 else
610 begin
611 result := nil;
612 end;
613 end;
616 var
617 zipf: TSFSZipVolumeFactory;
618 initialization
619 zipf := TSFSZipVolumeFactory.Create();
620 SFSRegisterVolumeFactory(zipf);
621 //finalization
622 // SFSUnregisterVolumeFactory(zipf);
623 end.