src/shared/xparser.pas

   1 (* Copyright (C)  DooM 2D:Forever Developers
   2  *
   3  * This program is free software: you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation, either version 3 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  15  *)
  16 {$INCLUDE a_modes.inc}
  17 unit xparser;
  18
  19 interface
  20
  21 uses
  22   Classes;
  23
  24
  25 // ////////////////////////////////////////////////////////////////////////// //
  26 type
  27   TTextParser = class
  28   public
  29     const
  30       TTNone = -1;
  31       TTEOF = 0;
  32       TTId = 1;
  33       TTInt = 2;
  34       //TTFloat = 3; // not yet
  35       TTStr = 4; // string
  36       TTComma = 5; // ','
  37       TTColon = 6; // ':'
  38       TTSemi = 7; // ';'
  39       TTBegin = 8; // left curly
  40       TTEnd = 9; // right curly
  41       TTDelim = 10; // other delimiters
  42
  43   private
  44     mLine, mCol: Integer;
  45     mCurChar, mNextChar: AnsiChar;
  46
  47     mAllowSignedNumbers: Boolean; // internal control
  48
  49     mTokLine, mTokCol: Integer; // token start
  50     mTokType: Integer;
  51     mTokStr: AnsiString; // string or identifier
  52     mTokChar: AnsiChar; // for delimiters
  53     mTokInt: Integer;
  54
  55   protected
  56     procedure warmup (); virtual; // called in constructor to warm up the system
  57     procedure loadNextChar (); virtual; abstract; // loads next char into mNextChar; #0 means 'eof'
  58
  59   public
  60     constructor Create (loadToken: Boolean=true);
  61     destructor Destroy (); override;
  62
  63     function isEOF (): Boolean; inline;
  64
  65     function skipChar (): Boolean; // returns `false` on eof
  66
  67     function skipBlanks (): Boolean; // ...and comments; returns `false` on eof
  68
  69     function skipToken (): Boolean; // returns `false` on eof
  70
  71     function expectId (): AnsiString;
  72     procedure expectId (const aid: AnsiString);
  73     function eatId (const aid: AnsiString): Boolean;
  74
  75     function expectStr (allowEmpty: Boolean=false): AnsiString;
  76     function expectInt (): Integer;
  77
  78     procedure expectTT (ttype: Integer);
  79     function eatTT (ttype: Integer): Boolean;
  80
  81     function expectDelim (const ch: AnsiChar): AnsiChar;
  82     function eatDelim (const ch: AnsiChar): Boolean;
  83
  84   public
  85     property col: Integer read mCol;
  86     property line: Integer read mLine;
  87
  88     property curChar: AnsiChar read mCurChar;
  89     property nextChar: AnsiChar read mNextChar;
  90
  91     // token start
  92     property tokCol: Integer read mTokCol;
  93     property tokLine: Integer read mTokLine;
  94
  95     property tokType: Integer read mTokType; // see TTXXX constants
  96     property tokStr: AnsiString read mTokStr; // string or identifier
  97     property tokChar: AnsiChar read mTokChar; // for delimiters
  98     property tokInt: Integer read mTokInt;
  99   end;
 100
 101
 102 // ////////////////////////////////////////////////////////////////////////// //
 103 type
 104   TFileTextParser = class(TTextParser)
 105   private
 106     const BufSize = 65536;
 107
 108   private
 109     mFile: TStream;
 110     mBuffer: PChar;
 111     mBufLen: Integer;
 112     mBufPos: Integer;
 113
 114   protected
 115     procedure loadNextChar (); override; // loads next char into mNextChar; #0 means 'eof'
 116
 117   public
 118     constructor Create (const fname: AnsiString; loadToken: Boolean=true);
 119     constructor Create (st: TStream; loadToken: Boolean=true); // will take ownership on st
 120     destructor Destroy (); override;
 121   end;
 122
 123   TStrTextParser = class(TTextParser)
 124   private
 125     mStr: AnsiString;
 126     mPos: Integer;
 127
 128   protected
 129     procedure loadNextChar (); override; // loads next char into mNextChar; #0 means 'eof'
 130
 131   public
 132     constructor Create (const astr: AnsiString; loadToken: Boolean=true);
 133     destructor Destroy (); override;
 134   end;
 135
 136
 137 // ////////////////////////////////////////////////////////////////////////// //
 138 type
 139   TTextWriter = class
 140   protected
 141     mIndent: Integer;
 142
 143   protected
 144     procedure putBuf (constref buf; len: SizeUInt); virtual; abstract;
 145
 146   public
 147     constructor Create ();
 148
 149     procedure put (const s: AnsiString); overload;
 150     procedure put (v: Byte); overload;
 151     procedure put (v: Integer); overload;
 152     procedure put (const fmt: AnsiString; args: array of const); overload;
 153     procedure putIndent ();
 154     procedure indent ();
 155     procedure unindent ();
 156   end;
 157
 158
 159 // ////////////////////////////////////////////////////////////////////////// //
 160 type
 161   TFileTextWriter = class(TTextWriter)
 162   private
 163     mFile: TStream;
 164
 165   protected
 166     procedure putBuf (constref buf; len: SizeUInt); override;
 167
 168   public
 169     constructor Create (const fname: AnsiString);
 170     destructor Destroy (); override;
 171   end;
 172
 173
 174 implementation
 175
 176 uses
 177   SysUtils, utils;
 178
 179
 180 var
 181   wc2shitmap: array[0..65535] of AnsiChar;
 182   wc2shitmapInited: Boolean = false;
 183
 184
 185 // ////////////////////////////////////////////////////////////////////////// //
 186 procedure initShitMap ();
 187 const
 188   cp1251: array[0..127] of Word = (
 189     $0402,$0403,$201A,$0453,$201E,$2026,$2020,$2021,$20AC,$2030,$0409,$2039,$040A,$040C,$040B,$040F,
 190     $0452,$2018,$2019,$201C,$201D,$2022,$2013,$2014,$003F,$2122,$0459,$203A,$045A,$045C,$045B,$045F,
 191     $00A0,$040E,$045E,$0408,$00A4,$0490,$00A6,$00A7,$0401,$00A9,$0404,$00AB,$00AC,$00AD,$00AE,$0407,
 192     $00B0,$00B1,$0406,$0456,$0491,$00B5,$00B6,$00B7,$0451,$2116,$0454,$00BB,$0458,$0405,$0455,$0457,
 193     $0410,$0411,$0412,$0413,$0414,$0415,$0416,$0417,$0418,$0419,$041A,$041B,$041C,$041D,$041E,$041F,
 194     $0420,$0421,$0422,$0423,$0424,$0425,$0426,$0427,$0428,$0429,$042A,$042B,$042C,$042D,$042E,$042F,
 195     $0430,$0431,$0432,$0433,$0434,$0435,$0436,$0437,$0438,$0439,$043A,$043B,$043C,$043D,$043E,$043F,
 196     $0440,$0441,$0442,$0443,$0444,$0445,$0446,$0447,$0448,$0449,$044A,$044B,$044C,$044D,$044E,$044F
 197   );
 198 var
 199   f: Integer;
 200 begin
 201   for f := 0 to High(wc2shitmap) do wc2shitmap[f] := '?';
 202   for f := 0 to 127 do wc2shitmap[f] := AnsiChar(f);
 203   for f := 0 to 127 do wc2shitmap[cp1251[f]] := AnsiChar(f+128);
 204   wc2shitmapInited := true;
 205 end;
 206
 207
 208 // ////////////////////////////////////////////////////////////////////////// //
 209 // TODO: make a hash or something
 210 function wcharTo1251 (wc: WideChar): AnsiChar; inline;
 211 begin
 212   if not wc2shitmapInited then initShitMap();
 213   if (LongWord(wc) > 65535) then result := '?' else result := wc2shitmap[LongWord(wc)];
 214 end;
 215
 216
 217 // ////////////////////////////////////////////////////////////////////////// //
 218 constructor TTextParser.Create (loadToken: Boolean=true);
 219 begin
 220   mLine := 1;
 221   mCol := 1;
 222   mCurChar := #0;
 223   mNextChar := #0;
 224   mTokType := TTNone;
 225   mTokStr := '';
 226   mTokChar := #0;
 227   mTokInt := 0;
 228   mAllowSignedNumbers := true;
 229   warmup(); // change `mAllowSignedNumbers` there, if necessary
 230   if loadToken then skipToken();
 231 end;
 232
 233
 234 destructor TTextParser.Destroy ();
 235 begin
 236   inherited;
 237 end;
 238
 239
 240 function TTextParser.isEOF (): Boolean; inline; begin result := (mCurChar = #0); end;
 241
 242
 243 procedure TTextParser.warmup ();
 244 begin
 245   mNextChar := ' ';
 246   loadNextChar();
 247   mCurChar := mNextChar;
 248   if (mNextChar <> #0) then loadNextChar();
 249 end;
 250
 251
 252 function TTextParser.skipChar (): Boolean;
 253 begin
 254   if (mCurChar = #0) then begin result := false; exit; end;
 255   if (mCurChar = #10) then begin mCol := 1; Inc(mLine); end else Inc(mCol);
 256   mCurChar := mNextChar;
 257   if (mCurChar = #0) then begin result := false; exit; end;
 258   loadNextChar();
 259   // skip CR in CR/LF
 260   if (mCurChar = #13) then
 261   begin
 262     if (mNextChar = #10) then loadNextChar();
 263     mCurChar := #10;
 264   end;
 265   result := true;
 266 end;
 267
 268
 269 function TTextParser.skipBlanks (): Boolean;
 270 var
 271   level: Integer;
 272 begin
 273   while not isEOF do
 274   begin
 275     if (curChar = '/') then
 276     begin
 277       // single-line comment
 278       if (nextChar = '/') then
 279       begin
 280         while not isEOF and (curChar <> #10) do skipChar();
 281         skipChar(); // skip EOL
 282         continue;
 283       end;
 284       // multline comment
 285       if (nextChar = '*') then
 286       begin
 287         // skip comment start
 288         skipChar();
 289         skipChar();
 290         while not isEOF do
 291         begin
 292           if (curChar = '*') and (nextChar = '/') then
 293           begin
 294             // skip comment end
 295             skipChar();
 296             skipChar();
 297             break;
 298           end;
 299           skipChar();
 300         end;
 301         continue;
 302       end;
 303       // nesting multline comment
 304       if (nextChar = '+') then
 305       begin
 306         // skip comment start
 307         skipChar();
 308         skipChar();
 309         level := 1;
 310         while not isEOF do
 311         begin
 312           if (curChar = '+') and (nextChar = '/') then
 313           begin
 314             // skip comment end
 315             skipChar();
 316             skipChar();
 317             Dec(level);
 318             if (level = 0) then break;
 319             continue;
 320           end;
 321           if (curChar = '/') and (nextChar = '+') then
 322           begin
 323             // skip comment start
 324             skipChar();
 325             skipChar();
 326             Inc(level);
 327             continue;
 328           end;
 329           skipChar();
 330         end;
 331         continue;
 332       end;
 333     end;
 334     if (curChar > ' ') then break;
 335     skipChar(); // skip blank
 336   end;
 337   result := not isEOF;
 338 end;
 339
 340
 341 function TTextParser.skipToken (): Boolean;
 342
 343   procedure parseInt ();
 344   var
 345     neg: Boolean = false;
 346     base: Integer = -1;
 347     n: Integer;
 348   begin
 349     if mAllowSignedNumbers then
 350     begin
 351       if (curChar = '+') or (curChar = '-') then
 352       begin
 353         neg := (curChar = '-');
 354         skipChar();
 355         if (curChar < '0') or (curChar > '9') then
 356         begin
 357           mTokType := TTDelim;
 358           if (neg) then mTokChar := '-' else mTokChar := '+';
 359           exit;
 360         end;
 361       end;
 362     end;
 363     if (curChar = '0') then
 364     begin
 365       case nextChar of
 366         'b','B': base := 2;
 367         'o','O': base := 8;
 368         'd','D': base := 10;
 369         'h','H': base := 16;
 370       end;
 371       if (base > 0) then
 372       begin
 373         // skip prefix
 374         skipChar();
 375         skipChar();
 376       end;
 377     end;
 378     // default base
 379     if (base < 0) then base := 10;
 380     if (digitInBase(curChar, base) < 0) then raise Exception.Create('invalid number');
 381     mTokType := TTInt;
 382     mTokInt := 0; // just in case
 383     while not isEOF do
 384     begin
 385       n := digitInBase(curChar, base);
 386       if (n < 0) then break;
 387       n := mTokInt*10+n;
 388       if (n < 0) or (n < mTokInt) then raise Exception.Create('integer overflow');
 389       mTokInt := n;
 390       skipChar();
 391     end;
 392     // check for valid number end
 393     if not isEOF then
 394     begin
 395       if (curChar = '.') then raise Exception.Create('floating numbers aren''t supported yet');
 396       if (curChar = '_') or ((curChar >= 'A') and (curChar <= 'Z')) or ((curChar >= 'a') and (curChar <= 'z')) or (curChar >= #128) then
 397       begin
 398         raise Exception.Create('invalid number');
 399       end;
 400     end;
 401     if neg then mTokInt := -mTokInt;
 402   end;
 403
 404   procedure parseString ();
 405   var
 406     qch, ch: AnsiChar;
 407     n: Integer;
 408   begin
 409     mTokType := TTStr;
 410     mTokStr := ''; // just in case
 411     qch := curChar;
 412     skipChar(); // skip starting quote
 413     while not isEOF do
 414     begin
 415       // escape
 416       if (qch = '"') and (curChar = '\') then
 417       begin
 418         if (nextChar = #0) then raise Exception.Create('unterminated string escape');
 419         ch := nextChar;
 420         // skip backslash and escape type
 421         skipChar();
 422         skipChar();
 423         case ch of
 424           't': mTokStr += #9;
 425           'n': mTokStr += #10;
 426           'r': mTokStr += #13;
 427           'z': mTokStr += #0;
 428           'e': mTokStr += #27;
 429           'x', 'X': // hex escape
 430             begin
 431               n := digitInBase(curChar, 16);
 432               if (n < 0) then raise Exception.Create('invalid hexstr escape');
 433               skipChar();
 434               if (digitInBase(curChar, 16) > 0) then
 435               begin
 436                 n := n*16+digitInBase(curChar, 16);
 437                 skipChar();
 438               end;
 439               mTokStr += AnsiChar(n);
 440             end;
 441           else mTokStr += ch;
 442         end;
 443         continue;
 444       end;
 445       // duplicate single quote (pascal style)
 446       if (qch = '''') and (curChar = '''') and (nextChar = '''') then
 447       begin
 448         // skip both quotes
 449         skipChar();
 450         skipChar();
 451         mTokStr += '''';
 452         continue;
 453       end;
 454       if (curChar = qch) then
 455       begin
 456         skipChar(); // skip ending quote
 457         break;
 458       end;
 459       mTokStr += curChar;
 460       skipChar();
 461     end;
 462   end;
 463
 464   procedure parseId ();
 465   begin
 466     mTokType := TTId;
 467     mTokStr := ''; // just in case
 468     while (curChar = '_') or ((curChar >= '0') and (curChar <= '9')) or
 469           ((curChar >= 'A') and (curChar <= 'Z')) or
 470           ((curChar >= 'a') and (curChar <= 'z')) or
 471           (curChar >= #128) do
 472     begin
 473       mTokStr += curChar;
 474       skipChar();
 475     end;
 476   end;
 477
 478 begin
 479   mTokType := TTEOF;
 480   mTokStr := '';
 481   mTokChar := #0;
 482   mTokInt := 0;
 483
 484   if not skipBlanks() then
 485   begin
 486     result := false;
 487     mTokLine := mLine;
 488     mTokCol := mCol;
 489     exit;
 490   end;
 491
 492   mTokLine := mLine;
 493   mTokCol := mCol;
 494
 495   result := true;
 496
 497   // number?
 498   if mAllowSignedNumbers and ((curChar = '+') or (curChar = '-')) then begin parseInt(); exit; end;
 499   if (curChar >= '0') and (curChar <= '9') then begin parseInt(); exit; end;
 500
 501   // string?
 502   if (curChar = '"') or (curChar = '''') then begin parseString(); exit; end;
 503
 504   // identifier?
 505   if (curChar = '_') or ((curChar >= 'A') and (curChar <= 'Z')) or ((curChar >= 'a') and (curChar <= 'z')) or (curChar >= #128) then begin parseId(); exit; end;
 506
 507   // known delimiters?
 508   case curChar of
 509     ',': mTokType := TTComma;
 510     ':': mTokType := TTColon;
 511     ';': mTokType := TTSemi;
 512     '{': mTokType := TTBegin;
 513     '}': mTokType := TTEnd;
 514     else mTokType := TTDelim;
 515   end;
 516   mTokChar := curChar;
 517   skipChar();
 518 end;
 519
 520
 521 function TTextParser.expectId (): AnsiString;
 522 begin
 523   if (mTokType <> TTId) then raise Exception.Create('identifier expected');
 524   result := mTokStr;
 525   skipToken();
 526 end;
 527
 528
 529 procedure TTextParser.expectId (const aid: AnsiString);
 530 begin
 531   if (mTokType <> TTId) or (CompareText(mTokStr, aid) <> 0) then raise Exception.Create('identifier '''+aid+''' expected');
 532   skipToken();
 533 end;
 534
 535
 536 function TTextParser.eatId (const aid: AnsiString): Boolean;
 537 begin
 538   result := false;
 539   if (mTokType <> TTId) or (CompareText(mTokStr, aid) <> 0) then exit;
 540   result := true;
 541   skipToken();
 542 end;
 543
 544
 545 function TTextParser.expectStr (allowEmpty: Boolean=false): AnsiString;
 546 begin
 547   if (mTokType <> TTStr) then raise Exception.Create('string expected');
 548   if (not allowEmpty) and (Length(mTokStr) = 0) then raise Exception.Create('non-empty string expected');
 549   result := mTokStr;
 550   skipToken();
 551 end;
 552
 553
 554 function TTextParser.expectInt (): Integer;
 555 begin
 556   if (mTokType <> TTInt) then raise Exception.Create('string expected');
 557   result := mTokInt;
 558   skipToken();
 559 end;
 560
 561
 562 procedure TTextParser.expectTT (ttype: Integer);
 563 begin
 564   if (mTokType <> ttype) then raise Exception.Create('unexpected token');
 565   skipToken();
 566 end;
 567
 568
 569 function TTextParser.eatTT (ttype: Integer): Boolean;
 570 begin
 571   result := (mTokType = ttype);
 572   if result then skipToken();
 573 end;
 574
 575
 576 function TTextParser.expectDelim (const ch: AnsiChar): AnsiChar;
 577 begin
 578   if (mTokType <> TTDelim) then raise Exception.Create(Format('delimiter ''%s'' expected', [ch]));
 579   result := mTokChar;
 580   skipToken();
 581 end;
 582
 583
 584 function TTextParser.eatDelim (const ch: AnsiChar): Boolean;
 585 begin
 586   result := false;
 587   if (mTokType <> TTDelim) or (mTokChar <> ch) then exit;
 588   result := true;
 589   skipToken();
 590 end;
 591
 592
 593 // ////////////////////////////////////////////////////////////////////////// //
 594 constructor TFileTextParser.Create (const fname: AnsiString; loadToken: Boolean=true);
 595 begin
 596   mBuffer := nil;
 597   mFile := openDiskFileRO(fname);
 598   GetMem(mBuffer, BufSize);
 599   mBufPos := 0;
 600   mBufLen := mFile.Read(mBuffer^, BufSize);
 601   if (mBufLen < 0) then raise Exception.Create('TFileTextParser: read error');
 602   inherited Create(loadToken);
 603 end;
 604
 605
 606 constructor TFileTextParser.Create (st: TStream; loadToken: Boolean=true);
 607 begin
 608   if (st = nil) then raise Exception.Create('cannot create parser for nil stream');
 609   mFile := st;
 610   GetMem(mBuffer, BufSize);
 611   mBufPos := 0;
 612   mBufLen := mFile.Read(mBuffer^, BufSize);
 613   if (mBufLen < 0) then raise Exception.Create('TFileTextParser: read error');
 614   inherited Create(loadToken);
 615 end;
 616
 617
 618 destructor TFileTextParser.Destroy ();
 619 begin
 620   if (mBuffer <> nil) then FreeMem(mBuffer);
 621   mFile.Free();
 622   inherited;
 623 end;
 624
 625
 626 procedure TFileTextParser.loadNextChar ();
 627 begin
 628   if (mBufLen = 0) then begin mNextChar := #0; exit; end;
 629   if (mBufPos >= mBufLen) then
 630   begin
 631     mBufLen := mFile.Read(mBuffer^, BufSize);
 632     if (mBufLen < 0) then raise Exception.Create('TFileTextParser: read error');
 633     if (mBufLen = 0) then begin mNextChar := #0; exit; end;
 634     mBufPos := 0;
 635   end;
 636   assert(mBufPos < mBufLen);
 637   mNextChar := mBuffer[mBufPos];
 638   Inc(mBufPos);
 639   if (mNextChar = #0) then mNextChar := ' ';
 640 end;
 641
 642
 643 // ////////////////////////////////////////////////////////////////////////// //
 644 constructor TStrTextParser.Create (const astr: AnsiString; loadToken: Boolean=true);
 645 begin
 646   mStr := astr;
 647   mPos := 1;
 648   inherited Create(loadToken);
 649 end;
 650
 651
 652 destructor TStrTextParser.Destroy ();
 653 begin
 654   mStr := '';
 655   inherited;
 656 end;
 657
 658
 659 procedure TStrTextParser.loadNextChar ();
 660 begin
 661   mNextChar := #0;
 662   if (mPos > Length(mStr)) then exit;
 663   mNextChar := mStr[mPos]; Inc(mPos);
 664   if (mNextChar = #0) then mNextChar := ' ';
 665 end;
 666
 667
 668 // ////////////////////////////////////////////////////////////////////////// //
 669 constructor TTextWriter.Create (); begin mIndent := 0; end;
 670 procedure TTextWriter.put (const s: AnsiString); overload; begin if (Length(s) > 0) then putBuf((@(s[1]))^, Length(s)); end;
 671 procedure TTextWriter.put (v: Byte); overload; begin put('%d', [v]); end;
 672 procedure TTextWriter.put (v: Integer); overload; begin put('%d', [v]); end;
 673 procedure TTextWriter.put (const fmt: AnsiString; args: array of const); overload; begin put(formatstrf(fmt, args)); end;
 674 procedure TTextWriter.putIndent (); var f: Integer; begin for f := 1 to mIndent do put(' '); end;
 675 procedure TTextWriter.indent (); begin Inc(mIndent, 2); end;
 676 procedure TTextWriter.unindent (); begin Dec(mIndent, 2); end;
 677
 678
 679 // ////////////////////////////////////////////////////////////////////////// //
 680 constructor TFileTextWriter.Create (const fname: AnsiString);
 681 begin
 682   mFile := createDiskFile(fname);
 683   inherited Create();
 684 end;
 685
 686
 687 destructor TFileTextWriter.Destroy ();
 688 begin
 689   mFile.Free();
 690   inherited;
 691 end;
 692
 693
 694 procedure TFileTextWriter.putBuf (constref buf; len: SizeUInt);
 695 var
 696   pc: PChar;
 697 begin
 698   if (len > 0) then
 699   begin
 700     pc := @buf;
 701     mFile.WriteBuffer(pc^, len);
 702     {
 703     while (len > 0) do
 704     begin
 705       write(pc^);
 706       Inc(pc);
 707       Dec(len);
 708     end;
 709     }
 710   end;
 711 end;
 712
 713
 714 end.