[JEDI.NET-commits] main/run Jedi.Math.Evaluation.Tokenization.pas,1.2,1.3

[Marcel B. <jed...@us...>]

Update of /cvsroot/jedidotnet/main/run
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv14380/main/run

Modified Files:
	Jedi.Math.Evaluation.Tokenization.pas 
Log Message:
Tokenizer didn't recognize identifiers

Index: Jedi.Math.Evaluation.Tokenization.pas
===================================================================
RCS file: /cvsroot/jedidotnet/main/run/Jedi.Math.Evaluation.Tokenization.pas,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** Jedi.Math.Evaluation.Tokenization.pas	25 Sep 2005 11:28:07 -0000	1.2
--- Jedi.Math.Evaluation.Tokenization.pas	27 Sep 2005 18:29:35 -0000	1.3
***************
*** 57,60 ****
--- 57,61 ----
    strict protected
      function CheckGreaterThanBasedTokens(out token: Token): Boolean;
+     function CheckIdentifierToken(out token: Token): Boolean;
      function CheckLessThanBasedTokens(out token: Token): Boolean;
      function CheckLogicalTokens(out token: Token): Boolean;
***************
*** 157,160 ****
--- 158,184 ----
  end;
  
+ function MathLexer.CheckIdentifierToken(out token: Token): Boolean;
+ var
+   sb: StringBuilder;
+   startLine: Integer;
+   startColumn: Integer;
+ begin
+   // When this method is entered, we already know it's because it hasn't recognized anything to do with an expression
+   // Therefore, it must be an identifier. We consider everything up to the next whitespace or expression related
+   // character part of the identifier.
+   Result := Input.Sequence.Length > 0;
+   if Result then
+   begin
+     sb := StringBuilder.Create;
+     startLine := Input.Line;
+     startColumn := Input.Column;
+     repeat
+       sb.Append(Input.Sequence);
+     until not Input.Next or ('+-*/<>=!&|^?:,'.IndexOf(Input.Sequence.Chars[0]) > -1);
+     Result := sb.Length > 0;
+     token := IdentifierToken.Create(Context, startLine, startColumn, sb.ToString);
+   end;
+ end;
+ 
  function MathLexer.CheckLessThanBasedTokens(out token: Token): Boolean;
  var
***************
*** 350,354 ****
  begin
    CheckWhitespace(Result) or CheckNumberToken(Result) or CheckGreaterThanBasedTokens(Result) or
!     CheckLessThanBasedTokens(Result) or CheckLogicalTokens(Result) or CheckSingleCharTokens(Result);
  end;
  
--- 374,379 ----
  begin
    CheckWhitespace(Result) or CheckNumberToken(Result) or CheckGreaterThanBasedTokens(Result) or
!     CheckLessThanBasedTokens(Result) or CheckLogicalTokens(Result) or CheckSingleCharTokens(Result) or
!     CheckIdentifierToken(Result);
  end;