[JEDI.NET-commits] main/run Jedi.Text.Tokenization.InputBase.pas,NONE,1.1 Jedi.Text.Tokenization.Pro

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Update of /cvsroot/jedidotnet/main/run
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv24439/main/run

Added Files:
	Jedi.Text.Tokenization.InputBase.pas 
	Jedi.Text.Tokenization.ProviderBase.pas 
	Jedi.Text.Tokenization.TokenBase.pas 
	Jedi.Text.Tokenization.TokenizerBase.pas 
Removed Files:
	Jedi.Text.Tokenization.Base.pas 
Log Message:
Redesigned tokenization

--- Jedi.Text.Tokenization.Base.pas DELETED ---

--- NEW FILE: Jedi.Text.Tokenization.TokenizerBase.pas ---
{---------------------------------------------------------------------------------------------------
The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you
may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.mozilla.org/MPL/MPL-1.1.html

Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
KIND, either expressed or implied. See the License for the specific language governing rights and
limitations under the License.

The Original Code is: Jedi.Text.Tokenization.TokenizerBase.pas, released on --.

The Initial Developer of the Original Code is Marcel Bestebroer
Portions created by Marcel Bestebroer are Copyright (C) 2004 Marcel Bestebroer
All Rights Reserved.

Contributor(s):

You may retrieve the latest version of this file at the JEDI.NET home page, located at
http://sf.net/projects/jedidotnet

Known Issues:
---------------------------------------------------------------------------------------------------}
// $Id: Jedi.Text.Tokenization.TokenizerBase.pas,v 1.1 2005/09/25 11:27:26 jedi_mbe Exp $
unit Jedi.Text.Tokenization.TokenizerBase;

interface

{$REGION 'uses'}
uses
  Jedi.System.SourceVersioning,
  Jedi.Text.Tokenization.InputBase,
  Jedi.Text.Tokenization.ProviderBase,
  Jedi.Text.Tokenization.TokenBase,
  System.Text;
{$ENDREGION}

{$REGION 'Tokenizer base class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenizerBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  Tokenizer = class abstract (&Object, ITokenProvider)
  {$REGION 'Constructors'}
  strict protected
    constructor Create(context: &Object; input: TokenizerInputProvider);
  {$ENDREGION}
  {$REGION 'TokenizerState enumeration'}
  strict protected
    type
      TokenizerState = (Initializing, Processing, Finalizing, Finalized);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FBuffer: TokenList;
    FContext: &Object;
    FCurrent: Token;
    FInput: TokenizerInputProvider;
    FState: TokenizerState;
  {$ENDREGION}
  {$REGION 'ITokenProvider methods'}
  strict protected
    procedure DiscardPointer;
    function get_Current: Token;
    function HasSavedPointer: Boolean;
    function Next: Boolean;
    procedure RestorePointer;
    procedure SavePointer;
  {$ENDREGION}
  {$REGION 'Abstract methods'}
  strict protected
    function InputToken: Token; virtual; abstract;
  {$ENDREGION}
  {$REGION 'Internal methods'}
  strict private
    function GetWhitespaceSequence: string;
  {$ENDREGION}
  {$REGION 'Whitespace checking methods'}
  strict protected
    function CheckLiteralWhitespace(out token: Token): Boolean;
    function CheckWhitespace(out token: Token): Boolean;
  {$ENDREGION}
  {$REGION 'Properties'}
  strict protected
    property Buffer: TokenList read FBuffer;
    property Input: TokenizerInputProvider read FInput;
    property State: TokenizerState read FState;
  public
    property Context: &Object read FContext;
  {$ENDREGION}
  end;
{$ENDREGION}

implementation

{$REGION 'Tokenizer'}
constructor Tokenizer.Create(context: &Object; input: TokenizerInputProvider);
begin
  inherited Create;
  FBuffer := TokenList.Create;
  FContext := context;
  FInput := input;
  FState := TokenizerState.Initializing;
end;

function Tokenizer.CheckLiteralWhitespace(out token: Token): Boolean;
var
  startLine: Integer;
  startColumn: Integer;
  seq: string;
begin
  startLine := Input.Line;
  startColumn := Input.Column;
  seq := GetWhitespaceSequence;
  Result := (seq.Length > 0);
  if Result then
    token := WhitespaceToken.CreateLiteral(seq, Context, startLine, startColumn)
  else
    token := nil;
end;

function Tokenizer.CheckWhitespace(out token: Token): Boolean;
var
  startLine: Integer;
  startColumn: Integer;
  seq: string;
begin
  startLine := Input.Line;
  startColumn := Input.Column;
  seq := GetWhitespaceSequence;
  Result := (seq.Length > 0);
  if Result then
    token := WhitespaceToken.CreateNonLiteral(Context, startLine, startColumn, Input.Line, Input.Column)
  else
    token := nil;
end;

procedure Tokenizer.DiscardPointer;
begin
  ITokenProvider(FBuffer).DiscardPointer;
end;

function Tokenizer.get_Current: Token;
begin
  if State = TokenizerState.Initializing then
    raise InvalidOperationException.Create('Before start');
  Result := FCurrent;
end;

function Tokenizer.GetWhitespaceSequence: string;
var
  sb: StringBuilder;
begin
  if (Input.Sequence.Length > 0) and System.Char.IsWhiteSpace(Input.Sequence, 0) then
  begin
    sb := StringBuilder.Create;
    repeat
      sb.Append(Input.Sequence);
    until not Input.Next or not System.Char.IsWhiteSpace(Input.Sequence, 0);
    Result := sb.ToString;
  end
  else
    Result := '';
end;

function Tokenizer.HasSavedPointer: Boolean;
begin
  Result := ITokenProvider(FBuffer).HasSavedPointer;
end;

function Tokenizer.Next: Boolean;
var
  nextToken: Token;
begin
  if HasSavedPointer and ITokenProvider(Buffer).Next then
  begin
    Result := True;
    FCurrent := ITokenProvider(Buffer).Current;
  end
  else
  begin
    if State = TokenizerState.Initializing then
    begin
      Result := True;
      FCurrent := TokenizerStartToken.Create(FContext);
      if FInput.Next then
        FState := TokenizerState.Processing
      else
        FState := TokenizerState.Finalizing;
    end
    else
    if State = TokenizerState.Finalizing then
    begin
      Result := True;
      FCurrent := TokenizerEndToken.Create(FContext, Input.Line, Input.Column);
      FState := TokenizerState.Finalized;
    end
    else
    if State = TokenizerState.Processing then
    begin
      nextToken := InputToken;
      Result := nextToken <> nil;
      if Result then
        FCurrent := nextToken
      else
      begin
        Result := True;
        FCurrent := UnknownCharacterToken.Create(FContext, Input.Sequence, Input.Line, Input.Column);
      end;
      if Input.Sequence = '' then
        FState := TokenizerState.Finalizing;
    end
    else
      Result := False;
    if Result and HasSavedPointer then
      Buffer.Add(FCurrent);
  end
end;

procedure Tokenizer.RestorePointer;
begin
  ITokenProvider(FBuffer).RestorePointer;
end;

procedure Tokenizer.SavePointer;
begin
  ITokenProvider(FBuffer).SavePointer;
end;
{$ENDREGION}

end.

--- NEW FILE: Jedi.Text.Tokenization.InputBase.pas ---
{---------------------------------------------------------------------------------------------------
The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you
may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.mozilla.org/MPL/MPL-1.1.html

Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
KIND, either expressed or implied. See the License for the specific language governing rights and
limitations under the License.

The Original Code is: Jedi.Text.Tokenization.InputBase.pas, released on --.

The Initial Developer of the Original Code is Marcel Bestebroer
Portions created by Marcel Bestebroer are Copyright (C) 2004 Marcel Bestebroer
All Rights Reserved.

Contributor(s):

You may retrieve the latest version of this file at the JEDI.NET home page, located at
http://sf.net/projects/jedidotnet

Known Issues:
---------------------------------------------------------------------------------------------------}
// $Id: Jedi.Text.Tokenization.InputBase.pas,v 1.1 2005/09/25 11:27:26 jedi_mbe Exp $
unit Jedi.Text.Tokenization.InputBase;

interface

{$REGION 'uses'}
uses
  Jedi.System.SourceVersioning,
  Jedi.System.Strings,
  System.Collections,
  System.IO,
  System.Text;
{$ENDREGION}

{$REGION 'Tokenizer input provider base class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.InputBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  TokenizerInputProvider = class abstract (&Object)
  {$REGION 'Constructors'}
  public
    constructor Create(tabSet: StringUtils.TabSet);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FBuffer: IList;
    FColumn: Integer;
    FIndex: Integer;
    FLine: Integer;
    FSavedInfos: Stack;
    FSequence: StringBuilder;
    FTabSet: StringUtils.TabSet;
  {$ENDREGION}
  {$REGION 'Abstract methods'}
  strict protected
    function PeekChar: Integer; virtual; abstract;
    function NextChar: Integer; virtual; abstract;
  {$ENDREGION}
  {$REGION 'Advance input'}
  public
    function Next: Boolean;
  {$ENDREGION}
  {$REGION 'Internal methods'}
  strict protected
    function ReadFromStack: Boolean;
  {$ENDREGION}
  {$REGION 'Location saving/restoring'}
  public
    procedure Discard;
    procedure Restore;
    procedure Save;
  {$ENDREGION}
  {$REGION 'Property accessors'}
  public
    function get_Sequence: string;
  {$ENDREGION}
  {$REGION 'Properties'}
  protected
    property Buffer: IList read FBuffer;
  public
    property Column: Integer read FColumn;
    property Line: Integer read FLine;
    property Sequence: string read get_Sequence;
    property TabSet: StringUtils.TabSet read FTabSet;
  {$ENDREGION}
  {$REGION 'TextReader-based input provider instantiation'}
  public
    class function FromReader(reader: TextReader): TokenizerInputProvider; overload; static;
    class function FromReader(reader: TextReader; tabSet: StringUtils.TabSet): TokenizerInputProvider; overload; static;
  {$ENDREGION}
  end;
{$ENDREGION}

implementation

{$REGION 'TextReader-based tokenizer input provider class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.InputBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  ReaderProvider = class (TokenizerInputProvider)
  {$REGION 'Constructors'}
  public
    constructor Create(reader: TextReader; tabSet: StringUtils.TabSet);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FReader: TextReader;
  {$ENDREGION}
  {$REGION 'Overrides'}
  strict protected
    function PeekChar: Integer; override;
    function NextChar: Integer; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Saved location information record'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.InputBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  SavedInfo = record
  {$REGION 'Constructors'}
  public
    constructor Create(provider: TokenizerInputProvider);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FColumn: Integer;
    FIndex: Integer;
    FLine: Integer;
    FSequence: string;
  {$ENDREGION}
  {$REGION 'Properties'}
  public
    property Column: Integer read FColumn;
    property &Index: Integer read FIndex;
    property Line: Integer read FLine;
    property Sequence: string read FSequence;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'ReaderProvider'}
constructor ReaderProvider.Create(reader: TextReader; tabSet: StringUtils.TabSet);
begin
  inherited Create(tabSet);
  FReader := reader;
end;

function ReaderProvider.PeekChar: Integer;
begin
  if FReader <> nil then
    Result := FReader.Peek
  else
    Result := -1;
end;

function ReaderProvider.NextChar: Integer;
begin
  if FReader <> nil then
  begin
    Result := FReader.Read;
    if Result = -1 then
    begin
      FReader.Close;
      FReader := nil;
    end;
  end
  else
    Result := -1;
end;
{$ENDREGION}

{$REGION 'SavedInfo'}
constructor SavedInfo.Create(provider: TokenizerInputProvider);
begin
  inherited Create;
  FColumn := provider.Column;
  FLine := provider.Line;
  FIndex := provider.Buffer.Count;
  FSequence := provider.Sequence;
end;
{$ENDREGION}

{$REGION 'TokenizerInputProvider'}
constructor TokenizerInputProvider.Create(tabSet: StringUtils.TabSet);
begin
  inherited Create;
  FTabSet := tabSet;
  FColumn := -1;
  FLine := -1;
  FBuffer := ArrayList.Create;
  FSavedInfos := Stack.Create;
  FSequence := StringBuilder.Create;
end;

procedure TokenizerInputProvider.Discard;
begin
  FSavedInfos.Pop;
end;

class function TokenizerInputProvider.FromReader(reader: TextReader): TokenizerInputProvider;
begin
  Result := ReaderProvider.Create(reader, StringUtils.TabSet.Create);
end;

class function TokenizerInputProvider.FromReader(reader: TextReader;
  tabSet: StringUtils.TabSet): TokenizerInputProvider;
begin
  Result := ReaderProvider.Create(reader, tabSet);
end;

function TokenizerInputProvider.get_Sequence: string;
begin
  Result := FSequence.ToString;
end;

function TokenizerInputProvider.Next: Boolean;
var
  cameFromStack: Boolean;
  ordinal: Integer;
begin
  if FSequence.Length > 0 then
  begin
    if FSequence.Chars[0] = #9 then
      FColumn := FTabSet.TabFrom(FColumn)
    else
    if (FSequence.Chars[0] = #10) or (FSequence.Chars[0] = #13) then
    begin
      Inc(FLine);
      FColumn := 0;
    end
    else
      Inc(FColumn);
  end;
  FSequence.Length := 0;
  cameFromStack := ReadFromStack;
  if cameFromStack then
  begin
    FSequence.Append(string(Buffer[FIndex]));
    Inc(FIndex);
  end
  else
  begin
    ordinal := NextChar;
    if ordinal > -1 then
    begin
      FSequence.Append(System.Char(ordinal));
      if ((ordinal = 10) and (PeekChar = 13)) or ((ordinal = 10) and (PeekChar = 13)) then
        FSequence.Append(System.Char(NextChar));
    end;
  end;
  Result := FSequence.Length > 0;
  if Result and (FLine < 0) then
  begin
    FLine := 0;
    FColumn := 0;
  end;
  if Result and not cameFromStack and (FSavedInfos.Count > 0) then
    FBuffer.Add(FSequence.ToString);
end;

function TokenizerInputProvider.ReadFromStack: Boolean;
begin
  Result := FIndex < FBuffer.Count;
end;

procedure TokenizerInputProvider.Restore;
var
  state: SavedInfo;
begin
  state := SavedInfo(FSavedInfos.Pop);
  FColumn := state.Column;
  FLine := state.Line;
  FSequence.Length := 0;
  FSequence.Append(state.Sequence);
  FIndex := state.Index;
end;

procedure TokenizerInputProvider.Save;
begin
  FSavedInfos.Push(Self);
end;
{$ENDREGION}

end.

--- NEW FILE: Jedi.Text.Tokenization.TokenBase.pas ---
{---------------------------------------------------------------------------------------------------
The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you
may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.mozilla.org/MPL/MPL-1.1.html

Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
KIND, either expressed or implied. See the License for the specific language governing rights and
limitations under the License.

The Original Code is: Jedi.Text.Tokenization.TokenBase.pas, released on --.

The Initial Developer of the Original Code is Marcel Bestebroer
Portions created by Marcel Bestebroer are Copyright (C) 2004 Marcel Bestebroer
All Rights Reserved.

Contributor(s):

You may retrieve the latest version of this file at the JEDI.NET home page, located at
http://sf.net/projects/jedidotnet

Known Issues:
---------------------------------------------------------------------------------------------------}
// $Id: Jedi.Text.Tokenization.TokenBase.pas,v 1.1 2005/09/25 11:27:26 jedi_mbe Exp $
unit Jedi.Text.Tokenization.TokenBase;

interface

{$REGION 'uses'}
uses
  Jedi.System.SourceVersioning,
  Jedi.System.Strings,
  System.Text;
{$ENDREGION}

{$REGION 'Token base class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  Token = class abstract (&Object)
  {$REGION 'Constructors'}
  strict protected
    constructor Create(context: &Object; line, column: Integer);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FContext: &Object;
    FColumn: Integer;
    FLine: Integer;
  {$ENDREGION}
  {$REGION 'Property accessors'}
  public
    function get_Literal: string; virtual; abstract;
  {$ENDREGION}
  {$REGION 'Properties'}
  public
    property Context: &Object read FContext;
    property Column: Integer read FColumn;
    property Line: Integer read FLine;
    property Literal: string read get_Literal;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Control tokens base class'}
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  ControlToken = class abstract (Token)
  {$REGION 'Constructors'}
  strict protected
    constructor Create(context: &Object; line, column: Integer);
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'End of tokenizer token'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  TokenizerEndToken = class sealed (ControlToken)
  {$REGION 'Constructors'}
  public
    constructor Create(context: &Object; line, column: Integer);
  {$ENDREGION}
  {$REGION 'Overrides'}
  public
    function get_Literal: string; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Start of tokenizer token'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  TokenizerStartToken = class sealed (ControlToken)
  {$REGION 'Constructors'}
  public
    constructor Create(context: &Object);
  {$ENDREGION}
  {$REGION 'Overrides'}
  public
    function get_Literal: string; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Unknown character sequence token'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  UnknownCharacterToken = class sealed (Token)
  {$REGION 'Constructors'}
  public
    constructor Create(context: &Object; sequence: string; line, column: Integer);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FSequence: string;
  {$ENDREGION}
  {$REGION 'Overrides'}
  public
    function get_Literal: string; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Whitespace token base class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  WhitespaceToken = class abstract (Token)
  {$REGION 'Constructors'}
  strict protected
    constructor Create(context: &Object; line, column: Integer);
  {$ENDREGION}
  {$REGION 'Property accessors'}
  public
    function get_IsNewline: Boolean; virtual; abstract;
  {$ENDREGION}
  {$REGION 'Properties'}
  public
    property IsNewline: Boolean read get_IsNewline;
  {$ENDREGION}
  {$REGION 'Whitespace token class instantiation methods'}
  public
    class function CreateLiteral(content: string; context: &Object; line, column: Integer): WhitespaceToken; static;
    class function CreateNonLiteral(context: &Object; line, column, endLine,
      endColumn: Integer): WhitespaceToken; static;
  {$ENDREGION}
  end;
{$ENDREGION}

implementation

{$REGION 'Literal whitespace token class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  LiteralWhitespace = class (WhitespaceToken)
  {$REGION 'Constructors'}
  public
    constructor Create(content: string; context: &Object; line, column: Integer);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FContent: string;
  {$ENDREGION}
  {$REGION 'Overrides'}
  public
    function get_IsNewline: Boolean; override;
    function get_Literal: string; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'Non-literal whitespace token class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.TokenBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  NonLiteralWhitespace = class (WhitespaceToken)
  {$REGION 'Constructors'}
  public
    constructor Create(context: &Object; line, column, endLine, endColumn: Integer);
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FEndColumn: Integer;
    FEndLine: Integer;
  {$ENDREGION}
  {$REGION 'Overrides'}
  public
    function get_IsNewline: Boolean; override;
    function get_Literal: string; override;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'ControlToken'}
constructor ControlToken.Create(context: &Object; line, column: Integer);
begin
  inherited Create(context, line, column);
end;
{$ENDREGION}

{$REGION 'LiteralWhitespace'}
constructor LiteralWhitespace.Create(content: string; context: &Object; line, column: Integer);
begin
  inherited Create(context, line, column);
  FContent := content;
end;

function LiteralWhitespace.get_IsNewline: Boolean;
begin
  Result := FContent.IndexOfAny([#10, #13]) > -1;
end;

function LiteralWhitespace.get_Literal: string;
begin
  Result := FContent;
end;
{$ENDREGION}

{$REGION 'NonLiteralWhitespace'}
constructor NonLiteralWhitespace.Create(context: &Object; line, column, endLine, endColumn: Integer);
begin
  inherited Create(context, line, column);
  FEndColumn := endColumn;
  FEndLine := endLine;
end;

function NonLiteralWhitespace.get_IsNewline: Boolean;
begin
  Result := FEndLine > Line;
end;

function NonLiteralWhitespace.get_Literal: string;
var
  sb: StringBuilder;
  i: Integer;
  col: Integer;
begin
  sb := StringBuilder.Create;
  for i := Line + 1 to FEndLine do
    sb.Append(Environment.NewLine);
  if FEndLine > Line then
    col := 0
  else
    col := Column;
  sb.Append(' ', FEndColumn - col);
  Result := sb.ToString;
end;
{$ENDREGION}

{$REGION 'Token'}
constructor Token.Create(context: &Object; line, column: Integer);
begin
  inherited Create;
  FContext := context;
  FLine := line;
  FColumn := column;
end;
{$ENDREGION}

{$REGION 'TokenizerEndToken'}
constructor TokenizerEndToken.Create(context: &Object; line, column: Integer);
begin
  inherited Create(context, line, column);
end;

function TokenizerEndToken.get_Literal: string;
begin
  Result := '';
end;
{$ENDREGION}

{$REGION 'TokenizerStartToken'}
constructor TokenizerStartToken.Create(context: &Object);
begin
  inherited Create(context, -1, -1);
end;

function TokenizerStartToken.get_Literal: string;
begin
  Result := '';
end;
{$ENDREGION}

{$REGION 'WhitespaceToken'}
constructor WhitespaceToken.Create(context: &Object; line, column: Integer);
begin
  inherited Create(context, line, column);
end;

class function WhitespaceToken.CreateLiteral(content: string; context: &Object; line, column: Integer): WhitespaceToken;
begin
  Result := LiteralWhitespace.Create(content, context, line, column);
end;

class function WhitespaceToken.CreateNonLiteral(context: &Object; line, column, endLine,
  endColumn: Integer): WhitespaceToken;
begin
  Result := NonLiteralWhitespace.Create(context, line, column, endLine, endColumn);
end;
{$ENDREGION}

{$REGION 'Unknown character sequence token'}
constructor UnknownCharacterToken.Create(context: &Object; sequence: string; line, column: Integer);
begin
  inherited Create(context, line, column);
  FSequence := sequence;
end;

function UnknownCharacterToken.get_Literal: string;
begin
  Result := FSequence;
end;
{$ENDREGION}

end.

--- NEW FILE: Jedi.Text.Tokenization.ProviderBase.pas ---
{---------------------------------------------------------------------------------------------------
The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you
may not use this file except in compliance with the License. You may obtain a copy of the License at
http://www.mozilla.org/MPL/MPL-1.1.html

Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
KIND, either expressed or implied. See the License for the specific language governing rights and
limitations under the License.

The Original Code is: Jedi.Text.Tokenization.ProviderBase.pas, released on --.

Contributor(s):

You may retrieve the latest version of this file at the JEDI.NET home page, located at
http://sf.net/projects/jedidotnet

Known Issues:
---------------------------------------------------------------------------------------------------}
// $Id: Jedi.Text.Tokenization.ProviderBase.pas,v 1.1 2005/09/25 11:27:26 jedi_mbe Exp $
unit Jedi.Text.Tokenization.ProviderBase;

interface

{$REGION 'uses'}
uses
  Jedi.System.SourceVersioning,
  Jedi.text.Tokenization.TokenBase,
  System.Collections;
{$ENDREGION}

{$REGION 'ITokenProvider interface'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.ProviderBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  ITokenProvider = interface
  {$REGION 'Methods'}
    procedure DiscardPointer;
    function HasSavedPointer: Boolean;
    function Next: Boolean;
    procedure RestorePointer;
    procedure SavePointer;
  {$ENDREGION}
  {$REGION 'Property accessors'}
    function get_Current: Token;
  {$ENDREGION}
  {$REGION 'Properties'}
    property Current: Token read get_Current;
  {$ENDREGION}
  end;
{$ENDREGION}

{$REGION 'List token provider class'}
type
  [JediSourceInfo(
    '$Source: /cvsroot/jedidotnet/main/run/Jedi.Text.Tokenization.ProviderBase.pas,v $',
    '$Revision: 1.1 $',
    '$Date: 2005/09/25 11:27:26 $')]
  TokenList = class (&Object, ITokenProvider)
  {$REGION 'Constructors'}
  public
    constructor Create;
  {$ENDREGION}
  {$REGION 'Data'}
  strict private
    FIndex: Integer;
    FList: IList;
    FSavePoints: Stack;
  {$ENDREGION}
  {$REGION 'ITokenProvider methods'}
  strict protected
    procedure DiscardPointer;
    function get_Current: Token;
    function HasSavedPointer: Boolean;
    function Next: Boolean;
    procedure RestorePointer;
    procedure SavePointer;
  {$ENDREGION}
  {$REGION 'List management methods'}
  public
    procedure Add(token: Token);
    procedure Clear;
  {$ENDREGION}
  end;
{$ENDREGION}

implementation

{$AUTOBOX ON}

{$REGION 'TokenList'}
constructor TokenList.Create;
begin
  inherited Create;
  FIndex := -1;
  FList := ArrayList.Create;
  FSavePoints := Stack.Create;
end;

procedure TokenList.Add(token: Token);
begin
  FList.Add(token);
end;

procedure TokenList.Clear;
begin
  FList.Clear;
end;

procedure TokenList.DiscardPointer;
begin
  FSavePoints.Pop;
end;

function TokenList.get_Current: Token;
begin
  if FIndex < 0 then
    raise InvalidOperationException.Create('Before start.');
  Result := Token(FList[FIndex]);
end;

function TokenList.HasSavedPointer: Boolean;
begin
  Result := FSavePoints.Count > 0;
end;

function TokenList.Next: Boolean;
begin
  Result := Succ(FIndex) < FList.Count;
  if Result then
    Inc(FIndex);
end;

procedure TokenList.RestorePointer;
begin
  FIndex := Integer(FSavePoints.Pop);
end;

procedure TokenList.SavePointer;
begin
  FSavePoints.Push(FIndex);
end;
{$ENDREGION}

end.

[JEDI.NET-commits] main/run Jedi.Text.Tokenization.InputBase.pas,NONE,1.1 Jedi.Text.Tokenization.Pro

[JEDI.NET-commits] main/run Jedi.Text.Tokenization.InputBase.pas,NONE,1.1 Jedi.Text.Tokenization.ProviderBase.pas,NONE,1.1 Jedi.Text.Tokenization.TokenBase.pas,NONE,1.1 Jedi.Text.Tokenization.TokenizerBase.pas,NONE,1.1 Jedi.Text.Tokenization.Base.pas,1.1,NONE