[Rubyeclipse-commit] SF.net SVN: rubyeclipse: [2161] branches/syntax/org.rubypeople.rdt.ui/src/ org/rubypeople/rdt

[<caw...@us...>]

Revision: 2161
          http://svn.sourceforge.net/rubyeclipse/?rev=2161&view=rev
Author:   cawilliams
Date:     2007-03-14 06:37:47 -0700 (Wed, 14 Mar 2007)

Log Message:
-----------
single lien comments must be handl;ed by the lexer, so we can't partition them into separate content types. Also, they get eaten up by the lexer in an odd way. 
When we grab a token, check if there was a comment eaten as well. If so, save the real token for next go-around and return the comment token first.

Modified Paths:
--------------
    branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
    branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
    branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java

Modified: branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
===================================================================
--- branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java	2007-03-14 08:16:07 UTC (rev 2160)
+++ branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java	2007-03-14 13:37:47 UTC (rev 2161)
@@ -22,10 +22,9 @@
 	protected int fPartitionOffset;
 
 	public final static String RUBY_MULTI_LINE_COMMENT = IRubyPartitions.RUBY_MULTI_LINE_COMMENT;
-	public static final String RUBY_SINGLE_LINE_COMMENT = IRubyPartitions.RUBY_SINGLE_LINE_COMMENT;
 
 	public static final String[] LEGAL_CONTENT_TYPES = {
-			RUBY_MULTI_LINE_COMMENT, RUBY_SINGLE_LINE_COMMENT,
+			RUBY_MULTI_LINE_COMMENT
 			};
 
 	public RubyPartitionScanner() {
@@ -35,14 +34,8 @@
 
 	protected void initialize() {
 		IToken multiLineComment = new Token(RUBY_MULTI_LINE_COMMENT);
-		IToken singleLineComment = new Token(RUBY_SINGLE_LINE_COMMENT);
 
 		List rules = new ArrayList();
-		// Single line comments
-		// ?# evaluates to the ascii value of #
-//		rules.add(new WordPatternRule(new NumberSignDetector(), "?", "#",
-//				Token.UNDEFINED));
-		rules.add(new EndOfLineRule("#", singleLineComment));
 		rules.add(new DocumentationCommentRule(multiLineComment));
 		IRule[] result = new IRule[rules.size()];
 		rules.toArray(result);

Modified: branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
===================================================================
--- branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java	2007-03-14 08:16:07 UTC (rev 2160)
+++ branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java	2007-03-14 13:37:47 UTC (rev 2161)
@@ -2,12 +2,14 @@
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.util.List;
 
 import org.eclipse.jface.preference.IPreferenceStore;
 import org.eclipse.jface.text.BadLocationException;
 import org.eclipse.jface.text.IDocument;
 import org.eclipse.jface.text.rules.IToken;
 import org.eclipse.jface.text.rules.Token;
+import org.jruby.ast.CommentNode;
 import org.jruby.lexer.yacc.LexState;
 import org.jruby.lexer.yacc.LexerSource;
 import org.jruby.lexer.yacc.RubyYaccLexer;
@@ -24,7 +26,10 @@
 
 	protected String[] keywords;
 
-	private static String[] fgTokenProperties = { IRubyColorConstants.RUBY_KEYWORD, IRubyColorConstants.RUBY_DEFAULT, IRubyColorConstants.RUBY_FIXNUM, IRubyColorConstants.RUBY_CHARACTER, IRubyColorConstants.RUBY_SYMBOL, IRubyColorConstants.RUBY_INSTANCE_VARIABLE, IRubyColorConstants.RUBY_GLOBAL, IRubyColorConstants.RUBY_STRING, IRubyColorConstants.RUBY_REGEXP, IRubyColorConstants.RUBY_ERROR
+	private static String[] fgTokenProperties = { IRubyColorConstants.RUBY_KEYWORD, IRubyColorConstants.RUBY_DEFAULT, 
+		IRubyColorConstants.RUBY_FIXNUM, IRubyColorConstants.RUBY_CHARACTER, IRubyColorConstants.RUBY_SYMBOL, 
+		IRubyColorConstants.RUBY_INSTANCE_VARIABLE, IRubyColorConstants.RUBY_GLOBAL, IRubyColorConstants.RUBY_STRING, 
+		IRubyColorConstants.RUBY_REGEXP, IRubyColorConstants.RUBY_ERROR, IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT
 	// TODO Add Ability to set colors for return and operators
 	// IRubyColorConstants.RUBY_METHOD_NAME,
 	// IRubyColorConstants.RUBY_KEYWORD_RETURN,
@@ -43,6 +48,12 @@
 	private int origLength;
 	private String contents;
 
+	private IToken fSavedToken = null;
+	private int fSavedLength = -1;
+	private int fSavedOffset = -1;
+
+	private boolean lastWasComment;
+
 	public RubyTokenScanner(IColorManager manager, IPreferenceStore store) {
 		super(manager, store);
 		lexer = new RubyYaccLexer();
@@ -55,14 +66,36 @@
 	}
 
 	public int getTokenLength() {
+		if (lastWasComment) {
+			lastWasComment = false;
+			return tokenLength;
+		}
+		if (fSavedLength != -1) {
+			int length = fSavedLength;
+			fSavedLength = -1;
+			return length;
+		}
 		return tokenLength;
 	}
 
 	public int getTokenOffset() {
+		if (lastWasComment) {			
+			return oldOffset;
+		}
+		if (fSavedOffset != -1) {
+			int offset = fSavedOffset;
+			fSavedOffset = -1;
+			return offset;
+		}
 		return oldOffset;
 	}
 
 	public IToken nextToken() {
+		if (fSavedToken != null) {
+			IToken returnToken = fSavedToken;
+			fSavedToken = null;
+			return returnToken;
+		}
 		oldOffset = getOffset();
 		tokenLength = 0;
 		IToken returnValue = getToken(IRubyColorConstants.RUBY_DEFAULT);
@@ -74,14 +107,23 @@
 			} else {
 				returnValue = token(lexer.token());
 			}
-			lexer.getStrTerm();
+			List comments = result.getCommentNodes();
+			if (comments != null && !comments.isEmpty()) {
+				CommentNode comment = (CommentNode) comments.remove(comments.size() - 1); // Grab last comment
+				tokenLength = comment.getContent().length() + 1;
+				fSavedToken = returnValue;
+				fSavedOffset = comment.getPosition().getEndOffset();
+				fSavedLength = lexerSource.getOffset() - fSavedOffset;			
+				lastWasComment = true;
+				return getToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
+			}
 		} catch (SyntaxException se) {
 			if (lexerSource.getOffset() - origLength == 0)
 				return Token.EOF; // return eof if we hit a problem found at
 									// end of parsing
 			else
 				tokenLength = getOffset() - oldOffset;
-			return new Token(IRubyColorConstants.RUBY_ERROR);
+			return getToken(IRubyColorConstants.RUBY_ERROR);
 		} catch (IOException e) {
 			RubyPlugin.log(e);
 		}

Modified: branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java
===================================================================
--- branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java	2007-03-14 08:16:07 UTC (rev 2160)
+++ branches/syntax/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java	2007-03-14 13:37:47 UTC (rev 2161)
@@ -87,7 +87,7 @@
 
 	protected AbstractRubyTokenScanner fCodeScanner;
 
-	protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner;
+	protected AbstractRubyScanner fMultilineCommentScanner;
 	private RubyDoubleClickSelector fRubyDoubleClickSelector;
 	private RubyCompletionProcessor fRubyCp;
 	
@@ -178,8 +178,7 @@
 	 */
 	public boolean affectsTextPresentation(PropertyChangeEvent event) {
 		return fCodeScanner.affectsBehavior(event)
-				|| fMultilineCommentScanner.affectsBehavior(event)
-				|| fSinglelineCommentScanner.affectsBehavior(event);
+				|| fMultilineCommentScanner.affectsBehavior(event);
 	}
 
 	/**
@@ -202,8 +201,6 @@
 			fCodeScanner.adaptToPreferenceChange(event);
 		if (fMultilineCommentScanner.affectsBehavior(event))
 			fMultilineCommentScanner.adaptToPreferenceChange(event);
-		if (fSinglelineCommentScanner.affectsBehavior(event))
-			fSinglelineCommentScanner.adaptToPreferenceChange(event);
 	}
 
 	/**
@@ -239,8 +236,6 @@
 		fCodeScanner = new RubyTokenScanner(getColorManager(), fPreferenceStore);
 		fMultilineCommentScanner = new RubyCommentScanner(getColorManager(),
 				fPreferenceStore, IRubyColorConstants.RUBY_MULTI_LINE_COMMENT);
-		fSinglelineCommentScanner = new RubyCommentScanner(getColorManager(),
-				fPreferenceStore, IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
 	}
 
 	/**
@@ -271,12 +266,6 @@
 		reconciler.setDamager(dr, IDocument.DEFAULT_CONTENT_TYPE);
 		reconciler.setRepairer(dr, IDocument.DEFAULT_CONTENT_TYPE);
 
-		dr = new DefaultDamagerRepairer(getSinglelineCommentScanner());
-		reconciler
-				.setDamager(dr, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
-		reconciler.setRepairer(dr,
-				RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
-
 		dr = new DefaultDamagerRepairer(getMultilineCommentScanner());
 		reconciler.setDamager(dr, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT);
 		reconciler
@@ -292,14 +281,9 @@
 		return fMultilineCommentScanner;
 	}
 
-	protected ITokenScanner getSinglelineCommentScanner() {
-		return fSinglelineCommentScanner;
-	}
-
 	public String[] getConfiguredContentTypes(ISourceViewer sourceViewer) {
 		return new String[] { IDocument.DEFAULT_CONTENT_TYPE,
-				RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT,
-				RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT };
+				RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT };
 	}
 
 	/*


This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.