Menu
▾
▴
[Rubyeclipse-commit] SF.net SVN: rubyeclipse: [2487] trunk/org.rubypeople.rdt.ui
|
From: <caw...@us...> - 2007-05-16 15:33:27
|
Revision: 2487
http://svn.sourceforge.net/rubyeclipse/?rev=2487&view=rev
Author: cawilliams
Date: 2007-05-16 08:33:25 -0700 (Wed, 16 May 2007)
Log Message:
-----------
change the partitionscanner to rip off the work I did on rubyTokenScanner. We pass it through the lexer to get the major partitions in the partition scanner now. This also lets us ignore comments in the RubyTokenScanner because it only deals with code (since the partition scanner breaks those into distinct partitions). Add some tests to make sure the partition scanner works.
Modified Paths:
--------------
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java
trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java 2007-05-16 15:21:22 UTC (rev 2486)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java 2007-05-16 15:33:25 UTC (rev 2487)
@@ -1,114 +1,181 @@
package org.rubypeople.rdt.internal.ui.text;
-import java.util.ArrayList;
+import java.io.IOException;
+import java.io.StringReader;
import java.util.List;
+import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
-import org.eclipse.jface.text.rules.BufferedRuleBasedScanner;
import org.eclipse.jface.text.rules.IPartitionTokenScanner;
-import org.eclipse.jface.text.rules.IPredicateRule;
-import org.eclipse.jface.text.rules.IRule;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.Token;
+import org.jruby.ast.CommentNode;
+import org.jruby.common.NullWarnings;
+import org.jruby.lexer.yacc.LexState;
+import org.jruby.lexer.yacc.LexerSource;
+import org.jruby.lexer.yacc.RubyYaccLexer;
+import org.jruby.lexer.yacc.SyntaxException;
+import org.jruby.parser.ParserSupport;
+import org.jruby.parser.RubyParserConfiguration;
+import org.jruby.parser.RubyParserResult;
+import org.rubypeople.rdt.internal.core.util.ASTUtil;
+import org.rubypeople.rdt.internal.ui.RubyPlugin;
-public class RubyPartitionScanner extends BufferedRuleBasedScanner implements
- IPartitionTokenScanner {
+public class RubyPartitionScanner implements IPartitionTokenScanner {
- /** The content type of the partition in which to resume scanning. */
- protected String fContentType;
-
- /** The offset of the partition inside which to resume. */
- protected int fPartitionOffset;
-
+ private RubyYaccLexer lexer;
+ private ParserSupport parserSupport;
+ private RubyParserResult result;
+ private boolean lastWasComment;
+ private int fSavedLength;
+ private IToken fSavedToken;
+ private int fSavedOffset;
+ private String contents;
+ private LexerSource lexerSource;
+ private int origOffset;
+ private int origLength;
+ private int tokenLength;
+ private int oldOffset;
+
+ // XXX Also do strings, regex partitions!
public final static String RUBY_MULTI_LINE_COMMENT = IRubyPartitions.RUBY_MULTI_LINE_COMMENT;
+ public final static String RUBY_SINGLE_LINE_COMMENT = IRubyPartitions.RUBY_SINGLE_LINE_COMMENT;
public static final String[] LEGAL_CONTENT_TYPES = {
- RUBY_MULTI_LINE_COMMENT
+ RUBY_MULTI_LINE_COMMENT, RUBY_SINGLE_LINE_COMMENT
};
public RubyPartitionScanner() {
- super();
- initialize();
+ lexer = new RubyYaccLexer();
+ parserSupport = new ParserSupport();
+ parserSupport.setConfiguration(new RubyParserConfiguration());
+ result = new RubyParserResult();
+ parserSupport.setResult(result);
+ lexer.setParserSupport(parserSupport);
+ lexer.setWarnings(new NullWarnings());
}
- protected void initialize() {
- IToken multiLineComment = new Token(RUBY_MULTI_LINE_COMMENT);
-
- List rules = new ArrayList();
- rules.add(new DocumentationCommentRule(multiLineComment));
- IRule[] result = new IRule[rules.size()];
- rules.toArray(result);
- setRules(result);
+ public void setPartialRange(IDocument document, int offset, int length,
+ String contentType, int partitionOffset) {
+ lexer.reset();
+ lexer.setState(LexState.EXPR_BEG);
+ parserSupport.initTopLocalVariables();
+ lastWasComment = false;
+ fSavedLength = -1;
+ fSavedToken = null;
+ fSavedOffset = -1;
+ try {
+ contents = document.get(offset, length);
+ lexerSource = new LexerSource("filename", new StringReader(contents));
+ lexer.setSource(lexerSource);
+ } catch (BadLocationException e) {
+ lexerSource = new LexerSource("filename", new StringReader(""));
+ lexer.setSource(lexerSource);
+ }
+ origOffset = offset;
+ origLength = length;
}
- /*
- * @see ITokenScanner#setRange(IDocument, int, int)
- */
- public void setRange(IDocument document, int offset, int length) {
- setPartialRange(document, offset, length, null, -1);
+ public int getTokenLength() {
+ if (lastWasComment) {
+ return tokenLength;
+ }
+ if (fSavedLength != -1) {
+ int length = fSavedLength;
+ fSavedLength = -1;
+ return length;
+ }
+ return tokenLength;
}
- /*
- * @see IPartitionTokenScanner#setPartialRange(IDocument, int, int, String,
- * int)
- */
- public void setPartialRange(IDocument document, int offset, int length,
- String contentType, int partitionOffset) {
- fContentType = contentType;
- fPartitionOffset = partitionOffset;
- if (partitionOffset > -1) {
- int delta = offset - partitionOffset;
- if (delta > 0) {
- super.setRange(document, partitionOffset, length + delta);
- fOffset = offset;
- return;
- }
+ public int getTokenOffset() {
+ if (lastWasComment) {
+ return oldOffset;
}
- super.setRange(document, offset, length);
+ if (fSavedOffset != -1) {
+ int offset = fSavedOffset;
+ fSavedOffset = -1;
+ return offset;
+ }
+ return oldOffset;
}
- /*
- * @see ITokenScanner#nextToken()
- */
public IToken nextToken() {
-
- if (fContentType == null || fRules == null) {
- // don't try to resume
- return super.nextToken();
+ if (lastWasComment) {
+ lastWasComment = false;
}
-
- // inside a partition
-
- fColumn = UNDEFINED;
- boolean resume = (fPartitionOffset > -1 && fPartitionOffset < fOffset);
- fTokenOffset = resume ? fPartitionOffset : fOffset;
-
- IRule rule;
- IToken token;
-
- for (int i = 0; i < fRules.length; i++) {
- rule = (IRule) fRules[i];
- if (rule instanceof IPredicateRule) {
- IPredicateRule predRule = (IPredicateRule) rule;
- token = predRule.getSuccessToken();
- if (fContentType.equals(token.getData())) {
- token = predRule.evaluate(this, resume);
- if (!token.isUndefined()) {
- fContentType = null;
- return token;
+ if (fSavedToken != null) {
+ IToken returnToken = fSavedToken;
+ fSavedToken = null;
+ return returnToken;
+ }
+ oldOffset = getOffset();
+ tokenLength = 0;
+ IToken returnValue = new Token(null);
+ boolean isEOF = false;
+ try {
+ isEOF = !lexer.advance();
+ if (isEOF) {
+ returnValue = Token.EOF;
+ } else {
+ returnValue = token(lexer.token());
+ }
+ List comments = result.getCommentNodes();
+ if (comments != null && !comments.isEmpty()) {
+ CommentNode comment;
+ boolean firstComment = true;
+ int endOffset = 0;
+ boolean multiline = false;
+ while (!comments.isEmpty()) {
+ comment = (CommentNode) comments.remove(0);
+ if (firstComment) {
+ String src = ASTUtil.getSource(contents, comment);
+ if (src != null && src.startsWith("=begin")) multiline = true;
+ firstComment = false;
+ oldOffset = origOffset + comment.getPosition().getStartOffset(); // correct start offset, since when a line with nothing but spaces on it appears before comment, we get messed up positions
}
+ endOffset = origOffset + comment.getPosition().getEndOffset();
}
- } else {
- token= rule.evaluate(this);
- if (!token.isUndefined())
- return token;
+ tokenLength = endOffset - oldOffset;
+ fSavedToken = returnValue;
+ fSavedOffset = oldOffset + tokenLength;
+ if (!isEOF) {
+ fSavedLength = getOffset() - fSavedOffset;
+ } else {
+ fSavedOffset--;
+ fSavedLength = 0;
+ }
+ lastWasComment = true;
+ // FIXME What about multiline comments?!
+ String contentType = RUBY_SINGLE_LINE_COMMENT;
+ if (multiline) contentType = RUBY_MULTI_LINE_COMMENT;
+ return new Token(contentType);
}
+ } catch (SyntaxException se) {
+ if (lexerSource.getOffset() - origLength == 0)
+ return Token.EOF; // return eof if we hit a problem found at
+ // end of parsing
+ else
+ tokenLength = getOffset() - oldOffset;
+ return new Token(null);
+ } catch (IOException e) {
+ RubyPlugin.log(e);
}
+ if (!isEOF)
+ tokenLength = getOffset() - oldOffset;
+ return returnValue;
+ }
+
+ private IToken token(int i) {
+ return new Token(null);
+ }
- // haven't found any rule for this type of partition
- fContentType = null;
- if (resume)
- fOffset = fPartitionOffset;
- return super.nextToken();
+ private int getOffset() {
+ return lexerSource.getOffset() + origOffset;
}
-}
\ No newline at end of file
+
+ public void setRange(IDocument document, int offset, int length) {
+ setPartialRange(document, offset, length, null, 0);
+ }
+
+}
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java 2007-05-16 15:21:22 UTC (rev 2486)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java 2007-05-16 15:33:25 UTC (rev 2487)
@@ -2,14 +2,12 @@
import java.io.IOException;
import java.io.StringReader;
-import java.util.List;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.rules.IToken;
import org.eclipse.jface.text.rules.Token;
-import org.jruby.ast.CommentNode;
import org.jruby.common.NullWarnings;
import org.jruby.lexer.yacc.LexState;
import org.jruby.lexer.yacc.LexerSource;
@@ -51,17 +49,12 @@
private boolean isInRegexp;
private boolean isInString;
private boolean isInSymbol;
+ private boolean inAlias;
private RubyParserResult result;
private int origOffset;
private int origLength;
- private String contents;
+ private String contents;
- private IToken fSavedToken = null;
- private int fSavedLength = -1;
- private int fSavedOffset = -1;
- private boolean lastWasComment;
- private boolean inAlias;
-
public RubyTokenScanner(IColorManager manager, IPreferenceStore store) {
super(manager, store);
lexer = new RubyYaccLexer();
@@ -75,38 +68,14 @@
}
public int getTokenLength() {
- if (lastWasComment) {
- return tokenLength;
- }
- if (fSavedLength != -1) {
- int length = fSavedLength;
- fSavedLength = -1;
- return length;
- }
return tokenLength;
}
public int getTokenOffset() {
- if (lastWasComment) {
- return oldOffset;
- }
- if (fSavedOffset != -1) {
- int offset = fSavedOffset;
- fSavedOffset = -1;
- return offset;
- }
return oldOffset;
}
public IToken nextToken() {
- if (lastWasComment) {
- lastWasComment = false;
- }
- if (fSavedToken != null) {
- IToken returnToken = fSavedToken;
- fSavedToken = null;
- return returnToken;
- }
oldOffset = getOffset();
tokenLength = 0;
IToken returnValue = getToken(IRubyColorConstants.RUBY_DEFAULT);
@@ -118,31 +87,6 @@
} else {
returnValue = token(lexer.token());
}
- List comments = result.getCommentNodes();
- if (comments != null && !comments.isEmpty()) {
- CommentNode comment;
- boolean firstComment = true;
- int endOffset = 0;
- while (!comments.isEmpty()) {
- comment = (CommentNode) comments.remove(0);
- if (firstComment) {
- firstComment = false;
- oldOffset = origOffset + comment.getPosition().getStartOffset(); // correct start offset, since when a line with nothing but spaces on it appears before comment, we get messed up positions
- }
- endOffset = origOffset + comment.getPosition().getEndOffset();
- }
- tokenLength = endOffset - oldOffset;
- fSavedToken = returnValue;
- fSavedOffset = oldOffset + tokenLength;
- if (!isEOF) {
- fSavedLength = getOffset() - fSavedOffset;
- } else {
- fSavedOffset--;
- fSavedLength = 0;
- }
- lastWasComment = true;
- return getToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
- }
} catch (SyntaxException se) {
if (lexerSource.getOffset() - origLength == 0)
return Token.EOF; // return eof if we hit a problem found at
@@ -283,10 +227,6 @@
lexer.reset();
lexer.setState(LexState.EXPR_BEG);
parserSupport.initTopLocalVariables();
- lastWasComment = false;
- fSavedLength = -1;
- fSavedToken = null;
- fSavedOffset = -1;
isInSymbol = false;
if (offset == 0) {
isInRegexp = false;
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java 2007-05-16 15:21:22 UTC (rev 2486)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java 2007-05-16 15:33:25 UTC (rev 2487)
@@ -29,10 +29,8 @@
import org.eclipse.swt.SWT;
import org.eclipse.swt.widgets.Shell;
import org.eclipse.ui.IEditorInput;
-import org.eclipse.ui.editors.text.EditorsUI;
import org.eclipse.ui.editors.text.TextSourceViewerConfiguration;
import org.eclipse.ui.texteditor.AbstractDecoratedTextEditorPreferenceConstants;
-import org.eclipse.ui.texteditor.ChainedPreferenceStore;
import org.eclipse.ui.texteditor.IDocumentProvider;
import org.eclipse.ui.texteditor.ITextEditor;
import org.rubypeople.rdt.core.IRubyElement;
@@ -47,7 +45,6 @@
import org.rubypeople.rdt.internal.ui.text.HTMLTextPresenter;
import org.rubypeople.rdt.internal.ui.text.IRubyColorConstants;
import org.rubypeople.rdt.internal.ui.text.IRubyPartitions;
-import org.rubypeople.rdt.internal.ui.text.PreferencesAdapter;
import org.rubypeople.rdt.internal.ui.text.RubyAnnotationHover;
import org.rubypeople.rdt.internal.ui.text.RubyCommentScanner;
import org.rubypeople.rdt.internal.ui.text.RubyDoubleClickSelector;
@@ -88,7 +85,7 @@
protected AbstractRubyTokenScanner fCodeScanner;
- protected AbstractRubyScanner fMultilineCommentScanner;
+ protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner;
private RubyDoubleClickSelector fRubyDoubleClickSelector;
private RubyCompletionProcessor fRubyCp;
@@ -166,7 +163,7 @@
* @since 0.8.0
*/
public boolean affectsTextPresentation(PropertyChangeEvent event) {
- return fCodeScanner.affectsBehavior(event) || fMultilineCommentScanner.affectsBehavior(event);
+ return fCodeScanner.affectsBehavior(event) || fMultilineCommentScanner.affectsBehavior(event) || fSinglelineCommentScanner.affectsBehavior(event);
}
/**
@@ -189,27 +186,11 @@
fCodeScanner.adaptToPreferenceChange(event);
if (fMultilineCommentScanner.affectsBehavior(event))
fMultilineCommentScanner.adaptToPreferenceChange(event);
+ if (fSinglelineCommentScanner.affectsBehavior(event))
+ fSinglelineCommentScanner.adaptToPreferenceChange(event);
}
/**
- * Creates and returns a preference store which combines the preference
- * stores from the text tools and which is read-only.
- *
- * @param rubyTextTools
- * the Ruby text tools
- * @return the combined read-only preference store
- * @since 0.8.0
- */
- private static final IPreferenceStore createPreferenceStore(RubyTextTools rubyTextTools) {
- Assert.isNotNull(rubyTextTools);
- IPreferenceStore generalTextStore = EditorsUI.getPreferenceStore();
- if (rubyTextTools.getCorePreferenceStore() == null)
- return new ChainedPreferenceStore(new IPreferenceStore[] { rubyTextTools.getPreferenceStore(), generalTextStore });
-
- return new ChainedPreferenceStore(new IPreferenceStore[] { rubyTextTools.getPreferenceStore(), new PreferencesAdapter(rubyTextTools.getCorePreferenceStore()), generalTextStore });
- }
-
- /**
* Initializes the scanners.
*
* @since 3.0
@@ -218,6 +199,7 @@
Assert.isTrue(isNewSetup());
fCodeScanner = new RubyTokenScanner(getColorManager(), fPreferenceStore);
fMultilineCommentScanner = new RubyCommentScanner(getColorManager(), fPreferenceStore, IRubyColorConstants.RUBY_MULTI_LINE_COMMENT);
+ fSinglelineCommentScanner = new RubyCommentScanner(getColorManager(), fPreferenceStore, IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
}
/**
@@ -250,6 +232,10 @@
dr = new DefaultDamagerRepairer(getMultilineCommentScanner());
reconciler.setDamager(dr, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT);
reconciler.setRepairer(dr, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT);
+
+ dr = new DefaultDamagerRepairer(getSinglelineCommentScanner());
+ reconciler.setDamager(dr, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
+ reconciler.setRepairer(dr, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
return reconciler;
}
@@ -260,9 +246,13 @@
protected ITokenScanner getMultilineCommentScanner() {
return fMultilineCommentScanner;
}
+
+ protected ITokenScanner getSinglelineCommentScanner() {
+ return fSinglelineCommentScanner;
+ }
public String[] getConfiguredContentTypes(ISourceViewer sourceViewer) {
- return new String[] { IDocument.DEFAULT_CONTENT_TYPE, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT };
+ return new String[] { IDocument.DEFAULT_CONTENT_TYPE, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT };
}
/*
Modified: trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java 2007-05-16 15:21:22 UTC (rev 2486)
+++ trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java 2007-05-16 15:33:25 UTC (rev 2487)
@@ -29,6 +29,14 @@
assert(true);
}
+ public void testPartitioningOfSingleLineComment() {
+ String source = "# This is a comment\n";
+
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 0));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 1));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 18));
+ }
+
public void testRecognizeSpecialCase() {
String source = "a,b=?#,'This is not a comment!'\n";
@@ -48,7 +56,7 @@
"=end";
assertEquals(RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, this.getContentType(source, 0));
assertEquals(RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, this.getContentType(source, source.length() / 2));
- assertEquals(RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, this.getContentType(source, source.length() - 1));
+ assertEquals(RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, this.getContentType(source, source.length() - 2));
}
public void testMultilineCommentNotOnFirstColumn() {
@@ -60,4 +68,31 @@
assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 10));
}
+ public void testRecognizeDivision() {
+ String source = "1/3 #This is a comment\n";
+
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 0));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 3));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 5));
+ }
+
+ public void testRecognizeOddballCharacters() {
+ String source = "?\" #comment\n";
+
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 0));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 2));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 5));
+
+ source = "?' #comment\n";
+
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 0));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 2));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 5));
+
+ source = "?/ #comment\n";
+
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 0));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(source, 2));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, 5));
+ }
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
