Menu
▾
▴
[Rubyeclipse-commit] SF.net SVN: rubyeclipse: [2639] trunk/org.rubypeople.rdt.ui
|
From: <caw...@us...> - 2007-06-20 15:33:24
|
Revision: 2639
http://svn.sourceforge.net/rubyeclipse/?rev=2639&view=rev
Author: cawilliams
Date: 2007-06-20 08:32:32 -0700 (Wed, 20 Jun 2007)
Log Message:
-----------
partition ruby files into code, single line comments, multi line comments and strings (new addition is strings). Adjust tests to meet new behavior/expectations.
This now allows us to embed code within strings (within code, ad infinitum).
Modified Paths:
--------------
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/IRubyPartitions.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubyTextTools.java
trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java
trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/ruby/TC_RubyTokenScanner.java
Added Paths:
-----------
trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/MergingPartitionScanner.java
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/IRubyPartitions.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/IRubyPartitions.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/IRubyPartitions.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -16,6 +16,11 @@
* @since 0.7.0
*/
public final static String RUBY_PARTITIONING= "___ruby_partitioning"; //$NON-NLS-1$
+
+ /**
+ * The identifier default ruby code partition content type.
+ */
+ String RUBY_DEFAULT= "__ruby_default"; //$NON-NLS-1$
/**
* The identifier of the single-line end comment partition content type.
Added: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/MergingPartitionScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/MergingPartitionScanner.java (rev 0)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/MergingPartitionScanner.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -0,0 +1,92 @@
+package org.rubypeople.rdt.internal.ui.text;
+
+import org.eclipse.core.runtime.Assert;
+import org.eclipse.jface.text.IDocument;
+import org.eclipse.jface.text.rules.IPartitionTokenScanner;
+import org.eclipse.jface.text.rules.IToken;
+import org.eclipse.jface.text.rules.Token;
+
+/**
+ * Wraps the Ruby PartitonScanner and merges consecutive tokens with teh same data/partition marking.
+ * So if we have 10 default tokens in a row, this will eat up the ten and return a token that spans all of them.
+ *
+ * @author Chris Williams
+ *
+ */
+public class MergingPartitionScanner implements IPartitionTokenScanner {
+
+ private RubyPartitionScanner fScanner;
+ private int fOffset;
+ private int fLength;
+ private int newOffset = 0;
+ private int newLength = 0;
+ private IToken lastToken;
+
+ public MergingPartitionScanner() {
+ fScanner = new RubyPartitionScanner();
+ }
+
+ public void setPartialRange(IDocument document, int offset, int length,
+ String contentType, int partitionOffset) {
+ clear();
+ fScanner.setPartialRange(document, offset, length, contentType, partitionOffset);
+ }
+
+ public int getTokenLength() {
+ return fLength;
+ }
+
+ public int getTokenOffset() {
+ return fOffset;
+ }
+
+ public IToken nextToken() {
+ fLength = newLength;
+ fOffset = newOffset;
+ if (lastToken != null && lastToken.isEOF()) {
+ return lastToken;
+ }
+
+ IToken token = null;
+ while(!(token = fScanner.nextToken()).isEOF()) {
+ if (lastToken != null && token.getData().equals(lastToken.getData())) {
+// fLength = (fScanner.getTokenOffset() - fOffset) + fScanner.getTokenLength();
+// Assert.isTrue(fLength >= 0);
+ } else if (lastToken == null) {
+ lastToken = token;
+ fOffset = fScanner.getTokenOffset();
+ fLength = fScanner.getTokenLength();
+ } else {
+ fLength = (fScanner.getTokenOffset() - fOffset);
+ newOffset = fScanner.getTokenOffset();
+ newLength = fScanner.getTokenLength();
+ Assert.isTrue(newLength >= 0);
+ IToken returnToken = lastToken; // make a copy of the last token
+ lastToken = token; // save new token
+ return returnToken;
+ }
+ }
+ if (lastToken == null) {
+ return Token.EOF;
+ }
+ newOffset = fScanner.getTokenOffset();
+ newLength = 0;
+ IToken returnToken = lastToken; // make a copy of the last token
+ lastToken = Token.EOF; // save new token
+ return returnToken;
+ }
+
+ public void setRange(IDocument document, int offset, int length) {
+ clear();
+ fScanner.setRange(document, offset, length);
+ }
+
+ private void clear() {
+ lastToken = null;
+ fLength = 0;
+ fOffset = 0;
+ newLength = 0;
+ newOffset = 0;
+ }
+
+}
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/RubyPartitionScanner.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -3,9 +3,12 @@
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import org.eclipse.core.runtime.Assert;
import org.eclipse.jface.text.BadLocationException;
+import org.eclipse.jface.text.Document;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.rules.IPartitionTokenScanner;
import org.eclipse.jface.text.rules.IToken;
@@ -19,11 +22,14 @@
import org.jruby.parser.ParserSupport;
import org.jruby.parser.RubyParserConfiguration;
import org.jruby.parser.RubyParserResult;
+import org.jruby.parser.Tokens;
import org.rubypeople.rdt.internal.core.util.ASTUtil;
import org.rubypeople.rdt.internal.ui.RubyPlugin;
public class RubyPartitionScanner implements IPartitionTokenScanner {
+ private static final String BEGIN = "=begin";
+
private static class QueuedToken {
private IToken token;
private int length;
@@ -51,21 +57,25 @@
private RubyYaccLexer lexer;
private ParserSupport parserSupport;
private RubyParserResult result;
- private String contents;
+ private String fContents;
private LexerSource lexerSource;
private int origOffset;
private int origLength;
- private int tokenLength;
- private int tokenOffset;
+ private int fLength;
+ private int fOffset;
- private List<QueuedToken> queue = new ArrayList<QueuedToken>();
+ private List<QueuedToken> fQueue = new ArrayList<QueuedToken>();
+ private String fContentType;
- // XXX Also do strings, regex partitions!
+ // XXX Also do regex partitions!
public final static String RUBY_MULTI_LINE_COMMENT = IRubyPartitions.RUBY_MULTI_LINE_COMMENT;
public final static String RUBY_SINGLE_LINE_COMMENT = IRubyPartitions.RUBY_SINGLE_LINE_COMMENT;
+ public final static String RUBY_STRING = IRubyPartitions.RUBY_STRING;
+ public final static String RUBY_REGULAR_EXPRESSION = IRubyPartitions.RUBY_REGULAR_EXPRESSION;
+ public static final String RUBY_DEFAULT = IDocument.DEFAULT_CONTENT_TYPE;
public static final String[] LEGAL_CONTENT_TYPES = {
- RUBY_MULTI_LINE_COMMENT, RUBY_SINGLE_LINE_COMMENT
+ RUBY_DEFAULT, RUBY_MULTI_LINE_COMMENT, RUBY_SINGLE_LINE_COMMENT, RUBY_REGULAR_EXPRESSION, RUBY_STRING
};
public RubyPartitionScanner() {
@@ -81,15 +91,22 @@
public void setPartialRange(IDocument document, int offset, int length,
String contentType, int partitionOffset) {
reset();
- try {
- contents = document.get(offset, length);
- lexerSource = new LexerSource("filename", new StringReader(contents), 0);
+ int myOffset = offset;
+ if (contentType != null) {
+ int diff = offset - partitionOffset;
+ myOffset = partitionOffset; // backtrack to beginning of partition so we don't get in weird state
+ length += diff;
+ }
+ if (myOffset == -1) myOffset = 0;
+ try {
+ fContents = document.get(myOffset, length);
+ lexerSource = new LexerSource("filename", new StringReader(fContents), 0);
lexer.setSource(lexerSource);
} catch (BadLocationException e) {
lexerSource = new LexerSource("filename", new StringReader(""), 0);
lexer.setSource(lexerSource);
}
- origOffset = offset;
+ origOffset = myOffset;
origLength = length;
}
@@ -97,84 +114,255 @@
lexer.reset();
lexer.setState(LexState.EXPR_BEG);
parserSupport.initTopLocalVariables();
- queue.clear();
+ fQueue.clear();
}
public int getTokenLength() {
- return tokenLength;
+ return fLength;
}
public int getTokenOffset() {
- return tokenOffset;
+ return fOffset;
}
public IToken nextToken() {
- if (!queue.isEmpty()) {
- QueuedToken token = queue.remove(0);
- tokenOffset = token.getOffset();
- tokenLength = token.getLength();
- return token.getToken();
+ if (!fQueue.isEmpty()) {
+ return popTokenOffQueue();
}
- tokenOffset = getOffset();
- tokenLength = 0;
+ fOffset = getOffset();
+ fLength = 0;
IToken returnValue = new Token(null);
boolean isEOF = false;
try {
isEOF = !lexer.advance();
if (isEOF) {
returnValue = Token.EOF;
+ } else {
+ int lexerToken = lexer.token();
+ if (lexerToken == Tokens.tSTRING_DVAR) { // we hit a single dynamic variable
+ addPoundToken();
+ scanDynamicVariable();
+ setLexerPastDynamicSectionOfString();
+ return popTokenOffQueue();
+ } else if (lexerToken == Tokens.tSTRING_DBEG) { // if we hit dynamic code inside a string
+ addPoundBraceToken();
+ scanTokensInsideDynamicPortion();
+ addClosingBraceToken();
+ setLexerPastDynamicSectionOfString();
+ return popTokenOffQueue();
+ }
+ returnValue = getToken(lexerToken);
}
List comments = result.getCommentNodes();
if (comments != null && !comments.isEmpty()) {
- CommentNode comment;
- boolean firstComment = true;
- int endOffset = 0;
- boolean multiline = false;
- while (!comments.isEmpty()) {
- comment = (CommentNode) comments.remove(0);
- if (firstComment) {
- String src = ASTUtil.getSource(contents, comment);
- if (src != null && src.startsWith("=begin")) multiline = true;
- firstComment = false;
- tokenOffset = origOffset + comment.getPosition().getStartOffset(); // correct start offset, since when a line with nothing but spaces on it appears before comment, we get messed up positions
- }
- endOffset = origOffset + comment.getPosition().getEndOffset();
- }
- tokenLength = endOffset - tokenOffset;
- int queuedOffset = tokenOffset + tokenLength;
- int queuedLength = 0;
- if (!isEOF) {
- queuedLength = getOffset() - queuedOffset;
- } else {
- queuedOffset--;
- }
- // Throw saved token onto queue
- queue.add(new QueuedToken(returnValue, queuedOffset, queuedLength));
- String contentType = RUBY_SINGLE_LINE_COMMENT;
- if (multiline) contentType = RUBY_MULTI_LINE_COMMENT;
- return new Token(contentType);
+ parseOutComments(comments);
+ addQueuedToken(returnValue, isEOF); // Queue the normal token we just ate up
+ comments.clear();
+ return popTokenOffQueue();
}
} catch (SyntaxException se) {
+ if (se.getMessage().equals("embedded document meets end of file")) {
+ // TODO recover somehow by removing this chunk out of the fContents?
+ setOffset(se.getPosition().getStartOffset());
+ fLength = fContents.length() - se.getPosition().getStartOffset();
+ return new Token(RUBY_MULTI_LINE_COMMENT);
+ }
+
if (lexerSource.getOffset() - origLength == 0)
return Token.EOF; // return eof if we hit a problem found at
// end of parsing
else
- tokenLength = getOffset() - tokenOffset;
- return new Token(null);
+ fLength = getOffset() - fOffset;
+ return new Token(RUBY_DEFAULT);
} catch (IOException e) {
RubyPlugin.log(e);
}
if (!isEOF)
- tokenLength = getOffset() - tokenOffset;
+ fLength = getOffset() - fOffset;
return returnValue;
}
+ private void setOffset(int offset) {
+// Assert.isTrue(offset > fOffset);
+ fOffset = offset;
+ }
+
+ private void addPoundToken() {
+ addStringToken(1);// add token for the #
+ }
+
+ private void scanDynamicVariable() {
+ int whitespace = fContents.indexOf(' ', fOffset - origOffset); // read until whitespace or '"'
+ if (whitespace == -1) whitespace = Integer.MAX_VALUE;
+ int doubleQuote = fContents.indexOf('"', fOffset - origOffset);
+ if (doubleQuote == -1) doubleQuote = Integer.MAX_VALUE;
+ int end = Math.min(whitespace, doubleQuote);
+ // FIXME If we can't find whitespace or doubleQuote, we are pretty screwed.
+ String possible = null;
+ if (end == -1) {
+ possible = fContents.substring(fOffset - origOffset);
+ } else {
+ possible = fContents.substring(fOffset - origOffset, end);
+ }
+ RubyPartitionScanner scanner = new RubyPartitionScanner();
+ IDocument document = new Document(possible);
+ scanner.setRange(document, 0, possible.length());
+ IToken token;
+ while (!(token = scanner.nextToken()).isEOF()) {
+ push(new QueuedToken(token, scanner.getTokenOffset() + (fOffset), scanner.getTokenLength()));
+ }
+ setOffset(fOffset + possible.length());
+ }
+
+ private void scanTokensInsideDynamicPortion() {
+ String possible = new String(fContents.substring(fOffset - origOffset));
+ int end = possible.indexOf('}');// TODO Find the end brace '}' in a proper way!
+ if (end != -1) {
+ possible = possible.substring(0, end);
+ } else {
+ possible = possible.substring(0);
+ }
+ RubyPartitionScanner scanner = new RubyPartitionScanner();
+ IDocument document = new Document(possible);
+ scanner.setRange(document, 0, possible.length());
+ IToken token;
+ while (!(token = scanner.nextToken()).isEOF()) {
+ push(new QueuedToken(token, scanner.getTokenOffset() + fOffset, scanner.getTokenLength()));
+ }
+ setOffset(fOffset + possible.length());
+ }
+
+ private void addPoundBraceToken() {
+ addStringToken(2); // add token for the #{
+ }
+
+ private void addStringToken(int length) {
+ push(new QueuedToken(new Token(RUBY_STRING), fOffset, length));
+ setOffset(fOffset + length); // move past token
+ }
+
+ private void addClosingBraceToken() {
+ addStringToken(1);
+ }
+
+ private void setLexerPastDynamicSectionOfString() throws IOException {
+ IDocument document;
+ StringBuffer fakeContents = new StringBuffer();
+ int start = fOffset - 1;
+ for (int i = 0; i < start; i++) {
+ fakeContents.append(" ");
+ }
+ fakeContents.append('"');
+ if ((fOffset - origOffset) < origLength) {
+ fakeContents.append(new String(fContents.substring((fOffset - origOffset)))); // BLAH removed + 1 from end here
+ }
+ document = new Document(fakeContents.toString());
+ List<QueuedToken> queueCopy = new ArrayList<QueuedToken>(fQueue);
+ setPartialRange(document, start, fakeContents.length() - start, null, start);
+ fQueue = new ArrayList<QueuedToken>(queueCopy);
+ lexer.advance();
+ }
+
+ private void parseOutComments(List comments) {
+ int i = 0;
+ for (Iterator iter = comments.iterator(); iter.hasNext();) {
+ CommentNode comment = (CommentNode) iter.next();
+ int offset = correctOffset(comment);
+ int length = comment.getContent().length();
+ Token token = new Token(getContentType(comment));
+ push(new QueuedToken(token, offset, length));
+ i++;
+ }
+ }
+
+ private IToken popTokenOffQueue() {
+ QueuedToken token = fQueue.remove(0);
+ setOffset(token.getOffset());
+ Assert.isTrue(token.getLength() >= 0);
+ fLength = token.getLength();
+ return token.getToken();
+ }
+
+ private IToken getToken(int i) {
+ // If we hit a 32 (space) inside a qword, just return string content type (not default)
+ // FIXME IF we're in qwords, we should inspect the contents because it may be a variable
+ if (i == 32) {
+ return new Token(fContentType);
+ }
+ switch (i) {
+ case Tokens.tSTRING_CONTENT:
+ return new Token(RUBY_STRING);
+ case Tokens.tSTRING_BEG:
+ return new Token(RUBY_STRING);
+ case Tokens.tQWORDS_BEG:
+ fContentType = RUBY_STRING;
+ return new Token(RUBY_STRING);
+ case Tokens.tSTRING_END:
+ fContentType = RUBY_DEFAULT;
+ return new Token(RUBY_STRING);
+ case Tokens.tREGEXP_BEG:
+ return new Token(RUBY_REGULAR_EXPRESSION);
+ case Tokens.tREGEXP_END:
+ return new Token(RUBY_REGULAR_EXPRESSION);
+ default:
+ return new Token(RUBY_DEFAULT);
+ }
+ }
+
+ /**
+ * Grabs the end of the comment
+ * @param comments
+ * @return
+ */
+ private int getEndOfComment(CommentNode comment) {
+ return origOffset + comment.getPosition().getEndOffset();
+ }
+
+ /**
+ * correct start offset, since when a line with nothing but spaces on it appears before comment,
+ * we get messed up positions
+ */
+ private int correctOffset(CommentNode comment) {
+ return origOffset + comment.getPosition().getStartOffset();
+ }
+
+ private boolean isCommentMultiLine(CommentNode comment) {
+ String src = ASTUtil.getSource(fContents, comment);
+ if (src != null && src.startsWith(BEGIN)) return true;
+ return false;
+ }
+
+ private String getContentType(CommentNode comment) {
+ if (isCommentMultiLine(comment)) return RUBY_MULTI_LINE_COMMENT;
+ return RUBY_SINGLE_LINE_COMMENT;
+ }
+
+ private void addQueuedToken(IToken returnValue, boolean isEOF) {
+ // grab end of last comment (last thing in queue)
+ QueuedToken token = peek();
+ setOffset(token.getOffset() + token.getLength());
+ int length = getOffset() - fOffset;
+ if (length < 0 ) {
+ length = 0;
+ }
+ push(new QueuedToken(returnValue, fOffset, length));
+ }
+
+ private QueuedToken peek() {
+ return fQueue.get(fQueue.size() - 1);
+ }
+
+ private void push(QueuedToken token) {
+ Assert.isTrue(token.getLength() >= 0);
+ fQueue.add(token);
+ }
+
private int getOffset() {
return lexerSource.getOffset() + origOffset;
}
public void setRange(IDocument document, int offset, int length) {
- setPartialRange(document, offset, length, null, 0);
+ setPartialRange(document, offset, length, null, -1);
}
}
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/internal/ui/text/ruby/RubyTokenScanner.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -33,7 +33,7 @@
private static String[] fgTokenProperties = { IRubyColorConstants.RUBY_KEYWORD, IRubyColorConstants.RUBY_DEFAULT,
IRubyColorConstants.RUBY_FIXNUM, IRubyColorConstants.RUBY_CHARACTER, IRubyColorConstants.RUBY_SYMBOL,
- IRubyColorConstants.RUBY_INSTANCE_VARIABLE, IRubyColorConstants.RUBY_GLOBAL, IRubyColorConstants.RUBY_STRING,
+ IRubyColorConstants.RUBY_INSTANCE_VARIABLE, IRubyColorConstants.RUBY_GLOBAL,
IRubyColorConstants.RUBY_REGEXP, IRubyColorConstants.RUBY_ERROR
// TODO Add Ability to set colors for return and operators
// IRubyColorConstants.RUBY_METHOD_NAME,
@@ -47,7 +47,6 @@
private int tokenLength;
private int oldOffset;
private boolean isInRegexp;
- private boolean isInString;
private boolean isInSymbol;
private boolean inAlias;
private RubyParserResult result;
@@ -81,7 +80,7 @@
IToken returnValue = getToken(IRubyColorConstants.RUBY_DEFAULT);
boolean isEOF = false;
try {
- isEOF = !lexer.advance();
+ isEOF = !lexer.advance(); // FIXME if we're assigning a string to a variable we may get a NumberformatException here!
if (isEOF) {
returnValue = Token.EOF;
} else {
@@ -90,10 +89,12 @@
} catch (SyntaxException se) {
if (lexerSource.getOffset() - origLength == 0)
return Token.EOF; // return eof if we hit a problem found at
- // end of parsing
- else
- tokenLength = getOffset() - oldOffset;
+ // end of parsing
+ tokenLength = getOffset() - oldOffset;
return getToken(IRubyColorConstants.RUBY_ERROR);
+ } catch (NumberFormatException nfe) {
+ tokenLength = getOffset() - oldOffset;
+ return returnValue;
} catch (IOException e) {
RubyPlugin.log(e);
}
@@ -111,8 +112,6 @@
return super.getToken(IRubyColorConstants.RUBY_SYMBOL);
if (isInRegexp)
return super.getToken(IRubyColorConstants.RUBY_REGEXP);
- if (isInString)
- return super.getToken(IRubyColorConstants.RUBY_STRING);
return super.getToken(key);
}
@@ -152,14 +151,6 @@
if ((((oldOffset - origOffset) + 1) < contents.length()) && (contents.charAt((oldOffset - origOffset) + 1) == '?'))
return doGetToken(IRubyColorConstants.RUBY_CHARACTER);
return doGetToken(IRubyColorConstants.RUBY_FIXNUM);
- case Tokens.tSTRING_CONTENT:
- return doGetToken(IRubyColorConstants.RUBY_STRING);
- case Tokens.tSTRING_BEG:
- isInString = true;
- return doGetToken(IRubyColorConstants.RUBY_STRING);
- case Tokens.tSTRING_END:
- isInString = false;
- return doGetToken(IRubyColorConstants.RUBY_STRING);
case Tokens.tREGEXP_BEG:
isInRegexp = true;
return doGetToken(IRubyColorConstants.RUBY_REGEXP);
@@ -228,7 +219,6 @@
isInSymbol = false;
if (offset == 0) {
isInRegexp = false;
- isInString = false;
}
try {
contents = document.get(offset, length);
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubySourceViewerConfiguration.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -64,6 +64,7 @@
import org.rubypeople.rdt.internal.ui.text.ruby.RubyFormattingStrategy;
import org.rubypeople.rdt.internal.ui.text.ruby.RubyReconcilingStrategy;
import org.rubypeople.rdt.internal.ui.text.ruby.RubyTokenScanner;
+import org.rubypeople.rdt.internal.ui.text.ruby.SingleTokenRubyScanner;
import org.rubypeople.rdt.internal.ui.text.ruby.hover.RubyEditorTextHoverDescriptor;
import org.rubypeople.rdt.internal.ui.text.ruby.hover.RubyEditorTextHoverProxy;
import org.rubypeople.rdt.internal.ui.text.ruby.hover.RubyInformationProvider;
@@ -89,7 +90,7 @@
protected AbstractRubyTokenScanner fCodeScanner;
- protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner;
+ protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner, fStringScanner;
private RubyDoubleClickSelector fRubyDoubleClickSelector;
private RubyCompletionProcessor fRubyCp;
@@ -167,7 +168,8 @@
* @since 0.8.0
*/
public boolean affectsTextPresentation(PropertyChangeEvent event) {
- return fCodeScanner.affectsBehavior(event) || fMultilineCommentScanner.affectsBehavior(event) || fSinglelineCommentScanner.affectsBehavior(event);
+ return fCodeScanner.affectsBehavior(event) || fMultilineCommentScanner.affectsBehavior(event)
+ || fSinglelineCommentScanner.affectsBehavior(event) || fStringScanner.affectsBehavior(event);
}
/**
@@ -192,6 +194,8 @@
fMultilineCommentScanner.adaptToPreferenceChange(event);
if (fSinglelineCommentScanner.affectsBehavior(event))
fSinglelineCommentScanner.adaptToPreferenceChange(event);
+ if (fStringScanner.affectsBehavior(event))
+ fStringScanner.adaptToPreferenceChange(event);
}
/**
@@ -204,6 +208,7 @@
fCodeScanner = new RubyTokenScanner(getColorManager(), fPreferenceStore);
fMultilineCommentScanner = new RubyCommentScanner(getColorManager(), fPreferenceStore, IRubyColorConstants.RUBY_MULTI_LINE_COMMENT);
fSinglelineCommentScanner = new RubyCommentScanner(getColorManager(), fPreferenceStore, IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
+ fStringScanner = new SingleTokenRubyScanner(getColorManager(), fPreferenceStore, IRubyColorConstants.RUBY_STRING);
}
/**
@@ -240,6 +245,10 @@
dr = new DefaultDamagerRepairer(getSinglelineCommentScanner());
reconciler.setDamager(dr, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
reconciler.setRepairer(dr, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT);
+
+ dr = new DefaultDamagerRepairer(getStringScanner());
+ reconciler.setDamager(dr, RubyPartitionScanner.RUBY_STRING);
+ reconciler.setRepairer(dr, RubyPartitionScanner.RUBY_STRING);
return reconciler;
}
@@ -254,9 +263,13 @@
protected ITokenScanner getSinglelineCommentScanner() {
return fSinglelineCommentScanner;
}
+
+ protected ITokenScanner getStringScanner() {
+ return fStringScanner;
+ }
public String[] getConfiguredContentTypes(ISourceViewer sourceViewer) {
- return new String[] { IDocument.DEFAULT_CONTENT_TYPE, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT };
+ return new String[] { IDocument.DEFAULT_CONTENT_TYPE, RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, RubyPartitionScanner.RUBY_STRING };
}
/*
Modified: trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubyTextTools.java
===================================================================
--- trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubyTextTools.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui/src/org/rubypeople/rdt/ui/text/RubyTextTools.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -11,11 +11,11 @@
import org.eclipse.jface.text.IDocumentPartitioner;
import org.eclipse.jface.text.rules.DefaultPartitioner;
import org.eclipse.jface.text.rules.IPartitionTokenScanner;
-import org.eclipse.jface.text.rules.ITokenScanner;
import org.eclipse.jface.util.IPropertyChangeListener;
import org.eclipse.jface.util.PropertyChangeEvent;
import org.rubypeople.rdt.internal.ui.RubyPlugin;
import org.rubypeople.rdt.internal.ui.rubyeditor.RubyEditorPreferences;
+import org.rubypeople.rdt.internal.ui.text.MergingPartitionScanner;
import org.rubypeople.rdt.internal.ui.text.IRubyColorConstants;
import org.rubypeople.rdt.internal.ui.text.RubyColorManager;
import org.rubypeople.rdt.internal.ui.text.RubyCommentScanner;
@@ -23,6 +23,7 @@
import org.rubypeople.rdt.internal.ui.text.ruby.AbstractRubyScanner;
import org.rubypeople.rdt.internal.ui.text.ruby.AbstractRubyTokenScanner;
import org.rubypeople.rdt.internal.ui.text.ruby.RubyTokenScanner;
+import org.rubypeople.rdt.internal.ui.text.ruby.SingleTokenRubyScanner;
public class RubyTextTools {
@@ -44,9 +45,9 @@
protected static String[] keywords;
protected RubyColorManager fColorManager;
- protected RubyPartitionScanner partitionScanner;
+ protected IPartitionTokenScanner partitionScanner;
protected AbstractRubyTokenScanner fCodeScanner;
- protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner;
+ protected AbstractRubyScanner fMultilineCommentScanner, fSinglelineCommentScanner, FStringScanner;
private IPreferenceStore fPreferenceStore;
private Preferences fCorePreferenceStore;
/** The preference change listener */
@@ -103,13 +104,15 @@
super();
fColorManager = new RubyColorManager(autoDisposeOnDisplayDispose);
- partitionScanner = new RubyPartitionScanner();
+ partitionScanner = new MergingPartitionScanner();
fCodeScanner = new RubyTokenScanner(fColorManager, store);
fMultilineCommentScanner = new RubyCommentScanner(fColorManager, store, coreStore,
IRubyColorConstants.RUBY_MULTI_LINE_COMMENT);
fSinglelineCommentScanner = new RubyCommentScanner(fColorManager, store, coreStore,
IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT);
+ FStringScanner = new SingleTokenRubyScanner(fColorManager, store,
+ IRubyColorConstants.RUBY_STRING);
fPreferenceStore = store;
fPreferenceStore.addPropertyChangeListener(fPreferenceListener);
@@ -133,7 +136,9 @@
if (fMultilineCommentScanner.affectsBehavior(event))
fMultilineCommentScanner.adaptToPreferenceChange(event);
if (fSinglelineCommentScanner.affectsBehavior(event))
- fSinglelineCommentScanner.adaptToPreferenceChange(event);
+ fSinglelineCommentScanner.adaptToPreferenceChange(event);
+ if (FStringScanner.affectsBehavior(event))
+ FStringScanner.adaptToPreferenceChange(event);
}
public IDocumentPartitioner createDocumentPartitioner() {
@@ -145,30 +150,6 @@
return partitionScanner;
}
- /**
- * @deprecated As of 0.8.0, replaced by
- * {@link RubySourceViewerConfiguration#getCodeScanner()}
- */
- public AbstractRubyTokenScanner getCodeScanner() {
- return fCodeScanner;
- }
-
- /**
- * @deprecated As of 0.8.0, replaced by
- * {@link RubySourceViewerConfiguration#getMultilineCommentScanner()}
- */
- public ITokenScanner getMultilineCommentScanner() {
- return fMultilineCommentScanner;
- }
-
- /**
- * @deprecated As of 0.8.0, replaced by
- * {@link RubySourceViewerConfiguration#getSingleineCommentScanner()}
- */
- public ITokenScanner getSinglelineCommentScanner() {
- return fSinglelineCommentScanner;
- }
-
public IPreferenceStore getPreferenceStore() {
return RubyPlugin.getDefault().getPreferenceStore();
}
@@ -192,7 +173,8 @@
public boolean affectsTextPresentation(PropertyChangeEvent event) {
return fCodeScanner.affectsBehavior(event)
|| fMultilineCommentScanner.affectsBehavior(event)
- || fSinglelineCommentScanner.affectsBehavior(event);
+ || fSinglelineCommentScanner.affectsBehavior(event)
+ || FStringScanner.affectsBehavior(event);
}
/**
Modified: trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/TC_RubyPartitionScanner.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -108,4 +108,73 @@
assertEquals(RubyPartitionScanner.RUBY_MULTI_LINE_COMMENT, this.getContentType(source, 10));
assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(source, source.length() - 5));
}
+
+ public void testMultipleCommentsInARow() {
+ String code = "# comment 1\n# comment 2\nclass Chris\nend\n";
+
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 6));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 17));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 26));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 29));
+ }
+
+ public void testCommentAfterEnd() {
+ String code = "class Chris\nend # comment\n";
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 12));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 17));
+ }
+
+ public void testCommentAfterEndWhileEditing() {
+ String code = "=begin\r\n" +
+"c\r\n" +
+"=end\r\n" +
+"#hmm\r\n" +
+"#comment here why is ths\r\n" +
+"class Chris\r\n" +
+" def thing\r\n" +
+" end #ocmm \r\n" +
+"end";
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 76));
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 83));
+ }
+
+ public void testCommentAtEndOfLineWithStringAtBeginning() {
+ String code = "hash = {\n" +
+ " \"string\" => { # comment\n" +
+ " 123\n" +
+ " }\n" +
+ "}";
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 0));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 4));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 6));
+
+ assertEquals(RubyPartitionScanner.RUBY_STRING, this.getContentType(code, 8));
+ assertEquals(RubyPartitionScanner.RUBY_STRING, this.getContentType(code, 12));
+ assertEquals(RubyPartitionScanner.RUBY_STRING, this.getContentType(code, 18));
+
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 19));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 22));
+
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 25));
+ }
+
+ public void testLinesWithJustSpaceBeforeComment() {
+ String code = " \n" +
+ " # comment\n" +
+ " def method\n" +
+ " \n" +
+ " end";
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 5));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 14));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 20));
+ }
+
+ public void testCommentsWithAlotOfPrecedingSpaces() {
+ String code = " # We \n" +
+ " # caller-requested until.\n" +
+ "return self\n";
+ assertEquals(RubyPartitionScanner.RUBY_SINGLE_LINE_COMMENT, this.getContentType(code, 16));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 63));
+ assertEquals(IDocument.DEFAULT_CONTENT_TYPE, this.getContentType(code, 70));
+ }
}
Modified: trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/ruby/TC_RubyTokenScanner.java
===================================================================
--- trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/ruby/TC_RubyTokenScanner.java 2007-06-20 13:35:18 UTC (rev 2638)
+++ trunk/org.rubypeople.rdt.ui.tests/src/org/rubypeople/rdt/internal/ui/text/ruby/TC_RubyTokenScanner.java 2007-06-20 15:32:32 UTC (rev 2639)
@@ -44,74 +44,6 @@
assertToken(IRubyColorConstants.RUBY_KEYWORD, 12, 3);
}
- public void testMultipleCommentsInARow() {
- String code = "# comment one\n#comment two\nclass Chris\nend\n";
- setUpScanner(code);
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 0, 26);
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 26, 6); // '\nclass'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 32, 6); // ' Chris'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 38, 1); // '\n'
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 39, 3); // 'end'
- }
-
- public void testCommentAfterEnd() {
- String code = "class Chris\nend # comment\n";
- setUpScanner(code);
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 0, 5); // 'class'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 5, 6); // ' Chris'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 11, 1); // '\n'
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 12, 3); // 'end'
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 16, 9); // '# comment'
- }
-
- public void testCommentAfterEndWhileEditing() {
- String code = "=begin\r\n" +
-"c\r\n" +
-"=end\r\n" +
-"#hmm\r\n" +
-"#comment here why is ths\r\n" +
-"class Chris\r\n" +
-" def thing\r\n" +
-" end #ocmm \r\n" +
-"end";
- setUpScanner(code, 75, 14);
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 75, 5); // ' end'
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 82, 5); // '#ocmm'
- }
-
- public void testCommentAtEndOfLineWithStringAtBeginning() {
- String code = "hash = {\n" +
- " \"string\" => { # comment\n" +
- " 123\n" +
- " }\n" +
- "}";
- setUpScanner(code);
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 0, 4); // 'hash'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 4, 2); // ' ='
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 6, 2); // ' {'
-
- assertToken(IRubyColorConstants.RUBY_STRING, 8, 4); // whitespace
- assertToken(IRubyColorConstants.RUBY_STRING, 12, 6);
- assertToken(IRubyColorConstants.RUBY_STRING, 18, 1);
-
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 19, 3);
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 22, 2);
-
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 25, 9);
- }
-
- public void testLinesWithJustSpaceBeforeComment() {
- String code = " \n" +
- " # comment\n" +
- " def method\n" +
- " \n" +
- " end";
- setUpScanner(code);
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 5, 9); // '# comment'
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 14, 6); // '\n def'
- assertToken(IRubyColorConstants.RUBY_DEFAULT, 20, 7); // ' method'
- }
-
public void testSymbolAtEndOfLine() {
String code = " helper_method :logged_in?\n" +
" def method\n" +
@@ -136,16 +68,6 @@
assertToken(IRubyColorConstants.RUBY_DEFAULT, 11, 1); // ']'
}
- public void testCommentsWithAlotOfPrecedingSpaces() {
- String code = " # We \n" +
- " # caller-requested until.\n" +
- "return self\n";
- setUpScanner(code);
- assertToken(IRubyColorConstants.RUBY_SINGLE_LINE_COMMENT, 16, 47); //
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 63, 7); // 'return'
- assertToken(IRubyColorConstants.RUBY_KEYWORD, 70, 5); // ' self'
- }
-
public void testSymbolInsideParentheses() {
String code = "Object.const_defined?(:RedCloth)";
setUpScanner(code);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
