Menu
▾
▴
[Jnode-svn-commits] SF.net SVN: jnode:[5016] trunk/shell/src
|
From: <cr...@us...> - 2009-02-10 14:24:06
|
Revision: 5016
http://jnode.svn.sourceforge.net/jnode/?rev=5016&view=rev
Author: crawley
Date: 2009-02-10 14:23:53 +0000 (Tue, 10 Feb 2009)
Log Message:
-----------
Implement \<newline> continuations.
Modified Paths:
--------------
trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTests.java
trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTests.java
trunk/shell/src/test/org/jnode/test/shell/bjorne/bjorne-shell-tests.xml
Modified: trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java
===================================================================
--- trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java 2009-02-09 16:04:03 UTC (rev 5015)
+++ trunk/shell/src/shell/org/jnode/shell/bjorne/BjorneTokenizer.java 2009-02-10 14:23:53 UTC (rev 5016)
@@ -66,6 +66,7 @@
import static org.jnode.shell.bjorne.BjorneToken.TOK_WHILE;
import static org.jnode.shell.bjorne.BjorneToken.TOK_WORD;
+import org.jnode.shell.IncompleteCommandException;
import org.jnode.shell.ShellFailureException;
public class BjorneTokenizer {
@@ -83,16 +84,79 @@
private final boolean debug;
- public BjorneTokenizer(String text) {
+ /**
+ * Create a tokenizer for the supplied shell input text.
+ * @param text the text to be tokenized
+ * @throws IncompleteCommandException if the text ends with a line continuation
+ */
+ public BjorneTokenizer(String text) throws IncompleteCommandException {
this(text, false);
}
- public BjorneTokenizer(String text, boolean debug) {
- chars = text.toCharArray();
+ /**
+ * Create a tokenizer for the supplied shell input text.
+ * @param text the text to be tokenized
+ * @param debug if {@code true}, produce debug output
+ * @throws IncompleteCommandException if the text ends with a line continuation
+ */
+ public BjorneTokenizer(String text, boolean debug) throws IncompleteCommandException {
+ chars = foldContinuations(text);
len = chars.length;
this.debug = debug;
}
+ /**
+ * Rewrite the supplied text to fold any line continuations.
+ *
+ * @param text the text to be processed
+ * @return the characters of text with any line continuations removed.
+ * @throws IncompleteCommandException
+ */
+ private char[] foldContinuations(String text) throws IncompleteCommandException {
+ if (text.indexOf('\\') == -1) {
+ return text.toCharArray();
+ }
+ int len = text.length();
+ StringBuilder sb = new StringBuilder(len);
+ boolean escape = false;
+ for (int i = 0; i < len; i++) {
+ char ch = text.charAt(i);
+ switch (ch) {
+ case '\\':
+ if (escape) {
+ sb.append('\\');
+ } else if (i == len - 1) {
+ // If we get a continuation sequence at the end of the
+ // text, the simplest thing is to ask for more input.
+ throw new IncompleteCommandException(
+ "More input required after '\\<newline>'", " > ");
+ }
+ escape = !escape;
+ break;
+ case '\n':
+ if (!escape) {
+ sb.append('\n');
+ } else {
+ escape = false;
+ }
+ break;
+ default:
+ if (escape) {
+ sb.append('\\');
+ escape = false;
+ }
+ sb.append(ch);
+ }
+ }
+ return sb.toString().toCharArray();
+ }
+
+ /**
+ * Get the next token without advancing. The default tokenization
+ * rules are used.
+ *
+ * @return the next token
+ */
public BjorneToken peek() {
if (current == null) {
current = advance();
@@ -103,6 +167,13 @@
return current;
}
+ /**
+ * Get the next token without advancing, using the tokenization
+ * rules corresponding to the supplied 'context'.
+ *
+ * @param context gives the tokenization rules
+ * @return the next token
+ */
public BjorneToken peek(int context) {
BjorneToken res = reinterpret(peek(), context);
if (debug) {
@@ -111,10 +182,20 @@
return res;
}
+ /**
+ * Test if {@link #next()} will return something other that EOS.
+ * @return <code>true</code> if there are more tokens to be delivered.
+ */
public boolean hasNext() {
return peek().getTokenType() != TOK_END_OF_STREAM;
}
+ /**
+ * Get the next token and advance. The default tokenization
+ * rules are used.
+ *
+ * @return the next token
+ */
public BjorneToken next() {
if (current == null) {
prev = advance();
@@ -129,6 +210,10 @@
return prev;
}
+ /**
+ * Backup one token in the token sequence. Calling this method twice without
+ * an intervening {@link #next()} call is invalid.
+ */
public void backup() {
if (prev == null) {
throw new ShellFailureException("incorrect backup");
@@ -142,6 +227,13 @@
prev = null;
}
+ /**
+ * Get the next token and advance, using the tokenization
+ * rules corresponding to the supplied 'context'.
+ *
+ * @param context gives the tokenization rules
+ * @return the next token
+ */
public BjorneToken next(int context) {
BjorneToken res = reinterpret(next(), context);
if (debug) {
@@ -150,6 +242,9 @@
return res;
}
+ /**
+ * This operation is not supported.
+ */
public void remove() {
throw new UnsupportedOperationException("remove not supported");
}
Modified: trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTests.java
===================================================================
--- trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTests.java 2009-02-09 16:04:03 UTC (rev 5015)
+++ trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneParserTests.java 2009-02-10 14:23:53 UTC (rev 5016)
@@ -30,7 +30,7 @@
private static final boolean DEBUG = true;
- public void testParser() {
+ public void testParser() throws ShellException {
new BjorneParser(new BjorneTokenizer(""), null);
}
Modified: trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTests.java
===================================================================
--- trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTests.java 2009-02-09 16:04:03 UTC (rev 5015)
+++ trunk/shell/src/test/org/jnode/test/shell/bjorne/BjorneTokenizerTests.java 2009-02-10 14:23:53 UTC (rev 5016)
@@ -67,16 +67,26 @@
import static org.jnode.shell.bjorne.BjorneToken.TOK_WORD;
import junit.framework.TestCase;
+import org.jnode.shell.IncompleteCommandException;
import org.jnode.shell.bjorne.BjorneToken;
import org.jnode.shell.bjorne.BjorneTokenizer;
public class BjorneTokenizerTests extends TestCase {
- public void testBjorneTokenizer() {
+ public void testBjorneTokenizer() throws IncompleteCommandException {
new BjorneTokenizer("hello");
}
- public void testEmpty() {
+ public void testBjorneTokenizer2() {
+ try {
+ new BjorneTokenizer("hello\\");
+ fail("no exception");
+ } catch (IncompleteCommandException ex) {
+ // expected
+ }
+ }
+
+ public void testEmpty() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("");
BjorneToken token = tokenizer.peek();
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
@@ -88,7 +98,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testNewline() {
+ public void testNewline() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("\n");
BjorneToken token = tokenizer.next();
assertEquals(TOK_END_OF_LINE, token.getTokenType());
@@ -96,7 +106,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testBlanksAndNewlines() {
+ public void testBlanksAndNewlines() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(" \n\t\n ");
BjorneToken token = tokenizer.next();
assertEquals(TOK_END_OF_LINE, token.getTokenType());
@@ -106,7 +116,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testComments() {
+ public void testComments() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"# comment\n #comment 2\n # comment # 3");
BjorneToken token = tokenizer.next();
@@ -117,7 +127,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols() {
+ public void testSymbols() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("; | & < > ( )");
BjorneToken token = tokenizer.next();
assertEquals(TOK_SEMI, token.getTokenType());
@@ -137,7 +147,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols2() {
+ public void testSymbols2() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"; ;; | || & && < << > >>");
BjorneToken token = tokenizer.next();
@@ -164,7 +174,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols3() {
+ public void testSymbols3() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(";;;|||&&&<<<>>>");
BjorneToken token = tokenizer.next();
assertEquals(TOK_DSEMI, token.getTokenType());
@@ -188,7 +198,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testSymbols4() {
+ public void testSymbols4() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"< << <<- <& <> > >> >| >&");
BjorneToken token = tokenizer.next();
@@ -213,7 +223,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords() {
+ public void testWords() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("hello there");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -225,7 +235,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords2() {
+ public void testWords2() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"hello\\ there\\\n friend");
BjorneToken token = tokenizer.next();
@@ -243,9 +253,20 @@
assertEquals("hello\\ there\\ friend", token.getText());
token = tokenizer.next();
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
+
+ tokenizer = new BjorneTokenizer(
+ "hello\\\nthere\\\n friend");
+ token = tokenizer.next();
+ assertEquals(TOK_WORD, token.getTokenType());
+ assertEquals("hellothere", token.getText());
+ token = tokenizer.next();
+ assertEquals(TOK_WORD, token.getTokenType());
+ assertEquals("friend", token.getText());
+ token = tokenizer.next();
+ assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords3() {
+ public void testWords3() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("'1 2' \"3 4\" `5 6`");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -260,7 +281,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords4() {
+ public void testWords4() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("'1 \"2\"' \"3\\\"4\"");
BjorneToken token = tokenizer.next();
assertEquals(TOK_WORD, token.getTokenType());
@@ -272,7 +293,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testWords5() {
+ public void testWords5() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("1<2>3&4;5|6)7");
BjorneToken token = tokenizer.next();
assertEquals(TOK_IO_NUMBER, token.getTokenType());
@@ -311,7 +332,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule1() {
+ public void testRule1() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"if then else elif fi for done while until "
+ "case { } ! do in esac");
@@ -351,7 +372,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule5() {
+ public void testRule5() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"if a a1 9a a_b a,b AB A=b");
BjorneToken token = tokenizer.next(RULE_5_CONTEXT);
@@ -374,7 +395,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule6() {
+ public void testRule6() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("if in do");
BjorneToken token = tokenizer.next(RULE_6_CONTEXT);
assertEquals(TOK_WORD, token.getTokenType());
@@ -386,7 +407,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule7a() {
+ public void testRule7a() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"if then else elif fi for done while until "
+ "case { } ! do in esac a= a=b 1a=b =c");
@@ -434,7 +455,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule7b() {
+ public void testRule7b() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"if then else elif fi for done while until "
+ "case { } ! do in esac a= a=b 1a=b =c");
@@ -482,7 +503,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRule8() {
+ public void testRule8() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer(
"if then else elif fi for done while until "
+ "case { } ! do in esac a a_b a= a=b 1a=b =c");
@@ -534,7 +555,7 @@
assertEquals(TOK_END_OF_STREAM, token.getTokenType());
}
- public void testRegress() {
+ public void testRegress() throws IncompleteCommandException {
BjorneTokenizer tokenizer = new BjorneTokenizer("ls -l");
BjorneToken token = tokenizer.peek(RULE_7a_CONTEXT);
assertEquals(TOK_WORD, token.getTokenType());
Modified: trunk/shell/src/test/org/jnode/test/shell/bjorne/bjorne-shell-tests.xml
===================================================================
--- trunk/shell/src/test/org/jnode/test/shell/bjorne/bjorne-shell-tests.xml 2009-02-09 16:04:03 UTC (rev 5015)
+++ trunk/shell/src/test/org/jnode/test/shell/bjorne/bjorne-shell-tests.xml 2009-02-10 14:23:53 UTC (rev 5016)
@@ -8,6 +8,14 @@
<output>HI
</output>
</testSpec>
+ <testSpec title="continuation" command="test" runMode="AS_SCRIPT" rc="0">
+ <script>#!bjorne
+ echo hello \
+mother
+ </script>
+ <output>hello mother
+</output>
+ </testSpec>
<testSpec title="$?" command="test" runMode="AS_SCRIPT" rc="0">
<script>#!bjorne
true
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
