[Jboss-svn-commits] JBL Code SVN: r1992 - in trunk/labs/jbossrules/drools-natural-dsl/src: main/java/org/drools/natural/lexer test/java/org/drools/natural/lexer test/java/org/drools/natural/ruledoc

[<jbo...@li...>]

Author: mic...@jb...
Date: 2006-01-04 19:32:02 -0500 (Wed, 04 Jan 2006)
New Revision: 1992

Modified:
   trunk/labs/jbossrules/drools-natural-dsl/src/main/java/org/drools/natural/lexer/SimpleSnippetLexer.java
   trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/lexer/SimpleSnippetLexerTest.java
   trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/ruledoc/RuleFragmentTest.java
Log:
normalised whitespace in tokens

Modified: trunk/labs/jbossrules/drools-natural-dsl/src/main/java/org/drools/natural/lexer/SimpleSnippetLexer.java
===================================================================
--- trunk/labs/jbossrules/drools-natural-dsl/src/main/java/org/drools/natural/lexer/SimpleSnippetLexer.java	2006-01-04 20:29:03 UTC (rev 1991)
+++ trunk/labs/jbossrules/drools-natural-dsl/src/main/java/org/drools/natural/lexer/SimpleSnippetLexer.java	2006-01-05 00:32:02 UTC (rev 1992)
@@ -142,9 +142,13 @@
 
     private void breakBySpace(int pos)
     {
-        if ( bracketStack.size( ) > 0 || inDoubleQuotes )
+        if (inDoubleQuotes) {
+            appendCurrentChar( pos );
+        } else if ( bracketStack.size( ) > 0)
         {
-            appendCurrentChar( pos );
+            if (previousIsNotSpace(pos)) {
+                appendCurrentChar( pos );
+            }
         }
         else
         {
@@ -153,6 +157,16 @@
 
     }
 
+    private boolean previousIsNotSpace(int pos){
+        
+        if (pos == 0) return true;
+        if (snippet[pos - 1] == ' ') {
+            return false;
+        } else {
+            return true;
+        }
+    }
+
     private void newToken()
     {
         String token = currentToken.toString( ).trim( );

Modified: trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/lexer/SimpleSnippetLexerTest.java
===================================================================
--- trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/lexer/SimpleSnippetLexerTest.java	2006-01-04 20:29:03 UTC (rev 1991)
+++ trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/lexer/SimpleSnippetLexerTest.java	2006-01-05 00:32:02 UTC (rev 1992)
@@ -124,7 +124,20 @@
         
         assertEquals(expected, rawTokens);
     }
+    
+    public void testNormaliseWhitespaceInTokens() {
+        String sample ="[Age  of][Age\n    of]";
+        NaturalSnippetLexer lex = new SimpleSnippetLexer(sample);
+        List rawTokens = lex.getRawTokens().getTokens();
         
+        List expected = new ArrayList();
+        expected.add("Age of");        
+        expected.add("Age of");
+        
+        assertEquals(expected, rawTokens);
+        
+    }
+        
     
     
 }

Modified: trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/ruledoc/RuleFragmentTest.java
===================================================================
--- trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/ruledoc/RuleFragmentTest.java	2006-01-04 20:29:03 UTC (rev 1991)
+++ trunk/labs/jbossrules/drools-natural-dsl/src/test/java/org/drools/natural/ruledoc/RuleFragmentTest.java	2006-01-05 00:32:02 UTC (rev 1992)
@@ -59,7 +59,7 @@
         RuleFragment helper = new RuleFragment(fragment, grammar);
         List cons = helper.getConsequences();
         assertEquals(1, cons.size());
-        assertEquals("cons  1", cons.get(0));
+        assertEquals("cons 1", cons.get(0));
     }