squirrel-sql-commits Mailing List for SQuirreL SQL Client (Page 14)

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Revision: 6410
          http://squirrel-sql.svn.sourceforge.net/squirrel-sql/?rev=6410&view=rev
Author:   wis775
Date:     2011-09-24 18:40:50 +0000 (Sat, 24 Sep 2011)
Log Message:
-----------
3408086: Non ASCII characters are highlighted as as invalid characters at the SQL-Editor pane.  Removed the limitation to ASCII - Added some test cases and fixed a IndexOutOfBoundException.

Modified Paths:
--------------
    trunk/sql12/app/src/main/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/Scanner.java

Added Paths:
-----------
    trunk/sql12/app/src/test/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/ScannerTest.java

Modified: trunk/sql12/app/src/main/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/Scanner.java
===================================================================

--- trunk/sql12/app/src/main/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/Scanner.java	2011-09-24 10:06:57 UTC (rev 6409)
+++ trunk/sql12/app/src/main/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/Scanner.java	2011-09-24 18:40:50 UTC (rev 6410)
@@ -73,7 +73,13 @@
 	private static final char CR  = '\r';
 	private static final char LF  = '\n';
 	private static final int noSym = 103;
-   private static final int[] start = {
+   
+	/**
+	 * The constants in this array are very confusing.
+	 * I don't think, that the constants don't mean, what their name said. e.g. for the '*' character, the constant ParsingConstants.KW_UNION is used.
+	 * Seems, that this is the result of a refactoring task.  
+	 */
+	private static final int[] start = {
     ParsingConstants.KW_AS,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
      0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
      0,  1,  4,  1,  1,  0,  0,  ParsingConstants.KIND_OPENING_BRAKET,  ParsingConstants.KW_UNION, 22, ParsingConstants.KW_ALL, ParsingConstants.KW_INSERT, ParsingConstants.KW_DISTINCT, ParsingConstants.KW_UPDATE,  2, ParsingConstants.KIND_EQUALS,
@@ -334,8 +340,14 @@
 		t = new Token();
 		t.pos = pos; t.col = pos - lineStart + 1; t.line = line;
 		StringBuffer buf = new StringBuffer();
-		int state = start[ch];
-		int apx = 0;
+		int state;
+		if(ch < start.length){
+			// the start state is only defined for the first 128 characters.
+			state = start[ch];
+		}else{
+			// for the rest, assume the value 1;
+			state = 1;
+		}
 		loop: for (;;) {
 			buf.append(strCh);
 			NextCh();

Added: trunk/sql12/app/src/test/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/ScannerTest.java
===================================================================
--- trunk/sql12/app/src/test/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/ScannerTest.java	                        (rev 0)
+++ trunk/sql12/app/src/test/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/ScannerTest.java	2011-09-24 18:40:50 UTC (rev 6410)
@@ -0,0 +1,193 @@
+/*
+ * Copyright (C) 2011 Stefan Willinger
+ * wi...@us...
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+package net.sourceforge.squirrel_sql.client.session.parser.kernel;
+
+import static org.mockito.Mockito.*;
+import static org.junit.Assert.*;
+
+import net.sourceforge.squirrel_sql.client.session.parser.kernel.Scanner.Buffer;
+import net.sourceforge.squirrel_sql.client.session.parser.kernel.Scanner.SBuffer;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+/**
+ * This test class is used, to make clear, how the scanner works :-)
+ * @author Stefan Willinger
+ *
+ */
+public class ScannerTest {
+	private Scanner classUnderTest;
+	
+	/**
+	 * @throws java.lang.Exception
+	 */
+	@Before
+	public void setUp() throws Exception {
+		
+	}
+
+	/**
+	 * Ensure, that a comment can contain non ASCII characters.
+	 * The previous behavior was to add an "invalid character in source file" error for each character greater than the ASCII range ('\u007f').
+	 */
+	@Test
+	public void testCommentWithNonASCII() {
+		SBuffer mockBuffer = new SBuffer("/*Ä*/ select");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token = classUnderTest.Scan();
+		
+		assertEquals("select", token.str);
+
+		verifyZeroInteractions(mockErrorStream);
+	}
+	
+	
+	/**
+	 * Ensure, that a statement can contain a non ASCII character
+	 */
+	@Test
+	public void testStatementWithNonASCII() {
+		SBuffer mockBuffer = new SBuffer("select sysdate as Ä from dual");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token;
+
+		token = classUnderTest.Scan();
+		assertEquals("select", token.str);
+		assertEquals(ParsingConstants.KW_SELECT, token.kind);
+		
+		token = classUnderTest.Scan();
+		assertEquals("sysdate", token.str);
+		assertEquals(1, token.kind);
+		
+		token = classUnderTest.Scan();
+		assertEquals("as", token.str);
+		assertEquals(ParsingConstants.KW_AS, token.kind);
+		
+		token = classUnderTest.Scan();
+		assertEquals("Ä", token.str);
+		assertEquals(1, token.kind);
+		
+		token = classUnderTest.Scan();
+		assertEquals("from", token.str);
+		assertEquals(ParsingConstants.KW_FROM, token.kind);
+		
+		token = classUnderTest.Scan();
+		assertEquals("dual", token.str);
+		assertEquals(1, token.kind);
+		
+		verifyZeroInteractions(mockErrorStream);
+	}
+	
+	
+	@Test
+	public void testTokenKind_Select() {
+		SBuffer mockBuffer = new SBuffer("select");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token = classUnderTest.Scan();
+		
+		assertEquals("select", token.str);
+		assertEquals(ParsingConstants.KW_SELECT, token.kind);
+
+		verifyZeroInteractions(mockErrorStream);
+	}
+	
+	/**
+	 * Just for playing with the Scanner to understand it.
+	 */
+	@Test
+	public void testTokenKind_WithNumbers() {
+		SBuffer mockBuffer = new SBuffer("select8");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token = classUnderTest.Scan();
+		
+		assertEquals("select8", token.str);
+		// Maybe, 1 means "No Keyword"
+		assertEquals(1, token.kind);
+
+		verifyZeroInteractions(mockErrorStream);
+	}
+	
+	/**
+	 * Ensure, that the open bracket is read correctly.
+	 */
+	@Test
+	public void testOpenBracket() {
+		SBuffer mockBuffer = new SBuffer("(");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token = classUnderTest.Scan();
+		
+		assertEquals("(", token.str);
+		assertEquals(ParsingConstants.KIND_OPENING_BRAKET, token.kind);
+
+		verifyZeroInteractions(mockErrorStream);
+	}
+	
+	@Test
+	public void testSemikolon() {
+		SBuffer mockBuffer = new SBuffer(";");
+		
+		ErrorStream mockErrorStream = Mockito.mock(ErrorStream.class);
+		classUnderTest = new Scanner(mockBuffer, mockErrorStream);
+		Token token = classUnderTest.Scan();
+		
+		assertEquals(";", token.str);
+		assertEquals(6, token.kind);
+
+		verifyZeroInteractions(mockErrorStream);
+	}
+
+	
+	
+	/**
+	 * For printing out the characters corresponding to the start array of the scanner.
+	 * Just for understanding this array.
+	 */
+	public static void main(String[] args) {
+		// This is a copy of the field at the scanner.
+		int[] start = {
+		    ParsingConstants.KW_AS,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+		     0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+		     0,  1,  4,  1,  1,  0,  0,  ParsingConstants.KIND_OPENING_BRAKET,  ParsingConstants.KW_UNION, 22, ParsingConstants.KW_ALL, ParsingConstants.KW_INSERT, ParsingConstants.KW_DISTINCT, ParsingConstants.KW_UPDATE,  2, ParsingConstants.KIND_EQUALS,
+		     ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT,  ParsingConstants.KW_EXCEPT, ParsingConstants.KW_SET,  ParsingConstants.KW_INTERSECT, ParsingConstants.KW_INTO, ParsingConstants.KW_MINUS, ParsingConstants.KW_FROM,  0,
+		     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+		     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+		     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+		     1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  0,  1,  1,  1,
+		     0};
+		 
+		 for (int i = 0; i < start.length; i++) {
+			System.out.println(((char) i) + "=" + start[i]);
+			 
+		}
+	}
+	
+
+}


Property changes on: trunk/sql12/app/src/test/java/net/sourceforge/squirrel_sql/client/session/parser/kernel/ScannerTest.java
___________________________________________________________________
Added: svn:mime-type
   + text/plain

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





2004	Jan	Feb	Mar	Apr (121)	May (188)	Jun (134)	Jul (115)	Aug (201)	Sep (22)	Oct (22)	Nov (15)	Dec (32)
2005	Jan (20)	Feb (4)	Mar (1)	Apr (84)	May (107)	Jun (101)	Jul (131)	Aug (66)	Sep (55)	Oct (189)	Nov (164)	Dec (60)
2006	Jan (75)	Feb (107)	Mar (117)	Apr (115)	May (91)	Jun (91)	Jul (225)	Aug (151)	Sep (117)	Oct (165)	Nov (263)	Dec (233)
2007	Jan (244)	Feb (116)	Mar (185)	Apr (263)	May (149)	Jun (292)	Jul (260)	Aug (306)	Sep (130)	Oct (61)	Nov (103)	Dec (144)
2008	Jan (112)	Feb (55)	Mar (116)	Apr (76)	May (69)	Jun (130)	Jul (75)	Aug (56)	Sep (76)	Oct (87)	Nov (403)	Dec (410)
2009	Jan (132)	Feb (168)	Mar (197)	Apr (67)	May (8)	Jun (44)	Jul (144)	Aug (126)	Sep (88)	Oct (3)	Nov (81)	Dec (97)
2010	Jan (238)	Feb (41)	Mar (61)	Apr (37)	May (31)	Jun (22)	Jul (92)	Aug (105)	Sep (46)	Oct (45)	Nov (55)	Dec (92)
2011	Jan (51)	Feb (21)	Mar (57)	Apr (20)	May (32)	Jun (29)	Jul (26)	Aug (50)	Sep (59)	Oct (25)	Nov (79)	Dec (2)
2012	Jan (14)	Feb (31)	Mar (11)	Apr (9)	May (13)	Jun (9)	Jul (6)	Aug (12)	Sep (16)	Oct (37)	Nov (12)	Dec (7)
2013	Jan (9)	Feb	Mar	Apr	May	Jun	Jul	Aug	Sep	Oct	Nov	Dec

squirrel-sql-commits Mailing List for SQuirreL SQL Client (Page 14)

A Java SQL client for any JDBC compliant database

squirrel-sql-commits — A list that broadcasts CVS commit activity