[Exist-commits] eXist-1.0/src/org/exist/storage/analysis SimpleTokenizer.java,1.7,1.8 TextToken.java,1.2,1.3

[Wolfgang M. M. <wol...@us...>]

Update of /cvsroot/exist/eXist-1.0/src/org/exist/storage/analysis
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv20167/src/org/exist/storage/analysis

Modified Files:
	SimpleTokenizer.java TextToken.java 
Log Message:
Improved page organization in dom.dbx to fix known XUpdate issues.

Index: TextToken.java
===================================================================
RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/analysis/TextToken.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** TextToken.java	13 Jun 2003 10:34:58 -0000	1.2
--- TextToken.java	14 Apr 2004 12:17:23 -0000	1.3
***************
*** 1,10 ****
  package org.exist.storage.analysis;
  
- /**
-  *  Description of the Class
-  *
-  *@author     Wolfgang Meier <me...@if...>
-  *@created    10. Oktober 2002
-  */
  public class TextToken {
  	
--- 1,5 ----
+ 
  package org.exist.storage.analysis;
  
  public class TextToken {
  	
***************
*** 25,29 ****
      public final static TextToken EOF_TOKEN = new TextToken(EOF);
  	
-     /**  Description of the Field */
      public final static String[] types = {"letter", "digit",
          "whitespace", "number",
--- 20,23 ----
***************
*** 98,122 ****
  	}
  	
!     /**  Description of the Method */
      public void consumeNext() {
          end++;
      }
  
- 
-     /**
-      *  Gets the char attribute of the Token object
-      *
-      *@return    The char value
-      */
      public char getChar() {
          return tokenText.charAt( start );
      }
  
- 
-     /**
-      *  Gets the text attribute of the Token object
-      *
-      *@return    The text value
-      */
      public String getText() {
          if(start >= tokenText.length() || end > tokenText.length())
--- 92,107 ----
  	}
  	
!     /**
!      * Consume the next character in the current buffer by incrementing
!      * the end offset.
!      */
      public void consumeNext() {
          end++;
      }
  
      public char getChar() {
          return tokenText.charAt( start );
      }
  
      public String getText() {
          if(start >= tokenText.length() || end > tokenText.length())
***************
*** 126,149 ****
      }
  
- 
-     /**
-      *  Gets the type attribute of the Token object
-      *
-      *@return    The type value
-      */
      public int getType() {
          return tokenType;
      }
  
- 
  	public void setType( int type ) {
  		tokenType = type;
  	}
  	
-     /**
-      *  Sets the text attribute of the Token object
-      *
-      *@param  text  The new text value
-      */
      public void setText( String text ) {
          tokenText = text;
--- 111,122 ----

Index: SimpleTokenizer.java
===================================================================
RCS file: /cvsroot/exist/eXist-1.0/src/org/exist/storage/analysis/SimpleTokenizer.java,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** SimpleTokenizer.java	31 Mar 2004 12:22:07 -0000	1.7
--- SimpleTokenizer.java	14 Apr 2004 12:17:23 -0000	1.8
***************
*** 12,16 ****
  public class SimpleTokenizer implements Tokenizer {
  	private int pos = 0;
- 	private int previous = 0;
  	private boolean stem = false;
  	private CharSequence text;
--- 12,15 ----
***************
*** 46,51 ****
  			if (ch == '\\' && isWildcard(LA(2))) {
  				break;
! 			}
! 			if (singleCharToken(ch)) {
  				// if this is a single char token and first in the sequence,
  				// consume it
--- 45,49 ----
  			if (ch == '\\' && isWildcard(LA(2))) {
  				break;
! 			} else if (ch > '\u2E80' && singleCharToken(ch)) {
  				// if this is a single char token and first in the sequence,
  				// consume it