[Pydev-cvs] org.python.pydev/src/org/python/pydev/editor/partitioner BufferedDocumentScanner.java, 1.2, 1.3 FastPythonPartitionScanner.java, 1.3, 1.4

[Fabio Z. <fa...@us...>]

Update of /cvsroot/pydev/org.python.pydev/src/org/python/pydev/editor/partitioner
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv4313/src/org/python/pydev/editor/partitioner

Modified Files:
	BufferedDocumentScanner.java FastPythonPartitionScanner.java 
Log Message:
Synching to latest changes:

Pydev
<ul>
<li><strong>Editor</strong>: Cursor settings no longer overridden</li>
<li><strong>Code-completion</strong>: If __all__ is defined with runtime elements (and not only in a single assign statement), it's ignored for code-completion purposes</li>
<li><strong>Debugger</strong>: Pythonpath the same in debug and regular modes (sys.path[0] is the same directory as the file run)</li>
<li><strong>Debugger</strong>: Persist choices done in the debugger when files from the debugger are not found</li>
<li><strong>Interpreter config</strong>: "email" automatically added to the "forced builtins"</li>
<li><strong>Parser</strong>: Correctly recognizing absolute import with 3 or more levels</li>
<li><strong>Syntax check</strong>: Option to do only on active editor</li>
</ul>

Also: tabs changed for spaces

Index: FastPythonPartitionScanner.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src/org/python/pydev/editor/partitioner/FastPythonPartitionScanner.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** FastPythonPartitionScanner.java	26 May 2006 14:31:25 -0000	1.3
--- FastPythonPartitionScanner.java	28 Sep 2008 12:45:47 -0000	1.4
***************
*** 16,217 ****
   */
  public class FastPythonPartitionScanner implements IPartitionTokenScanner, IPythonPartitions{
! 	
  
! 	// states
! 	private static final int PYTHON= 0;
! 	private static final int COMMENT= 1;
! 	private static final int SINGLE_LINE_STRING1= 2; //'
! 	private static final int SINGLE_LINE_STRING2= 3; //"
! 	private static final int MULTI_LINE_STRING1= 4;  //'
! 	private static final int MULTI_LINE_STRING2= 5;  //""
! 	private static final int BACKQUOTES= 6;  
! 	
! 	
! 	/** The scanner. */
! 	private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);	// faster implementation
  
! 	private final IToken[] fTokens= new IToken[] {
! 			new Token(null),
! 			new Token(PY_COMMENT),
! 			new Token(PY_SINGLELINE_STRING1),
! 			new Token(PY_SINGLELINE_STRING2),
! 			new Token(PY_MULTILINE_STRING1),
! 			new Token(PY_MULTILINE_STRING2),
! 			new Token(PY_BACKQUOTES),
! 		};
! 	private int fTokenOffset;
! 	private int fTokenLength;
! 	private String currContentType;
! 	
  
! 	public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
! 		if(partitionOffset != -1 && partitionOffset < offset){
! 			fScanner.setRange(document, partitionOffset, length+(offset-partitionOffset));
! 			fTokenOffset= partitionOffset;
! 			fTokenLength= 0;
! 			currContentType = null;
! 			
! 		}else{
! 			fScanner.setRange(document, offset, length);
! 			fTokenOffset= offset;
! 			fTokenLength= 0;
! 			currContentType = contentType;
! 		}
! 	}
  
! 	public void setRange(IDocument document, int offset, int length) {
! 		currContentType = null;
! 		fScanner.setRange(document, offset, length);
! 		fTokenOffset= offset;
! 		fTokenLength= 0;
! 	}
  
  
! 	public int getTokenOffset() {
! 		return fTokenOffset;
! 	}
  
! 	public int getTokenLength() {
! 		return fTokenLength;
! 	}
  
! 	/*
! 	 * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
! 	 */
! 	public IToken nextToken() {
! 		fTokenOffset += fTokenLength;
! 		fTokenLength= 0;
! 		
! 		int ch= fScanner.read();
! 		if(ch == ICharacterScanner.EOF){
! 			fTokenLength++;
! 			return Token.EOF;
! 		}
  
! 		if(currContentType != null){
! 			if(currContentType.equals(PY_COMMENT)){
! 				return handleComment(ch);
! 			}
! 			if(currContentType.equals(PY_SINGLELINE_STRING1)){
! 				return handleSingleQuotedString(ch);
! 				
! 			}
! 			if(currContentType.equals(PY_SINGLELINE_STRING2)){
! 				return handleSingleQuotedString(ch);
! 				
! 			}
! 			if(currContentType.equals(PY_MULTILINE_STRING1)){
! 				return handleSingleQuotedString(ch);
! 				
! 			}
! 			if(currContentType.equals(PY_MULTILINE_STRING2)){
! 				return handleSingleQuotedString(ch);
! 				
! 			}
! 			if(currContentType.equals(PY_BACKQUOTES)){
! 				
! 			}
! 			
! 		}
! 		
  
! 		
! 		// characters
!  		switch (ch) {
!  		case '#':
! 			return handleComment(ch);
!  	        
!  		case '"':
!  		case '\'':
!  			return handleSingleQuotedString(ch);
!  	        
!  		default:
!  			fTokenLength++;
!  			return fTokens[PYTHON];
!  		}
! 	}
  
! 	private IToken handleSingleQuotedString(int ch) {
! 		int initialChar = ch;
! 		int offsetEnd = fTokenOffset;
! 		
! 		if(isMultiLiteral(ch)){
! 			offsetEnd += 2; //ok, it is a multi-line with single quotes
! 			return handleMultiSingleQuotedString(ch, offsetEnd, initialChar);
! 		}
! 		
! 		//it is a single-line string
! 		ch = fScanner.read();
! 		offsetEnd++;
! 		
! 		while(ch!= '\n' && ch != '\r' && ch != initialChar && ch != ICharacterScanner.EOF){
! 			ch = fScanner.read();
! 			offsetEnd++;
! 		}
! 		
! 		offsetEnd++;
! 		fTokenLength = offsetEnd-fTokenOffset;
! 		if(initialChar == '\''){
! 			return fTokens[SINGLE_LINE_STRING1];
! 		}else{
! 			return fTokens[SINGLE_LINE_STRING2];
! 		}
! 	}
  
! 	
! 	private IToken handleMultiSingleQuotedString(int ch, int offsetEnd, int initialChar) {
! 		//it is a multi-line string
! 		ch = fScanner.read();
! 		offsetEnd++;
! 		
! 		while(ch != ICharacterScanner.EOF){
! 			if(ch == initialChar){
! 				if(isMultiLiteral(ch)){
! 					offsetEnd+=2;
! 					if(initialChar == '\''){
! 						return fTokens[MULTI_LINE_STRING1];
! 					}else{
! 						return fTokens[MULTI_LINE_STRING2];
! 					}
! 				}
! 			}
! 			ch = fScanner.read();
! 			offsetEnd++;
! 		}
! 		
! 		if(initialChar == '\''){
! 			return fTokens[SINGLE_LINE_STRING1];
! 		}else{
! 			return fTokens[SINGLE_LINE_STRING2];
! 		}
! 	}
  
! 	private boolean isMultiLiteral(int ch) {
! 		int c1 = fScanner.read();
! 		if(c1 == ch){
! 			int c2 = fScanner.read();
! 			if(c2 == ch){
! 				return true;
! 			}
! 			if(c2 != ICharacterScanner.EOF){
! 				fScanner.unread();
! 			}
! 		}
! 		if(c1 != ICharacterScanner.EOF){
! 			fScanner.unread();
! 		}
! 		return false;
! 	}
  
! 	private IToken handleComment(int ch) {
! 		int offsetEnd = fTokenOffset;
! 		
! 		while(ch!= '\n' && ch != '\r' && ch != ICharacterScanner.EOF){
! 			ch = fScanner.read();
! 			offsetEnd++;
! 		}
! 		fTokenLength = offsetEnd-fTokenOffset;
! 		return fTokens[COMMENT];
! 	}
  
  }
--- 16,217 ----
   */
  public class FastPythonPartitionScanner implements IPartitionTokenScanner, IPythonPartitions{
!     
  
!     // states
!     private static final int PYTHON= 0;
!     private static final int COMMENT= 1;
!     private static final int SINGLE_LINE_STRING1= 2; //'
!     private static final int SINGLE_LINE_STRING2= 3; //"
!     private static final int MULTI_LINE_STRING1= 4;  //'
!     private static final int MULTI_LINE_STRING2= 5;  //""
!     private static final int BACKQUOTES= 6;  
!     
!     
!     /** The scanner. */
!     private final BufferedDocumentScanner fScanner= new BufferedDocumentScanner(1000);    // faster implementation
  
!     private final IToken[] fTokens= new IToken[] {
!             new Token(null),
!             new Token(PY_COMMENT),
!             new Token(PY_SINGLELINE_STRING1),
!             new Token(PY_SINGLELINE_STRING2),
!             new Token(PY_MULTILINE_STRING1),
!             new Token(PY_MULTILINE_STRING2),
!             new Token(PY_BACKQUOTES),
!         };
!     private int fTokenOffset;
!     private int fTokenLength;
!     private String currContentType;
!     
  
!     public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
!         if(partitionOffset != -1 && partitionOffset < offset){
!             fScanner.setRange(document, partitionOffset, length+(offset-partitionOffset));
!             fTokenOffset= partitionOffset;
!             fTokenLength= 0;
!             currContentType = null;
!             
!         }else{
!             fScanner.setRange(document, offset, length);
!             fTokenOffset= offset;
!             fTokenLength= 0;
!             currContentType = contentType;
!         }
!     }
  
!     public void setRange(IDocument document, int offset, int length) {
!         currContentType = null;
!         fScanner.setRange(document, offset, length);
!         fTokenOffset= offset;
!         fTokenLength= 0;
!     }
  
  
!     public int getTokenOffset() {
!         return fTokenOffset;
!     }
  
!     public int getTokenLength() {
!         return fTokenLength;
!     }
  
!     /*
!      * @see org.eclipse.jface.text.rules.ITokenScanner#nextToken()
!      */
!     public IToken nextToken() {
!         fTokenOffset += fTokenLength;
!         fTokenLength= 0;
!         
!         int ch= fScanner.read();
!         if(ch == ICharacterScanner.EOF){
!             fTokenLength++;
!             return Token.EOF;
!         }
  
!         if(currContentType != null){
!             if(currContentType.equals(PY_COMMENT)){
!                 return handleComment(ch);
!             }
!             if(currContentType.equals(PY_SINGLELINE_STRING1)){
!                 return handleSingleQuotedString(ch);
!                 
!             }
!             if(currContentType.equals(PY_SINGLELINE_STRING2)){
!                 return handleSingleQuotedString(ch);
!                 
!             }
!             if(currContentType.equals(PY_MULTILINE_STRING1)){
!                 return handleSingleQuotedString(ch);
!                 
!             }
!             if(currContentType.equals(PY_MULTILINE_STRING2)){
!                 return handleSingleQuotedString(ch);
!                 
!             }
!             if(currContentType.equals(PY_BACKQUOTES)){
!                 
!             }
!             
!         }
!         
  
!         
!         // characters
!          switch (ch) {
!          case '#':
!             return handleComment(ch);
!              
!          case '"':
!          case '\'':
!              return handleSingleQuotedString(ch);
!              
!          default:
!              fTokenLength++;
!              return fTokens[PYTHON];
!          }
!     }
  
!     private IToken handleSingleQuotedString(int ch) {
!         int initialChar = ch;
!         int offsetEnd = fTokenOffset;
!         
!         if(isMultiLiteral(ch)){
!             offsetEnd += 2; //ok, it is a multi-line with single quotes
!             return handleMultiSingleQuotedString(ch, offsetEnd, initialChar);
!         }
!         
!         //it is a single-line string
!         ch = fScanner.read();
!         offsetEnd++;
!         
!         while(ch!= '\n' && ch != '\r' && ch != initialChar && ch != ICharacterScanner.EOF){
!             ch = fScanner.read();
!             offsetEnd++;
!         }
!         
!         offsetEnd++;
!         fTokenLength = offsetEnd-fTokenOffset;
!         if(initialChar == '\''){
!             return fTokens[SINGLE_LINE_STRING1];
!         }else{
!             return fTokens[SINGLE_LINE_STRING2];
!         }
!     }
  
!     
!     private IToken handleMultiSingleQuotedString(int ch, int offsetEnd, int initialChar) {
!         //it is a multi-line string
!         ch = fScanner.read();
!         offsetEnd++;
!         
!         while(ch != ICharacterScanner.EOF){
!             if(ch == initialChar){
!                 if(isMultiLiteral(ch)){
!                     offsetEnd+=2;
!                     if(initialChar == '\''){
!                         return fTokens[MULTI_LINE_STRING1];
!                     }else{
!                         return fTokens[MULTI_LINE_STRING2];
!                     }
!                 }
!             }
!             ch = fScanner.read();
!             offsetEnd++;
!         }
!         
!         if(initialChar == '\''){
!             return fTokens[SINGLE_LINE_STRING1];
!         }else{
!             return fTokens[SINGLE_LINE_STRING2];
!         }
!     }
  
!     private boolean isMultiLiteral(int ch) {
!         int c1 = fScanner.read();
!         if(c1 == ch){
!             int c2 = fScanner.read();
!             if(c2 == ch){
!                 return true;
!             }
!             if(c2 != ICharacterScanner.EOF){
!                 fScanner.unread();
!             }
!         }
!         if(c1 != ICharacterScanner.EOF){
!             fScanner.unread();
!         }
!         return false;
!     }
  
!     private IToken handleComment(int ch) {
!         int offsetEnd = fTokenOffset;
!         
!         while(ch!= '\n' && ch != '\r' && ch != ICharacterScanner.EOF){
!             ch = fScanner.read();
!             offsetEnd++;
!         }
!         fTokenLength = offsetEnd-fTokenOffset;
!         return fTokens[COMMENT];
!     }
  
  }

Index: BufferedDocumentScanner.java
===================================================================
RCS file: /cvsroot/pydev/org.python.pydev/src/org/python/pydev/editor/partitioner/BufferedDocumentScanner.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -C2 -d -r1.2 -r1.3
*** BufferedDocumentScanner.java	20 Oct 2007 19:30:36 -0000	1.2
--- BufferedDocumentScanner.java	28 Sep 2008 12:45:47 -0000	1.3
***************
*** 16,147 ****
  public final class BufferedDocumentScanner implements ICharacterScanner {
  
! 	/** The document being scanned. */
! 	private IDocument fDocument;
! 	/** The offset of the document range to scan. */
! 	private int fRangeOffset;
! 	/** The length of the document range to scan. */
! 	private int fRangeLength;
! 	/** The delimiters of the document. */
! 	private char[][] fDelimiters;
  
! 	/** The buffer. */
! 	private final char[] fBuffer;
! 	/** The offset of the buffer within the document. */
! 	private int fBufferOffset;
! 	/** The valid length of the buffer for access. */
! 	private int fBufferLength;
! 	/** The offset of the scanner within the buffer. */
! 	private int fOffset;
  
  
! 	/**
! 	 * Creates a new buffered document scanner.
! 	 * The buffer size is set to the given number of characters.
! 	 *
! 	 * @param size the buffer size
! 	 */
! 	public BufferedDocumentScanner(int size) {
! 		Assert.isTrue(size >= 1);
! 		fBuffer= new char[size];
! 	}
  
! 	/**
! 	 * Fills the buffer with the contens of the document starting at the given offset.
! 	 *
! 	 * @param offset the document offset at which the buffer starts
! 	 */
! 	private final void updateBuffer(int offset) {
  
! 		fBufferOffset= offset;
  
! 		if (fBufferOffset + fBuffer.length > fRangeOffset + fRangeLength)
! 			fBufferLength= fRangeLength - (fBufferOffset - fRangeOffset);
! 		else
! 			fBufferLength= fBuffer.length;
  
! 		try {
! 			final String content= fDocument.get(fBufferOffset, fBufferLength);
! 			content.getChars(0, fBufferLength, fBuffer, 0);
! 		} catch (BadLocationException e) {
! 		}
! 	}
  
! 	/**
! 	 * Configures the scanner by providing access to the document range over which to scan.
! 	 *
! 	 * @param document the document to scan
! 	 * @param offset the offset of the document range to scan
! 	 * @param length the length of the document range to scan
! 	 */
! 	public final void setRange(IDocument document, int offset, int length) {
  
! 		fDocument= document;
! 		fRangeOffset= offset;
! 		fRangeLength= length;
  
! 		String[] delimiters= document.getLegalLineDelimiters();
! 		fDelimiters= new char[delimiters.length][];
! 		for (int i= 0; i < delimiters.length; i++)
! 			fDelimiters[i]= delimiters[i].toCharArray();
  
! 		updateBuffer(offset);
! 		fOffset= 0;
! 	}
  
! 	/*
! 	 * @see ICharacterScanner#read()
! 	 */
! 	public final int read() {
  
! 		if (fOffset == fBufferLength) {
! 			if (fBufferOffset + fBufferLength == fDocument.getLength())
! 				return EOF;
! 			else {
! 				updateBuffer(fBufferOffset + fBufferLength);
! 				fOffset= 0;
! 			}
! 		}
  
! 		return fBuffer[fOffset++];
! 	}
  
! 	/*
! 	 * @see ICharacterScanner#unread
! 	 */
! 	public final void unread() {
  
! 		if (fOffset == 0) {
! 			if (fBufferOffset == fRangeOffset) {
! 				// error: BOF
! 			} else {
! 				updateBuffer(fBufferOffset - fBuffer.length);
! 				fOffset= fBuffer.length - 1;
! 			}
! 		} else {
! 			--fOffset;
! 		}
! 	}
  
! 	/*
! 	 * @see ICharacterScanner#getColumn()
! 	 */
! 	public final int getColumn() {
  
! 		try {
! 			final int offset= fBufferOffset + fOffset;
! 			final int line= fDocument.getLineOfOffset(offset);
! 			final int start= fDocument.getLineOffset(line);
! 			return offset - start;
! 		} catch (BadLocationException e) {
! 		}
  
! 		return -1;
! 	}
  
! 	/*
! 	 * @see ICharacterScanner#getLegalLineDelimiters()
! 	 */
! 	public final char[][] getLegalLineDelimiters() {
! 		return fDelimiters;
! 	}
  }
--- 16,147 ----
  public final class BufferedDocumentScanner implements ICharacterScanner {
  
!     /** The document being scanned. */
!     private IDocument fDocument;
!     /** The offset of the document range to scan. */
!     private int fRangeOffset;
!     /** The length of the document range to scan. */
!     private int fRangeLength;
!     /** The delimiters of the document. */
!     private char[][] fDelimiters;
  
!     /** The buffer. */
!     private final char[] fBuffer;
!     /** The offset of the buffer within the document. */
!     private int fBufferOffset;
!     /** The valid length of the buffer for access. */
!     private int fBufferLength;
!     /** The offset of the scanner within the buffer. */
!     private int fOffset;
  
  
!     /**
!      * Creates a new buffered document scanner.
!      * The buffer size is set to the given number of characters.
!      *
!      * @param size the buffer size
!      */
!     public BufferedDocumentScanner(int size) {
!         Assert.isTrue(size >= 1);
!         fBuffer= new char[size];
!     }
  
!     /**
!      * Fills the buffer with the contens of the document starting at the given offset.
!      *
!      * @param offset the document offset at which the buffer starts
!      */
!     private final void updateBuffer(int offset) {
  
!         fBufferOffset= offset;
  
!         if (fBufferOffset + fBuffer.length > fRangeOffset + fRangeLength)
!             fBufferLength= fRangeLength - (fBufferOffset - fRangeOffset);
!         else
!             fBufferLength= fBuffer.length;
  
!         try {
!             final String content= fDocument.get(fBufferOffset, fBufferLength);
!             content.getChars(0, fBufferLength, fBuffer, 0);
!         } catch (BadLocationException e) {
!         }
!     }
  
!     /**
!      * Configures the scanner by providing access to the document range over which to scan.
!      *
!      * @param document the document to scan
!      * @param offset the offset of the document range to scan
!      * @param length the length of the document range to scan
!      */
!     public final void setRange(IDocument document, int offset, int length) {
  
!         fDocument= document;
!         fRangeOffset= offset;
!         fRangeLength= length;
  
!         String[] delimiters= document.getLegalLineDelimiters();
!         fDelimiters= new char[delimiters.length][];
!         for (int i= 0; i < delimiters.length; i++)
!             fDelimiters[i]= delimiters[i].toCharArray();
  
!         updateBuffer(offset);
!         fOffset= 0;
!     }
  
!     /*
!      * @see ICharacterScanner#read()
!      */
!     public final int read() {
  
!         if (fOffset == fBufferLength) {
!             if (fBufferOffset + fBufferLength == fDocument.getLength())
!                 return EOF;
!             else {
!                 updateBuffer(fBufferOffset + fBufferLength);
!                 fOffset= 0;
!             }
!         }
  
!         return fBuffer[fOffset++];
!     }
  
!     /*
!      * @see ICharacterScanner#unread
!      */
!     public final void unread() {
  
!         if (fOffset == 0) {
!             if (fBufferOffset == fRangeOffset) {
!                 // error: BOF
!             } else {
!                 updateBuffer(fBufferOffset - fBuffer.length);
!                 fOffset= fBuffer.length - 1;
!             }
!         } else {
!             --fOffset;
!         }
!     }
  
!     /*
!      * @see ICharacterScanner#getColumn()
!      */
!     public final int getColumn() {
  
!         try {
!             final int offset= fBufferOffset + fOffset;
!             final int line= fDocument.getLineOfOffset(offset);
!             final int start= fDocument.getLineOffset(line);
!             return offset - start;
!         } catch (BadLocationException e) {
!         }
  
!         return -1;
!     }
  
!     /*
!      * @see ICharacterScanner#getLegalLineDelimiters()
!      */
!     public final char[][] getLegalLineDelimiters() {
!         return fDelimiters;
!     }
  }