Thread: [Pydev-cvs] org.python.pydev.parser/src/org/python/pydev/parser/jython ReaderCharStream.java,1.1,1.2
Brought to you by:
fabioz
From: Fabio Z. <fa...@us...> - 2006-04-27 20:01:09
|
Update of /cvsroot/pydev/org.python.pydev.parser/src/org/python/pydev/parser/jython In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv21233/src/org/python/pydev/parser/jython Modified Files: ReaderCharStream.java FastCharStream.java Log Message: Index: ReaderCharStream.java =================================================================== RCS file: /cvsroot/pydev/org.python.pydev.parser/src/org/python/pydev/parser/jython/ReaderCharStream.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** ReaderCharStream.java 20 Mar 2006 19:37:35 -0000 1.1 --- ReaderCharStream.java 27 Apr 2006 20:00:58 -0000 1.2 *************** *** 2,330 **** /** ! * An implementation of interface CharStream, where the data is read from ! * a Reader. ! * This file started life as a copy of ASCII_CharStream.java. */ ! public final class ReaderCharStream implements CharStream ! { ! public static final boolean staticFlag = false; ! int bufsize; ! int available; ! int tokenBegin; ! public int bufpos = -1; ! private int bufline[]; ! private int bufcolumn[]; ! private int column = 0; ! private int line = 1; ! private boolean prevCharIsCR = false; ! private boolean prevCharIsLF = false; ! private java.io.Reader inputStream; ! private char[] buffer; ! private int maxNextCharInd = 0; ! private int inBuf = 0; ! private final void ExpandBuff(boolean wrapAround) ! { ! char[] newbuffer = new char[bufsize + 2048]; ! int newbufline[] = new int[bufsize + 2048]; ! int newbufcolumn[] = new int[bufsize + 2048]; ! try ! { ! if (wrapAround) ! { ! System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); ! System.arraycopy(buffer, 0, newbuffer, ! bufsize - tokenBegin, bufpos); ! buffer = newbuffer; ! System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); ! System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); ! bufline = newbufline; ! System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); ! System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); ! bufcolumn = newbufcolumn; ! maxNextCharInd = (bufpos += (bufsize - tokenBegin)); ! } ! else ! { ! System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); ! buffer = newbuffer; ! System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); ! bufline = newbufline; ! System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); ! bufcolumn = newbufcolumn; ! maxNextCharInd = (bufpos -= tokenBegin); ! } ! } ! catch (Throwable t) ! { ! throw new Error(t.getMessage()); ! } ! bufsize += 2048; ! available = bufsize; ! tokenBegin = 0; ! } ! private final void FillBuff() throws java.io.IOException ! { ! if (maxNextCharInd == available) ! { ! if (available == bufsize) ! { ! if (tokenBegin > 2048) ! { ! bufpos = maxNextCharInd = 0; ! available = tokenBegin; ! } ! else if (tokenBegin < 0) ! bufpos = maxNextCharInd = 0; ! else ! ExpandBuff(false); ! } ! else if (available > tokenBegin) ! available = bufsize; ! else if ((tokenBegin - available) < 2048) ! ExpandBuff(true); ! else ! available = tokenBegin; ! } ! int i; ! try { ! if ((i = inputStream.read(buffer, maxNextCharInd, ! available - maxNextCharInd)) == -1) ! { ! inputStream.close(); ! throw new java.io.IOException(); ! } ! else ! maxNextCharInd += i; ! return; ! } ! catch(java.io.IOException e) { ! --bufpos; ! backup(0); ! if (tokenBegin == -1) ! tokenBegin = bufpos; ! throw e; ! } ! } ! public final char BeginToken() throws java.io.IOException ! { ! tokenBegin = -1; ! char c = readChar(); ! tokenBegin = bufpos; ! return c; ! } ! private final void UpdateLineColumn(char c) ! { ! column++; ! if (prevCharIsLF) ! { ! prevCharIsLF = false; ! line += (column = 1); ! } ! else if (prevCharIsCR) ! { ! prevCharIsCR = false; ! if (c == '\n') ! { ! prevCharIsLF = true; ! } ! else ! line += (column = 1); ! } ! switch (c) ! { ! case '\r' : ! prevCharIsCR = true; ! break; ! case '\n' : ! prevCharIsLF = true; ! break; ! //ok, this was commented out because the position would not reflect correctly the positions found in the ast. ! //this may have other problems, but they have to be analyzed better to see the problems this may bring ! //(files that mix tabs and spaces may suffer, but I could not find out very well the problems -- anyway, ! //restricting the analysis to files that have only tabs or only spaces seems reasonable -- shortcuts are available ! //so that we can convert a file from one type to another, so, what remains is making some lint analysis to be sure of it). ! // case '\t' : ! // column--; ! // column += (8 - (column & 07)); ! // break; ! default : ! break; ! } ! bufline[bufpos] = line; ! bufcolumn[bufpos] = column; ! } ! public final char readChar() throws java.io.IOException ! { ! if (inBuf > 0) ! { ! --inBuf; ! return buffer[(bufpos == bufsize - 1) ? (bufpos = 0) : ++bufpos]; ! } ! if (++bufpos >= maxNextCharInd) ! FillBuff(); ! char c = buffer[bufpos]; ! UpdateLineColumn(c); ! return (c); ! } ! /** ! * @deprecated ! * @see #getEndColumn ! */ ! public final int getColumn() { ! return bufcolumn[bufpos]; ! } ! /** ! * @deprecated ! * @see #getEndLine ! */ ! public final int getLine() { ! return bufline[bufpos]; ! } ! public final int getEndColumn() { ! return bufcolumn[bufpos]; ! } ! public final int getEndLine() { ! return bufline[bufpos]; ! } ! public final int getBeginColumn() { ! return bufcolumn[tokenBegin]; ! } ! public final int getBeginLine() { ! return bufline[tokenBegin]; ! } ! public final void backup(int amount) { ! inBuf += amount; ! if ((bufpos -= amount) < 0) ! bufpos += bufsize; ! } ! public ReaderCharStream(java.io.Reader dstream) ! { ! inputStream = dstream; ! line = 1; ! column = 0; ! available = bufsize = 4096; ! buffer = new char[bufsize]; ! bufline = new int[bufsize]; ! bufcolumn = new int[bufsize]; ! } ! public final String GetImage() ! { ! if (bufpos >= tokenBegin) ! return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); ! else ! return new String(buffer, tokenBegin, bufsize - tokenBegin) + ! new String(buffer, 0, bufpos + 1); ! } ! public final char[] GetSuffix(int len) ! { ! char[] ret = new char[len]; ! if ((bufpos + 1) >= len) ! System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); ! else ! { ! System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, ! len - bufpos - 1); ! System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); ! } ! return ret; ! } ! public void Done() ! { ! buffer = null; ! bufline = null; ! bufcolumn = null; ! } ! /** ! * Method to adjust line and column numbers for the start of a token.<BR> ! */ ! public void adjustBeginLineColumn(int newLine, int newCol) ! { ! int start = tokenBegin; ! int len; ! if (bufpos >= tokenBegin) ! { ! len = bufpos - tokenBegin + inBuf + 1; ! } ! else ! { ! len = bufsize - tokenBegin + bufpos + 1 + inBuf; ! } ! int i = 0, j = 0, k = 0; ! int nextColDiff = 0, columnDiff = 0; ! while (i < len && ! bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) ! { ! bufline[j] = newLine; ! nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; ! bufcolumn[j] = newCol + columnDiff; ! columnDiff = nextColDiff; ! i++; ! } ! if (i < len) ! { ! bufline[j] = newLine++; ! bufcolumn[j] = newCol + columnDiff; ! while (i++ < len) ! { ! if (bufline[j = start % bufsize] != bufline[++start % bufsize]) ! bufline[j] = newLine++; ! else ! bufline[j] = newLine; ! } ! } ! line = bufline[j]; ! column = bufcolumn[j]; ! } } --- 2,253 ---- /** ! * An implementation of interface CharStream, where the data is read from a Reader. This file started life as a copy of ASCII_CharStream.java. */ ! public final class ReaderCharStream implements CharStream { ! public static final boolean staticFlag = false; ! int bufsize; ! int available; ! int tokenBegin; ! public int bufpos = -1; ! private int bufline[]; ! private int bufcolumn[]; ! private int column = 0; ! private int line = 1; ! private boolean prevCharIsCR = false; ! private boolean prevCharIsLF = false; ! private java.io.Reader inputStream; ! private char[] buffer; + private int maxNextCharInd = 0; ! private int inBuf = 0; ! private final void ExpandBuff(boolean wrapAround) { ! char[] newbuffer = new char[bufsize + 2048]; ! int newbufline[] = new int[bufsize + 2048]; ! int newbufcolumn[] = new int[bufsize + 2048]; ! try { ! if (wrapAround) { ! System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); ! System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); ! buffer = newbuffer; ! System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); ! System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); ! bufline = newbufline; ! System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); ! System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); ! bufcolumn = newbufcolumn; ! maxNextCharInd = (bufpos += (bufsize - tokenBegin)); ! } else { ! System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); ! buffer = newbuffer; ! System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); ! bufline = newbufline; ! System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); ! bufcolumn = newbufcolumn; ! maxNextCharInd = (bufpos -= tokenBegin); ! } ! } catch (Throwable t) { ! throw new Error(t.getMessage()); ! } ! bufsize += 2048; ! available = bufsize; ! tokenBegin = 0; ! } ! private final void FillBuff() throws java.io.IOException { ! if (maxNextCharInd == available) { ! if (available == bufsize) { ! if (tokenBegin > 2048) { ! bufpos = maxNextCharInd = 0; ! available = tokenBegin; ! } else if (tokenBegin < 0) ! bufpos = maxNextCharInd = 0; ! else ! ExpandBuff(false); ! } else if (available > tokenBegin) ! available = bufsize; ! else if ((tokenBegin - available) < 2048) ! ExpandBuff(true); ! else ! available = tokenBegin; ! } ! int i; ! try { ! if ((i = inputStream.read(buffer, maxNextCharInd, available - maxNextCharInd)) == -1) { ! inputStream.close(); ! throw new java.io.IOException(); ! } else ! maxNextCharInd += i; ! return; ! } catch (java.io.IOException e) { ! --bufpos; ! backup(0); ! if (tokenBegin == -1) ! tokenBegin = bufpos; ! throw e; ! } ! } ! public final char BeginToken() throws java.io.IOException { ! tokenBegin = -1; ! char c = readChar(); ! tokenBegin = bufpos; ! return c; ! } ! private final void UpdateLineColumn(char c) { ! column++; ! if (prevCharIsLF) { ! prevCharIsLF = false; ! line += (column = 1); ! } else if (prevCharIsCR) { ! prevCharIsCR = false; ! if (c == '\n') { ! prevCharIsLF = true; ! } else ! line += (column = 1); ! } ! switch (c) { ! case '\r': ! prevCharIsCR = true; ! break; ! case '\n': ! prevCharIsLF = true; ! break; ! // ok, this was commented out because the position would not reflect correctly the positions found in the ast. ! // this may have other problems, but they have to be analyzed better to see the problems this may bring ! // (files that mix tabs and spaces may suffer, but I could not find out very well the problems -- anyway, ! // restricting the analysis to files that have only tabs or only spaces seems reasonable -- shortcuts are available ! // so that we can convert a file from one type to another, so, what remains is making some lint analysis to be sure of it). ! // case '\t' : ! // column--; ! // column += (8 - (column & 07)); ! // break; ! default: ! break; ! } ! bufline[bufpos] = line; ! bufcolumn[bufpos] = column; ! } ! public final char readChar() throws java.io.IOException { ! if (inBuf > 0) { ! --inBuf; ! return buffer[(bufpos == bufsize - 1) ? (bufpos = 0) : ++bufpos]; ! } ! if (++bufpos >= maxNextCharInd) ! FillBuff(); ! char c = buffer[bufpos]; ! UpdateLineColumn(c); ! return (c); ! } ! /** ! * @deprecated ! * @see #getEndColumn ! */ ! public final int getColumn() { ! return bufcolumn[bufpos]; ! } ! /** ! * @deprecated ! * @see #getEndLine ! */ ! public final int getLine() { ! return bufline[bufpos]; ! } ! public final int getEndColumn() { ! return bufcolumn[bufpos]; ! } ! public final int getEndLine() { ! return bufline[bufpos]; ! } ! public final int getBeginColumn() { ! return bufcolumn[tokenBegin]; ! } ! public final int getBeginLine() { ! return bufline[tokenBegin]; ! } ! public final void backup(int amount) { ! inBuf += amount; ! if ((bufpos -= amount) < 0) ! bufpos += bufsize; ! } ! public ReaderCharStream(java.io.Reader dstream) { ! inputStream = dstream; ! line = 1; ! column = 0; ! available = bufsize = 4096; ! buffer = new char[bufsize]; ! bufline = new int[bufsize]; ! bufcolumn = new int[bufsize]; ! } ! public final String GetImage() { ! if (bufpos >= tokenBegin) ! return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); ! else ! return new String(buffer, tokenBegin, bufsize - tokenBegin) + new String(buffer, 0, bufpos + 1); ! } ! public final char[] GetSuffix(int len) { ! char[] ret = new char[len]; ! if ((bufpos + 1) >= len) ! System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); ! else { ! System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, len - bufpos - 1); ! System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); ! } ! ! return ret; ! } ! ! public void Done() { ! buffer = null; ! bufline = null; ! bufcolumn = null; ! } } Index: FastCharStream.java =================================================================== RCS file: /cvsroot/pydev/org.python.pydev.parser/src/org/python/pydev/parser/jython/FastCharStream.java,v retrieving revision 1.1 retrieving revision 1.2 diff -C2 -d -r1.1 -r1.2 *** FastCharStream.java 27 Apr 2006 17:28:44 -0000 1.1 --- FastCharStream.java 27 Apr 2006 20:00:58 -0000 1.2 *************** *** 4,23 **** /** ! * An implementation of interface CharStream, where the data is read from ! * a Reader. ! * ! * Completely recreated so that we can read data directly from a String, as the initial implementation ! * was highly inneficient when working only with a string. */ ! public final class FastCharStream implements CharStream ! { private char[] buffer; private int bufpos; private int tokenBegin; public FastCharStream(String initialDoc) { this.buffer = initialDoc.toCharArray(); } --- 4,75 ---- /** ! * An implementation of interface CharStream, where the data is read from a Reader. Completely recreated so that we can read data directly from a String, as the ! * initial implementation was highly inneficient when working only with a string. */ ! public final class FastCharStream implements CharStream { private char[] buffer; + + private int bufline[]; + + private int bufcolumn[]; + + private boolean prevCharIsCR = false; + + private boolean prevCharIsLF = false; + + private int column = 0; + + private int line = 1; + private int bufpos; + + private int updatePos; + private int tokenBegin; + private final void UpdateLineColumn(char c) { + column++; + + if (prevCharIsLF) { + prevCharIsLF = false; + line += (column = 1); + } else if (prevCharIsCR) { + prevCharIsCR = false; + if (c == '\n') { + prevCharIsLF = true; + } else + line += (column = 1); + } + + switch (c) { + case '\r': + prevCharIsCR = true; + break; + case '\n': + prevCharIsLF = true; + break; + // ok, this was commented out because the position would not reflect correctly the positions found in the ast. + // this may have other problems, but they have to be analyzed better to see the problems this may bring + // (files that mix tabs and spaces may suffer, but I could not find out very well the problems -- anyway, + // restricting the analysis to files that have only tabs or only spaces seems reasonable -- shortcuts are available + // so that we can convert a file from one type to another, so, what remains is making some lint analysis to be sure of it). + // case '\t' : + // column--; + // column += (8 - (column & 07)); + // break; + default: + break; + } + + bufline[bufpos] = line; + bufcolumn[bufpos] = column; + } + public FastCharStream(String initialDoc) { this.buffer = initialDoc.toCharArray(); + this.bufline = new int[initialDoc.length()]; + this.bufcolumn = new int[initialDoc.length()]; } *************** *** 26,29 **** --- 78,85 ---- char r = this.buffer[bufpos]; bufpos++; + if(bufpos >= updatePos){ + updatePos++; + UpdateLineColumn(r); + } return r; } catch (ArrayIndexOutOfBoundsException e) { *************** *** 32,62 **** } ! public int getColumn() { ! return 0; } ! public int getLine() { ! return 0; } ! public int getEndColumn() { ! return 0; } ! public int getEndLine() { ! return 0; } ! public int getBeginColumn() { ! return 0; } ! public int getBeginLine() { ! return 0; } public void backup(int amount) { bufpos -= amount; ! if(bufpos < 0){ bufpos = 0; } --- 88,128 ---- } ! /** ! * @deprecated ! * @see #getEndColumn ! */ ! ! public final int getColumn() { ! return bufcolumn[bufpos]; } ! /** ! * @deprecated ! * @see #getEndLine ! */ ! ! public final int getLine() { ! return bufline[bufpos]; } ! public final int getEndColumn() { ! return bufcolumn[bufpos-1]; } ! public final int getEndLine() { ! return bufline[bufpos-1]; } ! public final int getBeginColumn() { ! return bufcolumn[tokenBegin]+1; } ! public final int getBeginLine() { ! return bufline[tokenBegin]-1; } public void backup(int amount) { bufpos -= amount; ! if (bufpos < 0) { bufpos = 0; } *************** *** 64,94 **** public char BeginToken() throws IOException { - char c = readChar(); tokenBegin = bufpos; return c; } public String GetImage() { ! if (bufpos >= tokenBegin) ! return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); ! else ! return new String(buffer, tokenBegin, buffer.length - tokenBegin) + new String(buffer, 0, bufpos + 1); } public char[] GetSuffix(int len) { ! if ((bufpos + len) > buffer.length){ ! len = buffer.length - bufpos; ! } ! ! char[] ret = new char[len]; ! if(len > 0){ ! try { System.arraycopy(buffer, bufpos - len, ret, 0, len); } catch (Exception e) { e.printStackTrace(); } ! } ! return ret; } --- 130,163 ---- public char BeginToken() throws IOException { tokenBegin = bufpos; + char c = readChar(); return c; } public String GetImage() { ! String s = null; ! if (bufpos >= tokenBegin) { ! s = new String(buffer, tokenBegin, bufpos - tokenBegin); ! } else { ! s = new String(buffer, tokenBegin, buffer.length - tokenBegin); ! } ! return s; } public char[] GetSuffix(int len) { ! if ((bufpos + len) > buffer.length) { ! len = buffer.length - bufpos; ! } ! ! char[] ret = new char[len]; ! if (len > 0) { ! try { System.arraycopy(buffer, bufpos - len, ret, 0, len); } catch (Exception e) { e.printStackTrace(); } ! } ! return ret; } |