[pywin32-checkins] pywin32/Pythonwin/pywin/idle AutoIndent.py, 1.7, 1.8 PyParse.py, 1.6, 1.7

[Mark H. <mha...@us...>]

Update of /cvsroot/pywin32/pywin32/Pythonwin/pywin/idle
In directory ddv4jf1.ch3.sourceforge.com:/tmp/cvs-serv12712/pywin/idle

Modified Files:
	AutoIndent.py PyParse.py 
Log Message:
correct unicode handling and make py3k friendly

Index: PyParse.py
===================================================================
RCS file: /cvsroot/pywin32/pywin32/Pythonwin/pywin/idle/PyParse.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** PyParse.py	14 Nov 2008 00:22:25 -0000	1.6
--- PyParse.py	3 Jan 2009 06:00:08 -0000	1.7
***************
*** 103,107 ****
  for ch in "\"'\\\n#":
      _tran[ord(ch)] = ch
! _tran = ''.join(_tran)
  del ch
  
--- 103,110 ----
  for ch in "\"'\\\n#":
      _tran[ord(ch)] = ch
! # We are called with unicode strings, and str.translate is one of the few
! # py2k functions which can't 'do the right thing' - so take care to ensure
! # _tran is full of unicode...
! _tran = u''.join(_tran)
  del ch
  

Index: AutoIndent.py
===================================================================
RCS file: /cvsroot/pywin32/pywin32/Pythonwin/pywin/idle/AutoIndent.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -C2 -d -r1.7 -r1.8
*** AutoIndent.py	6 Dec 2008 00:39:59 -0000	1.7
--- AutoIndent.py	3 Jan 2009 06:00:08 -0000	1.8
***************
*** 1,40 ****
! #from Tkinter import TclError
! #import tkMessageBox
! #import tkSimpleDialog
! 
! ###$ event <<newline-and-indent>>
! ###$ win <Key-Return>
! ###$ win <KP_Enter>
! ###$ unix <Key-Return>
! ###$ unix <KP_Enter>
! 
! ###$ event <<indent-region>>
! ###$ win <Control-bracketright>
! ###$ unix <Alt-bracketright>
! ###$ unix <Control-bracketright>
! 
! ###$ event <<dedent-region>>
! ###$ win <Control-bracketleft>
! ###$ unix <Alt-bracketleft>
! ###$ unix <Control-bracketleft>
! 
! ###$ event <<comment-region>>
! ###$ win <Alt-Key-3>
! ###$ unix <Alt-Key-3>
! 
! ###$ event <<uncomment-region>>
! ###$ win <Alt-Key-4>
! ###$ unix <Alt-Key-4>
! 
! ###$ event <<tabify-region>>
! ###$ win <Alt-Key-5>
! ###$ unix <Alt-Key-5>
! 
! ###$ event <<untabify-region>>
! ###$ win <Alt-Key-6>
! ###$ unix <Alt-Key-6>
! 
  import PyParse
  
  class AutoIndent:
  
--- 1,17 ----
! import sys
! import string, tokenize
  import PyParse
+ from pywin import default_scintilla_encoding
  
+ if sys.version_info < (3,):
+     # in py2k, tokenize() takes a 'token eater' callback, while
+     # generate_tokens is a generator that works with str objects.
+     token_generator = tokenize.generate_tokens
+ else:
+     # in py3k tokenize() is the generator working with 'byte' objects, and
+     # token_generator is the 'undocumented b/w compat' function that
+     # theoretically works with str objects - but actually seems to fail)
+     token_generator = tokenize.tokenize
+     
  class AutoIndent:
  
***************
*** 498,505 ****
      return raw, effective
  
- import tokenize
- _tokenize = tokenize
- del tokenize
- 
  class IndentSearcher:
  
--- 475,478 ----
***************
*** 525,553 ****
              else:
                  val = self.text.get(mark, mark + " lineend+1c")
          return val.encode(default_scintilla_encoding)
  
-     def tokeneater(self, type, token, start, end, line,
-                    INDENT=_tokenize.INDENT,
-                    NAME=_tokenize.NAME,
-                    OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
-         if self.finished:
-             pass
-         elif type == NAME and token in OPENERS:
-             self.blkopenline = line
-         elif type == INDENT and self.blkopenline:
-             self.indentedline = line
-             self.finished = 1
- 
      def run(self):
!         save_tabsize = _tokenize.tabsize
!         _tokenize.tabsize = self.tabwidth
          try:
              try:
!                 _tokenize.tokenize(self.readline, self.tokeneater)
!             except (_tokenize.TokenError, IndentationError):
                  # since we cut off the tokenizer early, we can trigger
                  # spurious errors
                  pass
          finally:
!             _tokenize.tabsize = save_tabsize
          return self.blkopenline, self.indentedline
--- 498,528 ----
              else:
                  val = self.text.get(mark, mark + " lineend+1c")
+         # hrm - not sure this is correct in py3k - the source code may have
+         # an encoding declared, but the data will *always* be in
+         # default_scintilla_encoding - so if anyone looks at the encoding decl
+         # in the source they will be wrong.  I think.  Maybe.  Or something...
          return val.encode(default_scintilla_encoding)
  
      def run(self):
!         OPENERS=('class', 'def', 'for', 'if', 'try', 'while')
!         INDENT=tokenize.INDENT
!         NAME=tokenize.NAME
!                    
!         save_tabsize = tokenize.tabsize
!         tokenize.tabsize = self.tabwidth
          try:
              try:
!                 for (typ, token, start, end, line) in token_generator(self.readline):
!                     if typ == NAME and token in OPENERS:
!                         self.blkopenline = line
!                     elif type == INDENT and self.blkopenline:
!                         self.indentedline = line
!                         break
! 
!             except (tokenize.TokenError, IndentationError):
                  # since we cut off the tokenizer early, we can trigger
                  # spurious errors
                  pass
          finally:
!             tokenize.tabsize = save_tabsize
          return self.blkopenline, self.indentedline