[PyWebMail-Checkins] webmail/webmail/utils/xhttp __init__.py, 1.26, 1.27

[Michael D. <du...@us...>]

Update of /cvsroot/pywebmail/webmail/webmail/utils/xhttp
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv28403

Modified Files:
	__init__.py 
Log Message:
Forward-port

Index: __init__.py
===================================================================
RCS file: /cvsroot/pywebmail/webmail/webmail/utils/xhttp/__init__.py,v
retrieving revision 1.26
retrieving revision 1.27
diff -C2 -d -r1.26 -r1.27
*** __init__.py	2 Apr 2008 23:11:03 -0000	1.26
--- __init__.py	2 Apr 2008 23:12:17 -0000	1.27
***************
*** 1,8 ****
  import sys, os, time, re, logging
  import socket, errno, urlparse, urllib2, httplib
  try: from cStringIO import StringIO
  except: from StringIO import StringIO
  from webmail.utils.xhttp.xCookie import CookieStorage
- from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, CookieAwareHTTPRedirectHandler
  import webmail.versions
  
--- 1,10 ----
  import sys, os, time, re, logging
  import socket, errno, urlparse, urllib2, httplib
+ try: cookielib
+ except: cookielib = None
  try: from cStringIO import StringIO
  except: from StringIO import StringIO
+ from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, DebugHTTPCookieProcessor, DebugHTTPRedirectHandler
  from webmail.utils.xhttp.xCookie import CookieStorage
  import webmail.versions
  
***************
*** 63,67 ****
  __default = ['__no_args__']
  def url2retrieve(url, filename=None, reporthook=None,
!                  data=None, content_type=None, cookies=CookieStorage(),
                   log=log, use_counter=0, blocking=0,
                   max_retries=__default, read_timeout=600,
--- 65,69 ----
  __default = ['__no_args__']
  def url2retrieve(url, filename=None, reporthook=None,
!                  data=None, content_type=None, cookies=None,
                   log=log, use_counter=0, blocking=0,
                   max_retries=__default, read_timeout=600,
***************
*** 71,81 ****
                   accept_charset=None, accept_language=None, override_ua=None,
                   retry_timeout=20, gai_retry_timeout=60, do_save_headers=0,
!                  parse_chunked=1, retry_any_result=0, max_redirects=20):
    if max_retries is __default: max_retries = default_retries
    req = urllib2.Request(url, data)
    req.max_redirections = max_redirects
!   path = urllib2.splitquery(req.get_selector())[0]
!   path = urllib2.splitattr(path)[0]
!   adv_headers = []
    if accept is not None:
      adv_headers.append(('Accept', accept))
--- 73,91 ----
                   accept_charset=None, accept_language=None, override_ua=None,
                   retry_timeout=20, gai_retry_timeout=60, do_save_headers=0,
!                  parse_chunked=1, retry_any_result=0, adv_headers=None,
!                  max_redirects=20):
    if max_retries is __default: max_retries = default_retries
+   if cookies is None:
+     if cookielib is not None:
+       cookies = cookielib.CookieJar()
+     else:
+       cookies = CookieStorage()
+   use_cookielib = not isinstance(cookies, CookieStorage)
    req = urllib2.Request(url, data)
    req.max_redirections = max_redirects
!   if adv_headers is None:
!     adv_headers = []
!   else:
!     adv_headers = adv_headers[:]
    if accept is not None:
      adv_headers.append(('Accept', accept))
***************
*** 91,105 ****
        req.add_header('Content-type', content_type)
        # Also urllib2 adds 'Content-length' automatically
!     host = req.get_host()
!     assert host, url
!     c = cookies.get_cookies(host, path)
!     if c:
!       log.debug('Sending cookies: '+c)
!       req.add_header('Cookie', c)
      if referer:
        req.add_header('Referer', referer)
      if adv_headers:
        map(lambda t:apply(req.add_header, t), adv_headers)
!   op = urllib2.build_opener(HTTPNoRedirectHandler)
    #op.addheaders.append(('Connection', 'close'))
    ua = userAgents.get(override_ua, override_ua)
--- 101,128 ----
        req.add_header('Content-type', content_type)
        # Also urllib2 adds 'Content-length' automatically
!     if not use_cookielib:
!       req_uri = req.get_selector()
!       path = urllib2.splitquery(req_uri)[0]
!       path = urllib2.splitattr(path)[0]
!       host = req.get_host()
!       assert host, req_uri
!       c = cookies.get_cookies(host, path)
!       if c:
!         log.debug('Sending cookies: '+c)
!         req.add_header('Cookie', c)
      if referer:
        req.add_header('Referer', referer)
      if adv_headers:
        map(lambda t:apply(req.add_header, t), adv_headers)
!   if use_cookielib:
!     ophl = [DebugHTTPCookieProcessor(cookies)]
!     if not use_redirect:
!       ophl.append(HTTPNoRedirectHandler)
!     else:
!       ophl.append(DebugHTTPRedirectHandler)
!   else:
!     ophl = [HTTPNoRedirectHandler]
!   ophl = filter(None, ophl) # filter-out undefined (debug) handlers
!   op = urllib2.build_opener(*tuple(ophl))
    #op.addheaders.append(('Connection', 'close'))
    ua = userAgents.get(override_ua, override_ua)
***************
*** 119,126 ****
    retries = 0
    redirects = 0
    for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v))
    for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v))
    while 1:
!     t = proxy.proxies.get(req.get_type())
      if t:
        log.info('Connecting to %s via proxy %s...', req.get_host(), t)
--- 142,155 ----
    retries = 0
    redirects = 0
+   log.debug('>%s %s HTTP/1.x' % (req.get_method(),req.get_selector()))
    for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v))
    for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v))
+   if data:
+     for line in data.splitlines()[:10]:
+       log.debug('>>'+line)
    while 1:
!     t = None
!     if proxy:
!       t = proxy.proxies.get(req.get_type())
      if t:
        log.info('Connecting to %s via proxy %s...', req.get_host(), t)
***************
*** 130,135 ****
        fp = op.open(req)
        headers = fp.info()
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
!       cookies.parse_headers(headers)
        tfp,fn,counter = _get_file(filename, use_counter, counter)
        if do_save_headers:
--- 159,166 ----
        fp = op.open(req)
        headers = fp.info()
+       log.debug('<HTTP/1.x %d %s' % (getattr(fp,'code',200), getattr(fp,'msg','?OK')))
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
!       if not use_cookielib:
!         cookies.parse_headers(headers)
        tfp,fn,counter = _get_file(filename, use_counter, counter)
        if do_save_headers:
***************
*** 139,144 ****
        if headers.has_key('content-length'):
          size = int(headers['Content-Length'])
!       _socketretrieve(fp, tfp, reporthook, size,
            blocking=blocking, read_timeout=read_timeout, log=log)
        fp.close()
        if fn:
--- 170,176 ----
        if headers.has_key('content-length'):
          size = int(headers['Content-Length'])
!       blocking = _socketretrieve(fp, tfp, reporthook, size,
            blocking=blocking, read_timeout=read_timeout, log=log)
+       if blocking: parse_chunked = 0 # specific of httplib.HTTPResponse
        fp.close()
        if fn:
***************
*** 159,176 ****
          else:
            log.debug('Content unchunked.')
-     except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e:
-       if max_retries is not None:
-         retry_str = '(%d/%d)' % (retries,max_retries)
-       if max_retries is not None and retries >= max_retries:
-         log.warn('%s%s: %s' % (e.__class__, retry_str, e))
-         raise
-       log.debug('Error on socket%s:', retry_str, exc_info=1)
-       retries = retries+1
-       log.info('Retry waiting %d seconds...', retry_timeout)
-       time.sleep(retry_timeout)
-       log.info('Retrying...')
-       continue
      except urllib2.HTTPError, e:
-       log.debug('Status %d: %s' % (e.code, e.msg))
        try:
          if hasattr(e, 'headers'):
--- 191,195 ----
***************
*** 181,185 ****
          pass
        if headers is None: headers = {}
!       if headers: cookies.parse_headers(headers)
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
        if use_redirect and e.code in (301,302) and headers.has_key('Location'):
--- 200,206 ----
          pass
        if headers is None: headers = {}
!       if not use_cookielib and headers:
!         cookies.parse_headers(headers)
!       log.debug('<HTTP/1.x %d %s' % (e.code, e.msg))
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
        if use_redirect and e.code in (301,302) and headers.has_key('Location'):
***************
*** 192,199 ****
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue
--- 213,226 ----
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue
***************
*** 216,224 ****
          log.debug('Failure retrieving URL: %s', e.url)
          raise
      except urllib2.URLError, e:
        URLError_retry = 0
        retry_str = ''
        if max_retries: retry_str = '(%d/%d)' % (retries,max_retries)
!       if hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror):
          log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args)
          code,text = e.reason.args
--- 243,265 ----
          log.debug('Failure retrieving URL: %s', e.url)
          raise
+     except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e:
+       if max_retries is not None:
+         retry_str = '(%d/%d)' % (retries,max_retries)
+       if max_retries is not None and retries >= max_retries:
+         log.warn('%s%s: %s' % (e.__class__, retry_str, e))
+         raise
+       log.debug('Error on socket%s:', retry_str, exc_info=1)
+       retries = retries+1
+       log.info('Retry waiting %d seconds...', retry_timeout)
+       time.sleep(retry_timeout)
+       log.info('Retrying...')
+       continue
      except urllib2.URLError, e:
        URLError_retry = 0
        retry_str = ''
        if max_retries: retry_str = '(%d/%d)' % (retries,max_retries)
!       if isinstance(e.reason, socket.error) and len(e.reason.args) != 2:
!         log.error('URLError%s: %s: %s', retry_str, e.reason.__class__, e.reason.args)
!       elif hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror):
          log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args)
          code,text = e.reason.args
***************
*** 239,243 ****
--- 280,291 ----
          code,text = e.reason.args
          #**** Cygwin (and hopefully POSIX) error codes
+         #-103 - Software caused connection abort
+         # 104 - Connection reset by peer
+         #-107 - Socket is not connected
+         #-110 - Operation timed out
          # 111 - Connection refused
+         # 113 - Software caused connection abort
+         #-115 - No route to host
+         # 116 - Connection timed out
          #**** Windows error codes
          # 10053 - Software caused connection abort
***************
*** 248,252 ****
          # 10065 - No route to host
          # 11001 - getaddrinfo failed
!         URLError_retry = code in (111,10053,10054,10057,10060,10061,10065,11001)
          URLError_reason = 'socket.error'
        if URLError_retry:
--- 296,301 ----
          # 10065 - No route to host
          # 11001 - getaddrinfo failed
!         URLError_retry = code in (104,111,113,116,7,
!                                   10053,10054,10057,10060,10061,10065,11001)
          URLError_reason = 'socket.error'
        if URLError_retry:
***************
*** 264,268 ****
          log.info('Retrying...')
          continue
!       log.error('URLError: %s', repr(e.reason))
        raise
      reslog.debug(str(url)+'\n'+s)
--- 313,317 ----
          log.info('Retrying...')
          continue
!       log.exception('URLError: %s', repr(e.reason))
        raise
      reslog.debug(str(url)+'\n'+s)
***************
*** 279,286 ****
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue
--- 328,341 ----
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue
***************
*** 296,303 ****
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue
--- 351,364 ----
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue