[PyWebMail-Checkins] webmail/webmail/utils/xhttp __init__.py, 1.25, 1.26

[Michael D. <du...@us...>]

Update of /cvsroot/pywebmail/webmail/webmail/utils/xhttp
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv27864

Modified Files:
	__init__.py 
Log Message:
Backport to 1.1.7

Index: __init__.py
===================================================================
RCS file: /cvsroot/pywebmail/webmail/webmail/utils/xhttp/__init__.py,v
retrieving revision 1.25
retrieving revision 1.26
diff -C2 -d -r1.25 -r1.26
*** __init__.py	9 Dec 2007 11:07:51 -0000	1.25
--- __init__.py	2 Apr 2008 23:11:03 -0000	1.26
***************
*** 1,10 ****
  import sys, os, time, re, logging
  import socket, errno, urlparse, urllib2, httplib
- try: cookielib
- except: cookielib = None
  try: from cStringIO import StringIO
  except: from StringIO import StringIO
- from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, DebugHTTPCookieProcessor, DebugHTTPRedirectHandler
  from webmail.utils.xhttp.xCookie import CookieStorage
  import webmail.versions
  
--- 1,8 ----
  import sys, os, time, re, logging
  import socket, errno, urlparse, urllib2, httplib
  try: from cStringIO import StringIO
  except: from StringIO import StringIO
  from webmail.utils.xhttp.xCookie import CookieStorage
+ from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, CookieAwareHTTPRedirectHandler
  import webmail.versions
  
***************
*** 65,69 ****
  __default = ['__no_args__']
  def url2retrieve(url, filename=None, reporthook=None,
!                  data=None, content_type=None, cookies=None,
                   log=log, use_counter=0, blocking=0,
                   max_retries=__default, read_timeout=600,
--- 63,67 ----
  __default = ['__no_args__']
  def url2retrieve(url, filename=None, reporthook=None,
!                  data=None, content_type=None, cookies=CookieStorage(),
                   log=log, use_counter=0, blocking=0,
                   max_retries=__default, read_timeout=600,
***************
*** 73,89 ****
                   accept_charset=None, accept_language=None, override_ua=None,
                   retry_timeout=20, gai_retry_timeout=60, do_save_headers=0,
!                  parse_chunked=1, retry_any_result=0, adv_headers=None):
    if max_retries is __default: max_retries = default_retries
-   if cookies is None:
-     if cookielib is not None:
-       cookies = cookielib.CookieJar()
-     else:
-       cookies = CookieStorage()
-   use_cookielib = not isinstance(cookies, CookieStorage)
    req = urllib2.Request(url, data)
!   if adv_headers is None:
!     adv_headers = []
!   else:
!     adv_headers = adv_headers[:]
    if accept is not None:
      adv_headers.append(('Accept', accept))
--- 71,81 ----
                   accept_charset=None, accept_language=None, override_ua=None,
                   retry_timeout=20, gai_retry_timeout=60, do_save_headers=0,
!                  parse_chunked=1, retry_any_result=0, max_redirects=20):
    if max_retries is __default: max_retries = default_retries
    req = urllib2.Request(url, data)
!   req.max_redirections = max_redirects
!   path = urllib2.splitquery(req.get_selector())[0]
!   path = urllib2.splitattr(path)[0]
!   adv_headers = []
    if accept is not None:
      adv_headers.append(('Accept', accept))
***************
*** 99,126 ****
        req.add_header('Content-type', content_type)
        # Also urllib2 adds 'Content-length' automatically
!     if not use_cookielib:
!       req_uri = req.get_selector()
!       path = urllib2.splitquery(req_uri)[0]
!       path = urllib2.splitattr(path)[0]
!       host = req.get_host()
!       assert host, req_uri
!       c = cookies.get_cookies(host, path)
!       if c:
!         log.debug('Sending cookies: '+c)
!         req.add_header('Cookie', c)
      if referer:
        req.add_header('Referer', referer)
      if adv_headers:
        map(lambda t:apply(req.add_header, t), adv_headers)
!   if use_cookielib:
!     ophl = [DebugHTTPCookieProcessor(cookies)]
!     if not use_redirect:
!       ophl.append(HTTPNoRedirectHandler)
!     else:
!       ophl.append(DebugHTTPRedirectHandler)
!   else:
!     ophl = [HTTPNoRedirectHandler]
!   ophl = filter(None, ophl) # filter-out undefined (debug) handlers
!   op = urllib2.build_opener(*tuple(ophl))
    #op.addheaders.append(('Connection', 'close'))
    ua = userAgents.get(override_ua, override_ua)
--- 91,105 ----
        req.add_header('Content-type', content_type)
        # Also urllib2 adds 'Content-length' automatically
!     host = req.get_host()
!     assert host, url
!     c = cookies.get_cookies(host, path)
!     if c:
!       log.debug('Sending cookies: '+c)
!       req.add_header('Cookie', c)
      if referer:
        req.add_header('Referer', referer)
      if adv_headers:
        map(lambda t:apply(req.add_header, t), adv_headers)
!   op = urllib2.build_opener(HTTPNoRedirectHandler)
    #op.addheaders.append(('Connection', 'close'))
    ua = userAgents.get(override_ua, override_ua)
***************
*** 139,152 ****
    counter = 0
    retries = 0
!   log.debug('>%s %s HTTP/1.x' % (req.get_method(),req.get_selector()))
    for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v))
    for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v))
-   if data:
-     for line in data.splitlines()[:10]:
-       log.debug('>>'+line)
    while 1:
!     t = None
!     if proxy:
!       t = proxy.proxies.get(req.get_type())
      if t:
        log.info('Connecting to %s via proxy %s...', req.get_host(), t)
--- 118,126 ----
    counter = 0
    retries = 0
!   redirects = 0
    for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v))
    for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v))
    while 1:
!     t = proxy.proxies.get(req.get_type())
      if t:
        log.info('Connecting to %s via proxy %s...', req.get_host(), t)
***************
*** 156,163 ****
        fp = op.open(req)
        headers = fp.info()
-       log.debug('<HTTP/1.x %d %s' % (getattr(fp,'code',200), getattr(fp,'msg','?OK')))
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
!       if not use_cookielib:
!         cookies.parse_headers(headers)
        tfp,fn,counter = _get_file(filename, use_counter, counter)
        if do_save_headers:
--- 130,135 ----
        fp = op.open(req)
        headers = fp.info()
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
!       cookies.parse_headers(headers)
        tfp,fn,counter = _get_file(filename, use_counter, counter)
        if do_save_headers:
***************
*** 167,173 ****
        if headers.has_key('content-length'):
          size = int(headers['Content-Length'])
!       blocking = _socketretrieve(fp, tfp, reporthook, size,
            blocking=blocking, read_timeout=read_timeout, log=log)
-       if blocking: parse_chunked = 0 # specific of httplib.HTTPResponse
        fp.close()
        if fn:
--- 139,144 ----
        if headers.has_key('content-length'):
          size = int(headers['Content-Length'])
!       _socketretrieve(fp, tfp, reporthook, size,
            blocking=blocking, read_timeout=read_timeout, log=log)
        fp.close()
        if fn:
***************
*** 188,192 ****
--- 159,176 ----
          else:
            log.debug('Content unchunked.')
+     except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e:
+       if max_retries is not None:
+         retry_str = '(%d/%d)' % (retries,max_retries)
+       if max_retries is not None and retries >= max_retries:
+         log.warn('%s%s: %s' % (e.__class__, retry_str, e))
+         raise
+       log.debug('Error on socket%s:', retry_str, exc_info=1)
+       retries = retries+1
+       log.info('Retry waiting %d seconds...', retry_timeout)
+       time.sleep(retry_timeout)
+       log.info('Retrying...')
+       continue
      except urllib2.HTTPError, e:
+       log.debug('Status %d: %s' % (e.code, e.msg))
        try:
          if hasattr(e, 'headers'):
***************
*** 197,203 ****
          pass
        if headers is None: headers = {}
!       if not use_cookielib and headers:
!         cookies.parse_headers(headers)
!       log.debug('<HTTP/1.x %d %s' % (e.code, e.msg))
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
        if use_redirect and e.code in (301,302) and headers.has_key('Location'):
--- 181,185 ----
          pass
        if headers is None: headers = {}
!       if headers: cookies.parse_headers(headers)
        for k,v in headers.items(): log.debug('<%s: %s' % (k,v))
        if use_redirect and e.code in (301,302) and headers.has_key('Location'):
***************
*** 205,220 ****
          log.info('Redirecting (%d: %s): %s' % (e.code, e.msg, url))
          req = urllib2.Request(url)
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue
--- 187,199 ----
          log.info('Redirecting (%d: %s): %s' % (e.code, e.msg, url))
          req = urllib2.Request(url)
+         req.max_redirections = max_redirects
+         redirects += 1
+         assert redirects <= max_redirects, "Redirection loop"
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue
***************
*** 237,259 ****
          log.debug('Failure retrieving URL: %s', e.url)
          raise
-     except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e:
-       if max_retries is not None:
-         retry_str = '(%d/%d)' % (retries,max_retries)
-       if max_retries is not None and retries >= max_retries:
-         log.warn('%s%s: %s' % (e.__class__, retry_str, e))
-         raise
-       log.debug('Error on socket%s:', retry_str, exc_info=1)
-       retries = retries+1
-       log.info('Retry waiting %d seconds...', retry_timeout)
-       time.sleep(retry_timeout)
-       log.info('Retrying...')
-       continue
      except urllib2.URLError, e:
        URLError_retry = 0
        retry_str = ''
        if max_retries: retry_str = '(%d/%d)' % (retries,max_retries)
!       if isinstance(e.reason, socket.error) and len(e.reason.args) != 2:
!         log.error('URLError%s: %s: %s', retry_str, e.reason.__class__, e.reason.args)
!       elif hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror):
          log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args)
          code,text = e.reason.args
--- 216,224 ----
          log.debug('Failure retrieving URL: %s', e.url)
          raise
      except urllib2.URLError, e:
        URLError_retry = 0
        retry_str = ''
        if max_retries: retry_str = '(%d/%d)' % (retries,max_retries)
!       if hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror):
          log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args)
          code,text = e.reason.args
***************
*** 274,285 ****
          code,text = e.reason.args
          #**** Cygwin (and hopefully POSIX) error codes
-         #-103 - Software caused connection abort
-         # 104 - Connection reset by peer
-         #-107 - Socket is not connected
-         #-110 - Operation timed out
          # 111 - Connection refused
-         # 113 - Software caused connection abort
-         #-115 - No route to host
-         # 116 - Connection timed out
          #**** Windows error codes
          # 10053 - Software caused connection abort
--- 239,243 ----
***************
*** 290,295 ****
          # 10065 - No route to host
          # 11001 - getaddrinfo failed
!         URLError_retry = code in (104,111,113,116,7,
!                                   10053,10054,10057,10060,10061,10065,11001)
          URLError_reason = 'socket.error'
        if URLError_retry:
--- 248,252 ----
          # 10065 - No route to host
          # 11001 - getaddrinfo failed
!         URLError_retry = code in (111,10053,10054,10057,10060,10061,10065,11001)
          URLError_reason = 'socket.error'
        if URLError_retry:
***************
*** 307,311 ****
          log.info('Retrying...')
          continue
!       log.exception('URLError: %s', repr(e.reason))
        raise
      reslog.debug(str(url)+'\n'+s)
--- 264,268 ----
          log.info('Retrying...')
          continue
!       log.error('URLError: %s', repr(e.reason))
        raise
      reslog.debug(str(url)+'\n'+s)
***************
*** 317,332 ****
          log.info('Redirecting (via Refresh header): '+url)
          req = urllib2.Request(url)
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue
--- 274,286 ----
          log.info('Redirecting (via Refresh header): '+url)
          req = urllib2.Request(url)
+         req.max_redirections = max_redirects
+         redirects += 1
+         assert redirects <= max_redirects, "Redirection loop"
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue
***************
*** 337,352 ****
          log.info('Redirecting (via META Refresh): '+url)
          req = urllib2.Request(url)
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         if not use_cookielib:
!           req_uri = req.get_selector()
!           path = urllib2.splitquery(req_uri)[0]
!           path = urllib2.splitattr(path)[0]
!           host = req.get_host()
!           assert host, req_uri
!           c = cookies.get_cookies(host, path)
!           if c:
!             log.debug('Sending cookies: '+c)
!             req.add_header('Cookie', c)
          retries = 0
          continue
--- 291,303 ----
          log.info('Redirecting (via META Refresh): '+url)
          req = urllib2.Request(url)
+         req.max_redirections = max_redirects
+         redirects += 1
+         assert redirects <= max_redirects, "Redirection loop"
          if adv_headers:
            map(lambda t:apply(req.add_header, t), adv_headers)
!         c = cookies.get_cookies(req.get_host(), path)
!         if c:
!           log.debug('Sending cookies: '+c)
!           req.add_header('Cookie', c)
          retries = 0
          continue