Thread: [PyWebMail-Checkins] webmail/webmail/utils/xhttp __init__.py, 1.25, 1.26
Status: Beta
Brought to you by:
dubnerm
From: Michael D. <du...@us...> - 2008-04-02 23:14:41
|
Update of /cvsroot/pywebmail/webmail/webmail/utils/xhttp In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv27864 Modified Files: __init__.py Log Message: Backport to 1.1.7 Index: __init__.py =================================================================== RCS file: /cvsroot/pywebmail/webmail/webmail/utils/xhttp/__init__.py,v retrieving revision 1.25 retrieving revision 1.26 diff -C2 -d -r1.25 -r1.26 *** __init__.py 9 Dec 2007 11:07:51 -0000 1.25 --- __init__.py 2 Apr 2008 23:11:03 -0000 1.26 *************** *** 1,10 **** import sys, os, time, re, logging import socket, errno, urlparse, urllib2, httplib - try: cookielib - except: cookielib = None try: from cStringIO import StringIO except: from StringIO import StringIO - from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, DebugHTTPCookieProcessor, DebugHTTPRedirectHandler from webmail.utils.xhttp.xCookie import CookieStorage import webmail.versions --- 1,8 ---- import sys, os, time, re, logging import socket, errno, urlparse, urllib2, httplib try: from cStringIO import StringIO except: from StringIO import StringIO from webmail.utils.xhttp.xCookie import CookieStorage + from webmail.utils.xhttp.utils import _socketretrieve, HTTPNoRedirectHandler, CookieAwareHTTPRedirectHandler import webmail.versions *************** *** 65,69 **** __default = ['__no_args__'] def url2retrieve(url, filename=None, reporthook=None, ! data=None, content_type=None, cookies=None, log=log, use_counter=0, blocking=0, max_retries=__default, read_timeout=600, --- 63,67 ---- __default = ['__no_args__'] def url2retrieve(url, filename=None, reporthook=None, ! data=None, content_type=None, cookies=CookieStorage(), log=log, use_counter=0, blocking=0, max_retries=__default, read_timeout=600, *************** *** 73,89 **** accept_charset=None, accept_language=None, override_ua=None, retry_timeout=20, gai_retry_timeout=60, do_save_headers=0, ! parse_chunked=1, retry_any_result=0, adv_headers=None): if max_retries is __default: max_retries = default_retries - if cookies is None: - if cookielib is not None: - cookies = cookielib.CookieJar() - else: - cookies = CookieStorage() - use_cookielib = not isinstance(cookies, CookieStorage) req = urllib2.Request(url, data) ! if adv_headers is None: ! adv_headers = [] ! else: ! adv_headers = adv_headers[:] if accept is not None: adv_headers.append(('Accept', accept)) --- 71,81 ---- accept_charset=None, accept_language=None, override_ua=None, retry_timeout=20, gai_retry_timeout=60, do_save_headers=0, ! parse_chunked=1, retry_any_result=0, max_redirects=20): if max_retries is __default: max_retries = default_retries req = urllib2.Request(url, data) ! req.max_redirections = max_redirects ! path = urllib2.splitquery(req.get_selector())[0] ! path = urllib2.splitattr(path)[0] ! adv_headers = [] if accept is not None: adv_headers.append(('Accept', accept)) *************** *** 99,126 **** req.add_header('Content-type', content_type) # Also urllib2 adds 'Content-length' automatically ! if not use_cookielib: ! req_uri = req.get_selector() ! path = urllib2.splitquery(req_uri)[0] ! path = urllib2.splitattr(path)[0] ! host = req.get_host() ! assert host, req_uri ! c = cookies.get_cookies(host, path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) if referer: req.add_header('Referer', referer) if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! if use_cookielib: ! ophl = [DebugHTTPCookieProcessor(cookies)] ! if not use_redirect: ! ophl.append(HTTPNoRedirectHandler) ! else: ! ophl.append(DebugHTTPRedirectHandler) ! else: ! ophl = [HTTPNoRedirectHandler] ! ophl = filter(None, ophl) # filter-out undefined (debug) handlers ! op = urllib2.build_opener(*tuple(ophl)) #op.addheaders.append(('Connection', 'close')) ua = userAgents.get(override_ua, override_ua) --- 91,105 ---- req.add_header('Content-type', content_type) # Also urllib2 adds 'Content-length' automatically ! host = req.get_host() ! assert host, url ! c = cookies.get_cookies(host, path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) if referer: req.add_header('Referer', referer) if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! op = urllib2.build_opener(HTTPNoRedirectHandler) #op.addheaders.append(('Connection', 'close')) ua = userAgents.get(override_ua, override_ua) *************** *** 139,152 **** counter = 0 retries = 0 ! log.debug('>%s %s HTTP/1.x' % (req.get_method(),req.get_selector())) for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v)) for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v)) - if data: - for line in data.splitlines()[:10]: - log.debug('>>'+line) while 1: ! t = None ! if proxy: ! t = proxy.proxies.get(req.get_type()) if t: log.info('Connecting to %s via proxy %s...', req.get_host(), t) --- 118,126 ---- counter = 0 retries = 0 ! redirects = 0 for k,v in req.headers.items(): log.debug('>%s: %s' % (k,v)) for k,v in op.addheaders: log.debug('+>%s: %s' % (k,v)) while 1: ! t = proxy.proxies.get(req.get_type()) if t: log.info('Connecting to %s via proxy %s...', req.get_host(), t) *************** *** 156,163 **** fp = op.open(req) headers = fp.info() - log.debug('<HTTP/1.x %d %s' % (getattr(fp,'code',200), getattr(fp,'msg','?OK'))) for k,v in headers.items(): log.debug('<%s: %s' % (k,v)) ! if not use_cookielib: ! cookies.parse_headers(headers) tfp,fn,counter = _get_file(filename, use_counter, counter) if do_save_headers: --- 130,135 ---- fp = op.open(req) headers = fp.info() for k,v in headers.items(): log.debug('<%s: %s' % (k,v)) ! cookies.parse_headers(headers) tfp,fn,counter = _get_file(filename, use_counter, counter) if do_save_headers: *************** *** 167,173 **** if headers.has_key('content-length'): size = int(headers['Content-Length']) ! blocking = _socketretrieve(fp, tfp, reporthook, size, blocking=blocking, read_timeout=read_timeout, log=log) - if blocking: parse_chunked = 0 # specific of httplib.HTTPResponse fp.close() if fn: --- 139,144 ---- if headers.has_key('content-length'): size = int(headers['Content-Length']) ! _socketretrieve(fp, tfp, reporthook, size, blocking=blocking, read_timeout=read_timeout, log=log) fp.close() if fn: *************** *** 188,192 **** --- 159,176 ---- else: log.debug('Content unchunked.') + except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e: + if max_retries is not None: + retry_str = '(%d/%d)' % (retries,max_retries) + if max_retries is not None and retries >= max_retries: + log.warn('%s%s: %s' % (e.__class__, retry_str, e)) + raise + log.debug('Error on socket%s:', retry_str, exc_info=1) + retries = retries+1 + log.info('Retry waiting %d seconds...', retry_timeout) + time.sleep(retry_timeout) + log.info('Retrying...') + continue except urllib2.HTTPError, e: + log.debug('Status %d: %s' % (e.code, e.msg)) try: if hasattr(e, 'headers'): *************** *** 197,203 **** pass if headers is None: headers = {} ! if not use_cookielib and headers: ! cookies.parse_headers(headers) ! log.debug('<HTTP/1.x %d %s' % (e.code, e.msg)) for k,v in headers.items(): log.debug('<%s: %s' % (k,v)) if use_redirect and e.code in (301,302) and headers.has_key('Location'): --- 181,185 ---- pass if headers is None: headers = {} ! if headers: cookies.parse_headers(headers) for k,v in headers.items(): log.debug('<%s: %s' % (k,v)) if use_redirect and e.code in (301,302) and headers.has_key('Location'): *************** *** 205,220 **** log.info('Redirecting (%d: %s): %s' % (e.code, e.msg, url)) req = urllib2.Request(url) if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! if not use_cookielib: ! req_uri = req.get_selector() ! path = urllib2.splitquery(req_uri)[0] ! path = urllib2.splitattr(path)[0] ! host = req.get_host() ! assert host, req_uri ! c = cookies.get_cookies(host, path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue --- 187,199 ---- log.info('Redirecting (%d: %s): %s' % (e.code, e.msg, url)) req = urllib2.Request(url) + req.max_redirections = max_redirects + redirects += 1 + assert redirects <= max_redirects, "Redirection loop" if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! c = cookies.get_cookies(req.get_host(), path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue *************** *** 237,259 **** log.debug('Failure retrieving URL: %s', e.url) raise - except socket_errors+(httplib.HTTPException, httplib.BadStatusLine, webmail.utils.xhttp.utils.ReadTimeoutError), e: - if max_retries is not None: - retry_str = '(%d/%d)' % (retries,max_retries) - if max_retries is not None and retries >= max_retries: - log.warn('%s%s: %s' % (e.__class__, retry_str, e)) - raise - log.debug('Error on socket%s:', retry_str, exc_info=1) - retries = retries+1 - log.info('Retry waiting %d seconds...', retry_timeout) - time.sleep(retry_timeout) - log.info('Retrying...') - continue except urllib2.URLError, e: URLError_retry = 0 retry_str = '' if max_retries: retry_str = '(%d/%d)' % (retries,max_retries) ! if isinstance(e.reason, socket.error) and len(e.reason.args) != 2: ! log.error('URLError%s: %s: %s', retry_str, e.reason.__class__, e.reason.args) ! elif hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror): log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args) code,text = e.reason.args --- 216,224 ---- log.debug('Failure retrieving URL: %s', e.url) raise except urllib2.URLError, e: URLError_retry = 0 retry_str = '' if max_retries: retry_str = '(%d/%d)' % (retries,max_retries) ! if hasattr(socket, 'sslerror') and isinstance(e.reason, socket.sslerror): log.error('URLError%s: socket.sslerror: %s', retry_str, e.reason.args) code,text = e.reason.args *************** *** 274,285 **** code,text = e.reason.args #**** Cygwin (and hopefully POSIX) error codes - #-103 - Software caused connection abort - # 104 - Connection reset by peer - #-107 - Socket is not connected - #-110 - Operation timed out # 111 - Connection refused - # 113 - Software caused connection abort - #-115 - No route to host - # 116 - Connection timed out #**** Windows error codes # 10053 - Software caused connection abort --- 239,243 ---- *************** *** 290,295 **** # 10065 - No route to host # 11001 - getaddrinfo failed ! URLError_retry = code in (104,111,113,116,7, ! 10053,10054,10057,10060,10061,10065,11001) URLError_reason = 'socket.error' if URLError_retry: --- 248,252 ---- # 10065 - No route to host # 11001 - getaddrinfo failed ! URLError_retry = code in (111,10053,10054,10057,10060,10061,10065,11001) URLError_reason = 'socket.error' if URLError_retry: *************** *** 307,311 **** log.info('Retrying...') continue ! log.exception('URLError: %s', repr(e.reason)) raise reslog.debug(str(url)+'\n'+s) --- 264,268 ---- log.info('Retrying...') continue ! log.error('URLError: %s', repr(e.reason)) raise reslog.debug(str(url)+'\n'+s) *************** *** 317,332 **** log.info('Redirecting (via Refresh header): '+url) req = urllib2.Request(url) if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! if not use_cookielib: ! req_uri = req.get_selector() ! path = urllib2.splitquery(req_uri)[0] ! path = urllib2.splitattr(path)[0] ! host = req.get_host() ! assert host, req_uri ! c = cookies.get_cookies(host, path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue --- 274,286 ---- log.info('Redirecting (via Refresh header): '+url) req = urllib2.Request(url) + req.max_redirections = max_redirects + redirects += 1 + assert redirects <= max_redirects, "Redirection loop" if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! c = cookies.get_cookies(req.get_host(), path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue *************** *** 337,352 **** log.info('Redirecting (via META Refresh): '+url) req = urllib2.Request(url) if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! if not use_cookielib: ! req_uri = req.get_selector() ! path = urllib2.splitquery(req_uri)[0] ! path = urllib2.splitattr(path)[0] ! host = req.get_host() ! assert host, req_uri ! c = cookies.get_cookies(host, path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue --- 291,303 ---- log.info('Redirecting (via META Refresh): '+url) req = urllib2.Request(url) + req.max_redirections = max_redirects + redirects += 1 + assert redirects <= max_redirects, "Redirection loop" if adv_headers: map(lambda t:apply(req.add_header, t), adv_headers) ! c = cookies.get_cookies(req.get_host(), path) ! if c: ! log.debug('Sending cookies: '+c) ! req.add_header('Cookie', c) retries = 0 continue |