From: <lu...@us...> - 2008-11-16 12:51:29
|
Revision: 261 http://s3tools.svn.sourceforge.net/s3tools/?rev=261&view=rev Author: ludvigm Date: 2008-11-16 12:51:24 +0000 (Sun, 16 Nov 2008) Log Message: ----------- * S3/Progress.py: Two progress meter implementations. * S3/Config.py, s3cmd: New --progress / --no-progress parameters and Config() members. * S3/S3.py: Call Progress() in send_file()/recv_file() * NEWS: Let everyone know ;-) Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Added Paths: ----------- s3cmd/trunk/S3/Progress.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/ChangeLog 2008-11-16 12:51:24 UTC (rev 261) @@ -1,3 +1,11 @@ +2008-11-17 Michal Ludvig <mi...@lo...> + + * S3/Progress.py: Two progress meter implementations. + * S3/Config.py, s3cmd: New --progress / --no-progress parameters + and Config() members. + * S3/S3.py: Call Progress() in send_file()/recv_file() + * NEWS: Let everyone know ;-) + 2008-11-16 Michal Ludvig <mi...@lo...> * NEWS: Fetch 0.9.8.4 release news from 0.9.8.x branch. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/NEWS 2008-11-16 12:51:24 UTC (rev 261) @@ -1,7 +1,6 @@ s3cmd 0.9.9 - ??? =========== -* Allow access to upper-case named buckets with - --use-old-connect-method parameter +* Implemented progress meter (--progress / --no-progress) * Removing of non-empty buckets with --force * Recursively remove objects from buckets with a given prefix with --recursive (-r) Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/S3/Config.py 2008-11-16 12:51:24 UTC (rev 261) @@ -6,6 +6,7 @@ import logging from logging import debug, info, warning, error import re +import Progress class Config(object): _instance = None @@ -17,6 +18,8 @@ host_bucket = "%(bucket)s.s3.amazonaws.com" simpledb_host = "sdb.amazonaws.com" verbosity = logging.WARNING + progress_meter = True + progress_class = Progress.ProgressANSI send_chunk = 4096 recv_chunk = 4096 human_readable_sizes = False Added: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py (rev 0) +++ s3cmd/trunk/S3/Progress.py 2008-11-16 12:51:24 UTC (rev 261) @@ -0,0 +1,109 @@ +## Amazon S3 manager +## Author: Michal Ludvig <mi...@lo...> +## http://www.logix.cz/michal +## License: GPL Version 2 + +import sys +import datetime +from Utils import formatSize + +class Progress(object): + def __init__(self, label, total_size): + self.new_file(label, total_size) + + def new_file(self, label, total_size): + self.label = label + self.total_size = total_size + self.current_position = 0 + self.time_start = datetime.datetime.now() + self.time_last = self.time_start + self.time_current = self.time_start + + self.display(new_file = True) + + def update(self, current_position = -1, delta_position = -1): + self.time_last = self.time_current + self.time_current = datetime.datetime.now() + if current_position > -1: + self.current_position = current_position + elif delta_position > -1: + self.current_position += delta_position + #else: + # no update, just call display() + self.display() + + def done(self, message): + self.display(done_message = message) + + def display(self, new_file = False, done_message = None): + """ + display(new_file = False[/True], done = False[/True]) + + Override this method to provide a nicer output. + """ + if new_file: + sys.stdout.write("%s " % self.label[:30].ljust(30)) + sys.stdout.flush() + self.last_milestone = 0 + return + + if self.current_position == self.total_size: + print_size = formatSize(self.current_position, True) + if print_size[1] != "": print_size[1] += "B" + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 + print_speed = formatSize(self.current_position / sec_elapsed, True, True) + sys.stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % + (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) + sys.stdout.flush() + return + + rel_position = selfself.current_position * 100 / self.total_size + if rel_position >= self.last_milestone: + self.last_milestone = (int(rel_position) / 5) * 5 + sys.stdout.write("%d%% ", self.last_milestone) + sys.stdout.flush() + return + +class ProgressANSI(Progress): + ## http://en.wikipedia.org/wiki/ANSI_escape_code + SCI = '\x1b[' + ANSI_hide_cursor = SCI + "?25l" + ANSI_show_cursor = SCI + "?25h" + ANSI_save_cursor_pos = SCI + "s" + ANSI_restore_cursor_pos = SCI + "u" + ANSI_move_cursor_to_column = SCI + "%uG" + ANSI_erase_to_eol = SCI + "0K" + + def display(self, new_file = False, done_message = None): + """ + display(new_file = False[/True], done_message = None) + """ + if new_file: + sys.stdout.write("%s " % self.label[:30].ljust(30)) + #sys.stdout.write(self.ANSI_hide_cursor) + sys.stdout.write(self.ANSI_save_cursor_pos) + sys.stdout.flush() + return + + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 + if (sec_elapsed > 0): + print_speed = formatSize(self.current_position / sec_elapsed, True, True) + else: + print_speed = (0, "") + sys.stdout.write(self.ANSI_restore_cursor_pos) + sys.stdout.write(self.ANSI_erase_to_eol) + sys.stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { + "current" : str(self.current_position).rjust(len(str(self.total_size))), + "total" : self.total_size, + "percent" : self.current_position * 100 / self.total_size, + "elapsed" : sec_elapsed, + "speed" : print_speed[0], + "speed_coeff" : print_speed[1] + }) + + if done_message: + sys.stdout.write(" %s\n" % done_message) + + sys.stdout.flush() Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/S3/S3.py 2008-11-16 12:51:24 UTC (rev 261) @@ -350,7 +350,12 @@ def send_file(self, request, file, throttle = 0, retries = 3): method_string, resource, headers = request - info("Sending file '%s', please wait..." % file.name) + size_left = size_total = headers.get("content-length") + if self.config.progress_meter: + progress = self.config.progress_class(file.name, size_total) + else: + info("Sending file '%s', please wait..." % file.name) + timestamp_start = time.time() conn = self.get_connection(resource['bucket']) conn.connect() conn.putrequest(method_string, self.format_uri(resource)) @@ -358,17 +363,20 @@ conn.putheader(header, str(headers[header])) conn.endheaders() file.seek(0) - timestamp_start = time.time() md5_hash = md5.new() - size_left = size_total = headers.get("content-length") while (size_left > 0): debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) data = file.read(self.config.send_chunk) md5_hash.update(data) - debug("SendFile: Sending %d bytes to the server" % len(data)) + if self.config.progress_meter: + progress.update(delta_position = len(data)) + else: + debug("SendFile: Sending %d bytes to the server" % len(data)) try: conn.send(data) except Exception, e: + if self.config.progress_meter: + progress.done("failed") ## When an exception occurs insert a if retries: conn.close() @@ -388,7 +396,6 @@ (size_total - size_left), (size_total - size_left) * 100 / size_total, size_total)) - timestamp_end = time.time() md5_computed = md5_hash.hexdigest() response = {} http_response = conn.getresponse() @@ -396,11 +403,20 @@ response["reason"] = http_response.reason response["headers"] = convertTupleListToDict(http_response.getheaders()) response["data"] = http_response.read() - response["elapsed"] = timestamp_end - timestamp_start response["size"] = size_total - response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) conn.close() + timestamp_end = time.time() + response["elapsed"] = timestamp_end - timestamp_start + response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) + + if self.config.progress_meter: + ## The above conn.close() takes some time -> update() progress meter + ## to correct the average speed. Otherwise people will complain that + ## 'progress' and response["speed"] are inconsistent ;-) + progress.update() + progress.done("done") + if response["status"] == 307: ## RedirectPermanent redir_bucket = getTextFromXml(response['data'], ".//Bucket") @@ -430,7 +446,11 @@ def recv_file(self, request, stream): method_string, resource, headers = request - info("Receiving file '%s', please wait..." % stream.name) + if self.config.progress_meter: + progress = self.config.progress_class(stream.name, 0) + else: + info("Receiving file '%s', please wait..." % stream.name) + timestamp_start = time.time() conn = self.get_connection(resource['bucket']) conn.connect() conn.putrequest(method_string, self.format_uri(resource)) @@ -457,8 +477,9 @@ md5_hash = md5.new() size_left = size_total = int(response["headers"]["content-length"]) + if self.config.progress_meter: + progress.total_size = size_total size_recvd = 0 - timestamp_start = time.time() while (size_recvd < size_total): this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk) @@ -468,11 +489,15 @@ md5_hash.update(data) size_recvd += len(data) ## Call progress meter from here... - debug("Received %d bytes (%d %% of %d)" % ( - size_recvd, - size_recvd * 100 / size_total, - size_total)) + if self.config.progress_meter: + progress.update(delta_position = len(data)) + else: + debug("Received %d bytes (%d %% of %d)" % ( + size_recvd, + size_recvd * 100 / size_total, + size_total)) conn.close() + progress.done("done") timestamp_end = time.time() response["md5"] = md5_hash.hexdigest() response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0 Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/s3cmd 2008-11-16 12:51:24 UTC (rev 261) @@ -957,6 +957,8 @@ optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.") + optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default).") + optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter.") optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |