From: <lu...@us...> - 2008-11-16 09:51:54
|
Revision: 258 http://s3tools.svn.sourceforge.net/s3tools/?rev=258&view=rev Author: ludvigm Date: 2008-11-16 09:51:48 +0000 (Sun, 16 Nov 2008) Log Message: ----------- Merge from 0.9.8.x branch, rel 249: * S3/S3.py, S3/Exception.py: Re-issue failed requests in S3.send_request() Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Exceptions.py s3cmd/trunk/S3/S3.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 09:47:26 UTC (rev 257) +++ s3cmd/trunk/ChangeLog 2008-11-16 09:51:48 UTC (rev 258) @@ -1,5 +1,7 @@ 2008-11-16 Michal Ludvig <mi...@lo...> + Merge from 0.9.8.x branch, rel 249: + * S3/S3.py, S3/Exception.py: Re-issue failed requests in S3.send_request() Merge from 0.9.8.x branch, rel 248: * s3cmd: Don't leak open filehandles in sync. Thx Patrick Linskey for report. Merge from 0.9.8.x branch, rel 247: Modified: s3cmd/trunk/S3/Exceptions.py =================================================================== --- s3cmd/trunk/S3/Exceptions.py 2008-11-16 09:47:26 UTC (rev 257) +++ s3cmd/trunk/S3/Exceptions.py 2008-11-16 09:51:48 UTC (rev 258) @@ -48,6 +48,9 @@ class S3DownloadError(S3Exception): pass +class S3RequestError(S3Exception): + pass + class InvalidFileError(S3Exception): pass Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-16 09:47:26 UTC (rev 257) +++ s3cmd/trunk/S3/S3.py 2008-11-16 09:51:48 UTC (rev 258) @@ -305,19 +305,26 @@ debug("CreateRequest: resource[uri]=" + resource['uri']) return (method_string, resource, headers) - def send_request(self, request, body = None): + def send_request(self, request, body = None, retries = 5): method_string, resource, headers = request debug("Processing request, please wait...") - conn = self.get_connection(resource['bucket']) - conn.request(method_string, self.format_uri(resource), body, headers) - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - response["data"] = http_response.read() - debug("Response: " + str(response)) - conn.close() + try: + conn = self.get_connection(resource['bucket']) + conn.request(method_string, self.format_uri(resource), body, headers) + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + response["data"] = http_response.read() + debug("Response: " + str(response)) + conn.close() + except Exception: + if retries: + warning("Retrying failed request: %s" % resource['uri']) + return self.send_request(request, body, retries - 1) + else: + raise S3RequestError("Request failed for: %s" % resource['uri']) if response["status"] == 307: ## RedirectPermanent @@ -328,7 +335,11 @@ return self.send_request(request, body) if response["status"] < 200 or response["status"] > 299: - raise S3Error(response) + if retries: + warning("Retrying failed request: %s" % resource['uri']) + return self.send_request(request, body, retries - 1) + else: + raise S3Error(response) return response def send_file(self, request, file, throttle = 0, retries = 3): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-16 09:54:13
|
Revision: 259 http://s3tools.svn.sourceforge.net/s3tools/?rev=259&view=rev Author: ludvigm Date: 2008-11-16 09:54:08 +0000 (Sun, 16 Nov 2008) Log Message: ----------- Merge from 0.9.8.x branch, rel 251: * S3/S3.py: Adjusting previous commit (orig 249) - it's not a good idea to retry ALL failures. Especially not those code=4xx where AmazonS3 servers are not happy with our requests. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 09:51:48 UTC (rev 258) +++ s3cmd/trunk/ChangeLog 2008-11-16 09:54:08 UTC (rev 259) @@ -1,5 +1,9 @@ 2008-11-16 Michal Ludvig <mi...@lo...> + Merge from 0.9.8.x branch, rel 251: + * S3/S3.py: Adjusting previous commit (orig 249) - it's not a good idea + to retry ALL failures. Especially not those code=4xx where AmazonS3 + servers are not happy with our requests. Merge from 0.9.8.x branch, rel 249: * S3/S3.py, S3/Exception.py: Re-issue failed requests in S3.send_request() Merge from 0.9.8.x branch, rel 248: Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-16 09:51:48 UTC (rev 258) +++ s3cmd/trunk/S3/S3.py 2008-11-16 09:54:08 UTC (rev 259) @@ -319,9 +319,9 @@ response["data"] = http_response.read() debug("Response: " + str(response)) conn.close() - except Exception: + except Exception, e: if retries: - warning("Retrying failed request: %s" % resource['uri']) + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) return self.send_request(request, body, retries - 1) else: raise S3RequestError("Request failed for: %s" % resource['uri']) @@ -334,12 +334,18 @@ warning("Redirected to: %s" % (redir_hostname)) return self.send_request(request, body) - if response["status"] < 200 or response["status"] > 299: + if response["status"] >= 500: + e = S3Error(response) if retries: - warning("Retrying failed request: %s" % resource['uri']) + warning(u"Retrying failed request: %s" % resource['uri']) + warning(unicode(e)) return self.send_request(request, body, retries - 1) else: - raise S3Error(response) + raise e + + if response["status"] < 200 or response["status"] > 299: + raise S3Error(response) + return response def send_file(self, request, file, throttle = 0, retries = 3): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-16 09:56:20
|
Revision: 260 http://s3tools.svn.sourceforge.net/s3tools/?rev=260&view=rev Author: ludvigm Date: 2008-11-16 09:56:14 +0000 (Sun, 16 Nov 2008) Log Message: ----------- * NEWS: Fetch 0.9.8.4 release news from 0.9.8.x branch. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 09:54:08 UTC (rev 259) +++ s3cmd/trunk/ChangeLog 2008-11-16 09:56:14 UTC (rev 260) @@ -1,5 +1,9 @@ 2008-11-16 Michal Ludvig <mi...@lo...> + * NEWS: Fetch 0.9.8.4 release news from 0.9.8.x branch. + +2008-11-16 Michal Ludvig <mi...@lo...> + Merge from 0.9.8.x branch, rel 251: * S3/S3.py: Adjusting previous commit (orig 249) - it's not a good idea to retry ALL failures. Especially not those code=4xx where AmazonS3 Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-16 09:54:08 UTC (rev 259) +++ s3cmd/trunk/NEWS 2008-11-16 09:56:14 UTC (rev 260) @@ -6,13 +6,19 @@ * Recursively remove objects from buckets with a given prefix with --recursive (-r) -s3cmd 0.9.8.4 - 2008-09-16 +s3cmd 0.9.8.4 - 2008-11-07 ============= -* Bugfix release: +* Stabilisation / bugfix release: * Restored access to upper-case named buckets. * Improved handling of filenames with Unicode characters. * Avoid ZeroDivisionError on ultrafast links (for instance on Amazon EC2) +* Re-issue failed requests (e.g. connection errors, internal + server errors, etc). +* Sync skips over files that can't be open instead of + terminating the sync completely. +* Doesn't run out of open files quota on sync with lots of + files. s3cmd 0.9.8.3 - 2008-07-29 ============= This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-16 12:51:29
|
Revision: 261 http://s3tools.svn.sourceforge.net/s3tools/?rev=261&view=rev Author: ludvigm Date: 2008-11-16 12:51:24 +0000 (Sun, 16 Nov 2008) Log Message: ----------- * S3/Progress.py: Two progress meter implementations. * S3/Config.py, s3cmd: New --progress / --no-progress parameters and Config() members. * S3/S3.py: Call Progress() in send_file()/recv_file() * NEWS: Let everyone know ;-) Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Added Paths: ----------- s3cmd/trunk/S3/Progress.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/ChangeLog 2008-11-16 12:51:24 UTC (rev 261) @@ -1,3 +1,11 @@ +2008-11-17 Michal Ludvig <mi...@lo...> + + * S3/Progress.py: Two progress meter implementations. + * S3/Config.py, s3cmd: New --progress / --no-progress parameters + and Config() members. + * S3/S3.py: Call Progress() in send_file()/recv_file() + * NEWS: Let everyone know ;-) + 2008-11-16 Michal Ludvig <mi...@lo...> * NEWS: Fetch 0.9.8.4 release news from 0.9.8.x branch. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/NEWS 2008-11-16 12:51:24 UTC (rev 261) @@ -1,7 +1,6 @@ s3cmd 0.9.9 - ??? =========== -* Allow access to upper-case named buckets with - --use-old-connect-method parameter +* Implemented progress meter (--progress / --no-progress) * Removing of non-empty buckets with --force * Recursively remove objects from buckets with a given prefix with --recursive (-r) Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/S3/Config.py 2008-11-16 12:51:24 UTC (rev 261) @@ -6,6 +6,7 @@ import logging from logging import debug, info, warning, error import re +import Progress class Config(object): _instance = None @@ -17,6 +18,8 @@ host_bucket = "%(bucket)s.s3.amazonaws.com" simpledb_host = "sdb.amazonaws.com" verbosity = logging.WARNING + progress_meter = True + progress_class = Progress.ProgressANSI send_chunk = 4096 recv_chunk = 4096 human_readable_sizes = False Added: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py (rev 0) +++ s3cmd/trunk/S3/Progress.py 2008-11-16 12:51:24 UTC (rev 261) @@ -0,0 +1,109 @@ +## Amazon S3 manager +## Author: Michal Ludvig <mi...@lo...> +## http://www.logix.cz/michal +## License: GPL Version 2 + +import sys +import datetime +from Utils import formatSize + +class Progress(object): + def __init__(self, label, total_size): + self.new_file(label, total_size) + + def new_file(self, label, total_size): + self.label = label + self.total_size = total_size + self.current_position = 0 + self.time_start = datetime.datetime.now() + self.time_last = self.time_start + self.time_current = self.time_start + + self.display(new_file = True) + + def update(self, current_position = -1, delta_position = -1): + self.time_last = self.time_current + self.time_current = datetime.datetime.now() + if current_position > -1: + self.current_position = current_position + elif delta_position > -1: + self.current_position += delta_position + #else: + # no update, just call display() + self.display() + + def done(self, message): + self.display(done_message = message) + + def display(self, new_file = False, done_message = None): + """ + display(new_file = False[/True], done = False[/True]) + + Override this method to provide a nicer output. + """ + if new_file: + sys.stdout.write("%s " % self.label[:30].ljust(30)) + sys.stdout.flush() + self.last_milestone = 0 + return + + if self.current_position == self.total_size: + print_size = formatSize(self.current_position, True) + if print_size[1] != "": print_size[1] += "B" + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 + print_speed = formatSize(self.current_position / sec_elapsed, True, True) + sys.stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % + (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) + sys.stdout.flush() + return + + rel_position = selfself.current_position * 100 / self.total_size + if rel_position >= self.last_milestone: + self.last_milestone = (int(rel_position) / 5) * 5 + sys.stdout.write("%d%% ", self.last_milestone) + sys.stdout.flush() + return + +class ProgressANSI(Progress): + ## http://en.wikipedia.org/wiki/ANSI_escape_code + SCI = '\x1b[' + ANSI_hide_cursor = SCI + "?25l" + ANSI_show_cursor = SCI + "?25h" + ANSI_save_cursor_pos = SCI + "s" + ANSI_restore_cursor_pos = SCI + "u" + ANSI_move_cursor_to_column = SCI + "%uG" + ANSI_erase_to_eol = SCI + "0K" + + def display(self, new_file = False, done_message = None): + """ + display(new_file = False[/True], done_message = None) + """ + if new_file: + sys.stdout.write("%s " % self.label[:30].ljust(30)) + #sys.stdout.write(self.ANSI_hide_cursor) + sys.stdout.write(self.ANSI_save_cursor_pos) + sys.stdout.flush() + return + + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 + if (sec_elapsed > 0): + print_speed = formatSize(self.current_position / sec_elapsed, True, True) + else: + print_speed = (0, "") + sys.stdout.write(self.ANSI_restore_cursor_pos) + sys.stdout.write(self.ANSI_erase_to_eol) + sys.stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { + "current" : str(self.current_position).rjust(len(str(self.total_size))), + "total" : self.total_size, + "percent" : self.current_position * 100 / self.total_size, + "elapsed" : sec_elapsed, + "speed" : print_speed[0], + "speed_coeff" : print_speed[1] + }) + + if done_message: + sys.stdout.write(" %s\n" % done_message) + + sys.stdout.flush() Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/S3/S3.py 2008-11-16 12:51:24 UTC (rev 261) @@ -350,7 +350,12 @@ def send_file(self, request, file, throttle = 0, retries = 3): method_string, resource, headers = request - info("Sending file '%s', please wait..." % file.name) + size_left = size_total = headers.get("content-length") + if self.config.progress_meter: + progress = self.config.progress_class(file.name, size_total) + else: + info("Sending file '%s', please wait..." % file.name) + timestamp_start = time.time() conn = self.get_connection(resource['bucket']) conn.connect() conn.putrequest(method_string, self.format_uri(resource)) @@ -358,17 +363,20 @@ conn.putheader(header, str(headers[header])) conn.endheaders() file.seek(0) - timestamp_start = time.time() md5_hash = md5.new() - size_left = size_total = headers.get("content-length") while (size_left > 0): debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) data = file.read(self.config.send_chunk) md5_hash.update(data) - debug("SendFile: Sending %d bytes to the server" % len(data)) + if self.config.progress_meter: + progress.update(delta_position = len(data)) + else: + debug("SendFile: Sending %d bytes to the server" % len(data)) try: conn.send(data) except Exception, e: + if self.config.progress_meter: + progress.done("failed") ## When an exception occurs insert a if retries: conn.close() @@ -388,7 +396,6 @@ (size_total - size_left), (size_total - size_left) * 100 / size_total, size_total)) - timestamp_end = time.time() md5_computed = md5_hash.hexdigest() response = {} http_response = conn.getresponse() @@ -396,11 +403,20 @@ response["reason"] = http_response.reason response["headers"] = convertTupleListToDict(http_response.getheaders()) response["data"] = http_response.read() - response["elapsed"] = timestamp_end - timestamp_start response["size"] = size_total - response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) conn.close() + timestamp_end = time.time() + response["elapsed"] = timestamp_end - timestamp_start + response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) + + if self.config.progress_meter: + ## The above conn.close() takes some time -> update() progress meter + ## to correct the average speed. Otherwise people will complain that + ## 'progress' and response["speed"] are inconsistent ;-) + progress.update() + progress.done("done") + if response["status"] == 307: ## RedirectPermanent redir_bucket = getTextFromXml(response['data'], ".//Bucket") @@ -430,7 +446,11 @@ def recv_file(self, request, stream): method_string, resource, headers = request - info("Receiving file '%s', please wait..." % stream.name) + if self.config.progress_meter: + progress = self.config.progress_class(stream.name, 0) + else: + info("Receiving file '%s', please wait..." % stream.name) + timestamp_start = time.time() conn = self.get_connection(resource['bucket']) conn.connect() conn.putrequest(method_string, self.format_uri(resource)) @@ -457,8 +477,9 @@ md5_hash = md5.new() size_left = size_total = int(response["headers"]["content-length"]) + if self.config.progress_meter: + progress.total_size = size_total size_recvd = 0 - timestamp_start = time.time() while (size_recvd < size_total): this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk) @@ -468,11 +489,15 @@ md5_hash.update(data) size_recvd += len(data) ## Call progress meter from here... - debug("Received %d bytes (%d %% of %d)" % ( - size_recvd, - size_recvd * 100 / size_total, - size_total)) + if self.config.progress_meter: + progress.update(delta_position = len(data)) + else: + debug("Received %d bytes (%d %% of %d)" % ( + size_recvd, + size_recvd * 100 / size_total, + size_total)) conn.close() + progress.done("done") timestamp_end = time.time() response["md5"] = md5_hash.hexdigest() response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0 Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-16 09:56:14 UTC (rev 260) +++ s3cmd/trunk/s3cmd 2008-11-16 12:51:24 UTC (rev 261) @@ -957,6 +957,8 @@ optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.") + optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default).") + optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter.") optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-20 13:57:52
|
Revision: 262 http://s3tools.svn.sourceforge.net/s3tools/?rev=262&view=rev Author: ludvigm Date: 2008-11-20 13:57:41 +0000 (Thu, 20 Nov 2008) Log Message: ----------- * s3cmd: Support for 'cp' command. * S3/S3.py: Added S3.object.copy() method. * s3cmd.1: Document 'cp' command. * NEWS: Let everyone know ;-) Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-16 12:51:24 UTC (rev 261) +++ s3cmd/trunk/ChangeLog 2008-11-20 13:57:41 UTC (rev 262) @@ -1,3 +1,12 @@ +2008-11-21 Michal Ludvig <mi...@lo...> + + * s3cmd: Support for 'cp' command. + * S3/S3.py: Added S3.object.copy() method. + * s3cmd.1: Document 'cp' command. + * NEWS: Let everyone know ;-) + Thanks Andrew Ryan for a patch proposal! + https://sourceforge.net/forum/forum.php?thread_id=2346987&forum_id=618865 + 2008-11-17 Michal Ludvig <mi...@lo...> * S3/Progress.py: Two progress meter implementations. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-16 12:51:24 UTC (rev 261) +++ s3cmd/trunk/NEWS 2008-11-20 13:57:41 UTC (rev 262) @@ -4,6 +4,7 @@ * Removing of non-empty buckets with --force * Recursively remove objects from buckets with a given prefix with --recursive (-r) +* Copying objects, within or between buckets. (Andrew Ryan) s3cmd 0.9.8.4 - 2008-11-07 ============= Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-16 12:51:24 UTC (rev 261) +++ s3cmd/trunk/S3/S3.py 2008-11-20 13:57:41 UTC (rev 262) @@ -199,6 +199,21 @@ response = self.send_request(request) return response + def object_copy(self, src_uri, dst_uri, extra_headers = None): + if src_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type) + if dst_uri.type != "s3": + raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type) + headers = SortedDict() + headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object())) + if self.config.acl_public: + headers["x-amz-acl"] = "public-read" + if extra_headers: + headers.update(extra_headers) + request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers) + response = self.send_request(request) + return response + def object_info(self, uri): request = self.create_request("OBJECT_HEAD", uri = uri) response = self.send_request(request) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-16 12:51:24 UTC (rev 261) +++ s3cmd/trunk/s3cmd 2008-11-20 13:57:41 UTC (rev 262) @@ -308,6 +308,25 @@ response = s3.object_delete(_uri) output("Object %s deleted" % _uri) +def cmd_cp(args): + s3 = S3(Config()) + src_uri = S3Uri(args.pop(0)) + dst_uri = S3Uri(args.pop(0)) + + if len(args): + raise ParameterError("Too many parameters! Expected: %s" % commands['cp']['param']) + + if src_uri.type != "s3" or dst_uri.type != "s3": + raise ParameterError("Parameters are not URIs! Expected: %s" % commands['cp']['param']) + + if dst_uri.object() == "": + dst_uri = S3Uri(dst_uri.uri() + src_uri.object()) + + response = s3.object_copy(src_uri, dst_uri) + output("Object is copied to %s" % dst_uri) + if Config().acl_public: + output("Public URL is: %s" % dst_uri.public_url()) + def cmd_info(args): s3 = S3(Config()) @@ -882,6 +901,7 @@ {"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2}, {"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0}, {"cmd":"info", "label":"Get various information about Buckets or Objects", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1}, + {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, #{"cmd":"setacl", "label":"Modify Access control list for Bucket or Object", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1}, ] Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2008-11-16 12:51:24 UTC (rev 261) +++ s3cmd/trunk/s3cmd.1 2008-11-20 13:57:41 UTC (rev 262) @@ -42,6 +42,10 @@ \fBsync\fR \fIs3://BUCKET[/PREFIX] LOCAL_DIR\fR Restore a tree from S3 to local directory .TP +\fBcp\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR +Make a copy of a file. Either in the same bucket with a different name +or in another bucket with the same or different name. +.TP \fBinfo\fR \fIs3://BUCKET[/OBJECT]\fR Get various information about a Bucket or Object .TP This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-24 09:43:49
|
Revision: 263 http://s3tools.svn.sourceforge.net/s3tools/?rev=263&view=rev Author: ludvigm Date: 2008-11-24 09:43:43 +0000 (Mon, 24 Nov 2008) Log Message: ----------- * S3/Utils.py: Common XML parser. * s3cmd, S3/Exeptions.py: Print info message on Error. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Exceptions.py s3cmd/trunk/S3/Utils.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-20 13:57:41 UTC (rev 262) +++ s3cmd/trunk/ChangeLog 2008-11-24 09:43:43 UTC (rev 263) @@ -1,3 +1,8 @@ +2008-11-24 Michal Ludvig <mi...@lo...> + + * S3/Utils.py: Common XML parser. + * s3cmd, S3/Exeptions.py: Print info message on Error. + 2008-11-21 Michal Ludvig <mi...@lo...> * s3cmd: Support for 'cp' command. Modified: s3cmd/trunk/S3/Exceptions.py =================================================================== --- s3cmd/trunk/S3/Exceptions.py 2008-11-20 13:57:41 UTC (rev 262) +++ s3cmd/trunk/S3/Exceptions.py 2008-11-24 09:43:43 UTC (rev 263) @@ -3,6 +3,7 @@ ## http://www.logix.cz/michal ## License: GPL Version 2 +from Utils import getRootTagName from logging import debug, info, warning, error try: @@ -27,7 +28,7 @@ if response.has_key("headers"): for header in response["headers"]: debug("HttpHeader: %s: %s" % (header, response["headers"][header])) - if response.has_key("data"): + if response.has_key("data") and getRootTagName(response["data"]) == "Error": tree = ET.fromstring(response["data"]) for child in tree.getchildren(): if child.text != "": Modified: s3cmd/trunk/S3/Utils.py =================================================================== --- s3cmd/trunk/S3/Utils.py 2008-11-20 13:57:41 UTC (rev 262) +++ s3cmd/trunk/S3/Utils.py 2008-11-24 09:43:43 UTC (rev 263) @@ -57,20 +57,27 @@ return "" return re.compile("^(\{[^}]+\})").match(element.tag).groups()[0] -def getListFromXml(xml, node): +def getTreeFromXml(xml): tree = ET.fromstring(xml) - xmlns = getNameSpace(tree) - nodes = tree.findall('.//%s%s' % (xmlns, node)) - return parseNodes(nodes, xmlns) + tree.xmlns = getNameSpace(tree) + return tree +def getListFromXml(xml, node): + tree = getTreeFromXml(xml) + nodes = tree.findall('.//%s%s' % (tree.xmlns, node)) + return parseNodes(nodes, tree.xmlns) + def getTextFromXml(xml, xpath): - tree = ET.fromstring(xml) - xmlns = getNameSpace(tree) + tree = getTreeFromXml(xml) if tree.tag.endswith(xpath): return tree.text else: - return tree.findtext(fixupXPath(xmlns, xpath)) + return tree.findtext(fixupXPath(tree.xmlns, xpath)) +def getRootTagName(xml): + tree = getTreeFromXml(xml) + return stripTagXmlns(tree.xmlns, tree.tag) + def dateS3toPython(date): date = re.compile("\.\d\d\dZ").sub(".000Z", date) return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-20 13:57:41 UTC (rev 262) +++ s3cmd/trunk/s3cmd 2008-11-24 09:43:43 UTC (rev 263) @@ -1108,6 +1108,8 @@ cmd_func(args) except S3Error, e: error("S3 error: %s" % e) + if e.info.has_key("Message"): + error(e.info['Message']) sys.exit(1) except ParameterError, e: error("Parameter problem: %s" % e) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 00:09:50
|
Revision: 264 http://s3tools.svn.sourceforge.net/s3tools/?rev=264&view=rev Author: ludvigm Date: 2008-11-25 00:09:47 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-24 09:43:43 UTC (rev 263) +++ s3cmd/trunk/ChangeLog 2008-11-25 00:09:47 UTC (rev 264) @@ -1,5 +1,9 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects + +2008-11-24 Michal Ludvig <mi...@lo...> + * S3/Utils.py: Common XML parser. * s3cmd, S3/Exeptions.py: Print info message on Error. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-24 09:43:43 UTC (rev 263) +++ s3cmd/trunk/NEWS 2008-11-25 00:09:47 UTC (rev 264) @@ -4,7 +4,8 @@ * Removing of non-empty buckets with --force * Recursively remove objects from buckets with a given prefix with --recursive (-r) -* Copying objects, within or between buckets. (Andrew Ryan) +* Copying and moving objects, within or between buckets. + (Andrew Ryan) s3cmd 0.9.8.4 - 2008-11-07 ============= Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-24 09:43:43 UTC (rev 263) +++ s3cmd/trunk/S3/S3.py 2008-11-25 00:09:47 UTC (rev 264) @@ -214,6 +214,14 @@ response = self.send_request(request) return response + def object_move(self, src_uri, dst_uri, extra_headers = None): + response_copy = self.object_copy(src_uri, dst_uri, extra_headers) + debug("Object %s copied to %s" % (src_uri, dst_uri)) + if getRootTagName(response_copy["data"]) == "CopyObjectResult": + response_delete = self.object_delete(src_uri) + debug("Object %s deleted" % src_uri) + return response_copy + def object_info(self, uri): request = self.create_request("OBJECT_HEAD", uri = uri) response = self.send_request(request) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-24 09:43:43 UTC (rev 263) +++ s3cmd/trunk/s3cmd 2008-11-25 00:09:47 UTC (rev 264) @@ -308,8 +308,7 @@ response = s3.object_delete(_uri) output("Object %s deleted" % _uri) -def cmd_cp(args): - s3 = S3(Config()) +def subcmd_cp_mv(args, process_fce, message): src_uri = S3Uri(args.pop(0)) dst_uri = S3Uri(args.pop(0)) @@ -322,11 +321,19 @@ if dst_uri.object() == "": dst_uri = S3Uri(dst_uri.uri() + src_uri.object()) - response = s3.object_copy(src_uri, dst_uri) - output("Object is copied to %s" % dst_uri) + response = process_fce(src_uri, dst_uri) + output(message % { "src" : src_uri, "dst" : dst_uri}) if Config().acl_public: output("Public URL is: %s" % dst_uri.public_url()) +def cmd_cp(args): + s3 = S3(Config()) + subcmd_cp_mv(args, s3.object_copy, "Object %(src)s copied to %(dst)s") + +def cmd_mv(args): + s3 = S3(Config()) + subcmd_cp_mv(args, s3.object_move, "Object %(src)s moved to %(dst)s") + def cmd_info(args): s3 = S3(Config()) @@ -902,6 +909,7 @@ {"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0}, {"cmd":"info", "label":"Get various information about Buckets or Objects", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1}, {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, + {"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2}, #{"cmd":"setacl", "label":"Modify Access control list for Bucket or Object", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1}, ] This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 01:15:45
|
Revision: 265 http://s3tools.svn.sourceforge.net/s3tools/?rev=265&view=rev Author: ludvigm Date: 2008-11-25 01:15:39 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * s3/s3.py: improved retrying in send_request() and send_file() Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 00:09:47 UTC (rev 264) +++ s3cmd/trunk/ChangeLog 2008-11-25 01:15:39 UTC (rev 265) @@ -1,5 +1,9 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * s3/s3.py: improved retrying in send_request() and send_file() + +2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects 2008-11-24 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-25 00:09:47 UTC (rev 264) +++ s3cmd/trunk/S3/S3.py 2008-11-25 01:15:39 UTC (rev 265) @@ -6,6 +6,7 @@ import sys import os, os.path import base64 +import time import md5 import sha import hmac @@ -58,6 +59,9 @@ ## S3 sometimes sends HTTP-307 response redir_map = {} + ## Maximum attempts of re-issuing failed requests + _max_retries = 5 + def __init__(self, config): self.config = config @@ -328,7 +332,11 @@ debug("CreateRequest: resource[uri]=" + resource['uri']) return (method_string, resource, headers) - def send_request(self, request, body = None, retries = 5): + def _fail_wait(self, retries): + # Wait a few seconds. The more it fails the more we wait. + return (self._max_retries - retries + 1) * 3 + + def send_request(self, request, body = None, retries = _max_retries): method_string, resource, headers = request debug("Processing request, please wait...") try: @@ -345,6 +353,8 @@ except Exception, e: if retries: warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) return self.send_request(request, body, retries - 1) else: raise S3RequestError("Request failed for: %s" % resource['uri']) @@ -362,6 +372,8 @@ if retries: warning(u"Retrying failed request: %s" % resource['uri']) warning(unicode(e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) return self.send_request(request, body, retries - 1) else: raise e @@ -371,7 +383,7 @@ return response - def send_file(self, request, file, throttle = 0, retries = 3): + def send_file(self, request, file, throttle = 0, retries = _max_retries): method_string, resource, headers = request size_left = size_total = headers.get("content-length") if self.config.progress_meter: @@ -379,56 +391,67 @@ else: info("Sending file '%s', please wait..." % file.name) timestamp_start = time.time() - conn = self.get_connection(resource['bucket']) - conn.connect() - conn.putrequest(method_string, self.format_uri(resource)) - for header in headers.keys(): - conn.putheader(header, str(headers[header])) - conn.endheaders() + try: + conn = self.get_connection(resource['bucket']) + conn.connect() + conn.putrequest(method_string, self.format_uri(resource)) + for header in headers.keys(): + conn.putheader(header, str(headers[header])) + conn.endheaders() + except Exception, e: + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + # Connection error -> same throttle value + return self.send_file(request, file, throttle, retries - 1) + else: + raise S3UploadError("Request failed for: %s" % resource['uri']) file.seek(0) md5_hash = md5.new() - while (size_left > 0): - debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) - data = file.read(self.config.send_chunk) - md5_hash.update(data) + try: + while (size_left > 0): + debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) + data = file.read(self.config.send_chunk) + md5_hash.update(data) + if self.config.progress_meter: + progress.update(delta_position = len(data)) + else: + debug("SendFile: Sending %d bytes to the server" % len(data)) + conn.send(data) + + size_left -= len(data) + if throttle: + time.sleep(throttle) + ## Call progress meter from here + debug("Sent %d bytes (%d %% of %d)" % ( + (size_total - size_left), + (size_total - size_left) * 100 / size_total, + size_total)) + md5_computed = md5_hash.hexdigest() + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + response["data"] = http_response.read() + response["size"] = size_total + conn.close() + except Exception, e: if self.config.progress_meter: - progress.update(delta_position = len(data)) + progress.done("failed") + if retries: + throttle = throttle and throttle * 5 or 0.01 + warning("Request failed: %s (%s)" % (resource['uri'], e)) + warning("Retrying on lower speed (throttle=%0.2f)" % throttle) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + # Connection error -> same throttle value + return self.send_file(request, file, throttle, retries - 1) else: - debug("SendFile: Sending %d bytes to the server" % len(data)) - try: - conn.send(data) - except Exception, e: - if self.config.progress_meter: - progress.done("failed") - ## When an exception occurs insert a - if retries: - conn.close() - warning("Upload of '%s' failed %s " % (file.name, e)) - throttle = throttle and throttle * 5 or 0.01 - warning("Retrying on lower speed (throttle=%0.2f)" % throttle) - return self.send_file(request, file, throttle, retries - 1) - else: - debug("Giving up on '%s' %s" % (file.name, e)) - raise S3UploadError + debug("Giving up on '%s' %s" % (file.name, e)) + raise S3UploadError("Request failed for: %s" % resource['uri']) - size_left -= len(data) - if throttle: - time.sleep(throttle) - ## Call progress meter from here - debug("Sent %d bytes (%d %% of %d)" % ( - (size_total - size_left), - (size_total - size_left) * 100 / size_total, - size_total)) - md5_computed = md5_hash.hexdigest() - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) - response["data"] = http_response.read() - response["size"] = size_total - conn.close() - timestamp_end = time.time() response["elapsed"] = timestamp_end - timestamp_start response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 05:12:53
|
Revision: 266 http://s3tools.svn.sourceforge.net/s3tools/?rev=266&view=rev Author: ludvigm Date: 2008-11-25 05:12:43 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * S3/Progress.py: Support for progress meter not starting in 0. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Progress.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 01:15:39 UTC (rev 265) +++ s3cmd/trunk/ChangeLog 2008-11-25 05:12:43 UTC (rev 266) @@ -1,6 +1,7 @@ 2008-11-24 Michal Ludvig <mi...@lo...> - * s3/s3.py: improved retrying in send_request() and send_file() + * S3/Progress.py: Support for progress meter not starting in 0. + * S3/S3.py: improved retrying in send_request() and send_file() 2008-11-24 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-11-25 01:15:39 UTC (rev 265) +++ s3cmd/trunk/S3/Progress.py 2008-11-25 05:12:43 UTC (rev 266) @@ -14,7 +14,13 @@ def new_file(self, label, total_size): self.label = label self.total_size = total_size - self.current_position = 0 + # Set initial_position to something in the + # case we're not counting from 0. For instance + # when appending to a partially downloaded file. + # Setting initial_position will let the speed + # be computed right. + self.initial_position = 0 + self.current_position = self.initial_position self.time_start = datetime.datetime.now() self.time_last = self.time_start self.time_current = self.time_start @@ -52,7 +58,7 @@ if print_size[1] != "": print_size[1] += "B" timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 - print_speed = formatSize(self.current_position / sec_elapsed, True, True) + print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) sys.stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) sys.stdout.flush() @@ -89,7 +95,7 @@ timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 if (sec_elapsed > 0): - print_speed = formatSize(self.current_position / sec_elapsed, True, True) + print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) else: print_speed = (0, "") sys.stdout.write(self.ANSI_restore_cursor_pos) @@ -97,7 +103,7 @@ sys.stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { "current" : str(self.current_position).rjust(len(str(self.total_size))), "total" : self.total_size, - "percent" : self.current_position * 100 / self.total_size, + "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0, "elapsed" : sec_elapsed, "speed" : print_speed[0], "speed_coeff" : print_speed[1] This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 05:38:06
|
Revision: 267 http://s3tools.svn.sourceforge.net/s3tools/?rev=267&view=rev Author: ludvigm Date: 2008-11-25 05:38:01 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for 'get' command, improved 'get' failure resiliency. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/ChangeLog 2008-11-25 05:38:01 UTC (rev 267) @@ -1,5 +1,7 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for + 'get' command, improved 'get' failure resiliency. * S3/Progress.py: Support for progress meter not starting in 0. * S3/S3.py: improved retrying in send_request() and send_file() Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/NEWS 2008-11-25 05:38:01 UTC (rev 267) @@ -6,6 +6,9 @@ prefix with --recursive (-r) * Copying and moving objects, within or between buckets. (Andrew Ryan) +* Continue getting partially downloaded files with --continue +* Improved resistance to communication errors (Connection + reset by peer, etc.) s3cmd 0.9.8.4 - 2008-11-07 ============= Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/S3/Config.py 2008-11-25 05:38:01 UTC (rev 267) @@ -24,6 +24,7 @@ recv_chunk = 4096 human_readable_sizes = False force = False + get_continue = False recursive = False acl_public = False proxy_host = "" Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/S3/S3.py 2008-11-25 05:38:01 UTC (rev 267) @@ -189,11 +189,11 @@ response = self.send_file(request, file) return response - def object_get(self, uri, stream): + def object_get(self, uri, stream, start_position): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) request = self.create_request("OBJECT_GET", uri = uri) - response = self.recv_file(request, stream) + response = self.recv_file(request, stream, start_position) return response def object_delete(self, uri): @@ -399,6 +399,8 @@ conn.putheader(header, str(headers[header])) conn.endheaders() except Exception, e: + if self.config.progress_meter: + progress.done("failed") if retries: warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) warning("Waiting %d sec..." % self._fail_wait(retries)) @@ -406,7 +408,7 @@ # Connection error -> same throttle value return self.send_file(request, file, throttle, retries - 1) else: - raise S3UploadError("Request failed for: %s" % resource['uri']) + raise S3UploadError("Upload failed for: %s" % resource['uri']) file.seek(0) md5_hash = md5.new() try: @@ -414,20 +416,12 @@ debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name)) data = file.read(self.config.send_chunk) md5_hash.update(data) + conn.send(data) if self.config.progress_meter: progress.update(delta_position = len(data)) - else: - debug("SendFile: Sending %d bytes to the server" % len(data)) - conn.send(data) - size_left -= len(data) if throttle: time.sleep(throttle) - ## Call progress meter from here - debug("Sent %d bytes (%d %% of %d)" % ( - (size_total - size_left), - (size_total - size_left) * 100 / size_total, - size_total)) md5_computed = md5_hash.hexdigest() response = {} http_response = conn.getresponse() @@ -442,7 +436,7 @@ progress.done("failed") if retries: throttle = throttle and throttle * 5 or 0.01 - warning("Request failed: %s (%s)" % (resource['uri'], e)) + warning("Upload failed: %s (%s)" % (resource['uri'], e)) warning("Retrying on lower speed (throttle=%0.2f)" % throttle) warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) @@ -450,7 +444,7 @@ return self.send_file(request, file, throttle, retries - 1) else: debug("Giving up on '%s' %s" % (file.name, e)) - raise S3UploadError("Request failed for: %s" % resource['uri']) + raise S3UploadError("Upload failed for: %s" % resource['uri']) timestamp_end = time.time() response["elapsed"] = timestamp_end - timestamp_start @@ -490,24 +484,40 @@ raise S3Error(response) return response - def recv_file(self, request, stream): + def recv_file(self, request, stream, start_position = 0, retries = _max_retries): method_string, resource, headers = request if self.config.progress_meter: progress = self.config.progress_class(stream.name, 0) else: info("Receiving file '%s', please wait..." % stream.name) timestamp_start = time.time() - conn = self.get_connection(resource['bucket']) - conn.connect() - conn.putrequest(method_string, self.format_uri(resource)) - for header in headers.keys(): - conn.putheader(header, str(headers[header])) - conn.endheaders() - response = {} - http_response = conn.getresponse() - response["status"] = http_response.status - response["reason"] = http_response.reason - response["headers"] = convertTupleListToDict(http_response.getheaders()) + try: + conn = self.get_connection(resource['bucket']) + conn.connect() + conn.putrequest(method_string, self.format_uri(resource)) + for header in headers.keys(): + conn.putheader(header, str(headers[header])) + if start_position > 0: + debug("Requesting Range: %d .. end" % start_position) + conn.putheader("Range", "bytes=%d-" % start_position) + conn.endheaders() + response = {} + http_response = conn.getresponse() + response["status"] = http_response.status + response["reason"] = http_response.reason + response["headers"] = convertTupleListToDict(http_response.getheaders()) + debug("Response: %s" % response) + except Exception, e: + if self.config.progress_meter: + progress.done("failed") + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + # Connection error -> same throttle value + return self.recv_file(request, stream, start_position, retries - 1) + else: + raise S3DownloadError("Download failed for: %s" % resource['uri']) if response["status"] == 307: ## RedirectPermanent @@ -521,38 +531,67 @@ if response["status"] < 200 or response["status"] > 299: raise S3Error(response) - md5_hash = md5.new() - size_left = size_total = int(response["headers"]["content-length"]) + if start_position == 0: + # Only compute MD5 on the fly if we're downloading from beginning + # Otherwise we'd get a nonsense. + md5_hash = md5.new() + size_left = int(response["headers"]["content-length"]) + size_total = start_position + size_left + current_position = start_position + if self.config.progress_meter: progress.total_size = size_total - size_recvd = 0 - while (size_recvd < size_total): - this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left - debug("ReceiveFile: Receiving up to %d bytes from the server" % this_chunk) - data = http_response.read(this_chunk) - debug("ReceiveFile: Writing %d bytes to file '%s'" % (len(data), stream.name)) - stream.write(data) - md5_hash.update(data) - size_recvd += len(data) - ## Call progress meter from here... + progress.initial_position = current_position + progress.current_position = current_position + + try: + while (current_position < size_total): + this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left + data = http_response.read(this_chunk) + stream.write(data) + if start_position == 0: + md5_hash.update(data) + current_position += len(data) + ## Call progress meter from here... + if self.config.progress_meter: + progress.update(delta_position = len(data)) + conn.close() + except Exception, e: if self.config.progress_meter: - progress.update(delta_position = len(data)) + progress.done("failed") + if retries: + warning("Retrying failed request: %s (%s)" % (resource['uri'], e)) + warning("Waiting %d sec..." % self._fail_wait(retries)) + time.sleep(self._fail_wait(retries)) + # Connection error -> same throttle value + return self.recv_file(request, stream, current_position, retries - 1) else: - debug("Received %d bytes (%d %% of %d)" % ( - size_recvd, - size_recvd * 100 / size_total, - size_total)) - conn.close() + raise S3DownloadError("Download failed for: %s" % resource['uri']) + + stream.flush() progress.done("done") timestamp_end = time.time() - response["md5"] = md5_hash.hexdigest() + + if start_position == 0: + # Only compute MD5 on the fly if we were downloading from the beginning + response["md5"] = md5_hash.hexdigest() + else: + # Otherwise try to compute MD5 of the output file + try: + response["md5"] = hash_file_md5(stream.name) + except IOError, e: + if e.errno != errno.ENOENT: + warning("Unable to open file: %s: %s" % (stream.name, e)) + warning("Unable to verify MD5. Assume it matches.") + response["md5"] = response["headers"]["etag"] + response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0 response["elapsed"] = timestamp_end - timestamp_start - response["size"] = size_recvd + response["size"] = current_position response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1) - if response["size"] != long(response["headers"]["content-length"]): + if response["size"] != start_position + long(response["headers"]["content-length"]): warning("Reported size (%s) does not match received size (%s)" % ( - response["headers"]["content-length"], response["size"])) + start_position + response["headers"]["content-length"], response["size"])) debug("ReceiveFile: Computed MD5 = %s" % response["md5"]) if not response["md5match"]: warning("MD5 signatures do not match: computed=%s, received=%s" % ( Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/s3cmd 2008-11-25 05:38:01 UTC (rev 267) @@ -253,6 +253,7 @@ uri_arg = args.pop(0) uri = S3Uri(uri_arg) + start_position = 0 if destination_file: destination = destination_file elif destination_dir: @@ -265,14 +266,21 @@ dst_stream = sys.stdout else: ## File - if not Config().force and os.path.exists(destination): - raise ParameterError("File %s already exists. Use --force to overwrite it" % destination) try: - dst_stream = open(destination, "wb") + dst_stream = open(destination, "ab") + if os.path.exists(destination): + if Config().get_continue: + start_position = dst_stream.tell() + elif Config().force: + start_position = 0L + dst_stream.seek(0L) + dst_stream.truncate() + else: + raise ParameterError("File %s already exists. Use either --force or --continue or give it a new name." % destination) except IOError, e: error("Skipping %s: %s" % (destination, e.strerror)) continue - response = s3.object_get(uri, dst_stream) + response = s3.object_get(uri, dst_stream, start_position = start_position) if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"]) response["size"] = os.stat(destination)[6] @@ -965,6 +973,7 @@ optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.") optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.") optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") + optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).") optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2008-11-25 05:12:43 UTC (rev 266) +++ s3cmd/trunk/s3cmd.1 2008-11-25 05:38:01 UTC (rev 267) @@ -113,6 +113,9 @@ \fB\-f\fR, \fB\-\-force\fR Force overwrite and other dangerous operations. .TP +\fB\-\-continue\fR +Continue getting a partially downloaded file (only for \fIget\fR command). This comes handy once download of a large file, say an ISO image, from a S3 bucket fails and a partially downloaded file is left on the disk. Unfortunately \fIput\fR command doesn't support restarting of failed upload due to Amazon S3 limitation. +.TP \fB\-P\fR, \fB\-\-acl\-public\fR Store objects with permissions allowing read for anyone. .TP This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 05:42:16
|
Revision: 268 http://s3tools.svn.sourceforge.net/s3tools/?rev=268&view=rev Author: ludvigm Date: 2008-11-25 05:42:06 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * s3cmd.1, TODO: Document 'mv' command. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/TODO s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 05:38:01 UTC (rev 267) +++ s3cmd/trunk/ChangeLog 2008-11-25 05:42:06 UTC (rev 268) @@ -1,5 +1,9 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd.1, TODO: Document 'mv' command. + +2008-11-24 Michal Ludvig <mi...@lo...> + * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for 'get' command, improved 'get' failure resiliency. * S3/Progress.py: Support for progress meter not starting in 0. Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-11-25 05:38:01 UTC (rev 267) +++ s3cmd/trunk/TODO 2008-11-25 05:42:06 UTC (rev 268) @@ -2,8 +2,6 @@ =========================== - For 0.9.9 - - Implement 'cp' and 'mv' - - Better upload / download progress display - Add --include/--include-from/--rinclude* for sync - Recursive processing / multiple sources with most commands. - Document --recursive and --force for buckets Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2008-11-25 05:38:01 UTC (rev 267) +++ s3cmd/trunk/s3cmd.1 2008-11-25 05:42:06 UTC (rev 268) @@ -43,8 +43,12 @@ Restore a tree from S3 to local directory .TP \fBcp\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR -Make a copy of a file. Either in the same bucket with a different name +\fBmv\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR +Make a copy of a file (\fIcp\fR) or move a file (\fImv\fR). +Destination can be in the same bucket with a different name or in another bucket with the same or different name. +Adding \fI\-\-acl\-public\fR will make the destination object +publicly accessible (see below). .TP \fBinfo\fR \fIs3://BUCKET[/OBJECT]\fR Get various information about a Bucket or Object This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 06:29:44
|
Revision: 269 http://s3tools.svn.sourceforge.net/s3tools/?rev=269&view=rev Author: ludvigm Date: 2008-11-25 06:29:39 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * s3cmd: Fixed 'get' conflict. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 05:42:06 UTC (rev 268) +++ s3cmd/trunk/ChangeLog 2008-11-25 06:29:39 UTC (rev 269) @@ -1,5 +1,6 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd: Fixed 'get' conflict. * s3cmd.1, TODO: Document 'mv' command. 2008-11-24 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-25 05:42:06 UTC (rev 268) +++ s3cmd/trunk/s3cmd 2008-11-25 06:29:39 UTC (rev 269) @@ -267,8 +267,9 @@ else: ## File try: + file_exists = os.path.exists(destination) dst_stream = open(destination, "ab") - if os.path.exists(destination): + if file_exists: if Config().get_continue: start_position = dst_stream.tell() elif Config().force: @@ -276,6 +277,7 @@ dst_stream.seek(0L) dst_stream.truncate() else: + dst_stream.close() raise ParameterError("File %s already exists. Use either --force or --continue or give it a new name." % destination) except IOError, e: error("Skipping %s: %s" % (destination, e.strerror)) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 06:35:07
|
Revision: 270 http://s3tools.svn.sourceforge.net/s3tools/?rev=270&view=rev Author: ludvigm Date: 2008-11-25 06:35:03 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * s3cmd, s3cmd.1, S3/S3.py: Display or don't display progress meter default depends on whether we're on TTY (console) or not. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 06:29:39 UTC (rev 269) +++ s3cmd/trunk/ChangeLog 2008-11-25 06:35:03 UTC (rev 270) @@ -1,5 +1,10 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd, s3cmd.1, S3/S3.py: Display or don't display progress meter + default depends on whether we're on TTY (console) or not. + +2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd: Fixed 'get' conflict. * s3cmd.1, TODO: Document 'mv' command. Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-25 06:29:39 UTC (rev 269) +++ s3cmd/trunk/S3/S3.py 2008-11-25 06:35:03 UTC (rev 270) @@ -569,9 +569,15 @@ raise S3DownloadError("Download failed for: %s" % resource['uri']) stream.flush() - progress.done("done") timestamp_end = time.time() + if self.config.progress_meter: + ## The above stream.flush() may take some time -> update() progress meter + ## to correct the average speed. Otherwise people will complain that + ## 'progress' and response["speed"] are inconsistent ;-) + progress.update() + progress.done("done") + if start_position == 0: # Only compute MD5 on the fly if we were downloading from the beginning response["md5"] = md5_hash.hexdigest() Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-25 06:29:39 UTC (rev 269) +++ s3cmd/trunk/s3cmd 2008-11-25 06:35:03 UTC (rev 270) @@ -996,8 +996,8 @@ optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.") - optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default).") - optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter.") + optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).") + optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).") optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.") optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.") optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version)) @@ -1044,6 +1044,10 @@ cfg.verbosity = options.verbosity logging.root.setLevel(cfg.verbosity) + ## Default to --progress on TTY devices, --no-progress elsewhere + ## Can be overriden by actual --(no-)progress parameter + cfg.update_option('progress_meter', sys.stdout.isatty()) + ## Update Config with other parameters for option in cfg.option_list(): try: Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2008-11-25 06:29:39 UTC (rev 269) +++ s3cmd/trunk/s3cmd.1 2008-11-25 06:35:03 UTC (rev 270) @@ -146,6 +146,9 @@ .\"\fB\-u\fR, \fB\-\-show\-uri\fR .\"Show complete S3 URI in listings. .TP +\fB\-\-progress\fR, \fB\-\-no\-progress\fR +Display or don't display progress meter. When running on TTY (e.g. console or xterm) the default is to display progress meter. If not on TTY (e.g. output is redirected somewhere or running from cron) the default is to not display progress meter. +.TP \fB\-v\fR, \fB\-\-verbose\fR Enable verbose output. .TP This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-25 08:19:24
|
Revision: 271 http://s3tools.svn.sourceforge.net/s3tools/?rev=271&view=rev Author: ludvigm Date: 2008-11-25 08:19:16 +0000 (Tue, 25 Nov 2008) Log Message: ----------- * Released version 0.9.9-pre2 ------------------------ * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2 * NEWS: Added 0.9.9-pre2 Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/PkgInfo.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 06:35:03 UTC (rev 270) +++ s3cmd/trunk/ChangeLog 2008-11-25 08:19:16 UTC (rev 271) @@ -1,5 +1,13 @@ 2008-11-24 Michal Ludvig <mi...@lo...> + * Released version 0.9.9-pre2 + ------------------------ + + * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2 + * NEWS: Added 0.9.9-pre2 + +2008-11-24 Michal Ludvig <mi...@lo...> + * s3cmd, s3cmd.1, S3/S3.py: Display or don't display progress meter default depends on whether we're on TTY (console) or not. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-25 06:35:03 UTC (rev 270) +++ s3cmd/trunk/NEWS 2008-11-25 08:19:16 UTC (rev 271) @@ -1,5 +1,5 @@ -s3cmd 0.9.9 - ??? -=========== +s3cmd 0.9.9-pre2 +================ * Implemented progress meter (--progress / --no-progress) * Removing of non-empty buckets with --force * Recursively remove objects from buckets with a given Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2008-11-25 06:35:03 UTC (rev 270) +++ s3cmd/trunk/S3/PkgInfo.py 2008-11-25 08:19:16 UTC (rev 271) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "0.9.9-pre1" +version = "0.9.9-pre2" url = "http://s3tools.logix.cz" license = "GPL version 2" short_description = "S3cmd is a tool for managing Amazon S3 storage space." This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-26 09:52:43
|
Revision: 272 http://s3tools.svn.sourceforge.net/s3tools/?rev=272&view=rev Author: ludvigm Date: 2008-11-26 09:52:40 +0000 (Wed, 26 Nov 2008) Log Message: ----------- * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2+svn Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/PkgInfo.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-25 08:19:16 UTC (rev 271) +++ s3cmd/trunk/ChangeLog 2008-11-26 09:52:40 UTC (rev 272) @@ -1,3 +1,7 @@ +2008-11-25 Michal Ludvig <mi...@lo...> + + * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2+svn + 2008-11-24 Michal Ludvig <mi...@lo...> * Released version 0.9.9-pre2 Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2008-11-25 08:19:16 UTC (rev 271) +++ s3cmd/trunk/S3/PkgInfo.py 2008-11-26 09:52:40 UTC (rev 272) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "0.9.9-pre2" +version = "0.9.9-pre2+svn" url = "http://s3tools.logix.cz" license = "GPL version 2" short_description = "S3cmd is a tool for managing Amazon S3 storage space." This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-26 10:10:52
|
Revision: 273 http://s3tools.svn.sourceforge.net/s3tools/?rev=273&view=rev Author: ludvigm Date: 2008-11-26 10:10:49 +0000 (Wed, 26 Nov 2008) Log Message: ----------- * upload-to-sf.sh: Helper script. Modified Paths: -------------- s3cmd/trunk/ChangeLog Added Paths: ----------- s3cmd/trunk/upload-to-sf.sh Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-26 09:52:40 UTC (rev 272) +++ s3cmd/trunk/ChangeLog 2008-11-26 10:10:49 UTC (rev 273) @@ -1,5 +1,6 @@ 2008-11-25 Michal Ludvig <mi...@lo...> + * upload-to-sf.sh: Helper script. * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2+svn 2008-11-24 Michal Ludvig <mi...@lo...> Added: s3cmd/trunk/upload-to-sf.sh =================================================================== --- s3cmd/trunk/upload-to-sf.sh (rev 0) +++ s3cmd/trunk/upload-to-sf.sh 2008-11-26 10:10:49 UTC (rev 273) @@ -0,0 +1,5 @@ +#!/bin/sh + +VERSION=$(./s3cmd --version | awk '{print $NF}') +echo -e "Uploading \033[32ms3cmd \033[31m${VERSION}\033[0m ..." +rsync -avP dist/s3cmd-${VERSION}.* lu...@fr...:uploads/ Property changes on: s3cmd/trunk/upload-to-sf.sh ___________________________________________________________________ Added: svn:executable + * This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-26 10:21:17
|
Revision: 274 http://s3tools.svn.sourceforge.net/s3tools/?rev=274&view=rev Author: ludvigm Date: 2008-11-26 10:21:15 +0000 (Wed, 26 Nov 2008) Log Message: ----------- * s3cmd, S3/Progress.py: Fixed Unicode output in Progress meter. * s3cmd: Fixed 'del --recursive' without prefix (i.e. all objects). * TODO: Updated list. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Progress.py s3cmd/trunk/TODO s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-26 10:10:49 UTC (rev 273) +++ s3cmd/trunk/ChangeLog 2008-11-26 10:21:15 UTC (rev 274) @@ -1,5 +1,8 @@ 2008-11-25 Michal Ludvig <mi...@lo...> + * s3cmd, S3/Progress.py: Fixed Unicode output in Progress meter. + * s3cmd: Fixed 'del --recursive' without prefix (i.e. all objects). + * TODO: Updated list. * upload-to-sf.sh: Helper script. * S3/PkgInfo.py: Bumped up version to 0.9.9-pre2+svn Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-11-26 10:10:49 UTC (rev 273) +++ s3cmd/trunk/S3/Progress.py 2008-11-26 10:21:15 UTC (rev 274) @@ -8,6 +8,8 @@ from Utils import formatSize class Progress(object): + _stdout = sys.stdout + def __init__(self, label, total_size): self.new_file(label, total_size) @@ -48,8 +50,8 @@ Override this method to provide a nicer output. """ if new_file: - sys.stdout.write("%s " % self.label[:30].ljust(30)) - sys.stdout.flush() + self._stdout.write("%s " % self.label[:30].ljust(30)) + self._stdout.flush() self.last_milestone = 0 return @@ -59,16 +61,16 @@ timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) - sys.stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % + self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) - sys.stdout.flush() + self._stdout.flush() return rel_position = selfself.current_position * 100 / self.total_size if rel_position >= self.last_milestone: self.last_milestone = (int(rel_position) / 5) * 5 - sys.stdout.write("%d%% ", self.last_milestone) - sys.stdout.flush() + self._stdout.write("%d%% ", self.last_milestone) + self._stdout.flush() return class ProgressANSI(Progress): @@ -86,10 +88,10 @@ display(new_file = False[/True], done_message = None) """ if new_file: - sys.stdout.write("%s " % self.label[:30].ljust(30)) - #sys.stdout.write(self.ANSI_hide_cursor) - sys.stdout.write(self.ANSI_save_cursor_pos) - sys.stdout.flush() + self._stdout.write("%s " % self.label[-30:].ljust(30)) + #self._stdout.write(self.ANSI_hide_cursor) + self._stdout.write(self.ANSI_save_cursor_pos) + self._stdout.flush() return timedelta = self.time_current - self.time_start @@ -98,9 +100,9 @@ print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) else: print_speed = (0, "") - sys.stdout.write(self.ANSI_restore_cursor_pos) - sys.stdout.write(self.ANSI_erase_to_eol) - sys.stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { + self._stdout.write(self.ANSI_restore_cursor_pos) + self._stdout.write(self.ANSI_erase_to_eol) + self._stdout.write("%(current)s of %(total)s %(percent)3d%% in %(elapsed)ds %(speed).2f %(speed_coeff)sB/s" % { "current" : str(self.current_position).rjust(len(str(self.total_size))), "total" : self.total_size, "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0, @@ -110,6 +112,6 @@ }) if done_message: - sys.stdout.write(" %s\n" % done_message) + self._stdout.write(" %s\n" % done_message) - sys.stdout.flush() + self._stdout.flush() Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-11-26 10:10:49 UTC (rev 273) +++ s3cmd/trunk/TODO 2008-11-26 10:21:15 UTC (rev 274) @@ -4,6 +4,8 @@ - For 0.9.9 - Add --include/--include-from/--rinclude* for sync - Recursive processing / multiple sources with most commands. + - Add 'setacl' command. + - Add commands for CloudFront. - Document --recursive and --force for buckets - Allow change /tmp to somewhere else Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-26 10:10:49 UTC (rev 273) +++ s3cmd/trunk/s3cmd 2008-11-26 10:21:15 UTC (rev 274) @@ -115,7 +115,7 @@ bucket = uri.bucket() object = uri.object() - output("Bucket 's3://%s':" % bucket) + debug("Bucket 's3://%s':" % bucket) if object.endswith('*'): object = object[:-1] try: @@ -295,9 +295,13 @@ while (len(args)): uri_arg = args.pop(0) uri = S3Uri(uri_arg) - if uri.type != "s3" or not uri.has_object(): + if uri.type != "s3": raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) - + if not uri.has_object(): + if Config().recursive and not Config().force: + raise ParameterError("Please use --force to delete ALL contents of %s" % uri) + elif not Config().recursive: + raise ParameterError("Object name required, not only the bucket name") subcmd_object_del_uri(uri) def subcmd_object_del_uri(uri, recursive = None): @@ -1048,6 +1052,9 @@ ## Can be overriden by actual --(no-)progress parameter cfg.update_option('progress_meter', sys.stdout.isatty()) + ## We may need a way to display progress meter on STDERR or somewhere else + Progress._stdout = _stdout + ## Update Config with other parameters for option in cfg.option_list(): try: @@ -1150,6 +1157,7 @@ from S3 import Utils from S3.Exceptions import * from S3.Utils import unicodise + from S3.Progress import Progress main() sys.exit(0) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-28 23:59:24
|
Revision: 275 http://s3tools.svn.sourceforge.net/s3tools/?rev=275&view=rev Author: ludvigm Date: 2008-11-28 23:59:20 +0000 (Fri, 28 Nov 2008) Log Message: ----------- * s3cmd: Delete local files with "sync --delete-removed" Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-26 10:21:15 UTC (rev 274) +++ s3cmd/trunk/ChangeLog 2008-11-28 23:59:20 UTC (rev 275) @@ -1,3 +1,7 @@ +2008-11-29 Michal Ludvig <mi...@lo...> + + * s3cmd: Delete local files with "sync --delete-removed" + 2008-11-25 Michal Ludvig <mi...@lo...> * s3cmd, S3/Progress.py: Fixed Unicode output in Progress meter. Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-26 10:21:15 UTC (rev 274) +++ s3cmd/trunk/s3cmd 2008-11-28 23:59:20 UTC (rev 275) @@ -503,6 +503,10 @@ src_uri = S3Uri(src) dst_uri = S3Uri(dst) + src_base = src_uri.uri() + dst_base = dst_uri.path() + if not src_base[-1] == "/": src_base += "/" + rem_list = _get_filelist_remote(src_uri) rem_count = len(rem_list) @@ -517,8 +521,8 @@ for file in loc_list: if cfg.delete_removed: - # os.unlink(file) - output("deleted '%s'" % file) + os.unlink(dst_base + file) + output("deleted '%s'" % (dst_base + file)) else: output("not-deleted '%s'" % file) @@ -528,9 +532,6 @@ timestamp_start = time.time() seq = 0 dir_cache = {} - src_base = src_uri.uri() - dst_base = dst_uri.path() - if not src_base[-1] == "/": src_base += "/" file_list = rem_list.keys() file_list.sort() for file in file_list: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-29 23:02:21
|
Revision: 277 http://s3tools.svn.sourceforge.net/s3tools/?rev=277&view=rev Author: ludvigm Date: 2008-11-29 23:02:11 +0000 (Sat, 29 Nov 2008) Log Message: ----------- * S3/S3.py: object_get() -- make start_position argument optional. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-29 00:02:29 UTC (rev 276) +++ s3cmd/trunk/ChangeLog 2008-11-29 23:02:11 UTC (rev 277) @@ -1,3 +1,7 @@ +2008-11-30 Michal Ludvig <mi...@lo...> + + * S3/S3.py: object_get() -- make start_position argument optional. + 2008-11-29 Michal Ludvig <mi...@lo...> * s3cmd: Delete local files with "sync --delete-removed" Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-29 00:02:29 UTC (rev 276) +++ s3cmd/trunk/S3/S3.py 2008-11-29 23:02:11 UTC (rev 277) @@ -189,7 +189,7 @@ response = self.send_file(request, file) return response - def object_get(self, uri, stream, start_position): + def object_get(self, uri, stream, start_position = 0): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) request = self.create_request("OBJECT_GET", uri = uri) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-30 14:15:18
|
Revision: 278 http://s3tools.svn.sourceforge.net/s3tools/?rev=278&view=rev Author: ludvigm Date: 2008-11-30 14:15:13 +0000 (Sun, 30 Nov 2008) Log Message: ----------- * run-tests.py: Added a lot of new tests. * testsuite/etc/logo.png: New file. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Added Paths: ----------- s3cmd/trunk/testsuite/etc/ s3cmd/trunk/testsuite/etc/logo.png Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-29 23:02:11 UTC (rev 277) +++ s3cmd/trunk/ChangeLog 2008-11-30 14:15:13 UTC (rev 278) @@ -1,3 +1,8 @@ +2008-12-01 Michal Ludvig <mi...@lo...> + + * run-tests.py: Added a lot of new tests. + * testsuite/etc/logo.png: New file. + 2008-11-30 Michal Ludvig <mi...@lo...> * S3/S3.py: object_get() -- make start_position argument optional. Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-11-29 23:02:11 UTC (rev 277) +++ s3cmd/trunk/run-tests.py 2008-11-30 14:15:13 UTC (rev 278) @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding=utf-8 -*- ## Amazon S3cmd - testsuite ## Author: Michal Ludvig <mi...@lo...> @@ -17,7 +18,7 @@ global count_fail if message: message = " (%s)" % message - print "FAIL%s" % (message) + print "\x1b[31;1mFAIL%s\x1b[0m" % (message) count_fail += 1 print "----" print " ".join([arg.find(" ")>=0 and "'%s'" % arg or arg for arg in cmd_args]) @@ -29,7 +30,7 @@ global count_pass if message: message = " (%s)" % message - print "OK%s" % (message) + print "\x1b[32;1mOK\x1b[0m%s" % (message) count_pass += 1 return 0 def compile_list(_list, regexps = False): @@ -37,7 +38,7 @@ _list = [_list] if regexps == False: - _list = [re.escape(item) for item in _list] + _list = [re.escape(item.encode("utf-8")) for item in _list] return [re.compile(item) for item in _list] @@ -52,18 +53,25 @@ find_list = [] find_list.extend(compile_list(must_find)) find_list.extend(compile_list(must_find_re, regexps = True)) + find_list_patterns = [] + find_list_patterns.extend(must_find) + find_list_patterns.extend(must_find_re) + not_find_list = [] not_find_list.extend(compile_list(must_not_find)) not_find_list.extend(compile_list(must_not_find_re, regexps = True)) + not_find_list_patterns = [] + not_find_list_patterns.extend(must_not_find) + not_find_list_patterns.extend(must_not_find_re) - for pattern in find_list: - match = pattern.search(stdout) + for index in range(len(find_list)): + match = find_list[index].search(stdout) if not match: - return failure("pattern not found: %s" % match.group()) - for pattern in not_find_list: - match = pattern.search(stdout) + return failure("pattern not found: %s" % find_list_patterns[index]) + for index in range(len(not_find_list)): + match = not_find_list[index].search(stdout) if match: - return failure("pattern found: %s" % match.group()) + return failure("pattern found: %s (match: %s)" % (not_find_list_patterns[index], match.group(0))) return success() def test_s3cmd(label, cmd_args = [], **kwargs): @@ -71,23 +79,74 @@ cmd_args.insert(0, "./s3cmd") return test(label, cmd_args, **kwargs) -test_s3cmd("Remove test buckets", ['rb', '-r', 's3://s3cmd-autotest-1', 's3://s3cmd-autotest-2', 's3://s3cmd-autotest-3'], +test_s3cmd("Remove test buckets", ['rb', '-r', 's3://s3cmd-autotest-1', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed", "Bucket 's3://s3cmd-autotest-2/' removed", - "Bucket 's3://s3cmd-autotest-3/' removed" ]) - -test_s3cmd("Create one bucket", ['mb', 's3://s3cmd-autotest-1'], + "Bucket 's3://s3cmd-Autotest-3/' removed" ]) + +test_s3cmd("Create one bucket (EU)", ['mb', '--bucket-location=EU', 's3://s3cmd-autotest-1'], must_find = "Bucket 's3://s3cmd-autotest-1/' created") -test_s3cmd("Create multiple buckets", ['mb', 's3://s3cmd-autotest-2', 's3://s3cmd-autotest-3'], - must_find = [ "Bucket 's3://s3cmd-autotest-2/' created", "Bucket 's3://s3cmd-autotest-3/' created" ]) +test_s3cmd("Create multiple buckets", ['mb', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], + must_find = [ "Bucket 's3://s3cmd-autotest-2/' created", "Bucket 's3://s3cmd-Autotest-3/' created" ]) -test_s3cmd("Invalid bucket name", ["mb", "s3://s3cmd-Autotest-.-"], +test_s3cmd("Invalid bucket name", ["mb", "--bucket-location=EU", "s3://s3cmd-Autotest-EU"], retcode = 1, - must_find = "ERROR: Parameter problem: Bucket name", + must_find = "ERROR: Parameter problem: Bucket name 's3cmd-Autotest-EU' contains disallowed character", must_not_find_re = "Bucket.*created") test_s3cmd("Buckets list", ["ls"], - must_find = [ "autotest-1", "autotest-2", "autotest-3" ], must_not_find_re = "Autotest") + must_find = [ "autotest-1", "autotest-2", "Autotest-3" ], must_not_find_re = "Autotest-EU") -test_s3cmd("Sync with exclude", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*/thousands/*', '--no-encrypt']) +test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) + +test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1'], + must_find = [ u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ], + must_not_find = [ "logo.png" ]) + +# test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) + +test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], + must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) + +test("Removing local target", ['rm', '-rf', 'testsuite-out']) + +test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = [ "stored as testsuite-out/etc/logo.png ", u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) + +test("Retrieve public URL", ['wget', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'], + must_find_re = [ 'logo.png.*saved \[22059/22059\]' ]) + +test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) + +test_s3cmd("Rename within S3", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], + must_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) + +test_s3cmd("Rename (NoSuchKey)", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], + retcode = 1, + must_find_re = [ 'ERROR:.*NoSuchKey' ], + must_not_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) + +test_s3cmd("Sync more from S3", ['sync', '--delete-removed', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = [ "deleted 'testsuite-out/etc/logo.png'", "stored as testsuite-out/etc2/Logo.PNG (22059 bytes", + "stored as testsuite-out/.svn/format " ], + must_not_find = [ "not-deleted etc/logo.png" ]) + +test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3'], + must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/Logo.PNG" ]) + +test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], + must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG deleted" ]) + +test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], + must_find_re = [ "Object.*unicode/ŪņЇЌœđЗ/.*deleted" ]) + +test_s3cmd("Recursive delete all", ['del', '--recursive', '--force', 's3://s3cmd-autotest-1'], + must_find_re = [ "Object.*binary/random-crap deleted" ]) + +test_s3cmd("Remove empty bucket", ['rb', 's3://s3cmd-autotest-1'], + must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed" ]) + +test_s3cmd("Remove remaining buckets", ['rb', '--recursive', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], + must_find = [ "Bucket 's3://s3cmd-autotest-2/' removed", + "Bucket 's3://s3cmd-Autotest-3/' removed" ]) Added: s3cmd/trunk/testsuite/etc/logo.png =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/etc/logo.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-11-30 14:20:25
|
Revision: 279 http://s3tools.svn.sourceforge.net/s3tools/?rev=279&view=rev Author: ludvigm Date: 2008-11-30 14:20:20 +0000 (Sun, 30 Nov 2008) Log Message: ----------- * Released version 0.9.9-pre3 --------------------------- * S3/PkgInfo.py: Bumped up version to 0.9.9-pre3 Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/PkgInfo.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-30 14:15:13 UTC (rev 278) +++ s3cmd/trunk/ChangeLog 2008-11-30 14:20:20 UTC (rev 279) @@ -1,5 +1,12 @@ 2008-12-01 Michal Ludvig <mi...@lo...> + * Released version 0.9.9-pre3 + --------------------------- + + * S3/PkgInfo.py: Bumped up version to 0.9.9-pre3 + +2008-12-01 Michal Ludvig <mi...@lo...> + * run-tests.py: Added a lot of new tests. * testsuite/etc/logo.png: New file. Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2008-11-30 14:15:13 UTC (rev 278) +++ s3cmd/trunk/S3/PkgInfo.py 2008-11-30 14:20:20 UTC (rev 279) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "0.9.9-pre2+svn" +version = "0.9.9-pre3" url = "http://s3tools.logix.cz" license = "GPL version 2" short_description = "S3cmd is a tool for managing Amazon S3 storage space." This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-01 13:07:43
|
Revision: 280 http://s3tools.svn.sourceforge.net/s3tools/?rev=280&view=rev Author: ludvigm Date: 2008-12-01 13:07:32 +0000 (Mon, 01 Dec 2008) Log Message: ----------- * s3cmd, S3/S3.py, NEWS: Support for (non-)recursive 'ls' Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-11-30 14:20:20 UTC (rev 279) +++ s3cmd/trunk/ChangeLog 2008-12-01 13:07:32 UTC (rev 280) @@ -1,3 +1,7 @@ +2008-12-02 Michal Ludvig <mi...@lo...> + + * s3cmd, S3/S3.py, NEWS: Support for (non-)recursive 'ls' + 2008-12-01 Michal Ludvig <mi...@lo...> * Released version 0.9.9-pre3 Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-11-30 14:20:20 UTC (rev 279) +++ s3cmd/trunk/NEWS 2008-12-01 13:07:32 UTC (rev 280) @@ -1,3 +1,13 @@ +s3cmd 0.9.9-pre4 +================ +* Support for non-recursive 'ls' + +s3cmd 0.9.9-pre3 +================ +* Bugfixes only + - Fixed sync from S3 to local + - Fixed progress meter with Unicode chars + s3cmd 0.9.9-pre2 ================ * Implemented progress meter (--progress / --no-progress) Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-11-30 14:20:20 UTC (rev 279) +++ s3cmd/trunk/S3/S3.py 2008-12-01 13:07:32 UTC (rev 280) @@ -114,20 +114,28 @@ def _get_contents(data): return getListFromXml(data, "Contents") - prefix = self.urlencode_string(prefix) - request = self.create_request("BUCKET_LIST", bucket = bucket, prefix = prefix) + def _get_common_prefixes(data): + return getListFromXml(data, "CommonPrefixes") + + uri_params = {} + if prefix: + uri_params['prefix'] = self.urlencode_string(prefix) + if not self.config.recursive: + uri_params['delimiter'] = "/" + request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) response = self.send_request(request) #debug(response) list = _get_contents(response["data"]) + prefixes = _get_common_prefixes(response["data"]) while _list_truncated(response["data"]): - marker = list[-1]["Key"] + uri_params['marker'] = self.urlencode_string(list[-1]["Key"]) debug("Listing continues after '%s'" % marker) - request = self.create_request("BUCKET_LIST", bucket = bucket, - prefix = prefix, - marker = self.urlencode_string(marker)) + request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) response = self.send_request(request) list += _get_contents(response["data"]) + prefixes += _get_common_prefixes(response["data"]) response['list'] = list + response['common_prefixes'] = prefixes return response def bucket_create(self, bucket, bucket_location = None): Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-11-30 14:20:20 UTC (rev 279) +++ s3cmd/trunk/s3cmd 2008-12-01 13:07:32 UTC (rev 280) @@ -113,19 +113,25 @@ def subcmd_bucket_list(s3, uri): bucket = uri.bucket() - object = uri.object() + prefix = uri.object() debug("Bucket 's3://%s':" % bucket) - if object.endswith('*'): - object = object[:-1] + if prefix.endswith('*'): + prefix = prefix[:-1] try: - response = s3.bucket_list(bucket, prefix = object) + response = s3.bucket_list(bucket, prefix = prefix) except S3Error, e: if S3.codes.has_key(e.info["Code"]): error(S3.codes[e.info["Code"]] % bucket) return else: raise + + for prefix in response['common_prefixes']: + output("%s %s" % ( + "D".rjust(28), + uri.compose_uri(bucket, prefix["Prefix"]))) + for object in response["list"]: size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) output("%s %s%s %s" % ( This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-01 13:57:45
|
Revision: 281 http://s3tools.svn.sourceforge.net/s3tools/?rev=281&view=rev Author: ludvigm Date: 2008-12-01 13:57:41 +0000 (Mon, 01 Dec 2008) Log Message: ----------- Fixes for non-recursive 'ls' support from previous commit. Modified Paths: -------------- s3cmd/trunk/S3/S3.py s3cmd/trunk/run-tests.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-12-01 13:07:32 UTC (rev 280) +++ s3cmd/trunk/S3/S3.py 2008-12-01 13:57:41 UTC (rev 281) @@ -105,7 +105,7 @@ response["list"] = getListFromXml(response["data"], "Bucket") return response - def bucket_list(self, bucket, prefix = None): + def bucket_list(self, bucket, prefix = None, recursive = None): def _list_truncated(data): ## <IsTruncated> can either be "true" or "false" or be missing completely is_truncated = getTextFromXml(data, ".//IsTruncated") or "false" @@ -120,7 +120,7 @@ uri_params = {} if prefix: uri_params['prefix'] = self.urlencode_string(prefix) - if not self.config.recursive: + if not self.config.recursive and not recursive: uri_params['delimiter'] = "/" request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) response = self.send_request(request) @@ -129,7 +129,7 @@ prefixes = _get_common_prefixes(response["data"]) while _list_truncated(response["data"]): uri_params['marker'] = self.urlencode_string(list[-1]["Key"]) - debug("Listing continues after '%s'" % marker) + debug("Listing continues after '%s'" % uri_params['marker']) request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params) response = self.send_request(request) list += _get_contents(response["data"]) Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-01 13:07:32 UTC (rev 280) +++ s3cmd/trunk/run-tests.py 2008-12-01 13:57:41 UTC (rev 281) @@ -100,10 +100,14 @@ test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) -test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1'], +test_s3cmd("Check bucket content (-r)", ['ls', '--recursive', 's3://s3cmd-autotest-1'], must_find = [ u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ], must_not_find = [ "logo.png" ]) +test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], + must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/unicode/$" ], + must_not_find = [ u"ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) + # test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-01 13:07:32 UTC (rev 280) +++ s3cmd/trunk/s3cmd 2008-12-01 13:57:41 UTC (rev 281) @@ -68,7 +68,7 @@ if object.endswith('*'): object = object[:-1] try: - response = s3.bucket_list(bucket, prefix = object) + response = s3.bucket_list(bucket, prefix = object, recursive = True) except S3Error, e: if S3.codes.has_key(e.Code): error(S3.codes[e.Code] % bucket) @@ -420,7 +420,7 @@ info("Retrieving list of remote files...") s3 = S3(Config()) - response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object()) + response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = True) rem_base = remote_uri.object() rem_base_len = len(rem_base) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-10 02:21:29
|
Revision: 282 http://s3tools.svn.sourceforge.net/s3tools/?rev=282&view=rev Author: ludvigm Date: 2008-12-10 02:21:25 +0000 (Wed, 10 Dec 2008) Log Message: ----------- * s3cmd: Updated email address for reporting bugs. * TODO: Updated list. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/TODO s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-01 13:57:41 UTC (rev 281) +++ s3cmd/trunk/ChangeLog 2008-12-10 02:21:25 UTC (rev 282) @@ -1,3 +1,8 @@ +2008-12-10 Michal Ludvig <mi...@lo...> + + * s3cmd: Updated email address for reporting bugs. + * TODO: Updated list. + 2008-12-02 Michal Ludvig <mi...@lo...> * s3cmd, S3/S3.py, NEWS: Support for (non-)recursive 'ls' Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-12-01 13:57:41 UTC (rev 281) +++ s3cmd/trunk/TODO 2008-12-10 02:21:25 UTC (rev 282) @@ -11,13 +11,11 @@ - After 1.0.0 - Speed up upload / download with multiple threads. + - Sync should be able to update metadata (UID, timstamps, etc) + if only these change (i.e. same content, different metainfo). + - Sync must backup non-files as well. At least directories, + symlinks and device nodes. -- Treat objects with "/" in their name as directories - - Will need local cache for bucket listings - - More user friendly 'del' operation that would work - with "directories" - - Recursion for some commands - - Implement GPG for sync (it's not that easy since it won't be easy to compare the encrypted-remote-object size with local file. @@ -26,6 +24,9 @@ the metadata encrypted in each object header where we'll have to do large number for object/HEAD requests. tough call). + Or we can only compare local timestamps with remote object + timestamps. If the local one is older we'll *assume* it + hasn't been changed. - Keep man page up to date and write some more documentation - Yeah, right ;-) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-01 13:57:41 UTC (rev 281) +++ s3cmd/trunk/s3cmd 2008-12-10 02:21:25 UTC (rev 282) @@ -1176,7 +1176,7 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! An unexpected error has occurred. Please report the following lines to: - s3t...@li... + s3t...@li... !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """) @@ -1188,7 +1188,7 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! An unexpected error has occurred. Please report the above lines to: - s3t...@li... + s3t...@li... !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """) sys.exit(1) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-10 05:05:21
|
Revision: 283 http://s3tools.svn.sourceforge.net/s3tools/?rev=283&view=rev Author: ludvigm Date: 2008-12-10 05:05:16 +0000 (Wed, 10 Dec 2008) Log Message: ----------- * s3cmd: Don't display download/upload completed message in --progress mode. * S3/S3.py: Pass src/dst names down to Progress class. * S3/Progress.py: added new class ProgressCR - apparently ProgressANSI doesn't work on MacOS-X (and perhaps elsewhere). * S3/Config.py: Default progress meter is now ProgressCR * s3cmd: Updated email address for reporting bugs. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/Progress.py s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-10 02:21:25 UTC (rev 282) +++ s3cmd/trunk/ChangeLog 2008-12-10 05:05:16 UTC (rev 283) @@ -1,5 +1,11 @@ 2008-12-10 Michal Ludvig <mi...@lo...> + * s3cmd: Don't display download/upload completed message + in --progress mode. + * S3/S3.py: Pass src/dst names down to Progress class. + * S3/Progress.py: added new class ProgressCR - apparently + ProgressANSI doesn't work on MacOS-X (and perhaps elsewhere). + * S3/Config.py: Default progress meter is now ProgressCR * s3cmd: Updated email address for reporting bugs. * TODO: Updated list. Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-12-10 02:21:25 UTC (rev 282) +++ s3cmd/trunk/S3/Config.py 2008-12-10 05:05:16 UTC (rev 283) @@ -19,7 +19,7 @@ simpledb_host = "sdb.amazonaws.com" verbosity = logging.WARNING progress_meter = True - progress_class = Progress.ProgressANSI + progress_class = Progress.ProgressCR send_chunk = 4096 recv_chunk = 4096 human_readable_sizes = False Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-12-10 02:21:25 UTC (rev 282) +++ s3cmd/trunk/S3/Progress.py 2008-12-10 05:05:16 UTC (rev 283) @@ -10,11 +10,11 @@ class Progress(object): _stdout = sys.stdout - def __init__(self, label, total_size): - self.new_file(label, total_size) + def __init__(self, labels, total_size): + self.new_file(labels, total_size) - def new_file(self, label, total_size): - self.label = label + def new_file(self, labels, total_size): + self.labels = labels self.total_size = total_size # Set initial_position to something in the # case we're not counting from 0. For instance @@ -43,6 +43,10 @@ def done(self, message): self.display(done_message = message) + def output_labels(self): + self._stdout.write("%s -> %s\n" % (self.labels['source'], self.labels['destination'])) + self._stdout.flush() + def display(self, new_file = False, done_message = None): """ display(new_file = False[/True], done = False[/True]) @@ -50,8 +54,7 @@ Override this method to provide a nicer output. """ if new_file: - self._stdout.write("%s " % self.label[:30].ljust(30)) - self._stdout.flush() + self.output_labels() self.last_milestone = 0 return @@ -82,14 +85,14 @@ ANSI_restore_cursor_pos = SCI + "u" ANSI_move_cursor_to_column = SCI + "%uG" ANSI_erase_to_eol = SCI + "0K" + ANSI_erase_current_line = SCI + "2K" def display(self, new_file = False, done_message = None): """ display(new_file = False[/True], done_message = None) """ if new_file: - self._stdout.write("%s " % self.label[-30:].ljust(30)) - #self._stdout.write(self.ANSI_hide_cursor) + self.output_labels() self._stdout.write(self.ANSI_save_cursor_pos) self._stdout.flush() return @@ -115,3 +118,36 @@ self._stdout.write(" %s\n" % done_message) self._stdout.flush() + +class ProgressCR(Progress): + ## Uses CR char (Carriage Return) just like other progress bars do. + CR_char = chr(13) + + def display(self, new_file = False, done_message = None): + """ + display(new_file = False[/True], done_message = None) + """ + if new_file: + self.output_labels() + return + + timedelta = self.time_current - self.time_start + sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 + if (sec_elapsed > 0): + print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + else: + print_speed = (0, "") + self._stdout.write(self.CR_char) + output = " %(current)s of %(total)s %(percent)3d%% in %(elapsed)4ds %(speed)7.2f %(speed_coeff)sB/s" % { + "current" : str(self.current_position).rjust(len(str(self.total_size))), + "total" : self.total_size, + "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0, + "elapsed" : sec_elapsed, + "speed" : print_speed[0], + "speed_coeff" : print_speed[1] + } + self._stdout.write(output) + if done_message: + self._stdout.write(" %s\n" % done_message) + + self._stdout.flush() Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-12-10 02:21:25 UTC (rev 282) +++ s3cmd/trunk/S3/S3.py 2008-12-10 05:05:16 UTC (rev 283) @@ -194,14 +194,16 @@ if self.config.acl_public: headers["x-amz-acl"] = "public-read" request = self.create_request("OBJECT_PUT", uri = uri, headers = headers) - response = self.send_file(request, file) + labels = { 'source' : file.name, 'destination' : uri } + response = self.send_file(request, file, labels) return response def object_get(self, uri, stream, start_position = 0): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) request = self.create_request("OBJECT_GET", uri = uri) - response = self.recv_file(request, stream, start_position) + labels = { 'source' : uri, 'destination' : stream.name } + response = self.recv_file(request, stream, labels, start_position) return response def object_delete(self, uri): @@ -391,11 +393,11 @@ return response - def send_file(self, request, file, throttle = 0, retries = _max_retries): + def send_file(self, request, file, labels, throttle = 0, retries = _max_retries): method_string, resource, headers = request size_left = size_total = headers.get("content-length") if self.config.progress_meter: - progress = self.config.progress_class(file.name, size_total) + progress = self.config.progress_class(labels, size_total) else: info("Sending file '%s', please wait..." % file.name) timestamp_start = time.time() @@ -414,7 +416,7 @@ warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) # Connection error -> same throttle value - return self.send_file(request, file, throttle, retries - 1) + return self.send_file(request, file, labels, throttle, retries - 1) else: raise S3UploadError("Upload failed for: %s" % resource['uri']) file.seek(0) @@ -449,7 +451,7 @@ warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) # Connection error -> same throttle value - return self.send_file(request, file, throttle, retries - 1) + return self.send_file(request, file, labels, throttle, retries - 1) else: debug("Giving up on '%s' %s" % (file.name, e)) raise S3UploadError("Upload failed for: %s" % resource['uri']) @@ -471,7 +473,7 @@ redir_hostname = getTextFromXml(response['data'], ".//Endpoint") self.set_hostname(redir_bucket, redir_hostname) warning("Redirected to: %s" % (redir_hostname)) - return self.send_file(request, file) + return self.send_file(request, file, labels) # S3 from time to time doesn't send ETag back in a response :-( # Force re-upload here. @@ -483,7 +485,7 @@ warning("MD5 Sums don't match!") if retries: warning("Retrying upload of %s" % (file.name)) - return self.send_file(request, file, throttle, retries - 1) + return self.send_file(request, file, labels, throttle, retries - 1) else: warning("Too many failures. Giving up on '%s'" % (file.name)) raise S3UploadError @@ -492,10 +494,10 @@ raise S3Error(response) return response - def recv_file(self, request, stream, start_position = 0, retries = _max_retries): + def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries): method_string, resource, headers = request if self.config.progress_meter: - progress = self.config.progress_class(stream.name, 0) + progress = self.config.progress_class(labels, 0) else: info("Receiving file '%s', please wait..." % stream.name) timestamp_start = time.time() @@ -523,7 +525,7 @@ warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) # Connection error -> same throttle value - return self.recv_file(request, stream, start_position, retries - 1) + return self.recv_file(request, stream, labels, start_position, retries - 1) else: raise S3DownloadError("Download failed for: %s" % resource['uri']) @@ -534,7 +536,7 @@ redir_hostname = getTextFromXml(response['data'], ".//Endpoint") self.set_hostname(redir_bucket, redir_hostname) warning("Redirected to: %s" % (redir_hostname)) - return self.recv_file(request, stream) + return self.recv_file(request, stream, labels) if response["status"] < 200 or response["status"] > 299: raise S3Error(response) @@ -572,7 +574,7 @@ warning("Waiting %d sec..." % self._fail_wait(retries)) time.sleep(self._fail_wait(retries)) # Connection error -> same throttle value - return self.recv_file(request, stream, current_position, retries - 1) + return self.recv_file(request, stream, labels, current_position, retries - 1) else: raise S3DownloadError("Download failed for: %s" % resource['uri']) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-10 02:21:25 UTC (rev 282) +++ s3cmd/trunk/s3cmd 2008-12-10 05:05:16 UTC (rev 283) @@ -219,9 +219,10 @@ warning("File can not be uploaded: %s" % e) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % - (file, uri_final, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total)) + if not Config().progress_meter: + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + (file, uri_final, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], + seq, total)) if Config().acl_public: output("Public URL of the object is: %s" % (uri_final.public_url())) @@ -292,9 +293,9 @@ if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"]) response["size"] = os.stat(destination)[6] - if destination != "-": + if not Config().progress_meter and destination != "-": speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output("Object %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % + output("File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1])) def cmd_object_del(args): @@ -603,9 +604,10 @@ error("%s: download failed too many times. Skipping that file." % file) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % - (uri, dst_file, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total_count)) + if not Config().progress_meter: + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + (uri, dst_file, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], + seq, total_count)) total_size += response["size"] total_elapsed = time.time() - timestamp_start @@ -702,9 +704,10 @@ warning("File can not be uploaded: %s" % e) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % - (src, uri, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total_count)) + if not cfg.progress_meter: + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + (src, uri, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], + seq, total_count)) total_size += response["size"] total_elapsed = time.time() - timestamp_start This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |