From: <lu...@us...> - 2008-12-10 05:06:32
|
Revision: 284 http://s3tools.svn.sourceforge.net/s3tools/?rev=284&view=rev Author: ludvigm Date: 2008-12-10 05:06:25 +0000 (Wed, 10 Dec 2008) Log Message: ----------- * TODO: Updated list. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/TODO Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-10 05:05:16 UTC (rev 283) +++ s3cmd/trunk/ChangeLog 2008-12-10 05:06:25 UTC (rev 284) @@ -1,5 +1,6 @@ 2008-12-10 Michal Ludvig <mi...@lo...> + * TODO: Updated list. * s3cmd: Don't display download/upload completed message in --progress mode. * S3/S3.py: Pass src/dst names down to Progress class. @@ -7,7 +8,6 @@ ProgressANSI doesn't work on MacOS-X (and perhaps elsewhere). * S3/Config.py: Default progress meter is now ProgressCR * s3cmd: Updated email address for reporting bugs. - * TODO: Updated list. 2008-12-02 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-12-10 05:05:16 UTC (rev 283) +++ s3cmd/trunk/TODO 2008-12-10 05:06:25 UTC (rev 284) @@ -4,9 +4,13 @@ - For 0.9.9 - Add --include/--include-from/--rinclude* for sync - Recursive processing / multiple sources with most commands. + - Incl. recursive cp/mv on remote "folders". + - Sanitize put/get behaviour. + For instance 'get s3://../blah/x.jpg' should save it to + x.jpg and not attempt blah/x.jpg. + - Document --recursive and --force for buckets - Add 'setacl' command. - Add commands for CloudFront. - - Document --recursive and --force for buckets - Allow change /tmp to somewhere else - After 1.0.0 @@ -15,6 +19,10 @@ if only these change (i.e. same content, different metainfo). - Sync must backup non-files as well. At least directories, symlinks and device nodes. + - Keep backup files remotely on put/sync-to if requested + (move the old 'object' to e.g. 'object~' and only then upload + the new one). Could be more advanced to keep, say, last 5 + copies, etc. - Implement GPG for sync (it's not that easy since it won't be easy to compare This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-11 06:18:03
|
Revision: 285 http://s3tools.svn.sourceforge.net/s3tools/?rev=285&view=rev Author: ludvigm Date: 2008-12-11 06:17:53 +0000 (Thu, 11 Dec 2008) Log Message: ----------- * s3cmd: Support for multiple sources in 'get' command. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3Uri.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-10 05:06:25 UTC (rev 284) +++ s3cmd/trunk/ChangeLog 2008-12-11 06:17:53 UTC (rev 285) @@ -1,3 +1,7 @@ +2008-12-11 Michal Ludvig <mi...@lo...> + + * s3cmd: Support for multiple sources in 'get' command. + 2008-12-10 Michal Ludvig <mi...@lo...> * TODO: Updated list. Modified: s3cmd/trunk/S3/S3Uri.py =================================================================== --- s3cmd/trunk/S3/S3Uri.py 2008-12-10 05:06:25 UTC (rev 284) +++ s3cmd/trunk/S3/S3Uri.py 2008-12-11 06:17:53 UTC (rev 285) @@ -42,6 +42,9 @@ def public_url(self): raise ValueError("This S3 URI does not have Anonymous URL representation") + def basename(self): + return self.__unicode__().split("/")[-1] + class S3UriS3(S3Uri): type = "s3" _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-10 05:06:25 UTC (rev 284) +++ s3cmd/trunk/s3cmd 2008-12-11 06:17:53 UTC (rev 285) @@ -231,43 +231,87 @@ os.remove(real_filename) def cmd_object_get(args): - s3 = S3(Config()) + cfg = Config() + s3 = S3(cfg) - if not S3Uri(args[0]).type == 's3': - raise ParameterError("Expecting S3 URI instead of '%s'" % args[0]) + ## Check arguments: + ## if not --recursive: + ## - first N arguments must be S3Uri + ## - if the last one is S3 make current dir the destination_base + ## - if the last one is a directory: + ## - take all 'basenames' of the remote objects and + ## make the destination name be 'destination_base'+'basename' + ## - if the last one is a file or not existing: + ## - if the number of sources (N, above) == 1 treat it + ## as a filename and save the object there. + ## - if there's more sources -> Error + ## if --recursive: + ## - first N arguments must be S3Uri + ## - for each Uri get a list of remote objects with that Uri as a prefix + ## - apply exclude/include rules + ## - each list item will have MD5sum, Timestamp and pointer to S3Uri + ## used as a prefix. + ## - the last arg may be a local directory - destination_base + ## - if the last one is S3 make current dir the destination_base + ## - if the last one doesn't exist check remote list: + ## - if there is only one item and its_prefix==its_name + ## download that item to the name given in last arg. + ## - if there are more remote items use the last arg as a destination_base + ## and try to create the directory (incl. all parents). + ## + ## In both cases we end up with a list mapping remote object names (keys) to local file names. - destination_dir = None - destination_file = None - if len(args) > 1: - if S3Uri(args[-1]).type == 's3': - # all S3, use object names to local dir - check_args_type(args, type="s3", verbose_type="S3 URI") # May raise ParameterError + ## Each item will contain the following attributes + # 'remote_uri' 'local_filename' 'remote_label' 'local_label' + download_list = [ ] + + remote_uris = [] + + if len(args) == 0: + raise ParameterError("Nothing to download. Expecting S3 URI.") + + if S3Uri(args[-1]).type != 's3': + destination_base = args.pop() + else: + destination_base = "." + + if len(args) == 0: + raise ParameterError("Nothing to download. Expecting S3 URI.") + + for arg in args: + uri = S3Uri(arg) + if not uri.type == 's3': + raise ParameterError("Expecting S3 URI instead of '%s'" % arg) + remote_uris.append(uri) + + if cfg.recursive: + raise NotImplementedError("Recursive get is not yet implemented") + else: + remote_keys = [] + if not os.path.isdir(destination_base) or destination_base == '-': + if len(remote_uris) > 1: + raise ParameterError("Destination must be a directory when downloading multiple sources.") + download_item = { + 'remote_uri' : remote_uris[0], + 'local_filename' : destination_base + } + remote_keys.append(download_item) else: - if (len(args) > 2): - # last must be dir, all preceding S3 - if not os.path.isdir(args[-1]): - raise ParameterError("Last parameter must be a directory") - destination_dir = args.pop() - check_args_type(args, type="s3", verbose_type="S3 URI") # May raise ParameterError - else: - # last must be a dir or a filename - if os.path.isdir(args[-1]): - destination_dir = args.pop() - else: - destination_file = args.pop() + if os.path.isdir(destination_base) and destination_base[-1] != os.path.sep: + destination_base += os.path.sep + for uri in remote_uris: + download_item = { + 'remote_uri' : uri, + 'local_filename' : destination_base + uri.basename(), + } + remote_keys.append(download_item) - while (len(args)): - uri_arg = args.pop(0) - uri = S3Uri(uri_arg) + for item in remote_keys: + uri = item['remote_uri'] + destination = item['local_filename'] start_position = 0 - if destination_file: - destination = destination_file - elif destination_dir: - destination = destination_dir + "/" + uri.object() - else: - # By default the destination filename is the object name - destination = uri.object() + if destination == "-": ## stdout dst_stream = sys.stdout @@ -418,7 +462,7 @@ return loc_list def _get_filelist_remote(remote_uri): - info("Retrieving list of remote files...") + info("Retrieving list of remote files for %s ..." % remote_uri) s3 = S3(Config()) response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = True) @@ -430,12 +474,12 @@ key = object['Key'][rem_base_len:] rem_list[key] = { 'size' : int(object['Size']), - # 'mtime' : dateS3toUnix(object['LastModified']), ## That's upload time, not our lastmod time :-( + 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( 'md5' : object['ETag'][1:-1], - 'object_key' : object['Key'] + 'object_key' : object['Key'], } return rem_list - + def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote): info("Verifying checksums...") cfg = Config() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-12 00:31:48
|
Revision: 286 http://s3tools.svn.sourceforge.net/s3tools/?rev=286&view=rev Author: ludvigm Date: 2008-12-12 00:31:38 +0000 (Fri, 12 Dec 2008) Log Message: ----------- * s3cmd: Better Exception output. Print sys.path on ImportError, don't print backtrace on KeyboardInterrupt Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-11 06:17:53 UTC (rev 285) +++ s3cmd/trunk/ChangeLog 2008-12-12 00:31:38 UTC (rev 286) @@ -1,3 +1,8 @@ +2008-12-12 Michal Ludvig <mi...@lo...> + + * s3cmd: Better Exception output. Print sys.path on ImportError, + don't print backtrace on KeyboardInterrupt + 2008-12-11 Michal Ludvig <mi...@lo...> * s3cmd: Support for multiple sources in 'get' command. Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-11 06:17:53 UTC (rev 285) +++ s3cmd/trunk/s3cmd 2008-12-12 00:31:38 UTC (rev 286) @@ -1215,9 +1215,14 @@ main() sys.exit(0) + except SystemExit, e: sys.exit(e.code) + except KeyboardInterrupt: + sys.stderr.write("See ya!\n") + sys.exit(1) + except Exception, e: sys.stderr.write(""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -1227,10 +1232,23 @@ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """) - sys.stderr.write("S3cmd: %s\n" % PkgInfo.version) - sys.stderr.write("Python: %s\n" % sys.version.replace('\n', ' ')) + tb = traceback.format_exc(sys.exc_info()) + e_class = str(e.__class__) + e_class = e_class[e_class.rfind(".")+1 : -2] + sys.stderr.write("Problem: %s: %s\n" % (e_class, str(e))) + try: + sys.stderr.write("S3cmd: %s\n" % PkgInfo.version) + except NameError: + sys.stderr.write("S3cmd: unknown version. Module import problem?\n") + sys.stderr.write("Python: %s\n" % sys.version.replace('\n', ' ')) sys.stderr.write("\n") - sys.stderr.write(traceback.format_exc(sys.exc_info())+"\n") + sys.stderr.write(tb) + if type(e) == ImportError: + sys.stderr.write("\n") + sys.stderr.write("Your sys.path contains these entries:\n") + for path in sys.path: + sys.stderr.write("\t%s\n" % path) + sys.stderr.write("Now the question is where has S3/S3.py been installed?\n") sys.stderr.write(""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! An unexpected error has occurred. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-13 13:00:04
|
Revision: 288 http://s3tools.svn.sourceforge.net/s3tools/?rev=288&view=rev Author: ludvigm Date: 2008-12-13 13:00:01 +0000 (Sat, 13 Dec 2008) Log Message: ----------- * S3/Progress.py: Restructured import Utils to avoid import conflicts. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Progress.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-13 12:51:58 UTC (rev 287) +++ s3cmd/trunk/ChangeLog 2008-12-13 13:00:01 UTC (rev 288) @@ -1,3 +1,8 @@ +2008-12-14 Michal Ludvig <mi...@lo...> + + * S3/Progress.py: Restructured import Utils to avoid import + conflicts. + 2008-12-12 Michal Ludvig <mi...@lo...> * s3cmd: Better Exception output. Print sys.path on ImportError, Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-12-13 12:51:58 UTC (rev 287) +++ s3cmd/trunk/S3/Progress.py 2008-12-13 13:00:01 UTC (rev 288) @@ -5,7 +5,7 @@ import sys import datetime -from Utils import formatSize +import Utils class Progress(object): _stdout = sys.stdout @@ -59,11 +59,11 @@ return if self.current_position == self.total_size: - print_size = formatSize(self.current_position, True) + print_size = Utils.formatSize(self.current_position, True) if print_size[1] != "": print_size[1] += "B" timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 - print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) self._stdout.write("100%% %s%s in %.2fs (%.2f %sB/s)\n" % (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1])) self._stdout.flush() @@ -100,7 +100,7 @@ timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 if (sec_elapsed > 0): - print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) else: print_speed = (0, "") self._stdout.write(self.ANSI_restore_cursor_pos) @@ -134,7 +134,7 @@ timedelta = self.time_current - self.time_start sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0 if (sec_elapsed > 0): - print_speed = formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) + print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True) else: print_speed = (0, "") self._stdout.write(self.CR_char) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-17 04:19:55
|
Revision: 293 http://s3tools.svn.sourceforge.net/s3tools/?rev=293&view=rev Author: ludvigm Date: 2008-12-17 04:19:46 +0000 (Wed, 17 Dec 2008) Log Message: ----------- * TODO: Updated Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/TODO Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-15 10:08:45 UTC (rev 292) +++ s3cmd/trunk/ChangeLog 2008-12-17 04:19:46 UTC (rev 293) @@ -1,3 +1,7 @@ +2008-12-17 Michal Ludvig <mi...@lo...> + + * TODO: Updated + 2008-12-14 Michal Ludvig <mi...@lo...> * S3/Progress.py: Restructured import Utils to avoid import Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-12-15 10:08:45 UTC (rev 292) +++ s3cmd/trunk/TODO 2008-12-17 04:19:46 UTC (rev 293) @@ -2,16 +2,20 @@ =========================== - For 0.9.9 - - Add --include/--include-from/--rinclude* for sync - Recursive processing / multiple sources with most commands. - Incl. recursive cp/mv on remote "folders". - Sanitize put/get behaviour. For instance 'get s3://../blah/x.jpg' should save it to x.jpg and not attempt blah/x.jpg. + Similar thing, 'put /foo/bar/xyz.jpg s3://bucket/dome/path/' + should save it to s3://bucket/dome/path/xyz.jpg - Document --recursive and --force for buckets + - Allow change /tmp to somewhere else + +- For 1.0.0 + - Add --include/--include-from/--rinclude* for sync - Add 'setacl' command. - Add commands for CloudFront. - - Allow change /tmp to somewhere else - After 1.0.0 - Speed up upload / download with multiple threads. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-22 01:09:52
|
Revision: 294 http://s3tools.svn.sourceforge.net/s3tools/?rev=294&view=rev Author: ludvigm Date: 2008-12-22 01:09:44 +0000 (Mon, 22 Dec 2008) Log Message: ----------- * s3cmd, S3/Config.py: Implemented recursive [get]. Added --skip-existing option for [get] and [sync]. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Config.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-17 04:19:46 UTC (rev 293) +++ s3cmd/trunk/ChangeLog 2008-12-22 01:09:44 UTC (rev 294) @@ -1,3 +1,8 @@ +2008-12-22 Michal Ludvig <mi...@lo...> + + * s3cmd, S3/Config.py: Implemented recursive [get]. + Added --skip-existing option for [get] and [sync]. + 2008-12-17 Michal Ludvig <mi...@lo...> * TODO: Updated Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-12-17 04:19:46 UTC (rev 293) +++ s3cmd/trunk/S3/Config.py 2008-12-22 01:09:44 UTC (rev 294) @@ -25,6 +25,7 @@ human_readable_sizes = False force = False get_continue = False + skip_existing = False recursive = False acl_public = False proxy_host = "" Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-17 04:19:46 UTC (rev 293) +++ s3cmd/trunk/s3cmd 2008-12-22 01:09:44 UTC (rev 294) @@ -263,14 +263,14 @@ ## Each item will contain the following attributes # 'remote_uri' 'local_filename' 'remote_label' 'local_label' - download_list = [ ] + download_list = [] remote_uris = [] if len(args) == 0: raise ParameterError("Nothing to download. Expecting S3 URI.") - if S3Uri(args[-1]).type != 's3': + if S3Uri(args[-1]).type == 'file': destination_base = args.pop() else: destination_base = "." @@ -284,10 +284,26 @@ raise ParameterError("Expecting S3 URI instead of '%s'" % arg) remote_uris.append(uri) + remote_keys = [] if cfg.recursive: - raise NotImplementedError("Recursive get is not yet implemented") + if not os.path.isdir(destination_base): + raise ParameterError("Destination must be a directory for recursive get.") + if destination_base[-1] != os.path.sep: + destination_base += os.path.sep + for uri in remote_uris: + objectlist = _get_filelist_remote(uri) + for key in objectlist.iterkeys(): + object = S3Uri(objectlist[key]['object_uri_str']) + ## Remove leading '/' from remote filenames + if key.find("/") == 0: + key = key[1:] + destination = destination_base + key + download_item = { + 'remote_uri' : object, + 'local_filename' : destination + } + remote_keys.append(download_item) else: - remote_keys = [] if not os.path.isdir(destination_base) or destination_base == '-': if len(remote_uris) > 1: raise ParameterError("Destination must be a directory when downloading multiple sources.") @@ -319,7 +335,16 @@ ## File try: file_exists = os.path.exists(destination) - dst_stream = open(destination, "ab") + try: + dst_stream = open(destination, "ab") + except IOError, e: + if e.errno == errno.ENOENT: + basename = destination[:destination.rindex(os.path.sep)] + info("Creating directory: %s" % basename) + os.makedirs(basename) + dst_stream = open(destination, "ab") + else: + raise if file_exists: if Config().get_continue: start_position = dst_stream.tell() @@ -327,9 +352,12 @@ start_position = 0L dst_stream.seek(0L) dst_stream.truncate() + elif Config().skip_existing: + info("Skipping over existing file: %s" % (destination)) + continue else: dst_stream.close() - raise ParameterError("File %s already exists. Use either --force or --continue or give it a new name." % destination) + raise ParameterError("File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) except IOError, e: error("Skipping %s: %s" % (destination, e.strerror)) continue @@ -470,14 +498,27 @@ rem_base = remote_uri.object() rem_base_len = len(rem_base) rem_list = {} + break_now = False for object in response['list']: - key = object['Key'][rem_base_len:] + if object['Key'] == rem_base and object['Key'][-1] != os.path.sep: + ## We asked for one file and we got that file :-) + key = os.path.basename(object['Key']) + object_uri_str = remote_uri.uri() + break_now = True + rem_list = {} ## Remove whatever has already been put to rem_list + else: + key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! + object_uri_str = remote_uri.uri() + key rem_list[key] = { 'size' : int(object['Size']), 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( 'md5' : object['ETag'][1:-1], 'object_key' : object['Key'], + 'object_uri_str' : object_uri_str, + 'base_uri' : remote_uri, } + if break_now: + break return rem_list def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote): @@ -510,6 +551,15 @@ else: debug("PASS: %s" % (os.sep + file)) if dst_list.has_key(file): + ## Was --skip-existing requested? + if cfg.skip_existing: + debug("IGNR: %s (used --skip-existing)" % (file)) + exists_list[file] = src_list[file] + del(src_list[file]) + ## Remove from destination-list, all that is left there will be deleted + del(dst_list[file]) + continue + ## Check size first if dst_list[file]['size'] == src_list[file]['size']: #debug("%s same size: %s" % (file, dst_list[file]['size'])) @@ -1034,6 +1084,7 @@ optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.") optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.") optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).") + optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).") optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-22 01:54:16
|
Revision: 295 http://s3tools.svn.sourceforge.net/s3tools/?rev=295&view=rev Author: ludvigm Date: 2008-12-22 01:54:14 +0000 (Mon, 22 Dec 2008) Log Message: ----------- * s3cmd, S3/S3.py, S3/Progress.py: Display "[X of Y]" in --progress mode. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Progress.py s3cmd/trunk/S3/S3.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-22 01:09:44 UTC (rev 294) +++ s3cmd/trunk/ChangeLog 2008-12-22 01:54:14 UTC (rev 295) @@ -1,5 +1,7 @@ 2008-12-22 Michal Ludvig <mi...@lo...> + * s3cmd, S3/S3.py, S3/Progress.py: Display "[X of Y]" + in --progress mode. * s3cmd, S3/Config.py: Implemented recursive [get]. Added --skip-existing option for [get] and [sync]. Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-12-22 01:09:44 UTC (rev 294) +++ s3cmd/trunk/S3/Progress.py 2008-12-22 01:54:14 UTC (rev 295) @@ -44,7 +44,7 @@ self.display(done_message = message) def output_labels(self): - self._stdout.write("%s -> %s\n" % (self.labels['source'], self.labels['destination'])) + self._stdout.write("%(source)s -> %(destination)s %(extra)s\n" % self.labels) self._stdout.flush() def display(self, new_file = False, done_message = None): Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-12-22 01:09:44 UTC (rev 294) +++ s3cmd/trunk/S3/S3.py 2008-12-22 01:54:14 UTC (rev 295) @@ -167,7 +167,7 @@ response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any" return response - def object_put(self, filename, uri, extra_headers = None): + def object_put(self, filename, uri, extra_headers = None, extra_label = ""): # TODO TODO # Make it consistent with stream-oriented object_get() if uri.type != "s3": @@ -194,15 +194,15 @@ if self.config.acl_public: headers["x-amz-acl"] = "public-read" request = self.create_request("OBJECT_PUT", uri = uri, headers = headers) - labels = { 'source' : file.name, 'destination' : uri } + labels = { 'source' : file.name, 'destination' : uri, 'extra' : extra_label } response = self.send_file(request, file, labels) return response - def object_get(self, uri, stream, start_position = 0): + def object_get(self, uri, stream, start_position = 0, extra_label = ""): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) request = self.create_request("OBJECT_GET", uri = uri) - labels = { 'source' : uri, 'destination' : stream.name } + labels = { 'source' : uri, 'destination' : stream.name, 'extra' : extra_label } response = self.recv_file(request, stream, labels, start_position) return response Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-22 01:09:44 UTC (rev 294) +++ s3cmd/trunk/s3cmd 2008-12-22 01:54:14 UTC (rev 295) @@ -208,10 +208,11 @@ uri_final = S3Uri(uri_arg_final) extra_headers = {} real_filename = file + seq_label = "[%d of %d]" % (seq, total) if Config().encrypt: exitcode, real_filename, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(file) try: - response = s3.object_put(real_filename, uri_final, extra_headers) + response = s3.object_put(real_filename, uri_final, extra_headers, extra_label = seq_label) except S3UploadError, e: error("Upload of '%s' failed too many times. Skipping that file." % real_filename) continue @@ -220,9 +221,9 @@ continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not Config().progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (file, uri_final, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total)) + seq_label)) if Config().acl_public: output("Public URL of the object is: %s" % (uri_final.public_url())) @@ -639,6 +640,7 @@ seq += 1 uri = S3Uri(src_base + file) dst_file = dst_base + file + seq_label = "[%d of %d]" % (seq, total_count) try: dst_dir = os.path.dirname(dst_file) if not dir_cache.has_key(dst_dir): @@ -658,7 +660,7 @@ os.close(os.open(dst_file, open_flags)) # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode. dst_stream = open(dst_file, "wb") - response = s3.object_get(uri, dst_stream) + response = s3.object_get(uri, dst_stream, extra_label = seq_label) dst_stream.close() if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs: attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs']) @@ -699,9 +701,9 @@ continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not Config().progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (uri, dst_file, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total_count)) + seq_label)) total_size += response["size"] total_elapsed = time.time() - timestamp_start @@ -786,11 +788,12 @@ seq += 1 src = loc_list[file]['full_name'] uri = S3Uri(dst_base + file) + seq_label = "[%d of %d]" % (seq, total_count) if cfg.preserve_attrs: attr_header = _build_attr_header(src) debug(attr_header) try: - response = s3.object_put(src, uri, attr_header) + response = s3.object_put(src, uri, attr_header, extra_label = seq_label) except S3UploadError, e: error("%s: upload failed too many times. Skipping that file." % src) continue @@ -799,9 +802,9 @@ continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not cfg.progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) [%d of %d]" % + output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (src, uri, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], - seq, total_count)) + seq_label)) total_size += response["size"] total_elapsed = time.time() - timestamp_start This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-27 12:30:46
|
Revision: 296 http://s3tools.svn.sourceforge.net/s3tools/?rev=296&view=rev Author: ludvigm Date: 2008-12-27 12:30:36 +0000 (Sat, 27 Dec 2008) Log Message: ----------- * setup.cfg: Remove explicit install prefix. That should fix Mac OS X and Windows "setup.py install" runs. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/setup.cfg Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-22 01:54:14 UTC (rev 295) +++ s3cmd/trunk/ChangeLog 2008-12-27 12:30:36 UTC (rev 296) @@ -1,3 +1,8 @@ +2008-12-26 Michal Ludvig <mi...@lo...> + + * setup.cfg: Remove explicit install prefix. That should fix + Mac OS X and Windows "setup.py install" runs. + 2008-12-22 Michal Ludvig <mi...@lo...> * s3cmd, S3/S3.py, S3/Progress.py: Display "[X of Y]" Modified: s3cmd/trunk/setup.cfg =================================================================== --- s3cmd/trunk/setup.cfg 2008-12-22 01:54:14 UTC (rev 295) +++ s3cmd/trunk/setup.cfg 2008-12-27 12:30:36 UTC (rev 296) @@ -1,5 +1,2 @@ [sdist] formats = gztar,zip - -[install] -prefix = /usr This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 02:00:36
|
Revision: 297 http://s3tools.svn.sourceforge.net/s3tools/?rev=297&view=rev Author: ludvigm Date: 2008-12-29 01:49:10 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * s3cmd, run-tests.py: Make it work on Windows. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-27 12:30:36 UTC (rev 296) +++ s3cmd/trunk/ChangeLog 2008-12-29 01:49:10 UTC (rev 297) @@ -1,3 +1,7 @@ +2008-12-29 Michal Ludvig <mi...@lo...> + + * s3cmd, run-tests.py: Make it work on Windows. + 2008-12-26 Michal Ludvig <mi...@lo...> * setup.cfg: Remove explicit install prefix. That should fix Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-27 12:30:36 UTC (rev 296) +++ s3cmd/trunk/run-tests.py 2008-12-29 01:49:10 UTC (rev 297) @@ -7,6 +7,7 @@ ## License: GPL Version 2 import sys +import os import re from subprocess import Popen, PIPE, STDOUT @@ -76,7 +77,9 @@ def test_s3cmd(label, cmd_args = [], **kwargs): if not cmd_args[0].endswith("s3cmd"): - cmd_args.insert(0, "./s3cmd") + cmd_args.insert(0, "python") + cmd_args.insert(1, "s3cmd") + return test(label, cmd_args, **kwargs) test_s3cmd("Remove test buckets", ['rb', '-r', 's3://s3cmd-autotest-1', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], @@ -98,31 +101,56 @@ test_s3cmd("Buckets list", ["ls"], must_find = [ "autotest-1", "autotest-2", "Autotest-3" ], must_not_find_re = "Autotest-EU") -test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) +if os.name != "nt": + ## Full testsuite - POSIX (Unix, Linux, ...) + test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) -test_s3cmd("Check bucket content (-r)", ['ls', '--recursive', 's3://s3cmd-autotest-1'], - must_find = [ u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ], - must_not_find = [ "logo.png" ]) + test_s3cmd("List bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], + must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/unicode/$" ], + must_not_find = [ u"ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) -test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], - must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/unicode/$" ], - must_not_find = [ u"ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) + test_s3cmd("List bucket recursive", ['ls', '--recursive', 's3://s3cmd-autotest-1'], + must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5", + u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ], + must_not_find = [ "logo.png" ]) -# test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) + # test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) -test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], - must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) + test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], + must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) -test("Removing local target", ['rm', '-rf', 'testsuite-out']) + test("Removing local target", ['rm', '-rf', 'testsuite-out']) -test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], - must_find = [ "stored as testsuite-out/etc/logo.png ", u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) + test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = [ "stored as testsuite-out/etc/logo.png ", u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) -test("Retrieve public URL", ['wget', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'], - must_find_re = [ 'logo.png.*saved \[22059/22059\]' ]) + test("Retrieve public URL", ['wget', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'], + must_find_re = [ 'logo.png.*saved \[22059/22059\]' ]) -test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) + test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) +else: + ## Reduced testsuite - Windows NT+ + test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--exclude', 'unicode/*', '--no-encrypt']) + + test_s3cmd("Check bucket content (-r)", ['ls', '--recursive', 's3://s3cmd-autotest-1'], + must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5" ], + must_not_find = [ "logo.png" ]) + + test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], + must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/binary/$" ], + must_not_find = [ u"random-crap.md5" ]) + + test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], + must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) + + if os.path.isdir("testsuite-out"): + test("Removing local target", ['rmdir', '/s/q', 'testsuite-out']) + + test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = [ "stored as testsuite-out/etc/logo.png " ]) + +## Common for POSIX and Win32 test_s3cmd("Rename within S3", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], must_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) @@ -142,8 +170,9 @@ test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG deleted" ]) -test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], - must_find_re = [ "Object.*unicode/ŪņЇЌœđЗ/.*deleted" ]) +if os.name != "nt": + test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], + must_find_re = [ "Object.*unicode/ŪņЇЌœđЗ/.*deleted" ]) test_s3cmd("Recursive delete all", ['del', '--recursive', '--force', 's3://s3cmd-autotest-1'], must_find_re = [ "Object.*binary/random-crap deleted" ]) Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-27 12:30:36 UTC (rev 296) +++ s3cmd/trunk/s3cmd 2008-12-29 01:49:10 UTC (rev 297) @@ -11,7 +11,6 @@ import os import re import errno -import pwd, grp import glob import traceback import codecs @@ -325,7 +324,7 @@ for item in remote_keys: uri = item['remote_uri'] - destination = item['local_filename'] + destination = item['local_filename'] start_position = 0 @@ -466,7 +465,7 @@ loc_base = os.path.join(local_path, "") filelist = os.walk(local_path) else: - loc_base = "./" + loc_base = "." + os.path.sep filelist = [( '.', [], [local_path] )] loc_base_len = len(loc_base) loc_list = {} @@ -718,7 +717,8 @@ info(outstr) def cmd_sync_local2remote(src, dst): - def _build_attr_header(src): + def _build_attr_header(src): + import pwd, grp attrs = {} st = os.stat_result(os.stat(src)) for attr in cfg.preserve_attrs_list: @@ -789,6 +789,7 @@ src = loc_list[file]['full_name'] uri = S3Uri(dst_base + file) seq_label = "[%d of %d]" % (seq, total_count) + attr_header = None if cfg.preserve_attrs: attr_header = _build_attr_header(src) debug(attr_header) @@ -1071,10 +1072,13 @@ default_verbosity = Config().verbosity optparser = OptionParser(option_class=OptionMimeType, formatter=MyHelpFormatter()) #optparser.disable_interspersed_args() - - if os.getenv("HOME"): - optparser.set_defaults(config=os.getenv("HOME")+"/.s3cfg") - + + if os.getenv("HOME"): + config_file = os.path.join(os.getenv("HOME"), ".s3cfg") + elif os.name == "nt" and os.getenv("USERPROFILE"): + config_file = os.path.join(os.getenv("USERPROFILE"), "Application Data", "s3cmd.ini") + + optparser.set_defaults(config=config_file) optparser.set_defaults(verbosity = default_verbosity) optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.") @@ -1162,6 +1166,15 @@ ## We may need a way to display progress meter on STDERR or somewhere else Progress._stdout = _stdout + + ## Unsupported features on Win32 platform + if os.name == "nt": + if cfg.preserve_attrs: + error("Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.") + cfg.preserve_attrs = False + if cfg.progress_meter: + error("Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.") + cfg.progress_meter = False ## Update Config with other parameters for option in cfg.option_list(): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 04:42:00
|
Revision: 298 http://s3tools.svn.sourceforge.net/s3tools/?rev=298&view=rev Author: ludvigm Date: 2008-12-29 04:41:56 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * run-tests.py: Improved testsuite, added parameters support to run only specified tests, cleaned up win/posix integration. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Added Paths: ----------- s3cmd/trunk/testsuite/etc/AtomicClockRadio.ttf s3cmd/trunk/testsuite/etc/TypeRa.ttf Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 01:49:10 UTC (rev 297) +++ s3cmd/trunk/ChangeLog 2008-12-29 04:41:56 UTC (rev 298) @@ -1,5 +1,10 @@ 2008-12-29 Michal Ludvig <mi...@lo...> + * run-tests.py: Improved testsuite, added parameters support + to run only specified tests, cleaned up win/posix integration. + +2008-12-29 Michal Ludvig <mi...@lo...> + * s3cmd, run-tests.py: Make it work on Windows. 2008-12-26 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-29 01:49:10 UTC (rev 297) +++ s3cmd/trunk/run-tests.py 2008-12-29 04:41:56 UTC (rev 298) @@ -13,7 +13,22 @@ count_pass = 0 count_fail = 0 +count_skip = 0 +test_counter = 0 +run_tests = [] +exclude_tests = [] + +if os.name == "posix": + have_unicode = True + have_wget = True +elif os.name == "nt": + have_unicode = False + have_wget = False +else: + print "Unknown platform: %s" % os.name + sys.exit(1) + def test(label, cmd_args = [], retcode = 0, must_find = [], must_not_find = [], must_find_re = [], must_not_find_re = []): def failure(message = ""): global count_fail @@ -34,6 +49,13 @@ print "\x1b[32;1mOK\x1b[0m%s" % (message) count_pass += 1 return 0 + def skip(message = ""): + global count_skip + if message: + message = " (%s)" % message + print "\x1b[33;1mSKIP\x1b[0m%s" % (message) + count_skip += 1 + return 0 def compile_list(_list, regexps = False): if type(_list) not in [ list, tuple ]: _list = [_list] @@ -41,11 +63,16 @@ if regexps == False: _list = [re.escape(item.encode("utf-8")) for item in _list] - return [re.compile(item) for item in _list] - - print (label + " ").ljust(30, "."), + return [re.compile(item, re.MULTILINE) for item in _list] + + global test_counter + test_counter += 1 + print ("%3d %s " % (test_counter, label)).ljust(30, "."), sys.stdout.flush() + if run_tests.count(test_counter) == 0 or exclude_tests.count(test_counter) > 0: + return skip() + p = Popen(cmd_args, stdout = PIPE, stderr = STDOUT, universal_newlines = True) stdout, stderr = p.communicate() if retcode != p.returncode: @@ -82,104 +109,202 @@ return test(label, cmd_args, **kwargs) +def test_mkdir(label, dir_name): + if os.name in ("posix", "nt"): + cmd = ['mkdir'] + else: + print "Unknown platform: %s" % os.name + sys.exit(1) + cmd.append(dir_name) + return test(label, cmd) + +def test_rmdir(label, dir_name): + if os.path.isdir(dir_name): + if os.name == "posix": + cmd = ['rm', '-rf'] + elif os.name == "nt": + cmd = ['rmdir', '/s/q'] + else: + print "Unknown platform: %s" % os.name + sys.exit(1) + cmd.append(dir_name) + return test(label, cmd) + + +argv = sys.argv[1:] +while argv: + arg = argv.pop(0) + if arg in ("-h", "--help"): + print "%s A B K..O -N" % sys.argv[0] + print "Run tests number A, B and K through to O, except for N" + sys.exit(0) + if arg in ("-l", "--list"): + exclude_tests = range(0, 999) + break + if arg.find("..") >= 0: + range_idx = arg.find("..") + range_start = arg[:range_idx] or 0 + range_end = arg[range_idx+2:] or 999 + run_tests.extend(range(int(range_start), int(range_end) + 1)) + elif arg.startswith("-"): + exclude_tests.append(int(arg[1:])) + else: + run_tests.append(int(arg)) + +if not run_tests: + run_tests = range(0, 999) + +## ====== Remove test buckets test_s3cmd("Remove test buckets", ['rb', '-r', 's3://s3cmd-autotest-1', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed", "Bucket 's3://s3cmd-autotest-2/' removed", "Bucket 's3://s3cmd-Autotest-3/' removed" ]) + +## ====== Create one bucket (EU) test_s3cmd("Create one bucket (EU)", ['mb', '--bucket-location=EU', 's3://s3cmd-autotest-1'], must_find = "Bucket 's3://s3cmd-autotest-1/' created") + + +## ====== Create multiple buckets test_s3cmd("Create multiple buckets", ['mb', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], must_find = [ "Bucket 's3://s3cmd-autotest-2/' created", "Bucket 's3://s3cmd-Autotest-3/' created" ]) + +## ====== Invalid bucket name test_s3cmd("Invalid bucket name", ["mb", "--bucket-location=EU", "s3://s3cmd-Autotest-EU"], retcode = 1, must_find = "ERROR: Parameter problem: Bucket name 's3cmd-Autotest-EU' contains disallowed character", must_not_find_re = "Bucket.*created") + +## ====== Buckets list test_s3cmd("Buckets list", ["ls"], must_find = [ "autotest-1", "autotest-2", "Autotest-3" ], must_not_find_re = "Autotest-EU") -if os.name != "nt": - ## Full testsuite - POSIX (Unix, Linux, ...) + +## ====== Sync to S3 +if have_unicode: test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) +else: + test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--exclude', 'unicode/*', '--no-encrypt']) - test_s3cmd("List bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], - must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/unicode/$" ], - must_not_find = [ u"ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) - test_s3cmd("List bucket recursive", ['ls', '--recursive', 's3://s3cmd-autotest-1'], - must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5", - u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ], - must_not_find = [ "logo.png" ]) +## ====== List bucket content +must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/binary/$", u"D s3://s3cmd-autotest-1/xyz/etc/$" ] +must_not_find = [ u"random-crap.md5", u".svn" ] +if have_unicode: + must_find_re.append(u"D s3://s3cmd-autotest-1/xyz/unicode/$") + must_not_find.append(u"ŪņЇЌœđЗ/☺ unicode € rocks ™") +test_s3cmd("List bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], + must_find_re = must_find_re, + must_not_find = must_not_find) - # test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) - test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], - must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) +## ====== List bucket recursive +must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5" ] +if have_unicode: + must_find.append(u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™") +test_s3cmd("List bucket recursive", ['ls', '--recursive', 's3://s3cmd-autotest-1'], + must_find = must_find, + must_not_find = [ "logo.png" ]) - test("Removing local target", ['rm', '-rf', 'testsuite-out']) +## ====== FIXME +# test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/']) - test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], - must_find = [ "stored as testsuite-out/etc/logo.png ", u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™" ]) - test("Retrieve public URL", ['wget', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'], - must_find_re = [ 'logo.png.*saved \[22059/22059\]' ]) +## ====== Put public, guess MIME +test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], + must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) - test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) -else: - ## Reduced testsuite - Windows NT+ - test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--exclude', 'unicode/*', '--no-encrypt']) +## ====== rmdir local +test_rmdir("Removing local target", 'testsuite-out') - test_s3cmd("Check bucket content (-r)", ['ls', '--recursive', 's3://s3cmd-autotest-1'], - must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5" ], - must_not_find = [ "logo.png" ]) - test_s3cmd("Check bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], - must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/binary/$" ], - must_not_find = [ u"random-crap.md5" ]) +## ====== Sync from S3 +must_find = [ "stored as testsuite-out/etc/logo.png " ] +if have_unicode: + must_find.append(u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™") +test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = must_find) - test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'], - must_find = [ "stored as s3://s3cmd-autotest-1/xyz/etc/logo.png" ]) - if os.path.isdir("testsuite-out"): - test("Removing local target", ['rmdir', '/s/q', 'testsuite-out']) +## ====== Retrieve from URL +if have_wget: + test("Retrieve from URL", ['wget', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'], + must_find_re = [ 'logo.png.*saved \[22059/22059\]' ]) - test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], - must_find = [ "stored as testsuite-out/etc/logo.png " ]) -## Common for POSIX and Win32 +## ====== Sync more to S3 +test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) + + +## ====== Rename within S3 test_s3cmd("Rename within S3", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], must_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) + +## ====== Rename (NoSuchKey) test_s3cmd("Rename (NoSuchKey)", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], retcode = 1, must_find_re = [ 'ERROR:.*NoSuchKey' ], must_not_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) + +## ====== Make dst dir for get +test_rmdir("Remove dst dir for get", "testsuite-out") + + +## ====== Get multiple files +test_s3cmd("Get multiple files", ['get', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-autotest-1/xyz/etc/AtomicClockRadio.ttf', 'testsuite-out'], + retcode = 1, + must_find = [ 'Destination must be a directory when downloading multiple sources.' ]) + + +## ====== Make dst dir for get +test_mkdir("Make dst dir for get", "testsuite-out") + + +## ====== Get multiple files +test_s3cmd("Get multiple files", ['get', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-autotest-1/xyz/etc/AtomicClockRadio.ttf', 'testsuite-out'], + must_find = [ u"saved as 'testsuite-out/Logo.PNG'", u"saved as 'testsuite-out/AtomicClockRadio.ttf'" ]) + + +## ====== Sync more from S3 test_s3cmd("Sync more from S3", ['sync', '--delete-removed', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], must_find = [ "deleted 'testsuite-out/etc/logo.png'", "stored as testsuite-out/etc2/Logo.PNG (22059 bytes", "stored as testsuite-out/.svn/format " ], - must_not_find = [ "not-deleted etc/logo.png" ]) + must_not_find_re = [ "not-deleted.*etc/logo.png" ]) + +## ====== Copy between buckets test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3'], must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/Logo.PNG" ]) + +## ====== Simple delete test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'], must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG deleted" ]) -if os.name != "nt": - test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], - must_find_re = [ "Object.*unicode/ŪņЇЌœđЗ/.*deleted" ]) +## ====== Recursive delete +test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], + must_find_re = [ "Object.*\.svn/format deleted" ]) + + +## ====== Recursive delete all test_s3cmd("Recursive delete all", ['del', '--recursive', '--force', 's3://s3cmd-autotest-1'], must_find_re = [ "Object.*binary/random-crap deleted" ]) + +## ====== Remove empty bucket test_s3cmd("Remove empty bucket", ['rb', 's3://s3cmd-autotest-1'], must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed" ]) + +## ====== Remove remaining buckets test_s3cmd("Remove remaining buckets", ['rb', '--recursive', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], must_find = [ "Bucket 's3://s3cmd-autotest-2/' removed", "Bucket 's3://s3cmd-Autotest-3/' removed" ]) Copied: s3cmd/trunk/testsuite/etc/AtomicClockRadio.ttf (from rev 293, s3cmd/trunk/artwork/AtomicClockRadio.ttf) =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/etc/AtomicClockRadio.ttf ___________________________________________________________________ Added: svn:mime-type + application/x-font-ttf Added: svn:mergeinfo + Copied: s3cmd/trunk/testsuite/etc/TypeRa.ttf (from rev 293, s3cmd/trunk/artwork/TypeRa.ttf) =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/etc/TypeRa.ttf ___________________________________________________________________ Added: svn:mime-type + application/x-font-ttf Added: svn:mergeinfo + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 05:12:17
|
Revision: 299 http://s3tools.svn.sourceforge.net/s3tools/?rev=299&view=rev Author: ludvigm Date: 2008-12-29 05:12:09 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * S3/Exception.py: Python 2.4 doesn't automatically set Exception.message. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Exceptions.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 04:41:56 UTC (rev 298) +++ s3cmd/trunk/ChangeLog 2008-12-29 05:12:09 UTC (rev 299) @@ -1,5 +1,7 @@ 2008-12-29 Michal Ludvig <mi...@lo...> + * S3/Exception.py: Python 2.4 doesn't automatically set + Exception.message. * run-tests.py: Improved testsuite, added parameters support to run only specified tests, cleaned up win/posix integration. Modified: s3cmd/trunk/S3/Exceptions.py =================================================================== --- s3cmd/trunk/S3/Exceptions.py 2008-12-29 04:41:56 UTC (rev 298) +++ s3cmd/trunk/S3/Exceptions.py 2008-12-29 05:12:09 UTC (rev 299) @@ -12,6 +12,9 @@ import elementtree.ElementTree as ET class S3Exception(Exception): + def __init__(self, message = ""): + self.message = message + def __str__(self): ## Is this legal? return unicode(self) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 05:18:51
|
Revision: 300 http://s3tools.svn.sourceforge.net/s3tools/?rev=300&view=rev Author: ludvigm Date: 2008-12-29 05:18:15 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * run-tests.py: Improved testsuite, added parameters support to run only specified tests, cleaned up win/posix integration. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 05:12:09 UTC (rev 299) +++ s3cmd/trunk/ChangeLog 2008-12-29 05:18:15 UTC (rev 300) @@ -1,9 +1,9 @@ 2008-12-29 Michal Ludvig <mi...@lo...> + * run-tests.py: Improved testsuite, added parameters support + to run only specified tests, cleaned up win/posix integration. * S3/Exception.py: Python 2.4 doesn't automatically set Exception.message. - * run-tests.py: Improved testsuite, added parameters support - to run only specified tests, cleaned up win/posix integration. 2008-12-29 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-29 05:12:09 UTC (rev 299) +++ s3cmd/trunk/run-tests.py 2008-12-29 05:18:15 UTC (rev 300) @@ -185,10 +185,10 @@ ## ====== Sync to S3 +exclude_unicode_args = [] if have_unicode: - test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt']) -else: - test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--exclude', 'unicode/*', '--no-encrypt']) + exclude_unicode_args = [ '--exclude', 'unicode/*' ] +test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt'] + exclude_unicode_args) ## ====== List bucket content @@ -238,7 +238,7 @@ ## ====== Sync more to S3 -test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt']) +test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt'] + exclude_unicode_args) ## ====== Rename within S3 This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 05:36:26
|
Revision: 301 http://s3tools.svn.sourceforge.net/s3tools/?rev=301&view=rev Author: ludvigm Date: 2008-12-29 05:36:22 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * s3cmd: Improved wildcard get. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 05:18:15 UTC (rev 300) +++ s3cmd/trunk/ChangeLog 2008-12-29 05:36:22 UTC (rev 301) @@ -1,5 +1,6 @@ 2008-12-29 Michal Ludvig <mi...@lo...> + * s3cmd: Improved wildcard get. * run-tests.py: Improved testsuite, added parameters support to run only specified tests, cleaned up win/posix integration. * S3/Exception.py: Python 2.4 doesn't automatically set Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-29 05:18:15 UTC (rev 300) +++ s3cmd/trunk/s3cmd 2008-12-29 05:36:22 UTC (rev 301) @@ -261,8 +261,8 @@ ## ## In both cases we end up with a list mapping remote object names (keys) to local file names. - ## Each item will contain the following attributes - # 'remote_uri' 'local_filename' 'remote_label' 'local_label' + ## Each item will be a dict with the following attributes + # {'remote_uri', 'local_filename'} download_list = [] remote_uris = [] @@ -316,15 +316,42 @@ if os.path.isdir(destination_base) and destination_base[-1] != os.path.sep: destination_base += os.path.sep for uri in remote_uris: - download_item = { - 'remote_uri' : uri, - 'local_filename' : destination_base + uri.basename(), - } - remote_keys.append(download_item) + uri_str = str(uri) + ## Wildcards used on remote side? + ## If yes we'll need a bucket listing... + if uri_str.find('*') > -1 or uri_str.find('?') > -1: + first_wildcard = uri_str.find('*') + first_questionmark = uri_str.find('?') + if first_questionmark > -1 and first_questionmark < first_wildcard: + first_wildcard = first_questionmark + prefix = uri_str[:first_wildcard] + rest = uri_str[first_wildcard+1:] + ## Only request recursive listing if the 'rest' of the URI, + ## i.e. the part after first wildcard, contains '/' + need_recursion = rest.find('/') > -1 + objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion) + if destination_base[-1] != os.path.sep: + destination_base += os.path.sep + for object in objectlist: + download_item = { + 'remote_uri' : S3Uri(objectlist[object]['object_uri_str']), + 'local_filename' : destination_base + object, + } + remote_keys.append(download_item) + else: + download_item = { + 'remote_uri' : uri, + 'local_filename' : destination_base + uri.basename(), + } + remote_keys.append(download_item) + total_count = len(remote_keys) + seq = 0 for item in remote_keys: + seq += 1 uri = item['remote_uri'] destination = item['local_filename'] + seq_label = "[%d of %d]" % (seq, total_count) start_position = 0 @@ -361,7 +388,7 @@ except IOError, e: error("Skipping %s: %s" % (destination, e.strerror)) continue - response = s3.object_get(uri, dst_stream, start_position = start_position) + response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label) if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"]) response["size"] = os.stat(destination)[6] @@ -489,21 +516,40 @@ } return loc_list -def _get_filelist_remote(remote_uri): +def _get_filelist_remote(remote_uri, recursive = True): + ## If remote_uri ends with '/' then all remote files will have + ## the remote_uri prefix removed in the relative path. + ## If, on the other hand, the remote_uri ends with something else + ## (probably alphanumeric symbol) we'll use the last path part + ## in the relative path. + ## + ## Complicated, eh? See an example: + ## _get_filelist_remote("s3://bckt/abc/def") may yield: + ## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} } + ## _get_filelist_remote("s3://bckt/abc/def/") will yield: + ## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} } + ## Furthermore a prefix-magic can restrict the return list: + ## _get_filelist_remote("s3://bckt/abc/def/x") yields: + ## { 'xyz/blah.txt' : {} } + info("Retrieving list of remote files for %s ..." % remote_uri) s3 = S3(Config()) - response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = True) + response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive) - rem_base = remote_uri.object() + rem_base_original = rem_base = remote_uri.object() + remote_uri_original = remote_uri + if rem_base != '' and rem_base[-1] != '/': + rem_base = rem_base[:rem_base.rfind('/')+1] + remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) rem_base_len = len(rem_base) rem_list = {} break_now = False for object in response['list']: - if object['Key'] == rem_base and object['Key'][-1] != os.path.sep: + if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: ## We asked for one file and we got that file :-) key = os.path.basename(object['Key']) - object_uri_str = remote_uri.uri() + object_uri_str = remote_uri_original.uri() break_now = True rem_list = {} ## Remove whatever has already been put to rem_list else: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 05:38:56
|
Revision: 302 http://s3tools.svn.sourceforge.net/s3tools/?rev=302&view=rev Author: ludvigm Date: 2008-12-29 05:38:46 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * TODO, NEWS: Updated Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/TODO Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 05:36:22 UTC (rev 301) +++ s3cmd/trunk/ChangeLog 2008-12-29 05:38:46 UTC (rev 302) @@ -1,5 +1,6 @@ 2008-12-29 Michal Ludvig <mi...@lo...> + * TODO, NEWS: Updated * s3cmd: Improved wildcard get. * run-tests.py: Improved testsuite, added parameters support to run only specified tests, cleaned up win/posix integration. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2008-12-29 05:36:22 UTC (rev 301) +++ s3cmd/trunk/NEWS 2008-12-29 05:38:46 UTC (rev 302) @@ -1,6 +1,11 @@ s3cmd 0.9.9-pre4 ================ -* Support for non-recursive 'ls' +* Support for non-recursive [ls] +* Support for multiple sources and recursive [get]. +* Improved wildcard [get]. +* New option --skip-existing for [get] and [sync]. +* Improved Progress class (fixes Mac OS X) +* Fixed installation on Windows and Mac OS X. s3cmd 0.9.9-pre3 ================ Modified: s3cmd/trunk/TODO =================================================================== --- s3cmd/trunk/TODO 2008-12-29 05:36:22 UTC (rev 301) +++ s3cmd/trunk/TODO 2008-12-29 05:38:46 UTC (rev 302) @@ -9,8 +9,11 @@ x.jpg and not attempt blah/x.jpg. Similar thing, 'put /foo/bar/xyz.jpg s3://bucket/dome/path/' should save it to s3://bucket/dome/path/xyz.jpg + - Sync should work for one file, for example + s3cmd sync /etc/passwd s3://bucket/passwd - Document --recursive and --force for buckets - Allow change /tmp to somewhere else + - With --guess-mime use 'magic' module if available. - For 1.0.0 - Add --include/--include-from/--rinclude* for sync This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 12:28:23
|
Revision: 303 http://s3tools.svn.sourceforge.net/s3tools/?rev=303&view=rev Author: ludvigm Date: 2008-12-29 12:28:19 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * s3cmd: Replace unknown Unicode characters with '?' to avoid UnicodeEncodeError's. Also make all output strings unicode. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 05:38:46 UTC (rev 302) +++ s3cmd/trunk/ChangeLog 2008-12-29 12:28:19 UTC (rev 303) @@ -1,3 +1,9 @@ +2008-12-30 Michal Ludvig <mi...@lo...> + + * s3cmd: Replace unknown Unicode characters with '?' + to avoid UnicodeEncodeError's. Also make all output strings + unicode. + 2008-12-29 Michal Ludvig <mi...@lo...> * TODO, NEWS: Updated Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-29 05:38:46 UTC (rev 302) +++ s3cmd/trunk/s3cmd 2008-12-29 12:28:19 UTC (rev 303) @@ -14,21 +14,17 @@ import glob import traceback import codecs +import locale from copy import copy from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter from logging import debug, info, warning, error from distutils.spawn import find_executable -## Output native on TTY, UTF-8 otherwise (redirects) -#_stdout = sys.stdout.isatty() and sys.stdout or codecs.getwriter("utf-8")(sys.stdout) -#_stderr = sys.stderr.isatty() and sys.stderr or codecs.getwriter("utf-8")(sys.stderr) ## Output UTF-8 in all cases -_stdout = codecs.getwriter("utf-8")(sys.stdout) -_stderr = codecs.getwriter("utf-8")(sys.stderr) -## Leave it to the terminal -#_stdout = sys.stdout -#_stderr = sys.stderr +encoding = locale.getpreferredencoding() or "utf-8" +_stdout = codecs.getwriter(encoding)(sys.stdout, "replace") +_stderr = codecs.getwriter(encoding)(sys.stderr, "replace") def output(message): _stdout.write(message + "\n") @@ -57,8 +53,8 @@ buckets_size += size total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes) total_size_str = str(total_size) + size_coeff - output("".rjust(8, "-")) - output("%s Total" % (total_size_str.ljust(8))) + output(u"".rjust(8, "-")) + output(u"%s Total" % (total_size_str.ljust(8))) def subcmd_bucket_usage(s3, uri): bucket = uri.bucket() @@ -80,7 +76,7 @@ bucket_size += size total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes) total_size_str = str(total_size) + size_coeff - output("%s %s" % (total_size_str.ljust(8), uri)) + output(u"%s %s" % (total_size_str.ljust(8), uri)) return bucket_size def cmd_ls(args): @@ -99,13 +95,13 @@ for bucket in response["list"]: subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"])) - output("") + output(u"") def subcmd_buckets_list_all(s3): response = s3.list_all_buckets() for bucket in response["list"]: - output("%s s3://%s" % ( + output(u"%s s3://%s" % ( formatDateTime(bucket["CreationDate"]), bucket["Name"], )) @@ -114,7 +110,7 @@ bucket = uri.bucket() prefix = uri.object() - debug("Bucket 's3://%s':" % bucket) + debug(u"Bucket 's3://%s':" % bucket) if prefix.endswith('*'): prefix = prefix[:-1] try: @@ -127,13 +123,13 @@ raise for prefix in response['common_prefixes']: - output("%s %s" % ( + output(u"%s %s" % ( "D".rjust(28), uri.compose_uri(bucket, prefix["Prefix"]))) for object in response["list"]: size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes) - output("%s %s%s %s" % ( + output(u"%s %s%s %s" % ( formatDateTime(object["LastModified"]), str(size).rjust(8), size_coeff.ljust(1), uri.compose_uri(bucket, object["Key"]), @@ -147,7 +143,7 @@ raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) try: response = s3.bucket_create(uri.bucket(), cfg.bucket_location) - output("Bucket '%s' created" % uri.uri()) + output(u"Bucket '%s' created" % uri.uri()) except S3Error, e: if S3.codes.has_key(e.info["Code"]): error(S3.codes[e.info["Code"]] % uri.bucket()) @@ -161,7 +157,7 @@ response = s3.bucket_delete(uri.bucket()) except S3Error, e: if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive): - warning("Bucket is not empty. Removing all the objects from it first. This may take some time...") + warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...") subcmd_object_del_uri(uri, recursive = True) return _bucket_delete_one(uri) elif S3.codes.has_key(e.info["Code"]): @@ -176,7 +172,7 @@ if not uri.type == "s3" or not uri.has_bucket() or uri.has_object(): raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg) _bucket_delete_one(uri) - output("Bucket '%s' removed" % uri.uri()) + output(u"Bucket '%s' removed" % uri.uri()) def cmd_object_put(args): s3 = S3(Config()) @@ -189,11 +185,11 @@ raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg) if len(args) > 1 and uri.object() != "" and not Config().force: - error("When uploading multiple files the last argument must") - error("be a S3 URI specifying just the bucket name") - error("WITHOUT object name!") - error("Alternatively use --force argument and the specified") - error("object name will be prefixed to all stored filenames.") + error(u"When uploading multiple files the last argument must") + error(u"be a S3 URI specifying just the bucket name") + error(u"WITHOUT object name!") + error(u"Alternatively use --force argument and the specified") + error(u"object name will be prefixed to all stored filenames.") sys.exit(1) seq = 0 @@ -213,21 +209,21 @@ try: response = s3.object_put(real_filename, uri_final, extra_headers, extra_label = seq_label) except S3UploadError, e: - error("Upload of '%s' failed too many times. Skipping that file." % real_filename) + error(u"Upload of '%s' failed too many times. Skipping that file." % real_filename) continue except InvalidFileError, e: - warning("File can not be uploaded: %s" % e) + warning(u"File can not be uploaded: %s" % e) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not Config().progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + output(u"File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (file, uri_final, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], seq_label)) if Config().acl_public: - output("Public URL of the object is: %s" % + output(u"Public URL of the object is: %s" % (uri_final.public_url())) if Config().encrypt and real_filename != file: - debug("Removing temporary encrypted file: %s" % real_filename) + debug(u"Removing temporary encrypted file: %s" % real_filename) os.remove(real_filename) def cmd_object_get(args): @@ -386,7 +382,7 @@ dst_stream.close() raise ParameterError("File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) except IOError, e: - error("Skipping %s: %s" % (destination, e.strerror)) + error(u"Skipping %s: %s" % (destination, e.strerror)) continue response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label) if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"): @@ -394,7 +390,7 @@ response["size"] = os.stat(destination)[6] if not Config().progress_meter and destination != "-": speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) - output("File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % + output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" % (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1])) def cmd_object_del(args): @@ -420,13 +416,13 @@ uri_base = 's3://' + uri.bucket() + "/" for idx in filelist: object = filelist[idx] - debug("Adding URI " + uri_base + object['object_key']) + debug(u"Adding URI " + uri_base + object['object_key']) uri_list.append(S3Uri(uri_base + object['object_key'])) else: uri_list.append(uri) for _uri in uri_list: response = s3.object_delete(_uri) - output("Object %s deleted" % _uri) + output(u"Object %s deleted" % _uri) def subcmd_cp_mv(args, process_fce, message): src_uri = S3Uri(args.pop(0)) @@ -444,7 +440,7 @@ response = process_fce(src_uri, dst_uri) output(message % { "src" : src_uri, "dst" : dst_uri}) if Config().acl_public: - output("Public URL is: %s" % dst_uri.public_url()) + output(u"Public URL is: %s" % dst_uri.public_url()) def cmd_cp(args): s3 = S3(Config()) @@ -466,18 +462,18 @@ try: if uri.has_object(): info = s3.object_info(uri) - output("%s (object):" % uri.uri()) - output(" File size: %s" % info['headers']['content-length']) - output(" Last mod: %s" % info['headers']['last-modified']) - output(" MIME type: %s" % info['headers']['content-type']) - output(" MD5 sum: %s" % info['headers']['etag'].strip('"')) + output(u"%s (object):" % uri.uri()) + output(u" File size: %s" % info['headers']['content-length']) + output(u" Last mod: %s" % info['headers']['last-modified']) + output(u" MIME type: %s" % info['headers']['content-type']) + output(u" MD5 sum: %s" % info['headers']['etag'].strip('"')) else: info = s3.bucket_info(uri) - output("%s (bucket):" % uri.uri()) - output(" Location: %s" % info['bucket-location']) + output(u"%s (bucket):" % uri.uri()) + output(u" Location: %s" % info['bucket-location']) acl = s3.get_acl(uri) for user in acl.keys(): - output(" ACL: %s: %s" % (user, acl[user])) + output(u" ACL: %s: %s" % (user, acl[user])) except S3Error, e: if S3.codes.has_key(e.info["Code"]): error(S3.codes[e.info["Code"]] % uri.bucket()) @@ -576,7 +572,7 @@ logging.root.setLevel(logging.DEBUG) for file in src_list.keys(): if not cfg.debug_syncmatch: - debug("CHECK: %s" % (os.sep + file)) + debug(u"CHECK: %s" % (os.sep + file)) excluded = False for r in cfg.exclude: ## all paths start with '/' from the base dir @@ -585,8 +581,8 @@ ## therefore this awkward excluded switch :-( excluded = True if cfg.debug_syncmatch: - debug("EXCL: %s" % (os.sep + file)) - debug("RULE: '%s'" % (cfg.debug_exclude[r])) + debug(u"EXCL: %s" % (os.sep + file)) + debug(u"RULE: '%s'" % (cfg.debug_exclude[r])) else: info("%s: excluded" % file) break @@ -595,11 +591,11 @@ del(src_list[file]) continue else: - debug("PASS: %s" % (os.sep + file)) + debug(u"PASS: %s" % (os.sep + file)) if dst_list.has_key(file): ## Was --skip-existing requested? if cfg.skip_existing: - debug("IGNR: %s (used --skip-existing)" % (file)) + debug(u"IGNR: %s (used --skip-existing)" % (file)) exists_list[file] = src_list[file] del(src_list[file]) ## Remove from destination-list, all that is left there will be deleted @@ -608,7 +604,7 @@ ## Check size first if dst_list[file]['size'] == src_list[file]['size']: - #debug("%s same size: %s" % (file, dst_list[file]['size'])) + #debug(u"%s same size: %s" % (file, dst_list[file]['size'])) ## ... same size, check MD5 if src_is_local_and_dst_is_remote: src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) @@ -617,22 +613,22 @@ src_md5 = src_list[file]['md5'] dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) if src_md5 == dst_md5: - #debug("%s md5 matches: %s" % (file, dst_md5)) + #debug(u"%s md5 matches: %s" % (file, dst_md5)) ## Checksums are the same. ## Remove from source-list, all that is left there will be transferred - debug("IGNR: %s (transfer not needed: MD5 OK, Size OK)" % file) + debug(u"IGNR: %s (transfer not needed: MD5 OK, Size OK)" % file) exists_list[file] = src_list[file] del(src_list[file]) else: - debug("XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) + debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) else: - debug("XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) + debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) ## Remove from destination-list, all that is left there will be deleted - #debug("%s removed from destination list" % file) + #debug(u"%s removed from destination list" % file) del(dst_list[file]) if cfg.debug_syncmatch: - warning("Exiting because of --debug-syncmatch") + warning(u"Exiting because of --debug-syncmatch") sys.exit(0) return src_list, dst_list, exists_list, exclude_list @@ -669,9 +665,9 @@ for file in loc_list: if cfg.delete_removed: os.unlink(dst_base + file) - output("deleted '%s'" % (dst_base + file)) + output(u"deleted '%s'" % (dst_base + file)) else: - output("not-deleted '%s'" % file) + output(u"not-deleted '%s'" % file) total_size = 0 total_count = len(rem_list) @@ -691,7 +687,7 @@ if not dir_cache.has_key(dst_dir): dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir) if dir_cache[dst_dir] == False: - warning("%s: destination directory not writable: %s" % (file, dst_dir)) + warning(u"%s: destination directory not writable: %s" % (file, dst_dir)) continue try: open_flags = os.O_CREAT @@ -700,7 +696,7 @@ else: open_flags |= os.O_EXCL - debug("dst_file=%s" % dst_file) + debug(u"dst_file=%s" % dst_file) # This will have failed should the file exist os.close(os.open(dst_file, open_flags)) # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode. @@ -720,21 +716,21 @@ try: dst_stream.close() except: pass if e.errno == errno.EEXIST: - warning("%s exists - not overwriting" % (dst_file)) + warning(u"%s exists - not overwriting" % (dst_file)) continue if e.errno in (errno.EPERM, errno.EACCES): - warning("%s not writable: %s" % (dst_file, e.strerror)) + warning(u"%s not writable: %s" % (dst_file, e.strerror)) continue raise e except KeyboardInterrupt: try: dst_stream.close() except: pass - warning("Exiting after keyboard interrupt") + warning(u"Exiting after keyboard interrupt") return except Exception, e: try: dst_stream.close() except: pass - error("%s: %s" % (file, e)) + error(u"%s: %s" % (file, e)) continue # We have to keep repeating this call because # Python 2.4 doesn't support try/except/finally @@ -742,11 +738,11 @@ try: dst_stream.close() except: pass except S3DownloadError, e: - error("%s: download failed too many times. Skipping that file." % file) + error(u"%s: download failed too many times. Skipping that file." % file) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not Config().progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + output(u"File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (uri, dst_file, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], seq_label)) total_size += response["size"] @@ -774,14 +770,14 @@ except KeyError: attr = "uid" val = st.st_uid - warning("%s: Owner username not known. Storing UID=%d instead." % (src, val)) + warning(u"%s: Owner username not known. Storing UID=%d instead." % (src, val)) elif attr == 'gname': try: val = grp.getgrgid(st.st_gid).gr_name except KeyError: attr = "gid" val = st.st_gid - warning("%s: Owner groupname not known. Storing GID=%d instead." % (src, val)) + warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (src, val)) else: val = getattr(st, 'st_' + attr) attrs[attr] = val @@ -792,9 +788,9 @@ s3 = S3(cfg) if cfg.encrypt: - error("S3cmd 'sync' doesn't support GPG encryption, sorry.") - error("Either use unconditional 's3cmd put --recursive'") - error("or disable encryption with --no-encrypt parameter.") + error(u"S3cmd 'sync' doesn't support GPG encryption, sorry.") + error(u"Either use unconditional 's3cmd put --recursive'") + error(u"or disable encryption with --no-encrypt parameter.") sys.exit(1) @@ -817,9 +813,9 @@ uri = S3Uri("s3://" + dst_uri.bucket()+"/"+rem_list[file]['object_key']) if cfg.delete_removed: response = s3.object_delete(uri) - output("deleted '%s'" % uri) + output(u"deleted '%s'" % uri) else: - output("not-deleted '%s'" % uri) + output(u"not-deleted '%s'" % uri) total_size = 0 total_count = len(loc_list) @@ -842,14 +838,14 @@ try: response = s3.object_put(src, uri, attr_header, extra_label = seq_label) except S3UploadError, e: - error("%s: upload failed too many times. Skipping that file." % src) + error(u"%s: upload failed too many times. Skipping that file." % src) continue except InvalidFileError, e: - warning("File can not be uploaded: %s" % e) + warning(u"File can not be uploaded: %s" % e) continue speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True) if not cfg.progress_meter: - output("File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % + output(u"File '%s' stored as %s (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" % (src, uri, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], seq_label)) total_size += response["size"] @@ -923,7 +919,7 @@ command = resolve_list(cfg.gpg_decrypt.split(" "), args) code = gpg_command(command, cfg.gpg_passphrase) if code == 0 and in_place: - debug("Renaming %s to %s" % (tmp_filename, filename)) + debug(u"Renaming %s to %s" % (tmp_filename, filename)) os.unlink(filename) os.rename(tmp_filename, filename) tmp_filename = filename @@ -952,8 +948,8 @@ try: while 1: - output("\nEnter new values or accept defaults in brackets with Enter.") - output("Refer to user manual for detailed description of all options.") + output(u"\nEnter new values or accept defaults in brackets with Enter.") + output(u"Refer to user manual for detailed description of all options.") for option in options: prompt = option[1] ## Option-specific handling @@ -974,7 +970,7 @@ pass if len(option) >= 3: - output("\n%s" % option[2]) + output(u"\n%s" % option[2]) val = raw_input(prompt + ": ") if val != "": @@ -982,19 +978,19 @@ # Turn 'Yes' into True, everything else into False val = val.lower().startswith('y') setattr(cfg, option[0], val) - output("\nNew settings:") + output(u"\nNew settings:") for option in options: - output(" %s: %s" % (option[1], getattr(cfg, option[0]))) + output(u" %s: %s" % (option[1], getattr(cfg, option[0]))) val = raw_input("\nTest access with supplied credentials? [Y/n] ") if val.lower().startswith("y") or val == "": try: - output("Please wait...") + output(u"Please wait...") S3(Config()).bucket_list("", "") - output("Success. Your access key and secret key worked fine :-)") + output(u"Success. Your access key and secret key worked fine :-)") - output("\nNow verifying that encryption works...") + output(u"\nNow verifying that encryption works...") if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"): - output("Not configured. Never mind.") + output(u"Not configured. Never mind.") else: if not getattr(cfg, "gpg_command"): raise Exception("Path to GPG program not set") @@ -1020,7 +1016,7 @@ raise Exception("Encryption verification error.") except Exception, e: - error("Test failed: %s" % (e)) + error(u"Test failed: %s" % (e)) val = raw_input("\nRetry configuration? [Y/n] ") if val.lower().startswith("y") or val == "": continue @@ -1044,14 +1040,14 @@ os.umask(old_mask) cfg.dump_config(f) f.close() - output("Configuration saved to '%s'" % config_file) + output(u"Configuration saved to '%s'" % config_file) except (EOFError, KeyboardInterrupt): - output("\nConfiguration aborted. Changes were NOT saved.") + output(u"\nConfiguration aborted. Changes were NOT saved.") return except IOError, e: - error("Writing config file failed: %s: %s" % (config_file, e.strerror)) + error(u"Writing config file failed: %s: %s" % (config_file, e.strerror)) sys.exit(1) def process_exclude_from_file(exf, exclude_array): @@ -1060,7 +1056,7 @@ ex = ex.strip() if re.match("^#", ex) or re.match("^\s*$", ex): continue - debug("adding rule: %s" % ex) + debug(u"adding rule: %s" % ex) exclude_array.append(ex) commands = {} @@ -1181,12 +1177,12 @@ stream = _stderr) if options.show_version: - output("s3cmd version %s" % PkgInfo.version) + output(u"s3cmd version %s" % PkgInfo.version) sys.exit(0) ## Now finally parse the config file if not options.config: - error("Can't find a config file. Please use --config option.") + error(u"Can't find a config file. Please use --config option.") sys.exit(1) try: @@ -1195,9 +1191,9 @@ if options.run_configure: cfg = Config() else: - error("%s: %s" % (options.config, e.strerror)) - error("Configuration file not available.") - error("Consider using --configure parameter to create one.") + error(u"%s: %s" % (options.config, e.strerror)) + error(u"Configuration file not available.") + error(u"Consider using --configure parameter to create one.") sys.exit(1) ## And again some logging level adjustments @@ -1216,17 +1212,17 @@ ## Unsupported features on Win32 platform if os.name == "nt": if cfg.preserve_attrs: - error("Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.") + error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.") cfg.preserve_attrs = False if cfg.progress_meter: - error("Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.") + error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.") cfg.progress_meter = False ## Update Config with other parameters for option in cfg.option_list(): try: if getattr(options, option) != None: - debug("Updating %s -> %s" % (option, getattr(options, option))) + debug(u"Updating %s -> %s" % (option, getattr(options, option))) cfg.update_option(option, getattr(options, option)) except AttributeError: ## Some Config() options are not settable from command line @@ -1238,12 +1234,12 @@ if options.exclude_from: for exf in options.exclude_from: - debug("processing --exclude-from %s" % exf) + debug(u"processing --exclude-from %s" % exf) process_exclude_from_file(exf, options.exclude) if options.exclude: for ex in options.exclude: - debug("processing rule: %s" % ex) + debug(u"processing rule: %s" % ex) exc = re.compile(glob.fnmatch.translate(ex)) cfg.exclude.append(exc) if options.debug_syncmatch: @@ -1255,20 +1251,20 @@ if options.rexclude_from: for exf in options.rexclude_from: - debug("processing --rexclude-from %s" % exf) + debug(u"processing --rexclude-from %s" % exf) process_exclude_from_file(exf, options.rexclude) if options.rexclude: for ex in options.rexclude: - debug("processing rule: %s" % ex) + debug(u"processing rule: %s" % ex) exc = re.compile(ex) cfg.exclude.append(exc) if options.debug_syncmatch: cfg.debug_exclude[exc] = ex if cfg.encrypt and cfg.gpg_passphrase == "": - error("Encryption requested but no passphrase set in config file.") - error("Please re-run 's3cmd --configure' and supply it.") + error(u"Encryption requested but no passphrase set in config file.") + error(u"Please re-run 's3cmd --configure' and supply it.") sys.exit(1) if options.dump_config: @@ -1280,7 +1276,7 @@ sys.exit(0) if len(args) < 1: - error("Missing command. Please run with --help for more information.") + error(u"Missing command. Please run with --help for more information.") sys.exit(1) ## Unicodise all remaining arguments: @@ -1288,28 +1284,28 @@ command = args.pop(0) try: - debug("Command: %s" % commands[command]["cmd"]) + debug(u"Command: %s" % commands[command]["cmd"]) ## We must do this lookup in extra step to ## avoid catching all KeyError exceptions ## from inner functions. cmd_func = commands[command]["func"] except KeyError, e: - error("Invalid command: %s" % e) + error(u"Invalid command: %s" % e) sys.exit(1) if len(args) < commands[command]["argc"]: - error("Not enough paramters for command '%s'" % command) + error(u"Not enough paramters for command '%s'" % command) sys.exit(1) try: cmd_func(args) except S3Error, e: - error("S3 error: %s" % e) + error(u"S3 error: %s" % e) if e.info.has_key("Message"): error(e.info['Message']) sys.exit(1) except ParameterError, e: - error("Parameter problem: %s" % e) + error(u"Parameter problem: %s" % e) sys.exit(1) if __name__ == '__main__': This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-29 12:29:58
|
Revision: 304 http://s3tools.svn.sourceforge.net/s3tools/?rev=304&view=rev Author: ludvigm Date: 2008-12-29 12:29:54 +0000 (Mon, 29 Dec 2008) Log Message: ----------- * run-tests.py: Exit on failed test. Fixed order of tests. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-29 12:28:19 UTC (rev 303) +++ s3cmd/trunk/ChangeLog 2008-12-29 12:29:54 UTC (rev 304) @@ -3,6 +3,7 @@ * s3cmd: Replace unknown Unicode characters with '?' to avoid UnicodeEncodeError's. Also make all output strings unicode. + * run-tests.py: Exit on failed test. Fixed order of tests. 2008-12-29 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-29 12:28:19 UTC (rev 303) +++ s3cmd/trunk/run-tests.py 2008-12-29 12:29:54 UTC (rev 304) @@ -41,7 +41,8 @@ print "----" print stdout print "----" - return 1 + #return 1 + sys.exit(1) def success(message = ""): global count_pass if message: @@ -186,7 +187,7 @@ ## ====== Sync to S3 exclude_unicode_args = [] -if have_unicode: +if not have_unicode: exclude_unicode_args = [ '--exclude', 'unicode/*' ] test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt'] + exclude_unicode_args) @@ -253,6 +254,13 @@ must_not_find = [ 'Object s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ]) +## ====== Sync more from S3 +test_s3cmd("Sync more from S3", ['sync', '--delete-removed', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], + must_find = [ "deleted 'testsuite-out/etc/logo.png'", "stored as testsuite-out/etc2/Logo.PNG (22059 bytes", + "stored as testsuite-out/.svn/format " ], + must_not_find_re = [ "not-deleted.*etc/logo.png" ]) + + ## ====== Make dst dir for get test_rmdir("Remove dst dir for get", "testsuite-out") @@ -272,13 +280,6 @@ must_find = [ u"saved as 'testsuite-out/Logo.PNG'", u"saved as 'testsuite-out/AtomicClockRadio.ttf'" ]) -## ====== Sync more from S3 -test_s3cmd("Sync more from S3", ['sync', '--delete-removed', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], - must_find = [ "deleted 'testsuite-out/etc/logo.png'", "stored as testsuite-out/etc2/Logo.PNG (22059 bytes", - "stored as testsuite-out/.svn/format " ], - must_not_find_re = [ "not-deleted.*etc/logo.png" ]) - - ## ====== Copy between buckets test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3'], must_find = [ "Object s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/Logo.PNG" ]) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 01:58:10
|
Revision: 306 http://s3tools.svn.sourceforge.net/s3tools/?rev=306&view=rev Author: ludvigm Date: 2008-12-31 01:54:26 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * s3cmd: Unicodised all info() output. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Progress.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-30 00:13:33 UTC (rev 305) +++ s3cmd/trunk/ChangeLog 2008-12-31 01:54:26 UTC (rev 306) @@ -1,3 +1,7 @@ +2008-12-31 Michal Ludvig <mi...@lo...> + + * s3cmd: Unicodised all info() output. + 2008-12-30 Michal Ludvig <mi...@lo...> * s3cmd: Replace unknown Unicode characters with '?' Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-12-30 00:13:33 UTC (rev 305) +++ s3cmd/trunk/S3/Progress.py 2008-12-31 01:54:26 UTC (rev 306) @@ -44,7 +44,7 @@ self.display(done_message = message) def output_labels(self): - self._stdout.write("%(source)s -> %(destination)s %(extra)s\n" % self.labels) + self._stdout.write(u"%(source)s -> %(destination)s %(extra)s\n" % self.labels) self._stdout.flush() def display(self, new_file = False, done_message = None): Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-30 00:13:33 UTC (rev 305) +++ s3cmd/trunk/s3cmd 2008-12-31 01:54:26 UTC (rev 306) @@ -363,7 +363,7 @@ except IOError, e: if e.errno == errno.ENOENT: basename = destination[:destination.rindex(os.path.sep)] - info("Creating directory: %s" % basename) + info(u"Creating directory: %s" % basename) os.makedirs(basename) dst_stream = open(destination, "ab") else: @@ -376,11 +376,11 @@ dst_stream.seek(0L) dst_stream.truncate() elif Config().skip_existing: - info("Skipping over existing file: %s" % (destination)) + info(u"Skipping over existing file: %s" % (destination)) continue else: dst_stream.close() - raise ParameterError("File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) + raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination) except IOError, e: error(u"Skipping %s: %s" % (destination, e.strerror)) continue @@ -482,7 +482,7 @@ raise def _get_filelist_local(local_uri): - info("Compiling list of local files...") + info(u"Compiling list of local files...") local_path = local_uri.path() if os.path.isdir(local_path): loc_base = os.path.join(local_path, "") @@ -528,7 +528,7 @@ ## _get_filelist_remote("s3://bckt/abc/def/x") yields: ## { 'xyz/blah.txt' : {} } - info("Retrieving list of remote files for %s ..." % remote_uri) + info(u"Retrieving list of remote files for %s ..." % remote_uri) s3 = S3(Config()) response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive) @@ -564,7 +564,7 @@ return rem_list def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote): - info("Verifying checksums...") + info(u"Verifying checksums...") cfg = Config() exists_list = {} exclude_list = {} @@ -584,7 +584,7 @@ debug(u"EXCL: %s" % (os.sep + file)) debug(u"RULE: '%s'" % (cfg.debug_exclude[r])) else: - info("%s: excluded" % file) + info(u"%s: excluded" % file) break if excluded: exclude_list = src_list[file] @@ -656,11 +656,11 @@ loc_list = _get_filelist_local(dst_uri) loc_count = len(loc_list) - info("Found %d remote files, %d local files" % (rem_count, loc_count)) + info(u"Found %d remote files, %d local files" % (rem_count, loc_count)) _compare_filelists(rem_list, loc_list, False) - info("Summary: %d remote files to download, %d local files to delete" % (len(rem_list), len(loc_list))) + info(u"Summary: %d remote files to download, %d local files to delete" % (len(rem_list), len(loc_list))) for file in loc_list: if cfg.delete_removed: @@ -803,11 +803,11 @@ rem_list = _get_filelist_remote(dst_uri) rem_count = len(rem_list) - info("Found %d local files, %d remote files" % (loc_count, rem_count)) + info(u"Found %d local files, %d remote files" % (loc_count, rem_count)) _compare_filelists(loc_list, rem_list, True) - info("Summary: %d local files to upload, %d remote files to delete" % (len(loc_list), len(rem_list))) + info(u"Summary: %d local files to upload, %d remote files to delete" % (len(loc_list), len(rem_list))) for file in rem_list: uri = S3Uri("s3://" + dst_uri.bucket()+"/"+rem_list[file]['object_key']) @@ -902,7 +902,7 @@ "input_file" : filename, "output_file" : tmp_filename, } - info("Encrypting file %(input_file)s to %(output_file)s..." % args) + info(u"Encrypting file %(input_file)s to %(output_file)s..." % args) command = resolve_list(cfg.gpg_encrypt.split(" "), args) code = gpg_command(command, cfg.gpg_passphrase) return (code, tmp_filename, "gpg") @@ -915,7 +915,7 @@ "input_file" : filename, "output_file" : tmp_filename, } - info("Decrypting file %(input_file)s to %(output_file)s..." % args) + info(u"Decrypting file %(input_file)s to %(output_file)s..." % args) command = resolve_list(cfg.gpg_decrypt.split(" "), args) code = gpg_command(command, cfg.gpg_passphrase) if code == 0 and in_place: @@ -1344,19 +1344,19 @@ tb = traceback.format_exc(sys.exc_info()) e_class = str(e.__class__) e_class = e_class[e_class.rfind(".")+1 : -2] - sys.stderr.write("Problem: %s: %s\n" % (e_class, str(e))) + sys.stderr.write(u"Problem: %s: %s\n" % (e_class, e)) try: sys.stderr.write("S3cmd: %s\n" % PkgInfo.version) except NameError: sys.stderr.write("S3cmd: unknown version. Module import problem?\n") sys.stderr.write("Python: %s\n" % sys.version.replace('\n', ' ')) sys.stderr.write("\n") - sys.stderr.write(tb) + sys.stderr.write(unicode(tb, errors="replace")) if type(e) == ImportError: sys.stderr.write("\n") sys.stderr.write("Your sys.path contains these entries:\n") for path in sys.path: - sys.stderr.write("\t%s\n" % path) + sys.stderr.write(u"\t%s\n" % path) sys.stderr.write("Now the question is where has S3/S3.py been installed?\n") sys.stderr.write(""" !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 02:37:07
|
Revision: 307 http://s3tools.svn.sourceforge.net/s3tools/?rev=307&view=rev Author: ludvigm Date: 2008-12-31 02:37:04 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * s3cmd: Avoid ZeroDivisionError on fast links. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 01:54:26 UTC (rev 306) +++ s3cmd/trunk/ChangeLog 2008-12-31 02:37:04 UTC (rev 307) @@ -1,5 +1,6 @@ 2008-12-31 Michal Ludvig <mi...@lo...> + * s3cmd: Avoid ZeroDivisionError on fast links. * s3cmd: Unicodised all info() output. 2008-12-30 Michal Ludvig <mi...@lo...> Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-31 01:54:26 UTC (rev 306) +++ s3cmd/trunk/s3cmd 2008-12-31 02:37:04 UTC (rev 307) @@ -851,7 +851,8 @@ total_size += response["size"] total_elapsed = time.time() - timestamp_start - speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True) + total_speed = total_elapsed and total_size/total_elapsed or 0.0 + speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True) # Only print out the result if any work has been done or # if the user asked for verbose output This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 02:42:36
|
Revision: 308 http://s3tools.svn.sourceforge.net/s3tools/?rev=308&view=rev Author: ludvigm Date: 2008-12-31 02:42:33 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, S3/Config.py, S3/S3.py: Added --encoding switch and Config.encoding variable. Don't assume utf-8 for filesystem and terminal output anymore. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/Exceptions.py s3cmd/trunk/S3/Progress.py s3cmd/trunk/S3/S3.py s3cmd/trunk/S3/Utils.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/ChangeLog 2008-12-31 02:42:33 UTC (rev 308) @@ -1,5 +1,9 @@ 2008-12-31 Michal Ludvig <mi...@lo...> + * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, + S3/Config.py, S3/S3.py: Added --encoding switch and + Config.encoding variable. Don't assume utf-8 for filesystem + and terminal output anymore. * s3cmd: Avoid ZeroDivisionError on fast links. * s3cmd: Unicodised all info() output. Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/S3/Config.py 2008-12-31 02:42:33 UTC (rev 308) @@ -59,6 +59,7 @@ exclude = [] # Dict mapping compiled REGEXPs back to their textual form debug_exclude = {} + encoding = "utf-8" ## Creating a singleton def __new__(self, configfile = None): Modified: s3cmd/trunk/S3/Exceptions.py =================================================================== --- s3cmd/trunk/S3/Exceptions.py 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/S3/Exceptions.py 2008-12-31 02:42:33 UTC (rev 308) @@ -13,11 +13,10 @@ class S3Exception(Exception): def __init__(self, message = ""): - self.message = message + self.message = unicodise(message) def __str__(self): - ## Is this legal? - return unicode(self) + return deunicodise(self.message) def __unicode__(self): return self.message Modified: s3cmd/trunk/S3/Progress.py =================================================================== --- s3cmd/trunk/S3/Progress.py 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/S3/Progress.py 2008-12-31 02:42:33 UTC (rev 308) @@ -11,6 +11,7 @@ _stdout = sys.stdout def __init__(self, labels, total_size): + self._stdout = sys.stdout self.new_file(labels, total_size) def new_file(self, labels, total_size): Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/S3/S3.py 2008-12-31 02:42:33 UTC (rev 308) @@ -174,12 +174,12 @@ raise ValueError("Expected URI type 's3', got '%s'" % uri.type) if not os.path.isfile(filename): - raise InvalidFileError("%s is not a regular file" % filename) + raise InvalidFileError(u"%s is not a regular file" % unicodise(filename)) try: file = open(filename, "rb") size = os.stat(filename)[ST_SIZE] except IOError, e: - raise InvalidFileError("%s: %s" % (filename, e.strerror)) + raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror)) headers = SortedDict() if extra_headers: headers.update(extra_headers) @@ -194,7 +194,7 @@ if self.config.acl_public: headers["x-amz-acl"] = "public-read" request = self.create_request("OBJECT_PUT", uri = uri, headers = headers) - labels = { 'source' : file.name, 'destination' : uri, 'extra' : extra_label } + labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label } response = self.send_file(request, file, labels) return response Modified: s3cmd/trunk/S3/Utils.py =================================================================== --- s3cmd/trunk/S3/Utils.py 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/S3/Utils.py 2008-12-31 02:42:33 UTC (rev 308) @@ -13,6 +13,8 @@ from logging import debug, info, warning, error +import Config + try: import xml.etree.ElementTree as ET except ImportError: @@ -181,21 +183,44 @@ return False return True -def unicodise(string): +def unicodise(string, encoding = None, errors = "replace"): """ Convert 'string' to Unicode or raise an exception. """ - debug("Unicodising %r" % string) + + if not encoding: + encoding = Config.Config().encoding + + debug("Unicodising %r using %s" % (string, encoding)) if type(string) == unicode: return string try: - return string.decode("utf-8") + return string.decode(encoding, errors) except UnicodeDecodeError: raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) -def try_unicodise(string): +def deunicodise(string, encoding = None, errors = "replace"): + """ + Convert unicode 'string' to <type str>, by default replacing + all invalid characters with '?' or raise an exception. + """ + + if not encoding: + encoding = Config.Config().encoding + + debug("DeUnicodising %r using %s" % (string, encoding)) + if type(string) != unicode: + return str(string) try: - return unicodise(string) - except UnicodeDecodeError: - return string + return string.encode(encoding, errors) + except UnicodeEncodeError: + raise UnicodeEncodeError("Conversion from unicode failed: %r" % string) +def unicodise_safe(string, encoding = None): + """ + Convert 'string' to Unicode according to current encoding + and replace all invalid characters with '?' + """ + + return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?') + Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2008-12-31 02:37:04 UTC (rev 307) +++ s3cmd/trunk/s3cmd 2008-12-31 02:42:33 UTC (rev 308) @@ -21,13 +21,8 @@ from logging import debug, info, warning, error from distutils.spawn import find_executable -## Output UTF-8 in all cases -encoding = locale.getpreferredencoding() or "utf-8" -_stdout = codecs.getwriter(encoding)(sys.stdout, "replace") -_stderr = codecs.getwriter(encoding)(sys.stderr, "replace") - def output(message): - _stdout.write(message + "\n") + sys.stdout.write(message + "\n") def check_args_type(args, type, verbose_type): for arg in args: @@ -353,7 +348,7 @@ if destination == "-": ## stdout - dst_stream = sys.stdout + dst_stream = sys.__stdout__ else: ## File try: @@ -1121,7 +1116,10 @@ elif os.name == "nt" and os.getenv("USERPROFILE"): config_file = os.path.join(os.getenv("USERPROFILE"), "Application Data", "s3cmd.ini") - optparser.set_defaults(config=config_file) + preferred_encoding = locale.getpreferredencoding() or "UTF-8" + + optparser.set_defaults(encoding = preferred_encoding) + optparser.set_defaults(config = config_file) optparser.set_defaults(verbosity = default_verbosity) optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool.") @@ -1153,6 +1151,8 @@ optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.") optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option") + optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding) + optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form.") optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).") @@ -1175,7 +1175,7 @@ ## debugging/verbose output for config file parser on request logging.basicConfig(level=options.verbosity, format='%(levelname)s: %(message)s', - stream = _stderr) + stream = sys.stderr) if options.show_version: output(u"s3cmd version %s" % PkgInfo.version) @@ -1207,9 +1207,6 @@ ## Can be overriden by actual --(no-)progress parameter cfg.update_option('progress_meter', sys.stdout.isatty()) - ## We may need a way to display progress meter on STDERR or somewhere else - Progress._stdout = _stdout - ## Unsupported features on Win32 platform if os.name == "nt": if cfg.preserve_attrs: @@ -1229,6 +1226,10 @@ ## Some Config() options are not settable from command line pass + ## Set output and filesystem encoding for printing out filenames. + sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace") + sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace") + ## Process GLOB (shell wildcard style) excludes if options.exclude is None: options.exclude = [] This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 02:47:29
|
Revision: 309 http://s3tools.svn.sourceforge.net/s3tools/?rev=309&view=rev Author: ludvigm Date: 2008-12-31 02:47:20 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * run-tests.py: Don't assume utf-8, use preferred encoding instead. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 02:42:33 UTC (rev 308) +++ s3cmd/trunk/ChangeLog 2008-12-31 02:47:20 UTC (rev 309) @@ -1,5 +1,7 @@ 2008-12-31 Michal Ludvig <mi...@lo...> + * run-tests.py: Don't assume utf-8, use preferred encoding + instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, S3/Config.py, S3/S3.py: Added --encoding switch and Config.encoding variable. Don't assume utf-8 for filesystem @@ -9,6 +11,14 @@ 2008-12-30 Michal Ludvig <mi...@lo...> + * Released version 0.9.9-pre4 + --------------------------- + + * S3/PkgInfo.py: Bumped up version to 0.9.9-pre4 + * NEWS: Updated. + +2008-12-30 Michal Ludvig <mi...@lo...> + * s3cmd: Replace unknown Unicode characters with '?' to avoid UnicodeEncodeError's. Also make all output strings unicode. Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-31 02:42:33 UTC (rev 308) +++ s3cmd/trunk/run-tests.py 2008-12-31 02:47:20 UTC (rev 309) @@ -10,6 +10,7 @@ import os import re from subprocess import Popen, PIPE, STDOUT +import locale count_pass = 0 count_fail = 0 @@ -33,7 +34,7 @@ def failure(message = ""): global count_fail if message: - message = " (%s)" % message + message = " (%r)" % message print "\x1b[31;1mFAIL%s\x1b[0m" % (message) count_fail += 1 print "----" @@ -46,14 +47,14 @@ def success(message = ""): global count_pass if message: - message = " (%s)" % message + message = " (%r)" % message print "\x1b[32;1mOK\x1b[0m%s" % (message) count_pass += 1 return 0 def skip(message = ""): global count_skip if message: - message = " (%s)" % message + message = " (%r)" % message print "\x1b[33;1mSKIP\x1b[0m%s" % (message) count_skip += 1 return 0 @@ -62,7 +63,7 @@ _list = [_list] if regexps == False: - _list = [re.escape(item.encode("utf-8")) for item in _list] + _list = [re.escape(item.encode(locale.getpreferredencoding(), "replace")) for item in _list] return [re.compile(item, re.MULTILINE) for item in _list] This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 03:01:29
|
Revision: 310 http://s3tools.svn.sourceforge.net/s3tools/?rev=310&view=rev Author: ludvigm Date: 2008-12-31 03:01:25 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * testsuite: reorganised UTF-8 files Modified Paths: -------------- s3cmd/trunk/ChangeLog Added Paths: ----------- s3cmd/trunk/testsuite/encodings/ s3cmd/trunk/testsuite/encodings/GBK/ s3cmd/trunk/testsuite/encodings/UTF-8/ s3cmd/trunk/testsuite/encodings/UTF-8/?\197?\170?\197?\134?\208?\135?\208?\140?\197?\147?\196?\145?\208?\151/ Removed Paths: ------------- s3cmd/trunk/testsuite/unicode/?\197?\170?\197?\134?\208?\135?\208?\140?\197?\147?\196?\145?\208?\151/ Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 02:47:20 UTC (rev 309) +++ s3cmd/trunk/ChangeLog 2008-12-31 03:01:25 UTC (rev 310) @@ -1,5 +1,6 @@ 2008-12-31 Michal Ludvig <mi...@lo...> + * testsuite: reorganised UTF-8 files * run-tests.py: Don't assume utf-8, use preferred encoding instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, Property changes on: s3cmd/trunk/testsuite/encodings/UTF-8/?\197?\170?\197?\134?\208?\135?\208?\140?\197?\147?\196?\145?\208?\151 ___________________________________________________________________ Added: svn:mergeinfo + This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 03:09:42
|
Revision: 311 http://s3tools.svn.sourceforge.net/s3tools/?rev=311&view=rev Author: ludvigm Date: 2008-12-31 03:09:38 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * testsuite: added GBK encoding files. Modified Paths: -------------- s3cmd/trunk/ChangeLog Added Paths: ----------- s3cmd/trunk/testsuite/encodings/GBK/12?\230?\156?\13631?\230?\151?\165/ s3cmd/trunk/testsuite/encodings/GBK/12?\230?\156?\13631?\230?\151?\165/1-?\231?\137?\185?\232?\137?\178?\230?\162?\157?\231?\155?\174 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 03:01:25 UTC (rev 310) +++ s3cmd/trunk/ChangeLog 2008-12-31 03:09:38 UTC (rev 311) @@ -1,6 +1,6 @@ 2008-12-31 Michal Ludvig <mi...@lo...> - * testsuite: reorganised UTF-8 files + * testsuite: reorganised UTF-8 files, added GBK encoding files. * run-tests.py: Don't assume utf-8, use preferred encoding instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, Added: s3cmd/trunk/testsuite/encodings/GBK/12?\230?\156?\13631?\230?\151?\165/1-?\231?\137?\185?\232?\137?\178?\230?\162?\157?\231?\155?\174 =================================================================== This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 03:58:42
|
Revision: 312 http://s3tools.svn.sourceforge.net/s3tools/?rev=312&view=rev Author: ludvigm Date: 2008-12-31 03:58:38 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * testsuite: moved encoding-specific files to 'tar' archives. Modified Paths: -------------- s3cmd/trunk/ChangeLog Added Paths: ----------- s3cmd/trunk/testsuite/encodings/GBK.tar Removed Paths: ------------- s3cmd/trunk/testsuite/encodings/GBK/ Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 03:09:38 UTC (rev 311) +++ s3cmd/trunk/ChangeLog 2008-12-31 03:58:38 UTC (rev 312) @@ -1,6 +1,7 @@ 2008-12-31 Michal Ludvig <mi...@lo...> - * testsuite: reorganised UTF-8 files, added GBK encoding files. + * testsuite: reorganised UTF-8 files, added GBK encoding files, + moved encoding-specific files to 'tar' archives. * run-tests.py: Don't assume utf-8, use preferred encoding instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, Added: s3cmd/trunk/testsuite/encodings/GBK.tar =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/encodings/GBK.tar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 04:01:48
|
Revision: 313 http://s3tools.svn.sourceforge.net/s3tools/?rev=313&view=rev Author: ludvigm Date: 2008-12-31 04:01:45 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * testsuite: moved encoding-specific files to 'tar.gz' archives. Modified Paths: -------------- s3cmd/trunk/ChangeLog Added Paths: ----------- s3cmd/trunk/testsuite/encodings/GBK.tar.gz s3cmd/trunk/testsuite/encodings/UTF-8.tar.gz Removed Paths: ------------- s3cmd/trunk/testsuite/encodings/GBK.tar s3cmd/trunk/testsuite/encodings/UTF-8/ Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 03:58:38 UTC (rev 312) +++ s3cmd/trunk/ChangeLog 2008-12-31 04:01:45 UTC (rev 313) @@ -1,7 +1,7 @@ 2008-12-31 Michal Ludvig <mi...@lo...> * testsuite: reorganised UTF-8 files, added GBK encoding files, - moved encoding-specific files to 'tar' archives. + moved encoding-specific files to 'tar.gz' archives. * run-tests.py: Don't assume utf-8, use preferred encoding instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, Deleted: s3cmd/trunk/testsuite/encodings/GBK.tar =================================================================== (Binary files differ) Added: s3cmd/trunk/testsuite/encodings/GBK.tar.gz =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/encodings/GBK.tar.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: s3cmd/trunk/testsuite/encodings/UTF-8.tar.gz =================================================================== (Binary files differ) Property changes on: s3cmd/trunk/testsuite/encodings/UTF-8.tar.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2008-12-31 04:09:33
|
Revision: 314 http://s3tools.svn.sourceforge.net/s3tools/?rev=314&view=rev Author: ludvigm Date: 2008-12-31 04:09:24 +0000 (Wed, 31 Dec 2008) Log Message: ----------- * run-tests.py: Adapted to the above change. * run-tests.sh: removed. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py Removed Paths: ------------- s3cmd/trunk/run-tests.sh Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2008-12-31 04:01:45 UTC (rev 313) +++ s3cmd/trunk/ChangeLog 2008-12-31 04:09:24 UTC (rev 314) @@ -2,6 +2,8 @@ * testsuite: reorganised UTF-8 files, added GBK encoding files, moved encoding-specific files to 'tar.gz' archives. + * run-tests.py: Adapted to the above change. + * run-tests.sh: removed. * run-tests.py: Don't assume utf-8, use preferred encoding instead. * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py, Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2008-12-31 04:01:45 UTC (rev 313) +++ s3cmd/trunk/run-tests.py 2008-12-31 04:09:24 UTC (rev 314) @@ -21,15 +21,33 @@ exclude_tests = [] if os.name == "posix": - have_unicode = True have_wget = True elif os.name == "nt": - have_unicode = False have_wget = False else: print "Unknown platform: %s" % os.name sys.exit(1) +## Patterns for Unicode tests +patterns = {} +patterns['UTF-8'] = u"ŪņЇЌœđЗ/☺ unicode € rocks ™" +patterns['GBK'] = u"12月31日/1-特色條目" + +encoding = locale.getpreferredencoding() +if not encoding: + print "Guessing current system encoding failed. Consider setting $LANG variable." + sys.exit(1) + +have_encoding = os.path.isdir('testsuite/encodings/' + encoding) +if not have_encoding and os.path.isfile('testsuite/encodings/%s.tar.gz' % encoding): + os.system("tar xvz -C testsuite/encodings -f testsuite/encodings/UTF-8.tar.gz") + have_encoding = os.path.isdir('testsuite/encodings/' + encoding) + +if have_encoding: + enc_base_remote = "s3://s3cmd-autotest-1/xyz/%s/" % encoding + enc_pattern = patterns[encoding] + print "System encoding: " + encoding + def test(label, cmd_args = [], retcode = 0, must_find = [], must_not_find = [], must_find_re = [], must_not_find_re = []): def failure(message = ""): global count_fail @@ -63,7 +81,7 @@ _list = [_list] if regexps == False: - _list = [re.escape(item.encode(locale.getpreferredencoding(), "replace")) for item in _list] + _list = [re.escape(item.encode(encoding, "replace")) for item in _list] return [re.compile(item, re.MULTILINE) for item in _list] @@ -187,18 +205,19 @@ ## ====== Sync to S3 -exclude_unicode_args = [] -if not have_unicode: - exclude_unicode_args = [ '--exclude', 'unicode/*' ] -test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt'] + exclude_unicode_args) +test_s3cmd("Sync to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ]) +if have_encoding: + ## ====== Sync UTF-8 / GBK / ... to S3 + test_s3cmd("Sync %s to S3" % encoding, ['sync', 'testsuite/encodings/' + encoding, enc_base_remote, '--exclude', '.svn/*', '--no-encrypt' ]) + ## ====== List bucket content must_find_re = [ u"D s3://s3cmd-autotest-1/xyz/binary/$", u"D s3://s3cmd-autotest-1/xyz/etc/$" ] must_not_find = [ u"random-crap.md5", u".svn" ] -if have_unicode: - must_find_re.append(u"D s3://s3cmd-autotest-1/xyz/unicode/$") - must_not_find.append(u"ŪņЇЌœđЗ/☺ unicode € rocks ™") +if have_encoding: + must_find_re.append(u"D %s$" % enc_base_remote) + must_not_find.append(enc_pattern) test_s3cmd("List bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'], must_find_re = must_find_re, must_not_find = must_not_find) @@ -206,8 +225,8 @@ ## ====== List bucket recursive must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5" ] -if have_unicode: - must_find.append(u"s3://s3cmd-autotest-1/xyz/unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™") +if have_encoding: + must_find.append(enc_base_remote + enc_pattern) test_s3cmd("List bucket recursive", ['ls', '--recursive', 's3://s3cmd-autotest-1'], must_find = must_find, must_not_find = [ "logo.png" ]) @@ -227,8 +246,8 @@ ## ====== Sync from S3 must_find = [ "stored as testsuite-out/etc/logo.png " ] -if have_unicode: - must_find.append(u"unicode/ŪņЇЌœđЗ/☺ unicode € rocks ™") +if have_encoding: + must_find.append("stored as testsuite-out/" + encoding + "/" + enc_pattern) test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'], must_find = must_find) @@ -240,7 +259,7 @@ ## ====== Sync more to S3 -test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt'] + exclude_unicode_args) +test_s3cmd("Sync more to S3", ['sync', 'testsuite', 's3://s3cmd-autotest-1/xyz/', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ]) ## ====== Rename within S3 @@ -292,7 +311,7 @@ ## ====== Recursive delete -test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/unicode'], +test_s3cmd("Recursive delete", ['del', '--recursive', 's3://s3cmd-autotest-1/xyz/etc'], must_find_re = [ "Object.*\.svn/format deleted" ]) Deleted: s3cmd/trunk/run-tests.sh =================================================================== --- s3cmd/trunk/run-tests.sh 2008-12-31 04:01:45 UTC (rev 313) +++ s3cmd/trunk/run-tests.sh 2008-12-31 04:09:24 UTC (rev 314) @@ -1,41 +0,0 @@ -#!/bin/sh -set -e -x - -./s3cmd mb s3://s3cmd-autotest -./s3cmd ls s3://s3cmd-autotest -./s3cmd put s3cmd s3cmd.1 s3://s3cmd-autotest -./s3cmd ls s3://s3cmd-autotest -./s3cmd del s3://s3cmd-autotest/s3cmd.1 -./s3cmd get s3://s3cmd-autotest/s3cmd s3cmd.get -diff s3cmd s3cmd.get -rm -fv s3cmd.get - -set +x -echo; echo -echo "=== Now running 'sync' tests ===" -echo; echo -set -x - -VER=$(./s3cmd --version | cut -d\ -f3) -tar xvfz dist/s3cmd-${VER}.tar.gz -echo "Will be removed" > s3cmd-${VER}/file.to.remove -./s3cmd sync s3cmd-${VER} s3://s3cmd-autotest/sync-test -echo "Added file" > s3cmd-${VER}/added.file -rm -f s3cmd-${VER}/file.to.remove -./s3cmd sync --delete s3cmd-${VER} s3://s3cmd-autotest/sync-test -rm -f s3cmd-${VER}/S3/PkgInfo.py -rm -f s3cmd-${VER}/s3cmd -./s3cmd sync --delete --exclude "/s3cmd-${VER}/S3/S3*" s3://s3cmd-autotest/sync-test s3cmd-${VER} -rm -rf s3cmd-${VER} - -./s3cmd rb s3://s3cmd-autotest/ || true -# ERROR: S3 error: 409 (Conflict): BucketNotEmpty - -./s3cmd rb --force s3://s3cmd-autotest/ - -set +x - -echo; echo -echo; echo -echo "=== All good. Ready for release :-) ===" -echo This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |