From: <lu...@us...> - 2010-11-12 07:05:15
|
Revision: 451 http://s3tools.svn.sourceforge.net/s3tools/?rev=451&view=rev Author: ludvigm Date: 2010-11-12 07:05:04 +0000 (Fri, 12 Nov 2010) Log Message: ----------- * format-manpage.pl: new manpage auto-formatter * s3cmd.1: Updated using the above helper script * setup.py: Warn if manpage is too old. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd.1 s3cmd/trunk/setup.py Added Paths: ----------- s3cmd/trunk/format-manpage.pl Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-10-26 12:02:35 UTC (rev 450) +++ s3cmd/trunk/ChangeLog 2010-11-12 07:05:04 UTC (rev 451) @@ -1,3 +1,9 @@ +2010-11-12 Michal Ludvig <ml...@lo...> + + * format-manpage.pl: new manpage auto-formatter + * s3cmd.1: Updated using the above helper script + * setup.py: Warn if manpage is too old. + 2010-10-27 Michal Ludvig <ml...@lo...> * run-tests.py, testsuite.tar.gz: Keep the testsuite in Added: s3cmd/trunk/format-manpage.pl =================================================================== --- s3cmd/trunk/format-manpage.pl (rev 0) +++ s3cmd/trunk/format-manpage.pl 2010-11-12 07:05:04 UTC (rev 451) @@ -0,0 +1,185 @@ +#!/usr/bin/perl + +# Format s3cmd.1 manpage +# Usage: +# s3cmd --help | format-manpage.pl > s3cmd.1 + +use strict; + +my $commands = ""; +my $cfcommands = ""; +my $options = ""; + +while (<>) { + if (/^Commands:/) { + while (<>) { + last if (/^\s*$/); + my ($desc, $cmd, $cmdline); + ($desc = $_) =~ s/^\s*(.*?)\s*$/$1/; + ($cmdline = <>) =~ s/^\s*s3cmd (.*?) (.*?)\s*$/s3cmd \\fB$1\\fR \\fI$2\\fR/; + $cmd = $1; + if ($cmd =~ /^cf/) { + $cfcommands .= ".TP\n$cmdline\n$desc\n"; + } else { + $commands .= ".TP\n$cmdline\n$desc\n"; + } + } + } + if (/^Options:/) { + my ($opt, $desc); + while (<>) { + last if (/^\s*$/); + $_ =~ s/\s*(.*?)\s*$/$1/; + $desc = ""; + $opt = ""; + if (/^(-.*)/) { + $opt = $1; + if ($opt =~ / /) { + ($opt, $desc) = split(/\s\s+/, $opt, 2); + } + $opt =~ s/(-[^ ,=]+)/\\fB$1\\fR/g; + $opt =~ s/-/\\-/g; + $options .= ".TP\n$opt\n"; + } else { + $desc .= $_; + } + if ($desc) { + $options .= "$desc\n"; + } + } + } +} +print " +.TH s3cmd 1 +.SH NAME +s3cmd \\- tool for managing Amazon S3 storage space and Amazon CloudFront content delivery network +.SH SYNOPSIS +.B s3cmd +[\\fIOPTIONS\\fR] \\fICOMMAND\\fR [\\fIPARAMETERS\\fR] +.SH DESCRIPTION +.PP +.B s3cmd +is a command line client for copying files to/from +Amazon S3 (Simple Storage Service) and performing other +related tasks, for instance creating and removing buckets, +listing objects, etc. + +.SH COMMANDS +.PP +.B s3cmd +can do several \\fIactions\\fR specified by the following \\fIcommands\\fR. +$commands + +.PP +Commands for CloudFront management +$cfcommands + +.SH OPTIONS +.PP +Some of the below specified options can have their default +values set in +.B s3cmd +config file (by default \$HOME/.s3cmd). As it's a simple text file +feel free to open it with your favorite text editor and do any +changes you like. +$options + +.SH EXAMPLES +One of the most powerful commands of \\fIs3cmd\\fR is \\fBs3cmd sync\\fR used for +synchronising complete directory trees to or from remote S3 storage. To some extent +\\fBs3cmd put\\fR and \\fBs3cmd get\\fR share a similar behaviour with \\fBsync\\fR. +.PP +Basic usage common in backup scenarios is as simple as: +.nf + s3cmd sync /local/path/ s3://test-bucket/backup/ +.fi +.PP +This command will find all files under /local/path directory and copy them +to corresponding paths under s3://test-bucket/backup on the remote side. +For example: +.nf + /local/path/\\fBfile1.ext\\fR \\-> s3://bucket/backup/\\fBfile1.ext\\fR + /local/path/\\fBdir123/file2.bin\\fR \\-> s3://bucket/backup/\\fBdir123/file2.bin\\fR +.fi +.PP +However if the local path doesn't end with a slash the last directory's name +is used on the remote side as well. Compare these with the previous example: +.nf + s3cmd sync /local/path s3://test-bucket/backup/ +.fi +will sync: +.nf + /local/\\fBpath/file1.ext\\fR \\-> s3://bucket/backup/\\fBpath/file1.ext\\fR + /local/\\fBpath/dir123/file2.bin\\fR \\-> s3://bucket/backup/\\fBpath/dir123/file2.bin\\fR +.fi +.PP +To retrieve the files back from S3 use inverted syntax: +.nf + s3cmd sync s3://test-bucket/backup/ /tmp/restore/ +.fi +that will download files: +.nf + s3://bucket/backup/\\fBfile1.ext\\fR \\-> /tmp/restore/\\fBfile1.ext\\fR + s3://bucket/backup/\\fBdir123/file2.bin\\fR \\-> /tmp/restore/\\fBdir123/file2.bin\\fR +.fi +.PP +Without the trailing slash on source the behaviour is similar to +what has been demonstrated with upload: +.nf + s3cmd sync s3://test-bucket/backup /tmp/restore/ +.fi +will download the files as: +.nf + s3://bucket/\\fBbackup/file1.ext\\fR \\-> /tmp/restore/\\fBbackup/file1.ext\\fR + s3://bucket/\\fBbackup/dir123/file2.bin\\fR \\-> /tmp/restore/\\fBbackup/dir123/file2.bin\\fR +.fi +.PP +All source file names, the bold ones above, are matched against \\fBexclude\\fR +rules and those that match are then re\\-checked against \\fBinclude\\fR rules to see +whether they should be excluded or kept in the source list. +.PP +For the purpose of \\fB\\-\\-exclude\\fR and \\fB\\-\\-include\\fR matching only the +bold file names above are used. For instance only \\fBpath/file1.ext\\fR is tested +against the patterns, not \\fI/local/\\fBpath/file1.ext\\fR +.PP +Both \\fB\\-\\-exclude\\fR and \\fB\\-\\-include\\fR work with shell-style wildcards (a.k.a. GLOB). +For a greater flexibility s3cmd provides Regular-expression versions of the two exclude options +named \\fB\\-\\-rexclude\\fR and \\fB\\-\\-rinclude\\fR. +The options with ...\\fB\\-from\\fR suffix (eg \\-\\-rinclude\\-from) expect a filename as +an argument. Each line of such a file is treated as one pattern. +.PP +There is only one set of patterns built from all \\fB\\-\\-(r)exclude(\\-from)\\fR options +and similarly for include variant. Any file excluded with eg \\-\\-exclude can +be put back with a pattern found in \\-\\-rinclude\\-from list. +.PP +Run s3cmd with \\fB\\-\\-dry\\-run\\fR to verify that your rules work as expected. +Use together with \\fB\\-\\-debug\\fR get detailed information +about matching file names against exclude and include rules. +.PP +For example to exclude all files with \".jpg\" extension except those beginning with a number use: +.PP + \\-\\-exclude '*.jpg' \\-\\-rinclude '[0-9].*\\.jpg' + +.SH SEE ALSO +For the most up to date list of options run +.B s3cmd \\-\\-help +.br +For more info about usage, examples and other related info visit project homepage at +.br +.B http://s3tools.org + +.SH AUTHOR +Written by Michal Ludvig <mludvig\@logix.net.nz> +.SH CONTACT, SUPPORT +Prefered way to get support is our mailing list: +.I s3tools\\-general\@lists.sourceforge.net +.SH REPORTING BUGS +Report bugs to +.I s3tools\\-bugs\@lists.sourceforge.net +.SH COPYRIGHT +Copyright \\(co 2007,2008,2009,2010 Michal Ludvig <http://www.logix.cz/michal> +.br +This is free software. You may redistribute copies of it under the terms of +the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>. +There is NO WARRANTY, to the extent permitted by law. +"; Property changes on: s3cmd/trunk/format-manpage.pl ___________________________________________________________________ Added: svn:executable + * Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2010-10-26 12:02:35 UTC (rev 450) +++ s3cmd/trunk/s3cmd.1 2010-11-12 07:05:04 UTC (rev 451) @@ -1,3 +1,4 @@ + .TH s3cmd 1 .SH NAME s3cmd \- tool for managing Amazon S3 storage space and Amazon CloudFront content delivery network @@ -11,72 +12,80 @@ Amazon S3 (Simple Storage Service) and performing other related tasks, for instance creating and removing buckets, listing objects, etc. + +.SH COMMANDS .PP .B s3cmd can do several \fIactions\fR specified by the following \fIcommands\fR. .TP -\fBmb\fR \fIs3://BUCKET\fR +s3cmd \fBmb\fR \fIs3://BUCKET\fR Make bucket .TP -\fBrb\fR \fIs3://BUCKET\fR +s3cmd \fBrb\fR \fIs3://BUCKET\fR Remove bucket .TP -\fBls\fR \fI[s3://BUCKET[/PREFIX]]\fR +s3cmd \fBls\fR \fI[s3://BUCKET[/PREFIX]]\fR List objects or buckets .TP -\fBla\fR +s3cmd \fBla\fR \fI\fR List all object in all buckets .TP -\fBput\fR \fIFILE [FILE...] s3://BUCKET[/PREFIX]\fR -Put file into bucket (i.e. upload to S3) +s3cmd \fBput\fR \fIFILE [FILE...] s3://BUCKET[/PREFIX]\fR +Put file into bucket .TP -\fBget\fR \fIs3://BUCKET/OBJECT LOCAL_FILE\fR -Get file from bucket (i.e. download from S3) +s3cmd \fBget\fR \fIs3://BUCKET/OBJECT LOCAL_FILE\fR +Get file from bucket .TP -\fBdel\fR \fIs3://BUCKET/OBJECT\fR +s3cmd \fBdel\fR \fIs3://BUCKET/OBJECT\fR Delete file from bucket .TP -\fBsync\fR \fILOCAL_DIR s3://BUCKET[/PREFIX]\fR -Backup a directory tree to S3 +s3cmd \fBsync\fR \fILOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR\fR +Synchronize a directory tree to S3 .TP -\fBsync\fR \fIs3://BUCKET[/PREFIX] LOCAL_DIR\fR -Restore a tree from S3 to local directory +s3cmd \fBdu\fR \fI[s3://BUCKET[/PREFIX]]\fR +Disk usage by buckets .TP -\fBcp\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR, \fBmv\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR -Make a copy of a file (\fIcp\fR) or move a file (\fImv\fR). -Destination can be in the same bucket with a different name -or in another bucket with the same or different name. -Adding \fI\-\-acl\-public\fR will make the destination object -publicly accessible (see below). +s3cmd \fBinfo\fR \fIs3://BUCKET[/OBJECT]\fR +Get various information about Buckets or Files .TP -\fBsetacl\fR \fIs3://BUCKET[/OBJECT]\fR -Modify \fIAccess control list\fI for Bucket or Files. Use with -\fI\-\-acl\-public\fR or \fI\-\-acl\-private\fR -.TP -\fBinfo\fR \fIs3://BUCKET[/OBJECT]\fR -Get various information about a Bucket or Object +s3cmd \fBcp\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR +Copy object .TP -\fBdu\fR \fI[s3://BUCKET[/PREFIX]]\fR -Disk usage \- amount of data stored in S3 +s3cmd \fBmv\fR \fIs3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]\fR +Move object +.TP +s3cmd \fBsetacl\fR \fIs3://BUCKET[/OBJECT]\fR +Modify Access control list for Bucket or Files +.TP +s3cmd \fBaccesslog\fR \fIs3://BUCKET\fR +Enable/disable bucket access logging +.TP +s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR +Sign arbitrary string using the secret key +.TP +s3cmd \fBfixbucket\fR \fIs3://BUCKET[/PREFIX]\fR +Fix invalid file names in a bucket + .PP Commands for CloudFront management .TP -\fBcflist\fR +s3cmd \fBcflist\fR \fI\fR List CloudFront distribution points .TP -\fBcfinfo\fR [\fIcf://DIST_ID\fR] +s3cmd \fBcfinfo\fR \fI[cf://DIST_ID]\fR Display CloudFront distribution point parameters .TP -\fBcfcreate\fR \fIs3://BUCKET\fR +s3cmd \fBcfcreate\fR \fIs3://BUCKET\fR Create CloudFront distribution point .TP -\fBcfdelete\fR \fIcf://DIST_ID\fR +s3cmd \fBcfdelete\fR \fIcf://DIST_ID\fR Delete CloudFront distribution point .TP -\fBcfmodify\fR \fIcf://DIST_ID\fR +s3cmd \fBcfmodify\fR \fIcf://DIST_ID\fR Change CloudFront distribution point parameters + .SH OPTIONS .PP Some of the below specified options can have their default @@ -85,145 +94,201 @@ config file (by default $HOME/.s3cmd). As it's a simple text file feel free to open it with your favorite text editor and do any changes you like. -.PP -\fIConfig file related options.\fR .TP +\fB\-h\fR, \fB\-\-help\fR +show this help message and exit +.TP \fB\-\-configure\fR -Invoke interactive (re)configuration tool. Don't worry, you won't -lose your settings on subsequent runs. +Invoke interactive (re)configuration tool. .TP \fB\-c\fR FILE, \fB\-\-config\fR=FILE -Config file name. Defaults to $HOME/.s3cfg +Config file name. Defaults to /home/mludvig/.s3cfg .TP \fB\-\-dump\-config\fR Dump current configuration after parsing config files and command line options and exit. -.PP -\fIOptions specific for \fIfile transfer commands\fR (\fBsync\fR, \fBput\fR and \fBget\fR): .TP \fB\-n\fR, \fB\-\-dry\-run\fR -Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other in -formation though. +Only show what should be uploaded or downloaded but +don't actually do it. May still perform S3 requests to +get bucket listings and other information though (only +for file transfer commands) .TP +\fB\-e\fR, \fB\-\-encrypt\fR +Encrypt files before uploading to S3. +.TP +\fB\-\-no\-encrypt\fR +Don't encrypt files. +.TP +\fB\-f\fR, \fB\-\-force\fR +Force overwrite and other dangerous operations. +.TP +\fB\-\-continue\fR +Continue getting a partially downloaded file (only for +[get] command). +.TP +\fB\-\-skip\-existing\fR +Skip over files that exist at the destination (only +for [get] and [sync] commands). +.TP +\fB\-r\fR, \fB\-\-recursive\fR +Recursive upload, download or removal. +.TP +\fB\-P\fR, \fB\-\-acl\-public\fR +Store objects with ACL allowing read for anyone. +.TP +\fB\-\-acl\-private\fR +Store objects with default ACL allowing access for you +only. +.TP +\fB\-\-acl\-grant\fR=PERMISSION:EMAIL or USER_CANONICAL_ID +Grant stated permission to a given amazon user. +Permission is one of: read, write, read_acp, +write_acp, full_control, all +.TP +\fB\-\-acl\-revoke\fR=PERMISSION:USER_CANONICAL_ID +Revoke stated permission for a given amazon user. +Permission is one of: read, write, read_acp, wr +ite_acp, full_control, all +.TP \fB\-\-delete\-removed\fR -Delete remote objects with no corresponding local file when \fIsync\fRing \fBto\fR S3 or delete local files with no corresponding object in S3 when \fIsync\fRing \fBfrom\fR S3. +Delete remote objects with no corresponding local file +[sync] .TP \fB\-\-no\-delete\-removed\fR -Don't delete remote objects. Default for \fIsync\fR command. +Don't delete remote objects. .TP \fB\-p\fR, \fB\-\-preserve\fR -Preserve filesystem attributes (mode, ownership, timestamps). Default for \fIsync\fR command. +Preserve filesystem attributes (mode, ownership, +timestamps). Default for [sync] command. .TP \fB\-\-no\-preserve\fR -Don't store filesystem attributes with uploaded files. +Don't store FS attributes .TP -\fB\-\-exclude GLOB\fR -Exclude files matching GLOB (a.k.a. shell-style wildcard) from \fIsync\fI. See FILE TRANSFERS section and \fIhttp://s3tools.org/s3cmd-sync\fR for more information. +\fB\-\-exclude\fR=GLOB +Filenames and paths matching GLOB will be excluded +from sync .TP -\fB\-\-exclude\-from FILE\fR -Same as \-\-exclude but reads GLOBs from the given FILE instead of expecting them on the command line. +\fB\-\-exclude\-from\fR=FILE +Read --exclude GLOBs from FILE .TP -\fB\-\-rexclude REGEXP\fR -Same as \-\-exclude but works with REGEXPs (Regular expressions). +\fB\-\-rexclude\fR=REGEXP +Filenames and paths matching REGEXP (regular +expression) will be excluded from sync .TP -\fB\-\-rexclude\-from FILE\fR -Same as \-\-exclude\-from but works with REGEXPs. +\fB\-\-rexclude\-from\fR=FILE +Read --rexclude REGEXPs from FILE .TP -\fB\-\-include=GLOB\fR, \fB\-\-include\-from=FILE\fR, \fB\-\-rinclude=REGEXP\fR, \fB\-\-rinclude\-from=FILE\fR -Filenames and paths matching GLOB or REGEXP will be included even if previously excluded by one of \-\-(r)exclude(\-from) patterns +\fB\-\-include\fR=GLOB +Filenames and paths matching GLOB will be included +even if previously excluded by one of .TP -\fB\-\-continue\fR -Continue getting a partially downloaded file (only for \fIget\fR command). This comes handy once download of a large file, say an ISO image, from a S3 bucket fails and a partially downloaded file is left on the disk. Unfortunately \fIput\fR command doesn't support restarting of failed upload due to Amazon S3 limitations. +\fB\-\-(r)exclude(\-from)\fR patterns .TP -\fB\-\-skip\-existing\fR -Skip over files that exist at the destination (only for \fIget\fR and \fIsync\fR commands). +\fB\-\-include\-from\fR=FILE +Read --include GLOBs from FILE .TP -\fB\-\-follow\-symlinks\fR -Treat local symbolic links as if they are regular files, -copying their targets to the remote. (Only for \fIput\fR and \fIsync\fR commands). +\fB\-\-rinclude\fR=REGEXP +Same as --include but uses REGEXP (regular expression) +instead of GLOB .TP -\fB\-m\fR MIME/TYPE, \fB\-\-mime\-type\fR=MIME/TYPE -Default MIME\-type to be set for objects stored. +\fB\-\-rinclude\-from\fR=FILE +Read --rinclude REGEXPs from FILE .TP -\fB\-M\fR, \fB\-\-guess\-mime\-type\fR -Guess MIME\(hytype of files by their extension. Falls -back to default MIME\(hyType as specified by \fB\-\-mime\-type\fR -option +\fB\-\-bucket\-location\fR=BUCKET_LOCATION +Datacentre to create bucket in. As of now the +datacenters are: US (default), EU, us-west-1, and ap- +southeast-1 .TP -\fB\-\-add\-header=NAME:VALUE\fR -Add a given HTTP header to the upload request. Can be used multiple times with different header names. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like. +\fB\-\-reduced\-redundancy\fR, \fB\-\-rr\fR +Store object with 'Reduced redundancy'. Lower per-GB +price. [put, cp, mv] .TP -\fB\-P\fR, \fB\-\-acl\-public\fR -Store objects with permissions allowing read for anyone. See \fIhttp://s3tools.org/s3cmd-public\fR for details and hints for storing publicly accessible files. +\fB\-\-access\-logging\-target\-prefix\fR=LOG_TARGET_PREFIX +Target prefix for access logs (S3 URI) (for [cfmodify] +and [accesslog] commands) .TP -\fB\-\-acl\-private\fR -Store objects with default ACL allowing access for you only. +\fB\-\-no\-access\-logging\fR +Disable access logging (for [cfmodify] and [accesslog] +commands) .TP -\fB\-e\fR, \fB\-\-encrypt\fR -Use GPG encryption to protect stored objects from unauthorized access. See \fIhttp://s3tools.org/s3cmd-public\fR for details about encryption. +\fB\-m\fR MIME/TYPE, \fB\-\-mime\-type\fR=MIME/TYPE +Default MIME-type to be set for objects stored. .TP -\fB\-\-no\-encrypt\fR -Don't encrypt files. -.PP -\fIOptions for CloudFront commands\fR: -.PP -See \fIhttp://s3tools.org/s3cmd-cloudfront\fR for more details. +\fB\-M\fR, \fB\-\-guess\-mime\-type\fR +Guess MIME-type of files by their extension. Falls +back to default MIME-Type as specified by --mime-type +option .TP -\fB\-\-enable\fR -Enable given CloudFront distribution (only for \fIcfmodify\fR command) +\fB\-\-add\-header\fR=NAME:VALUE +Add a given HTTP header to the upload request. Can be +used multiple times. For instance set 'Expires' or +'Cache-Control' headers (or both) using this options +if you like. .TP -\fB\-\-disable\fR -Enable given CloudFront distribution (only for \fIcfmodify\fR command) +\fB\-\-encoding\fR=ENCODING +Override autodetected terminal and filesystem encoding +(character set). Autodetected: UTF-8 .TP -\fB\-\-cf\-add\-cname=CNAME\fR -Add given CNAME to a CloudFront distribution (only for \fIcfcreate\fR and \fIcfmodify\fR commands) +\fB\-\-verbatim\fR +Use the S3 name as given on the command line. No pre- +processing, encoding, etc. Use with caution! .TP -\fB\-\-cf\-remove\-cname=CNAME\fR -Remove given CNAME from a CloudFront distribution (only for \fIcfmodify\fR command) +\fB\-\-list\-md5\fR +Include MD5 sums in bucket listings (only for 'ls' +command). .TP -\fB\-\-cf\-comment=COMMENT\fR -Set COMMENT for a given CloudFront distribution (only for \fIcfcreate\fR and \fIcfmodify\fR commands) -.PP -\fIOptions common for all commands\fR (where it makes sense indeed): +\fB\-H\fR, \fB\-\-human\-readable\-sizes\fR +Print sizes in human readable form (eg 1kB instead of +1234). .TP -\fB\-r\fR, \fB\-\-recursive\fR -Recursive upload, download or removal. When used with \fIdel\fR it can -remove all the files in a bucket. +\fB\-\-progress\fR +Display progress meter (default on TTY). .TP -\fB\-f\fR, \fB\-\-force\fR -Force overwrite and other dangerous operations. Can be used to remove -a non\-empty buckets with \fIs3cmd rb \-\-force s3://bkt\fR +\fB\-\-no\-progress\fR +Don't display progress meter (default on non-TTY). .TP -\fB\-\-bucket\-location\fR=BUCKET_LOCATION -Specify datacentre where to create the bucket. Possible values are \fIUS\fR (default) or \fIEU\fR. +\fB\-\-enable\fR +Enable given CloudFront distribution (only for +[cfmodify] command) .TP -\fB\-H\fR, \fB\-\-human\-readable\-sizes\fR -Print sizes in human readable form. +\fB\-\-disable\fR +Enable given CloudFront distribution (only for +[cfmodify] command) .TP -\fB\-\-list\-md5\fR -Include MD5 sums in bucket listings (only for \fIls\fR command). +\fB\-\-cf\-add\-cname\fR=CNAME +Add given CNAME to a CloudFront distribution (only for +[cfcreate] and [cfmodify] commands) .TP -\fB\-\-progress\fR, \fB\-\-no\-progress\fR -Display or don't display progress meter. When running on TTY (e.g. console or xterm) the default is to display progress meter. If not on TTY (e.g. output is redirected somewhere or running from cron) the default is to not display progress meter. +\fB\-\-cf\-remove\-cname\fR=CNAME +Remove given CNAME from a CloudFront distribution +(only for [cfmodify] command) .TP -\fB\-\-encoding=ENCODING\fR -Override autodetected terminal and filesystem encoding (character set). +\fB\-\-cf\-comment\fR=COMMENT +Set COMMENT for a given CloudFront distribution (only +for [cfcreate] and [cfmodify] commands) .TP +\fB\-\-cf\-default\-root\-object\fR=DEFAULT_ROOT_OBJECT +Set the default root object to return when no object +is specified in the URL. Use a relative path, i.e. +default/index.html instead of /default/index.html or +s3://bucket/default/index.html (only for [cfcreate] +and [cfmodify] commands) +.TP \fB\-v\fR, \fB\-\-verbose\fR Enable verbose output. .TP \fB\-d\fR, \fB\-\-debug\fR Enable debug output. .TP -\fB\-h\fR, \fB\-\-help\fR -Show the help message and exit +\fB\-\-version\fR +Show s3cmd version (1.0.0-rc1) and exit. .TP -\fB\-\-version\fR -Show -.B s3cmd -version and exit. +\fB\-F\fR, \fB\-\-follow\-symlinks\fR +Follow symbolic links as if they are regular files -.SH FILE TRANSFERS + +.SH EXAMPLES One of the most powerful commands of \fIs3cmd\fR is \fBs3cmd sync\fR used for synchronising complete directory trees to or from remote S3 storage. To some extent \fBs3cmd put\fR and \fBs3cmd get\fR share a similar behaviour with \fBsync\fR. @@ -308,7 +373,7 @@ .B http://s3tools.org .SH AUTHOR -Written by Michal Ludvig <mi...@lo...> +Written by Michal Ludvig <ml...@lo...> .SH CONTACT, SUPPORT Prefered way to get support is our mailing list: .I s3tools\-ge...@li... @@ -316,7 +381,7 @@ Report bugs to .I s3tools\-b...@li... .SH COPYRIGHT -Copyright \(co 2007,2008,2009 Michal Ludvig <http://www.logix.cz/michal> +Copyright \(co 2007,2008,2009,2010 Michal Ludvig <http://www.logix.cz/michal> .br This is free software. You may redistribute copies of it under the terms of the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>. Modified: s3cmd/trunk/setup.py =================================================================== --- s3cmd/trunk/setup.py 2010-10-26 12:02:35 UTC (rev 450) +++ s3cmd/trunk/setup.py 2010-11-12 07:05:04 UTC (rev 451) @@ -33,6 +33,14 @@ except: pass +## Re-create the manpage +## (Beware! Perl script on the loose!!) +if sys.argv[1] == "sdist": + if os.stat_result(os.stat("s3cmd.1")).st_mtime < os.stat_result(os.stat("s3cmd")).st_mtime: + sys.stderr.write("Re-create man page first!\n") + sys.stderr.write("Run: ./s3cmd --help | ./format-manpage.pl > s3cmd.1\n") + sys.exit(1) + ## Don't install manpages and docs when $S3CMD_PACKAGING is set ## This was a requirement of Debian package maintainer. if not os.getenv("S3CMD_PACKAGING"): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-11-12 09:02:38
|
Revision: 452 http://s3tools.svn.sourceforge.net/s3tools/?rev=452&view=rev Author: ludvigm Date: 2010-11-12 09:02:27 +0000 (Fri, 12 Nov 2010) Log Message: ----------- * s3cmd: Fixed typo in "s3cmd du" error path. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-11-12 07:05:04 UTC (rev 451) +++ s3cmd/trunk/ChangeLog 2010-11-12 09:02:27 UTC (rev 452) @@ -1,5 +1,9 @@ 2010-11-12 Michal Ludvig <ml...@lo...> + * s3cmd: Fixed typo in "s3cmd du" error path. + +2010-11-12 Michal Ludvig <ml...@lo...> + * format-manpage.pl: new manpage auto-formatter * s3cmd.1: Updated using the above helper script * setup.py: Warn if manpage is too old. Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-11-12 07:05:04 UTC (rev 451) +++ s3cmd/trunk/s3cmd 2010-11-12 09:02:27 UTC (rev 452) @@ -102,8 +102,8 @@ try: response = s3.bucket_list(bucket, prefix = object, recursive = True) except S3Error, e: - if S3.codes.has_key(e.Code): - error(S3.codes[e.Code] % bucket) + if S3.codes.has_key(e.info["Code"]): + error(S3.codes[e.info["Code"]] % bucket) return else: raise This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-11-12 11:04:21
|
Revision: 453 http://s3tools.svn.sourceforge.net/s3tools/?rev=453&view=rev Author: ludvigm Date: 2010-11-12 11:04:10 +0000 (Fri, 12 Nov 2010) Log Message: ----------- * s3cmd: Added support for remote-to-remote sync. (Based on patch from Sundar Raman - thanks!) * run-tests.py: Testsuite for the above. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/run-tests.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-11-12 09:02:27 UTC (rev 452) +++ s3cmd/trunk/ChangeLog 2010-11-12 11:04:10 UTC (rev 453) @@ -1,3 +1,9 @@ +2010-11-13 Michal Ludvig <ml...@lo...> + + * s3cmd: Added support for remote-to-remote sync. + (Based on patch from Sundar Raman - thanks!) + * run-tests.py: Testsuite for the above. + 2010-11-12 Michal Ludvig <ml...@lo...> * s3cmd: Fixed typo in "s3cmd du" error path. Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2010-11-12 09:02:27 UTC (rev 452) +++ s3cmd/trunk/run-tests.py 2010-11-12 11:04:10 UTC (rev 453) @@ -423,12 +423,29 @@ must_find = [ "File %s/xyz/etc2/Logo.PNG copied to %s/xyz/etc2/logo.png" % (pbucket(1), pbucket(3)) ]) ## ====== Recursive copy -test_s3cmd("Recursive copy, set ACL", ['cp', '-r', '--acl-public', '%s/xyz/' % pbucket(1), '%s/copy' % pbucket(2), '--exclude', 'demo/*', '--exclude', 'non-printables*'], +test_s3cmd("Recursive copy, set ACL", ['cp', '-r', '--acl-public', '%s/xyz/' % pbucket(1), '%s/copy' % pbucket(2), '--exclude', 'demo/dir?/*.txt', '--exclude', 'non-printables*'], must_find = [ "File %s/xyz/etc2/Logo.PNG copied to %s/copy/etc2/Logo.PNG" % (pbucket(1), pbucket(2)), "File %s/xyz/blahBlah/Blah.txt copied to %s/copy/blahBlah/Blah.txt" % (pbucket(1), pbucket(2)), "File %s/xyz/blahBlah/blah.txt copied to %s/copy/blahBlah/blah.txt" % (pbucket(1), pbucket(2)) ], - must_not_find = [ "demo/" ]) + must_not_find = [ "demo/dir1/file1-1.txt" ]) +## ====== Verify ACL and MIME type +test_s3cmd("Verify ACL and MIME type", ['info', '%s/copy/etc2/Logo.PNG' % pbucket(2) ], + must_find_re = [ "MIME type:.*image/png", + "ACL:.*\*anon\*: READ", + "URL:.*http://%s.s3.amazonaws.com/copy/etc2/Logo.PNG" % bucket(2) ]) + +## ====== Rename within S3 +test_s3cmd("Rename within S3", ['mv', '%s/copy/etc2/Logo.PNG' % pbucket(2), '%s/copy/etc/logo.png' % pbucket(2)], + must_find = [ 'File %s/copy/etc2/Logo.PNG moved to %s/copy/etc/logo.png' % (pbucket(2), pbucket(2))]) + +## ====== Sync between buckets +test_s3cmd("Sync remote2remote", ['sync', '%s/xyz/' % pbucket(1), '%s/copy/' % pbucket(2), '--delete-removed', '--exclude', 'non-printables*'], + must_find = [ "File %s/xyz/demo/dir1/file1-1.txt copied to %s/copy/demo/dir1/file1-1.txt" % (pbucket(1), pbucket(2)), + "File %s/xyz/etc2/Logo.PNG copied to %s/copy/etc2/Logo.PNG" % (pbucket(1), pbucket(2)), + "deleted: '%s/copy/etc/logo.png'" % pbucket(2) ], + must_not_find = [ "blah.txt" ]) + ## ====== Don't Put symbolic link test_s3cmd("Don't put symbolic links", ['put', 'testsuite/etc/linked1.png', 's3://%s/xyz/' % bucket(1),], must_not_find_re = [ "linked1.png"]) @@ -445,12 +462,6 @@ "etc/brokenlink.png"], ) -## ====== Verify ACL and MIME type -test_s3cmd("Verify ACL and MIME type", ['info', '%s/copy/etc2/Logo.PNG' % pbucket(2) ], - must_find_re = [ "MIME type:.*image/png", - "ACL:.*\*anon\*: READ", - "URL:.*http://%s.s3.amazonaws.com/copy/etc2/Logo.PNG" % bucket(2) ]) - ## ====== Multi source move test_s3cmd("Multi-source move", ['mv', '-r', '%s/copy/blahBlah/Blah.txt' % pbucket(2), '%s/copy/etc/' % pbucket(2), '%s/moved/' % pbucket(2)], must_find = [ "File %s/copy/blahBlah/Blah.txt moved to %s/moved/Blah.txt" % (pbucket(2), pbucket(2)), Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-11-12 09:02:27 UTC (rev 452) +++ s3cmd/trunk/s3cmd 2010-11-12 11:04:10 UTC (rev 453) @@ -770,12 +770,18 @@ debug(u"PASS: %s" % (file)) return src_list, exclude_list -def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote): +def _compare_filelists(src_list, dst_list, src_remote, dst_remote): + def __direction_str(is_remote): + return is_remote and "remote" or "local" + + # We don't support local->local sync, use 'rsync' or something like that instead ;-) + assert(not(src_remote == False and dst_remote == False)) + info(u"Verifying attributes...") cfg = Config() exists_list = SortedDict(ignore_case = False) - debug("Comparing filelists (src_is_local_and_dst_is_remote=%s)" % src_is_local_and_dst_is_remote) + debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote))) debug("src_list.keys: %s" % src_list.keys()) debug("dst_list.keys: %s" % dst_list.keys()) @@ -799,12 +805,16 @@ if attribs_match and 'md5' in cfg.sync_checks: ## ... same size, check MD5 - if src_is_local_and_dst_is_remote: + if src_remote == False and dst_remote == True: src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) dst_md5 = dst_list[file]['md5'] - else: + elif src_remote == True and dst_remote == False: src_md5 = src_list[file]['md5'] dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) + elif src_remote == True and dst_remote == True: + src_md5 = src_list[file]['md5'] + dst_md5 = dst_list[file]['md5'] + if src_md5 != dst_md5: ## Checksums are different. attribs_match = False @@ -821,6 +831,80 @@ return src_list, dst_list, exists_list +def cmd_sync_remote2remote(args): + s3 = S3(Config()) + + # Normalise s3://uri (e.g. assert trailing slash) + destination_base = unicode(S3Uri(args[-1])) + + src_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True) + dst_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True) + + src_count = len(src_list) + dst_count = len(dst_list) + + info(u"Found %d source files, %d destination files" % (src_count, dst_count)) + + src_list, exclude_list = _filelist_filter_exclude_include(src_list) + + src_list, dst_list, existing_list = _compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True) + + src_count = len(src_list) + dst_count = len(dst_list) + + print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count)) + + if src_count > 0: + ### Populate 'remote_uri' only if we've got something to sync from src to dst + for key in src_list: + src_list[key]['target_uri'] = destination_base + key + + if cfg.dry_run: + for key in exclude_list: + output(u"exclude: %s" % unicodise(key)) + if cfg.delete_removed: + for key in dst_list: + output(u"delete: %s" % dst_list[key]['object_uri_str']) + for key in src_list: + output(u"Sync: %s -> %s" % (src_list[key]['object_uri_str'], src_list[key]['target_uri'])) + warning(u"Exitting now because of --dry-run") + return + + # Delete items in destination that are not in source + if cfg.delete_removed: + if cfg.dry_run: + for key in dst_list: + output(u"delete: %s" % dst_list[key]['object_uri_str']) + else: + for key in dst_list: + uri = S3Uri(dst_list[key]['object_uri_str']) + s3.object_delete(uri) + output(u"deleted: '%s'" % uri) + + # Perform the synchronization of files + timestamp_start = time.time() + seq = 0 + file_list = src_list.keys() + file_list.sort() + for file in file_list: + seq += 1 + item = src_list[file] + src_uri = S3Uri(item['object_uri_str']) + dst_uri = S3Uri(item['target_uri']) + seq_label = "[%d of %d]" % (seq, src_count) + extra_headers = copy(cfg.extra_headers) + try: + response = s3.object_copy(src_uri, dst_uri, extra_headers) + output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri }) + except S3Error, e: + error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e }) + total_elapsed = time.time() - timestamp_start + outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed) + if seq > 0: + output(outstr) + else: + info(outstr) + def cmd_sync_remote2local(args): def _parse_attrs_header(attrs_header): attrs = {} @@ -842,7 +926,7 @@ remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) - remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, False) + remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False) local_count = len(local_list) remote_count = len(remote_list) @@ -1031,7 +1115,7 @@ # Flush remote_list, by the way remote_list = { local_list.keys()[0] : remote_list_entry } - local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, True) + local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True) local_count = len(local_list) remote_count = len(remote_list) @@ -1118,6 +1202,8 @@ return cmd_sync_local2remote(args) if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file": return cmd_sync_remote2local(args) + if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3": + return cmd_sync_remote2remote(args) raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args)) def cmd_setacl(args): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-08 13:06:18
|
Revision: 454 http://s3tools.svn.sourceforge.net/s3tools/?rev=454&view=rev Author: ludvigm Date: 2010-12-08 13:06:12 +0000 (Wed, 08 Dec 2010) Log Message: ----------- * Released version 1.0.0-rc2 -------------------------- * S3/PkgInfo.py: Updated to 1.0.0-rc2 * NEWS, TODO, s3cmd.1: Updated. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/PkgInfo.py s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-11-12 11:04:10 UTC (rev 453) +++ s3cmd/trunk/ChangeLog 2010-12-08 13:06:12 UTC (rev 454) @@ -1,3 +1,11 @@ +2010-12-09 Michal Ludvig <ml...@lo...> + + * Released version 1.0.0-rc2 + -------------------------- + + * S3/PkgInfo.py: Updated to 1.0.0-rc2 + * NEWS, TODO, s3cmd.1: Updated. + 2010-11-13 Michal Ludvig <ml...@lo...> * s3cmd: Added support for remote-to-remote sync. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2010-11-12 11:04:10 UTC (rev 453) +++ s3cmd/trunk/NEWS 2010-12-08 13:06:12 UTC (rev 454) @@ -1,3 +1,7 @@ +s3cmd 1.0.0-rc2 - 2010-??-?? +=============== +* [sync] now supports bucket-to-bucket synchronisation + s3cmd 1.0.0-rc1 - 2010-10-26 =============== * Added [accesslog] command. (needs manpage!) Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2010-11-12 11:04:10 UTC (rev 453) +++ s3cmd/trunk/S3/PkgInfo.py 2010-12-08 13:06:12 UTC (rev 454) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "1.0.0-rc1" +version = "1.0.0-rc2" url = "http://s3tools.org" license = "GPL version 2" short_description = "Command line tool for managing Amazon S3 and CloudFront services" Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2010-11-12 11:04:10 UTC (rev 453) +++ s3cmd/trunk/s3cmd.1 2010-12-08 13:06:12 UTC (rev 454) @@ -282,7 +282,7 @@ Enable debug output. .TP \fB\-\-version\fR -Show s3cmd version (1.0.0-rc1) and exit. +Show s3cmd version (1.0.0-rc2) and exit. .TP \fB\-F\fR, \fB\-\-follow\-symlinks\fR Follow symbolic links as if they are regular files This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-23 12:34:17
|
Revision: 457 http://s3tools.svn.sourceforge.net/s3tools/?rev=457&view=rev Author: ludvigm Date: 2010-12-23 12:34:09 +0000 (Thu, 23 Dec 2010) Log Message: ----------- * s3cmd: Don't crash when file disappears before checking MD5. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-12-08 13:09:03 UTC (rev 456) +++ s3cmd/trunk/ChangeLog 2010-12-23 12:34:09 UTC (rev 457) @@ -1,3 +1,8 @@ +2010-12-24 Michal Ludvig <ml...@lo...> + + * s3cmd: Don't crash when file disappears before + checking MD5. + 2010-12-09 Michal Ludvig <ml...@lo...> * Released version 1.0.0-rc2 Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-12-08 13:09:03 UTC (rev 456) +++ s3cmd/trunk/s3cmd 2010-12-23 12:34:09 UTC (rev 457) @@ -805,15 +805,23 @@ if attribs_match and 'md5' in cfg.sync_checks: ## ... same size, check MD5 - if src_remote == False and dst_remote == True: - src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) - dst_md5 = dst_list[file]['md5'] - elif src_remote == True and dst_remote == False: - src_md5 = src_list[file]['md5'] - dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) - elif src_remote == True and dst_remote == True: - src_md5 = src_list[file]['md5'] - dst_md5 = dst_list[file]['md5'] + try: + if src_remote == False and dst_remote == True: + src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) + dst_md5 = dst_list[file]['md5'] + elif src_remote == True and dst_remote == False: + src_md5 = src_list[file]['md5'] + dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) + elif src_remote == True and dst_remote == True: + src_md5 = src_list[file]['md5'] + dst_md5 = dst_list[file]['md5'] + except: + # MD5 sum verification failed - ignore that file altogether + debug(u"IGNR: %s (disappeared)" % (file)) + warning(u"%s: file disappeared, ignoring." % (file)) + del(src_list[file]) + del(dst_list[file]) + continue if src_md5 != dst_md5: ## Checksums are different. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-23 13:03:28
|
Revision: 458 http://s3tools.svn.sourceforge.net/s3tools/?rev=458&view=rev Author: ludvigm Date: 2010-12-23 13:03:19 +0000 (Thu, 23 Dec 2010) Log Message: ----------- * s3cmd: Added --(no-)check-md5 for [sync]. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-12-23 12:34:09 UTC (rev 457) +++ s3cmd/trunk/ChangeLog 2010-12-23 13:03:19 UTC (rev 458) @@ -1,5 +1,6 @@ 2010-12-24 Michal Ludvig <ml...@lo...> + * s3cmd: Added --(no-)check-md5 for [sync]. * s3cmd: Don't crash when file disappears before checking MD5. Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-12-23 12:34:09 UTC (rev 457) +++ s3cmd/trunk/s3cmd 2010-12-23 13:03:19 UTC (rev 458) @@ -1714,6 +1714,8 @@ optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).") optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).") optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") + optparser.add_option( "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)") + optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size be compared. May significantly speed up transfer but may also miss some changed files.") optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all") @@ -1841,6 +1843,15 @@ for grant in options.acl_revokes: cfg.acl_revokes.append(grant) + ## Process --(no-)check-md5 + if options.check_md5 == False: + try: + cfg.sync_checks.remove("md5") + except: + pass + if options.check_md5 == True and cfg.sync_checks.count("md5") == 0: + cfg.sync_checks.append("md5") + ## Update Config with other parameters for option in cfg.option_list(): try: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-23 13:24:33
|
Revision: 459 http://s3tools.svn.sourceforge.net/s3tools/?rev=459&view=rev Author: ludvigm Date: 2010-12-23 13:24:24 +0000 (Thu, 23 Dec 2010) Log Message: ----------- * s3cmd: Added --(no-)check-md5 for [sync]. * run-tests.py, testsuite.tar.gz: Added testsuite for the above. * NEWS: Document the above. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/run-tests.py s3cmd/trunk/testsuite.tar.gz Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-12-23 13:03:19 UTC (rev 458) +++ s3cmd/trunk/ChangeLog 2010-12-23 13:24:24 UTC (rev 459) @@ -1,6 +1,9 @@ 2010-12-24 Michal Ludvig <ml...@lo...> * s3cmd: Added --(no-)check-md5 for [sync]. + * run-tests.py, testsuite.tar.gz: Added testsuite for + the above. + * NEWS: Document the above. * s3cmd: Don't crash when file disappears before checking MD5. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2010-12-23 13:03:19 UTC (rev 458) +++ s3cmd/trunk/NEWS 2010-12-23 13:24:24 UTC (rev 459) @@ -1,3 +1,7 @@ +s3cmd 1.0.0 +=========== +* [sync] now supports --no-check-md5 + s3cmd 1.0.0-rc2 - 2010-12-09 =============== * [sync] now supports bucket-to-bucket synchronisation Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2010-12-23 13:03:19 UTC (rev 458) +++ s3cmd/trunk/run-tests.py 2010-12-23 13:24:24 UTC (rev 459) @@ -189,6 +189,18 @@ test_rmdir(label + "(rm)", dir_name) return test_mkdir(label + "(mk)", dir_name) +def test_copy(label, src_file, dst_file): + if os.name == "posix": + cmd = ['cp', '-f'] + elif os.name == "nt": + cmd = ['copy'] + else: + print "Unknown platform: %s" % os.name + sys.exit(1) + cmd.append(src_file) + cmd.append(dst_file) + return test(label, cmd) + try: pwd = pwd.getpwuid(os.getuid()) bucket_prefix = "%s.%s-" % (pwd.pw_name, pwd.pw_uid) @@ -287,6 +299,19 @@ must_find = [ u"File 'testsuite/encodings/%(encoding)s/%(pattern)s' stored as '%(pbucket)s/xyz/encodings/%(encoding)s/%(pattern)s'" % { 'encoding' : encoding, 'pattern' : enc_pattern , 'pbucket' : pbucket(1)} ]) +## ====== Don't check MD5 sum on Sync +test_copy("Change file cksum1.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum1.txt") +test_copy("Change file cksum33.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum33.txt") +test_s3cmd("Don't check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--no-check-md5'], + must_find = [ "cksum33.txt" ], + must_not_find = [ "cksum1.txt" ]) + + +## ====== Check MD5 sum on Sync +test_s3cmd("Check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--check-md5'], + must_find = [ "cksum1.txt" ]) + + ## ====== List bucket content test_s3cmd("List bucket content", ['ls', '%s/xyz/' % pbucket(1) ], must_find_re = [ u"DIR %s/xyz/binary/$" % pbucket(1) , u"DIR %s/xyz/etc/$" % pbucket(1) ], @@ -375,7 +400,6 @@ must_not_find = [ "File 'testsuite/etc/linked.png' stored as '%s/xyz/etc/linked.png" % pbucket(1) ]) - ## ====== Rename within S3 test_s3cmd("Rename within S3", ['mv', '%s/xyz/etc/logo.png' % pbucket(1), '%s/xyz/etc2/Logo.PNG' % pbucket(1)], must_find = [ 'File %s/xyz/etc/logo.png moved to %s/xyz/etc2/Logo.PNG' % (pbucket(1), pbucket(1))]) Modified: s3cmd/trunk/testsuite.tar.gz =================================================================== (Binary files differ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-23 13:47:29
|
Revision: 460 http://s3tools.svn.sourceforge.net/s3tools/?rev=460&view=rev Author: ludvigm Date: 2010-12-23 13:47:23 +0000 (Thu, 23 Dec 2010) Log Message: ----------- Fixed testsuite for --(no-)check-md5 Modified Paths: -------------- s3cmd/trunk/run-tests.py s3cmd/trunk/testsuite.tar.gz Modified: s3cmd/trunk/run-tests.py =================================================================== --- s3cmd/trunk/run-tests.py 2010-12-23 13:24:24 UTC (rev 459) +++ s3cmd/trunk/run-tests.py 2010-12-23 13:47:23 UTC (rev 460) @@ -38,6 +38,11 @@ print "Something went wrong while unpacking testsuite.tar.gz" sys.exit(1) +os.system("tar -xf testsuite/checksum.tar -C testsuite") +if not os.path.isfile('testsuite/checksum/cksum33.txt'): + print "Something went wrong while unpacking testsuite/checkum.tar" + sys.exit(1) + ## Fix up permissions for permission-denied tests os.chmod("testsuite/permission-tests/permission-denied-dir", 0444) os.chmod("testsuite/permission-tests/permission-denied.txt", 0000) @@ -299,19 +304,6 @@ must_find = [ u"File 'testsuite/encodings/%(encoding)s/%(pattern)s' stored as '%(pbucket)s/xyz/encodings/%(encoding)s/%(pattern)s'" % { 'encoding' : encoding, 'pattern' : enc_pattern , 'pbucket' : pbucket(1)} ]) -## ====== Don't check MD5 sum on Sync -test_copy("Change file cksum1.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum1.txt") -test_copy("Change file cksum33.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum33.txt") -test_s3cmd("Don't check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--no-check-md5'], - must_find = [ "cksum33.txt" ], - must_not_find = [ "cksum1.txt" ]) - - -## ====== Check MD5 sum on Sync -test_s3cmd("Check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--check-md5'], - must_find = [ "cksum1.txt" ]) - - ## ====== List bucket content test_s3cmd("List bucket content", ['ls', '%s/xyz/' % pbucket(1) ], must_find_re = [ u"DIR %s/xyz/binary/$" % pbucket(1) , u"DIR %s/xyz/etc/$" % pbucket(1) ], @@ -400,6 +392,19 @@ must_not_find = [ "File 'testsuite/etc/linked.png' stored as '%s/xyz/etc/linked.png" % pbucket(1) ]) +## ====== Don't check MD5 sum on Sync +test_copy("Change file cksum1.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum1.txt") +test_copy("Change file cksum33.txt", "testsuite/checksum/cksum2.txt", "testsuite/checksum/cksum33.txt") +test_s3cmd("Don't check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--no-check-md5'], + must_find = [ "cksum33.txt" ], + must_not_find = [ "cksum1.txt" ]) + + +## ====== Check MD5 sum on Sync +test_s3cmd("Check MD5", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--check-md5'], + must_find = [ "cksum1.txt" ]) + + ## ====== Rename within S3 test_s3cmd("Rename within S3", ['mv', '%s/xyz/etc/logo.png' % pbucket(1), '%s/xyz/etc2/Logo.PNG' % pbucket(1)], must_find = [ 'File %s/xyz/etc/logo.png moved to %s/xyz/etc2/Logo.PNG' % (pbucket(1), pbucket(1))]) Modified: s3cmd/trunk/testsuite.tar.gz =================================================================== (Binary files differ) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2010-12-23 14:01:38
|
Revision: 461 http://s3tools.svn.sourceforge.net/s3tools/?rev=461&view=rev Author: ludvigm Date: 2010-12-23 14:01:32 +0000 (Thu, 23 Dec 2010) Log Message: ----------- * s3cmd: Set 10s socket timeout for read()/write(). Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/Config.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-12-23 13:47:23 UTC (rev 460) +++ s3cmd/trunk/ChangeLog 2010-12-23 14:01:32 UTC (rev 461) @@ -1,5 +1,6 @@ 2010-12-24 Michal Ludvig <ml...@lo...> + * s3cmd: Set 10s socket timeout for read()/write(). * s3cmd: Added --(no-)check-md5 for [sync]. * run-tests.py, testsuite.tar.gz: Added testsuite for the above. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2010-12-23 13:47:23 UTC (rev 460) +++ s3cmd/trunk/NEWS 2010-12-23 14:01:32 UTC (rev 461) @@ -1,6 +1,7 @@ s3cmd 1.0.0 =========== * [sync] now supports --no-check-md5 +* Network connections now have 10s timeout s3cmd 1.0.0-rc2 - 2010-12-09 =============== Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2010-12-23 13:47:23 UTC (rev 460) +++ s3cmd/trunk/S3/Config.py 2010-12-23 14:01:32 UTC (rev 461) @@ -74,7 +74,8 @@ urlencoding_mode = "normal" log_target_prefix = "" reduced_redundancy = False - follow_symlinks=False + follow_symlinks = False + socket_timeout = 10 ## Creating a singleton def __new__(self, configfile = None): Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-12-23 13:47:23 UTC (rev 460) +++ s3cmd/trunk/s3cmd 2010-12-23 14:01:32 UTC (rev 461) @@ -22,6 +22,7 @@ import locale import subprocess import htmlentitydefs +import socket from copy import copy from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter @@ -1906,9 +1907,12 @@ cfg.include.extend(patterns_list) cfg.debug_include.update(patterns_textual) - ## Process --follow-symlinks + ## Process --follow-symlinks cfg.update_option("follow_symlinks", options.follow_symlinks) + ## Set socket read()/write() timeout + socket.setdefaulttimeout(cfg.socket_timeout) + if cfg.encrypt and cfg.gpg_passphrase == "": error(u"Encryption requested but no passphrase set in config file.") error(u"Please re-run 's3cmd --configure' and supply it.") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-01-02 02:33:58
|
Revision: 462 http://s3tools.svn.sourceforge.net/s3tools/?rev=462&view=rev Author: ludvigm Date: 2011-01-02 02:33:50 +0000 (Sun, 02 Jan 2011) Log Message: ----------- * s3cmd, s3cmd.1, format-manpage.pl: Improved --help text and manpage. * s3cmd: Removed explicit processing of --follow-symlinks (is cought by the default / main loop). Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/format-manpage.pl s3cmd/trunk/s3cmd s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2010-12-23 14:01:32 UTC (rev 461) +++ s3cmd/trunk/ChangeLog 2011-01-02 02:33:50 UTC (rev 462) @@ -1,3 +1,10 @@ +2011-01-02 Michal Ludvig <ml...@lo...> + + * s3cmd, s3cmd.1, format-manpage.pl: Improved --help text + and manpage. + * s3cmd: Removed explicit processing of --follow-symlinks + (is cought by the default / main loop). + 2010-12-24 Michal Ludvig <ml...@lo...> * s3cmd: Set 10s socket timeout for read()/write(). Modified: s3cmd/trunk/format-manpage.pl =================================================================== --- s3cmd/trunk/format-manpage.pl 2010-12-23 14:01:32 UTC (rev 461) +++ s3cmd/trunk/format-manpage.pl 2011-01-02 02:33:50 UTC (rev 462) @@ -159,7 +159,6 @@ For example to exclude all files with \".jpg\" extension except those beginning with a number use: .PP \\-\\-exclude '*.jpg' \\-\\-rinclude '[0-9].*\\.jpg' - .SH SEE ALSO For the most up to date list of options run .B s3cmd \\-\\-help @@ -167,9 +166,12 @@ For more info about usage, examples and other related info visit project homepage at .br .B http://s3tools.org - +.SH DONATIONS +Please consider a donation if you have found s3cmd useful: +.br +.B http://s3tools.org/donate .SH AUTHOR -Written by Michal Ludvig <mludvig\@logix.net.nz> +Written by Michal Ludvig <mludvig\@logix.net.nz> and 15+ contributors .SH CONTACT, SUPPORT Prefered way to get support is our mailing list: .I s3tools\\-general\@lists.sourceforge.net @@ -177,7 +179,7 @@ Report bugs to .I s3tools\\-bugs\@lists.sourceforge.net .SH COPYRIGHT -Copyright \\(co 2007,2008,2009,2010 Michal Ludvig <http://www.logix.cz/michal> +Copyright \\(co 2007,2008,2009,2010,2011 Michal Ludvig <http://www.logix.cz/michal> .br This is free software. You may redistribute copies of it under the terms of the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>. Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2010-12-23 14:01:32 UTC (rev 461) +++ s3cmd/trunk/s3cmd 2011-01-02 02:33:50 UTC (rev 462) @@ -1716,7 +1716,7 @@ optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).") optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.") optparser.add_option( "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)") - optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size be compared. May significantly speed up transfer but may also miss some changed files.") + optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size will be compared. May significantly speed up transfer but may also miss some changed files.") optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.") optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.") optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all") @@ -1771,7 +1771,8 @@ '"buckets" and uploading, downloading and removing '+ '"objects" from these buckets.') optparser.epilog = format_commands(optparser.get_prog_name(), commands_list) - optparser.epilog += ("\nSee program homepage for more information at\n%s\n" % PkgInfo.url) + optparser.epilog += ("\nFor more informations see the progect homepage:\n%s\n" % PkgInfo.url) + optparser.epilog += ("\nConsider a donation if you have found s3cmd useful:\n%s/donate\n" % PkgInfo.url) (options, args) = optparser.parse_args() @@ -1907,9 +1908,6 @@ cfg.include.extend(patterns_list) cfg.debug_include.update(patterns_textual) - ## Process --follow-symlinks - cfg.update_option("follow_symlinks", options.follow_symlinks) - ## Set socket read()/write() timeout socket.setdefaulttimeout(cfg.socket_timeout) Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2010-12-23 14:01:32 UTC (rev 461) +++ s3cmd/trunk/s3cmd.1 2011-01-02 02:33:50 UTC (rev 462) @@ -134,6 +134,15 @@ \fB\-r\fR, \fB\-\-recursive\fR Recursive upload, download or removal. .TP +\fB\-\-check\-md5\fR +Check MD5 sums when comparing files for [sync]. +(default) +.TP +\fB\-\-no\-check\-md5\fR +Do not check MD5 sums when comparing files for [sync]. +Only size will be compared. May significantly speed up +transfer but may also miss some changed files. +.TP \fB\-P\fR, \fB\-\-acl\-public\fR Store objects with ACL allowing read for anyone. .TP @@ -282,7 +291,7 @@ Enable debug output. .TP \fB\-\-version\fR -Show s3cmd version (1.0.0-rc2) and exit. +Show s3cmd version (1.0.0) and exit. .TP \fB\-F\fR, \fB\-\-follow\-symlinks\fR Follow symbolic links as if they are regular files @@ -363,7 +372,6 @@ For example to exclude all files with ".jpg" extension except those beginning with a number use: .PP \-\-exclude '*.jpg' \-\-rinclude '[0-9].*\.jpg' - .SH SEE ALSO For the most up to date list of options run .B s3cmd \-\-help @@ -371,9 +379,12 @@ For more info about usage, examples and other related info visit project homepage at .br .B http://s3tools.org - +.SH DONATIONS +Please consider a donation if you have found s3cmd useful: +.br +.B http://s3tools.org/donate .SH AUTHOR -Written by Michal Ludvig <ml...@lo...> +Written by Michal Ludvig <ml...@lo...> and 15+ contributors .SH CONTACT, SUPPORT Prefered way to get support is our mailing list: .I s3tools\-ge...@li... @@ -381,7 +392,7 @@ Report bugs to .I s3tools\-b...@li... .SH COPYRIGHT -Copyright \(co 2007,2008,2009,2010 Michal Ludvig <http://www.logix.cz/michal> +Copyright \(co 2007,2008,2009,2010,2011 Michal Ludvig <http://www.logix.cz/michal> .br This is free software. You may redistribute copies of it under the terms of the GNU General Public License version 2 <http://www.gnu.org/licenses/gpl.html>. This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-01-02 05:52:19
|
Revision: 463 http://s3tools.svn.sourceforge.net/s3tools/?rev=463&view=rev Author: ludvigm Date: 2011-01-02 05:52:13 +0000 (Sun, 02 Jan 2011) Log Message: ----------- * s3cmd: Improved r457 (Don't crash when file disappears before checking MD5). Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-01-02 02:33:50 UTC (rev 462) +++ s3cmd/trunk/ChangeLog 2011-01-02 05:52:13 UTC (rev 463) @@ -1,5 +1,7 @@ 2011-01-02 Michal Ludvig <ml...@lo...> + * s3cmd: Improved r457 (Don't crash when file disappears + before checking MD5). * s3cmd, s3cmd.1, format-manpage.pl: Improved --help text and manpage. * s3cmd: Removed explicit processing of --follow-symlinks Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2011-01-02 02:33:50 UTC (rev 462) +++ s3cmd/trunk/s3cmd 2011-01-02 05:52:13 UTC (rev 463) @@ -816,7 +816,7 @@ elif src_remote == True and dst_remote == True: src_md5 = src_list[file]['md5'] dst_md5 = dst_list[file]['md5'] - except: + except (IOError,OSError), e: # MD5 sum verification failed - ignore that file altogether debug(u"IGNR: %s (disappeared)" % (file)) warning(u"%s: file disappeared, ignoring." % (file)) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-01-09 09:01:54
|
Revision: 464 http://s3tools.svn.sourceforge.net/s3tools/?rev=464&view=rev Author: ludvigm Date: 2011-01-09 09:01:48 +0000 (Sun, 09 Jan 2011) Log Message: ----------- * Released version 1.0.0 ---------------------- * S3/PkgInfo.py: Updated to 1.0.0 * NEWS: Updated. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/PkgInfo.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-01-02 05:52:13 UTC (rev 463) +++ s3cmd/trunk/ChangeLog 2011-01-09 09:01:48 UTC (rev 464) @@ -1,3 +1,11 @@ +2011-01-09 Michal Ludvig <ml...@lo...> + + * Released version 1.0.0 + ---------------------- + + * S3/PkgInfo.py: Updated to 1.0.0 + * NEWS: Updated. + 2011-01-02 Michal Ludvig <ml...@lo...> * s3cmd: Improved r457 (Don't crash when file disappears Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2011-01-02 05:52:13 UTC (rev 463) +++ s3cmd/trunk/NEWS 2011-01-09 09:01:48 UTC (rev 464) @@ -1,18 +1,12 @@ -s3cmd 1.0.0 +s3cmd 1.0.0 - 2011-01-18 =========== * [sync] now supports --no-check-md5 * Network connections now have 10s timeout - -s3cmd 1.0.0-rc2 - 2010-12-09 -=============== * [sync] now supports bucket-to-bucket synchronisation - -s3cmd 1.0.0-rc1 - 2010-10-26 -=============== -* Added [accesslog] command. (needs manpage!) +* Added [accesslog] command. * Added access logging for CloudFront distributions using [cfmodify --log] -* Added --acl-grant and --acl-revoke [Timothee Linden] +* Added --acl-grant and --acl-revoke [Timothee Groleau] * Allow s3:// URI as well as cf:// URI as a distribution name for most CloudFront related commands. * Support for Reduced Redundancy Storage (--reduced-redundancy) Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2011-01-02 05:52:13 UTC (rev 463) +++ s3cmd/trunk/S3/PkgInfo.py 2011-01-09 09:01:48 UTC (rev 464) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "1.0.0-rc2" +version = "1.0.0" url = "http://s3tools.org" license = "GPL version 2" short_description = "Command line tool for managing Amazon S3 and CloudFront services" This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-01-13 10:09:19
|
Revision: 467 http://s3tools.svn.sourceforge.net/s3tools/?rev=467&view=rev Author: ludvigm Date: 2011-01-13 10:09:12 +0000 (Thu, 13 Jan 2011) Log Message: ----------- * s3cmd, S3/FileLists.py: Move file/object listing functions to S3/FileLists.py Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/s3cmd Added Paths: ----------- s3cmd/trunk/S3/FileLists.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-01-09 09:05:37 UTC (rev 466) +++ s3cmd/trunk/ChangeLog 2011-01-13 10:09:12 UTC (rev 467) @@ -1,3 +1,8 @@ +2011-01-13 Michal Ludvig <ml...@lo...> + + * s3cmd, S3/FileLists.py: Move file/object listing functions + to S3/FileLists.py + 2011-01-09 Michal Ludvig <ml...@lo...> * Released version 1.0.0 Added: s3cmd/trunk/S3/FileLists.py =================================================================== --- s3cmd/trunk/S3/FileLists.py (rev 0) +++ s3cmd/trunk/S3/FileLists.py 2011-01-13 10:09:12 UTC (rev 467) @@ -0,0 +1,339 @@ +## Create and compare lists of files/objects +## Author: Michal Ludvig <mi...@lo...> +## http://www.logix.cz/michal +## License: GPL Version 2 + +from S3 import S3 +from Config import Config +from S3Uri import S3Uri +from SortedDict import SortedDict +from Utils import * + +from logging import debug, info, warning, error + +import os +import glob + +__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"] + +def _fswalk_follow_symlinks(path): + ''' + Walk filesystem, following symbolic links (but without recursion), on python2.4 and later + + If a recursive directory link is detected, emit a warning and skip. + ''' + assert os.path.isdir(path) # only designed for directory argument + walkdirs = set([path]) + targets = set() + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + current = os.path.join(dirpath, dirname) + target = os.path.realpath(current) + if os.path.islink(current): + if target in targets: + warning("Skipping recursively symlinked directory %s" % dirname) + else: + walkdirs.add(current) + targets.add(target) + for walkdir in walkdirs: + for value in os.walk(walkdir): + yield value + +def _fswalk(path, follow_symlinks): + ''' + Directory tree generator + + path (str) is the root of the directory tree to walk + + follow_symlinks (bool) indicates whether to descend into symbolically linked directories + ''' + if follow_symlinks: + return _fswalk_follow_symlinks(path) + return os.walk(path) + +def filter_exclude_include(src_list): + info(u"Applying --exclude/--include") + cfg = Config() + exclude_list = SortedDict(ignore_case = False) + for file in src_list.keys(): + debug(u"CHECK: %s" % file) + excluded = False + for r in cfg.exclude: + if r.search(file): + excluded = True + debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r])) + break + if excluded: + ## No need to check for --include if not excluded + for r in cfg.include: + if r.search(file): + excluded = False + debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r])) + break + if excluded: + ## Still excluded - ok, action it + debug(u"EXCLUDE: %s" % file) + exclude_list[file] = src_list[file] + del(src_list[file]) + continue + else: + debug(u"PASS: %s" % (file)) + return src_list, exclude_list + +def fetch_local_list(args, recursive = None): + def _get_filelist_local(local_uri): + info(u"Compiling list of local files...") + if local_uri.isdir(): + local_base = deunicodise(local_uri.basename()) + local_path = deunicodise(local_uri.path()) + filelist = _fswalk(local_path, cfg.follow_symlinks) + single_file = False + else: + local_base = "" + local_path = deunicodise(local_uri.dirname()) + filelist = [( local_path, [], [deunicodise(local_uri.basename())] )] + single_file = True + loc_list = SortedDict(ignore_case = False) + for root, dirs, files in filelist: + rel_root = root.replace(local_path, local_base, 1) + for f in files: + full_name = os.path.join(root, f) + if not os.path.isfile(full_name): + continue + if os.path.islink(full_name): + if not cfg.follow_symlinks: + continue + relative_file = unicodise(os.path.join(rel_root, f)) + if os.path.sep != "/": + # Convert non-unix dir separators to '/' + relative_file = "/".join(relative_file.split(os.path.sep)) + if cfg.urlencoding_mode == "normal": + relative_file = replace_nonprintables(relative_file) + if relative_file.startswith('./'): + relative_file = relative_file[2:] + sr = os.stat_result(os.lstat(full_name)) + loc_list[relative_file] = { + 'full_name_unicode' : unicodise(full_name), + 'full_name' : full_name, + 'size' : sr.st_size, + 'mtime' : sr.st_mtime, + ## TODO: Possibly more to save here... + } + return loc_list, single_file + + cfg = Config() + local_uris = [] + local_list = SortedDict(ignore_case = False) + single_file = False + + if type(args) not in (list, tuple): + args = [args] + + if recursive == None: + recursive = cfg.recursive + + for arg in args: + uri = S3Uri(arg) + if not uri.type == 'file': + raise ParameterError("Expecting filename or directory instead of: %s" % arg) + if uri.isdir() and not recursive: + raise ParameterError("Use --recursive to upload a directory: %s" % arg) + local_uris.append(uri) + + for uri in local_uris: + list_for_uri, single_file = _get_filelist_local(uri) + local_list.update(list_for_uri) + + ## Single file is True if and only if the user + ## specified one local URI and that URI represents + ## a FILE. Ie it is False if the URI was of a DIR + ## and that dir contained only one FILE. That's not + ## a case of single_file==True. + if len(local_list) > 1: + single_file = False + + return local_list, single_file + +def fetch_remote_list(args, require_attribs = False, recursive = None): + def _get_filelist_remote(remote_uri, recursive = True): + ## If remote_uri ends with '/' then all remote files will have + ## the remote_uri prefix removed in the relative path. + ## If, on the other hand, the remote_uri ends with something else + ## (probably alphanumeric symbol) we'll use the last path part + ## in the relative path. + ## + ## Complicated, eh? See an example: + ## _get_filelist_remote("s3://bckt/abc/def") may yield: + ## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} } + ## _get_filelist_remote("s3://bckt/abc/def/") will yield: + ## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} } + ## Furthermore a prefix-magic can restrict the return list: + ## _get_filelist_remote("s3://bckt/abc/def/x") yields: + ## { 'xyz/blah.txt' : {} } + + info(u"Retrieving list of remote files for %s ..." % remote_uri) + + s3 = S3(Config()) + response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive) + + rem_base_original = rem_base = remote_uri.object() + remote_uri_original = remote_uri + if rem_base != '' and rem_base[-1] != '/': + rem_base = rem_base[:rem_base.rfind('/')+1] + remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) + rem_base_len = len(rem_base) + rem_list = SortedDict(ignore_case = False) + break_now = False + for object in response['list']: + if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: + ## We asked for one file and we got that file :-) + key = os.path.basename(object['Key']) + object_uri_str = remote_uri_original.uri() + break_now = True + rem_list = {} ## Remove whatever has already been put to rem_list + else: + key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! + object_uri_str = remote_uri.uri() + key + rem_list[key] = { + 'size' : int(object['Size']), + 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( + 'md5' : object['ETag'][1:-1], + 'object_key' : object['Key'], + 'object_uri_str' : object_uri_str, + 'base_uri' : remote_uri, + } + if break_now: + break + return rem_list + + cfg = Config() + remote_uris = [] + remote_list = SortedDict(ignore_case = False) + + if type(args) not in (list, tuple): + args = [args] + + if recursive == None: + recursive = cfg.recursive + + for arg in args: + uri = S3Uri(arg) + if not uri.type == 's3': + raise ParameterError("Expecting S3 URI instead of '%s'" % arg) + remote_uris.append(uri) + + if recursive: + for uri in remote_uris: + objectlist = _get_filelist_remote(uri) + for key in objectlist: + remote_list[key] = objectlist[key] + else: + for uri in remote_uris: + uri_str = str(uri) + ## Wildcards used in remote URI? + ## If yes we'll need a bucket listing... + if uri_str.find('*') > -1 or uri_str.find('?') > -1: + first_wildcard = uri_str.find('*') + first_questionmark = uri_str.find('?') + if first_questionmark > -1 and first_questionmark < first_wildcard: + first_wildcard = first_questionmark + prefix = uri_str[:first_wildcard] + rest = uri_str[first_wildcard+1:] + ## Only request recursive listing if the 'rest' of the URI, + ## i.e. the part after first wildcard, contains '/' + need_recursion = rest.find('/') > -1 + objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion) + for key in objectlist: + ## Check whether the 'key' matches the requested wildcards + if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): + remote_list[key] = objectlist[key] + else: + ## No wildcards - simply append the given URI to the list + key = os.path.basename(uri.object()) + if not key: + raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) + remote_item = { + 'base_uri': uri, + 'object_uri_str': unicode(uri), + 'object_key': uri.object() + } + if require_attribs: + response = S3(cfg).object_info(uri) + remote_item.update({ + 'size': int(response['headers']['content-length']), + 'md5': response['headers']['etag'].strip('"\''), + 'timestamp' : dateRFC822toUnix(response['headers']['date']) + }) + remote_list[key] = remote_item + return remote_list + +def compare_filelists(src_list, dst_list, src_remote, dst_remote): + def __direction_str(is_remote): + return is_remote and "remote" or "local" + + # We don't support local->local sync, use 'rsync' or something like that instead ;-) + assert(not(src_remote == False and dst_remote == False)) + + info(u"Verifying attributes...") + cfg = Config() + exists_list = SortedDict(ignore_case = False) + + debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote))) + debug("src_list.keys: %s" % src_list.keys()) + debug("dst_list.keys: %s" % dst_list.keys()) + + for file in src_list.keys(): + debug(u"CHECK: %s" % file) + if dst_list.has_key(file): + ## Was --skip-existing requested? + if cfg.skip_existing: + debug(u"IGNR: %s (used --skip-existing)" % (file)) + exists_list[file] = src_list[file] + del(src_list[file]) + ## Remove from destination-list, all that is left there will be deleted + del(dst_list[file]) + continue + + attribs_match = True + ## Check size first + if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: + debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) + attribs_match = False + + if attribs_match and 'md5' in cfg.sync_checks: + ## ... same size, check MD5 + try: + if src_remote == False and dst_remote == True: + src_md5 = hash_file_md5(src_list[file]['full_name']) + dst_md5 = dst_list[file]['md5'] + elif src_remote == True and dst_remote == False: + src_md5 = src_list[file]['md5'] + dst_md5 = hash_file_md5(dst_list[file]['full_name']) + elif src_remote == True and dst_remote == True: + src_md5 = src_list[file]['md5'] + dst_md5 = dst_list[file]['md5'] + except (IOError,OSError), e: + # MD5 sum verification failed - ignore that file altogether + debug(u"IGNR: %s (disappeared)" % (file)) + warning(u"%s: file disappeared, ignoring." % (file)) + del(src_list[file]) + del(dst_list[file]) + continue + + if src_md5 != dst_md5: + ## Checksums are different. + attribs_match = False + debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) + + if attribs_match: + ## Remove from source-list, all that is left there will be transferred + debug(u"IGNR: %s (transfer not needed)" % file) + exists_list[file] = src_list[file] + del(src_list[file]) + + ## Remove from destination-list, all that is left there will be deleted + del(dst_list[file]) + + return src_list, dst_list, exists_list + + Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2011-01-09 09:05:37 UTC (rev 466) +++ s3cmd/trunk/s3cmd 2011-01-13 10:09:12 UTC (rev 467) @@ -37,41 +37,6 @@ if S3Uri(arg).type != type: raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg)) -def _fswalk_follow_symlinks(path): - ''' - Walk filesystem, following symbolic links (but without recursion), on python2.4 and later - - If a recursive directory link is detected, emit a warning and skip. - ''' - assert os.path.isdir(path) # only designed for directory argument - walkdirs = set([path]) - targets = set() - for dirpath, dirnames, filenames in os.walk(path): - for dirname in dirnames: - current = os.path.join(dirpath, dirname) - target = os.path.realpath(current) - if os.path.islink(current): - if target in targets: - warning("Skipping recursively symlinked directory %s" % dirname) - else: - walkdirs.add(current) - targets.add(target) - for walkdir in walkdirs: - for value in os.walk(walkdir): - yield value - -def fswalk(path, follow_symlinks): - ''' - Directory tree generator - - path (str) is the root of the directory tree to walk - - follow_symlinks (bool) indicates whether to descend into symbolically linked directories - ''' - if follow_symlinks: - return _fswalk_follow_symlinks(path) - return os.walk(path) - def cmd_du(args): s3 = S3(Config()) if len(args) > 0: @@ -222,100 +187,6 @@ _bucket_delete_one(uri) output(u"Bucket '%s' removed" % uri.uri()) -def fetch_local_list(args, recursive = None): - local_uris = [] - local_list = SortedDict(ignore_case = False) - single_file = False - - if type(args) not in (list, tuple): - args = [args] - - if recursive == None: - recursive = cfg.recursive - - for arg in args: - uri = S3Uri(arg) - if not uri.type == 'file': - raise ParameterError("Expecting filename or directory instead of: %s" % arg) - if uri.isdir() and not recursive: - raise ParameterError("Use --recursive to upload a directory: %s" % arg) - local_uris.append(uri) - - for uri in local_uris: - list_for_uri, single_file = _get_filelist_local(uri) - local_list.update(list_for_uri) - - ## Single file is True if and only if the user - ## specified one local URI and that URI represents - ## a FILE. Ie it is False if the URI was of a DIR - ## and that dir contained only one FILE. That's not - ## a case of single_file==True. - if len(local_list) > 1: - single_file = False - - return local_list, single_file - -def fetch_remote_list(args, require_attribs = False, recursive = None): - remote_uris = [] - remote_list = SortedDict(ignore_case = False) - - if type(args) not in (list, tuple): - args = [args] - - if recursive == None: - recursive = cfg.recursive - - for arg in args: - uri = S3Uri(arg) - if not uri.type == 's3': - raise ParameterError("Expecting S3 URI instead of '%s'" % arg) - remote_uris.append(uri) - - if recursive: - for uri in remote_uris: - objectlist = _get_filelist_remote(uri) - for key in objectlist: - remote_list[key] = objectlist[key] - else: - for uri in remote_uris: - uri_str = str(uri) - ## Wildcards used in remote URI? - ## If yes we'll need a bucket listing... - if uri_str.find('*') > -1 or uri_str.find('?') > -1: - first_wildcard = uri_str.find('*') - first_questionmark = uri_str.find('?') - if first_questionmark > -1 and first_questionmark < first_wildcard: - first_wildcard = first_questionmark - prefix = uri_str[:first_wildcard] - rest = uri_str[first_wildcard+1:] - ## Only request recursive listing if the 'rest' of the URI, - ## i.e. the part after first wildcard, contains '/' - need_recursion = rest.find('/') > -1 - objectlist = _get_filelist_remote(S3Uri(prefix), recursive = need_recursion) - for key in objectlist: - ## Check whether the 'key' matches the requested wildcards - if glob.fnmatch.fnmatch(objectlist[key]['object_uri_str'], uri_str): - remote_list[key] = objectlist[key] - else: - ## No wildcards - simply append the given URI to the list - key = os.path.basename(uri.object()) - if not key: - raise ParameterError(u"Expecting S3 URI with a filename or --recursive: %s" % uri.uri()) - remote_item = { - 'base_uri': uri, - 'object_uri_str': unicode(uri), - 'object_key': uri.object() - } - if require_attribs: - response = S3(cfg).object_info(uri) - remote_item.update({ - 'size': int(response['headers']['content-length']), - 'md5': response['headers']['etag'].strip('"\''), - 'timestamp' : Utils.dateRFC822toUnix(response['headers']['date']) - }) - remote_list[key] = remote_item - return remote_list - def cmd_object_put(args): cfg = Config() s3 = S3(cfg) @@ -334,7 +205,7 @@ local_list, single_file_local = fetch_local_list(args) - local_list, exclude_list = _filelist_filter_exclude_include(local_list) + local_list, exclude_list = filter_exclude_include(local_list) local_count = len(local_list) @@ -437,7 +308,7 @@ raise ParameterError("Nothing to download. Expecting S3 URI.") remote_list = fetch_remote_list(args, require_attribs = False) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list, exclude_list = filter_exclude_include(remote_list) remote_count = len(remote_list) @@ -538,7 +409,7 @@ recursive = cfg.recursive remote_list = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list, exclude_list = filter_exclude_include(remote_list) remote_count = len(remote_list) @@ -567,7 +438,7 @@ destination_base = dst_base_uri.uri() remote_list = fetch_remote_list(args, require_attribs = False) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list, exclude_list = filter_exclude_include(remote_list) remote_count = len(remote_list) @@ -651,195 +522,6 @@ else: raise -def _get_filelist_local(local_uri): - info(u"Compiling list of local files...") - if local_uri.isdir(): - local_base = deunicodise(local_uri.basename()) - local_path = deunicodise(local_uri.path()) - filelist = fswalk(local_path, cfg.follow_symlinks) - single_file = False - else: - local_base = "" - local_path = deunicodise(local_uri.dirname()) - filelist = [( local_path, [], [deunicodise(local_uri.basename())] )] - single_file = True - loc_list = SortedDict(ignore_case = False) - for root, dirs, files in filelist: - rel_root = root.replace(local_path, local_base, 1) - for f in files: - full_name = os.path.join(root, f) - if not os.path.isfile(full_name): - continue - if os.path.islink(full_name): - if not cfg.follow_symlinks: - continue - relative_file = unicodise(os.path.join(rel_root, f)) - if os.path.sep != "/": - # Convert non-unix dir separators to '/' - relative_file = "/".join(relative_file.split(os.path.sep)) - if cfg.urlencoding_mode == "normal": - relative_file = replace_nonprintables(relative_file) - if relative_file.startswith('./'): - relative_file = relative_file[2:] - sr = os.stat_result(os.lstat(full_name)) - loc_list[relative_file] = { - 'full_name_unicode' : unicodise(full_name), - 'full_name' : full_name, - 'size' : sr.st_size, - 'mtime' : sr.st_mtime, - ## TODO: Possibly more to save here... - } - return loc_list, single_file - -def _get_filelist_remote(remote_uri, recursive = True): - ## If remote_uri ends with '/' then all remote files will have - ## the remote_uri prefix removed in the relative path. - ## If, on the other hand, the remote_uri ends with something else - ## (probably alphanumeric symbol) we'll use the last path part - ## in the relative path. - ## - ## Complicated, eh? See an example: - ## _get_filelist_remote("s3://bckt/abc/def") may yield: - ## { 'def/file1.jpg' : {}, 'def/xyz/blah.txt' : {} } - ## _get_filelist_remote("s3://bckt/abc/def/") will yield: - ## { 'file1.jpg' : {}, 'xyz/blah.txt' : {} } - ## Furthermore a prefix-magic can restrict the return list: - ## _get_filelist_remote("s3://bckt/abc/def/x") yields: - ## { 'xyz/blah.txt' : {} } - - info(u"Retrieving list of remote files for %s ..." % remote_uri) - - s3 = S3(Config()) - response = s3.bucket_list(remote_uri.bucket(), prefix = remote_uri.object(), recursive = recursive) - - rem_base_original = rem_base = remote_uri.object() - remote_uri_original = remote_uri - if rem_base != '' and rem_base[-1] != '/': - rem_base = rem_base[:rem_base.rfind('/')+1] - remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base)) - rem_base_len = len(rem_base) - rem_list = SortedDict(ignore_case = False) - break_now = False - for object in response['list']: - if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep: - ## We asked for one file and we got that file :-) - key = os.path.basename(object['Key']) - object_uri_str = remote_uri_original.uri() - break_now = True - rem_list = {} ## Remove whatever has already been put to rem_list - else: - key = object['Key'][rem_base_len:] ## Beware - this may be '' if object['Key']==rem_base !! - object_uri_str = remote_uri.uri() + key - rem_list[key] = { - 'size' : int(object['Size']), - 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-( - 'md5' : object['ETag'][1:-1], - 'object_key' : object['Key'], - 'object_uri_str' : object_uri_str, - 'base_uri' : remote_uri, - } - if break_now: - break - return rem_list - -def _filelist_filter_exclude_include(src_list): - info(u"Applying --exclude/--include") - cfg = Config() - exclude_list = SortedDict(ignore_case = False) - for file in src_list.keys(): - debug(u"CHECK: %s" % file) - excluded = False - for r in cfg.exclude: - if r.search(file): - excluded = True - debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r])) - break - if excluded: - ## No need to check for --include if not excluded - for r in cfg.include: - if r.search(file): - excluded = False - debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r])) - break - if excluded: - ## Still excluded - ok, action it - debug(u"EXCLUDE: %s" % file) - exclude_list[file] = src_list[file] - del(src_list[file]) - continue - else: - debug(u"PASS: %s" % (file)) - return src_list, exclude_list - -def _compare_filelists(src_list, dst_list, src_remote, dst_remote): - def __direction_str(is_remote): - return is_remote and "remote" or "local" - - # We don't support local->local sync, use 'rsync' or something like that instead ;-) - assert(not(src_remote == False and dst_remote == False)) - - info(u"Verifying attributes...") - cfg = Config() - exists_list = SortedDict(ignore_case = False) - - debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote))) - debug("src_list.keys: %s" % src_list.keys()) - debug("dst_list.keys: %s" % dst_list.keys()) - - for file in src_list.keys(): - debug(u"CHECK: %s" % file) - if dst_list.has_key(file): - ## Was --skip-existing requested? - if cfg.skip_existing: - debug(u"IGNR: %s (used --skip-existing)" % (file)) - exists_list[file] = src_list[file] - del(src_list[file]) - ## Remove from destination-list, all that is left there will be deleted - del(dst_list[file]) - continue - - attribs_match = True - ## Check size first - if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']: - debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size'])) - attribs_match = False - - if attribs_match and 'md5' in cfg.sync_checks: - ## ... same size, check MD5 - try: - if src_remote == False and dst_remote == True: - src_md5 = Utils.hash_file_md5(src_list[file]['full_name']) - dst_md5 = dst_list[file]['md5'] - elif src_remote == True and dst_remote == False: - src_md5 = src_list[file]['md5'] - dst_md5 = Utils.hash_file_md5(dst_list[file]['full_name']) - elif src_remote == True and dst_remote == True: - src_md5 = src_list[file]['md5'] - dst_md5 = dst_list[file]['md5'] - except (IOError,OSError), e: - # MD5 sum verification failed - ignore that file altogether - debug(u"IGNR: %s (disappeared)" % (file)) - warning(u"%s: file disappeared, ignoring." % (file)) - del(src_list[file]) - del(dst_list[file]) - continue - - if src_md5 != dst_md5: - ## Checksums are different. - attribs_match = False - debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5)) - - if attribs_match: - ## Remove from source-list, all that is left there will be transferred - debug(u"IGNR: %s (transfer not needed)" % file) - exists_list[file] = src_list[file] - del(src_list[file]) - - ## Remove from destination-list, all that is left there will be deleted - del(dst_list[file]) - - return src_list, dst_list, exists_list - def cmd_sync_remote2remote(args): s3 = S3(Config()) @@ -854,9 +536,9 @@ info(u"Found %d source files, %d destination files" % (src_count, dst_count)) - src_list, exclude_list = _filelist_filter_exclude_include(src_list) + src_list, exclude_list = filter_exclude_include(src_list) - src_list, dst_list, existing_list = _compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True) + src_list, dst_list, existing_list = compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True) src_count = len(src_list) dst_count = len(dst_list) @@ -933,9 +615,9 @@ info(u"Found %d remote files, %d local files" % (remote_count, local_count)) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list, exclude_list = filter_exclude_include(remote_list) - remote_list, local_list, existing_list = _compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False) + remote_list, local_list, existing_list = compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False) local_count = len(local_list) remote_count = len(remote_list) @@ -1116,7 +798,7 @@ info(u"Found %d local files, %d remote files" % (local_count, remote_count)) - local_list, exclude_list = _filelist_filter_exclude_include(local_list) + local_list, exclude_list = filter_exclude_include(local_list) if single_file_local and len(local_list) == 1 and len(remote_list) == 1: ## Make remote_key same as local_key for comparison if we're dealing with only one file @@ -1124,7 +806,7 @@ # Flush remote_list, by the way remote_list = { local_list.keys()[0] : remote_list_entry } - local_list, remote_list, existing_list = _compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True) + local_list, remote_list, existing_list = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True) local_count = len(local_list) remote_count = len(remote_list) @@ -1275,7 +957,7 @@ args.append(arg) remote_list = fetch_remote_list(args) - remote_list, exclude_list = _filelist_filter_exclude_include(remote_list) + remote_list, exclude_list = filter_exclude_include(remote_list) remote_count = len(remote_list) @@ -2002,6 +1684,7 @@ from S3.Utils import * from S3.Progress import Progress from S3.CloudFront import Cmd as CfCmd + from S3.FileLists import * main() sys.exit(0) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-03-30 09:29:34
|
Revision: 470 http://s3tools.svn.sourceforge.net/s3tools/?rev=470&view=rev Author: ludvigm Date: 2011-03-30 09:29:26 +0000 (Wed, 30 Mar 2011) Log Message: ----------- * S3/CloudFront.py: Cmd._get_dist_name_for_bucket() moved to CloudFront class. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/CloudFront.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-01-20 08:21:22 UTC (rev 469) +++ s3cmd/trunk/ChangeLog 2011-03-30 09:29:26 UTC (rev 470) @@ -1,3 +1,8 @@ +2011-03-30 Michal Ludvig <ml...@lo...> + + * S3/CloudFront.py: Cmd._get_dist_name_for_bucket() moved to + CloudFront class. + 2011-01-13 Michal Ludvig <ml...@lo...> * s3cmd, S3/FileLists.py: Move file/object listing functions Modified: s3cmd/trunk/S3/CloudFront.py =================================================================== --- s3cmd/trunk/S3/CloudFront.py 2011-01-20 08:21:22 UTC (rev 469) +++ s3cmd/trunk/S3/CloudFront.py 2011-03-30 09:29:26 UTC (rev 470) @@ -190,6 +190,7 @@ ## Maximum attempts of re-issuing failed requests _max_retries = 5 + dist_list = None def __init__(self, config): self.config = config @@ -388,6 +389,21 @@ # Wait a few seconds. The more it fails the more we wait. return (self._max_retries - retries + 1) * 3 + def get_dist_name_for_bucket(self, uri): + if (uri.type == "cf"): + return uri + if (uri.type != "s3"): + raise ParameterError("CloudFront or S3 URI required instead of: %s" % arg) + + debug("_get_dist_name_for_bucket(%r)" % uri) + if CloudFront.dist_list is None: + response = self.GetList() + CloudFront.dist_list = {} + for d in response['dist_list'].dist_summs: + CloudFront.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri() + debug("dist_list: %s" % CloudFront.dist_list) + return CloudFront.dist_list[uri.bucket()] + class Cmd(object): """ Class that implements CloudFront commands @@ -408,34 +424,17 @@ setattr(Cmd.options, option, value) options = Options() - dist_list = None @staticmethod - def _get_dist_name_for_bucket(uri): - cf = CloudFront(Config()) - debug("_get_dist_name_for_bucket(%r)" % uri) - assert(uri.type == "s3") - if Cmd.dist_list is None: - response = cf.GetList() - Cmd.dist_list = {} - for d in response['dist_list'].dist_summs: - Cmd.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri() - debug("dist_list: %s" % Cmd.dist_list) - return Cmd.dist_list[uri.bucket()] - - @staticmethod def _parse_args(args): + cf = CloudFront(Config()) cfuris = [] for arg in args: - uri = S3Uri(arg) - if uri.type == 's3': - try: - uri = Cmd._get_dist_name_for_bucket(uri) - except Exception, e: - debug(e) - raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % uri) - if uri.type != 'cf': - raise ParameterError("CloudFront URI required instead of: %s" % arg) + try: + uri = cf.get_dist_name_for_bucket(S3Uri(arg)) + except Exception, e: + debug(e) + raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % arg) cfuris.append(uri) return cfuris This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-03-30 10:32:38
|
Revision: 471 http://s3tools.svn.sourceforge.net/s3tools/?rev=471&view=rev Author: ludvigm Date: 2011-03-30 10:32:31 +0000 (Wed, 30 Mar 2011) Log Message: ----------- * S3/CloudFront.py: Fix warning with Python 2.7 Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/CloudFront.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-03-30 09:29:26 UTC (rev 470) +++ s3cmd/trunk/ChangeLog 2011-03-30 10:32:31 UTC (rev 471) @@ -1,5 +1,6 @@ 2011-03-30 Michal Ludvig <ml...@lo...> + * S3/CloudFront.py: Fix warning with Python 2.7 * S3/CloudFront.py: Cmd._get_dist_name_for_bucket() moved to CloudFront class. Modified: s3cmd/trunk/S3/CloudFront.py =================================================================== --- s3cmd/trunk/S3/CloudFront.py 2011-03-30 09:29:26 UTC (rev 470) +++ s3cmd/trunk/S3/CloudFront.py 2011-03-30 10:32:31 UTC (rev 471) @@ -123,10 +123,10 @@ EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>" xmlns = "http://cloudfront.amazonaws.com/doc/2010-07-15/" def __init__(self, xml = None, tree = None): - if not xml: + if xml is None: xml = DistributionConfig.EMPTY_CONFIG - if not tree: + if tree is None: tree = getTreeFromXml(xml) if tree.tag != "DistributionConfig": This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-08 13:45:14
|
Revision: 472 http://s3tools.svn.sourceforge.net/s3tools/?rev=472&view=rev Author: ludvigm Date: 2011-04-08 13:45:08 +0000 (Fri, 08 Apr 2011) Log Message: ----------- * S3/Utils.py: getDictFromTree() now recurses into sub-trees. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Utils.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-03-30 10:32:31 UTC (rev 471) +++ s3cmd/trunk/ChangeLog 2011-04-08 13:45:08 UTC (rev 472) @@ -1,3 +1,8 @@ +2011-04-10 Michal Ludvig <ml...@lo...> + + * S3/Utils.py: getDictFromTree() now recurses into + sub-trees. + 2011-03-30 Michal Ludvig <ml...@lo...> * S3/CloudFront.py: Fix warning with Python 2.7 Modified: s3cmd/trunk/S3/Utils.py =================================================================== --- s3cmd/trunk/S3/Utils.py 2011-03-30 10:32:31 UTC (rev 471) +++ s3cmd/trunk/S3/Utils.py 2011-04-08 13:45:08 UTC (rev 472) @@ -82,14 +82,16 @@ ret_dict = {} for child in tree.getchildren(): if child.getchildren(): - ## Complex-type child. We're not interested - continue + ## Complex-type child. Recurse + content = getDictFromTree(child) + else: + content = child.text if ret_dict.has_key(child.tag): if not type(ret_dict[child.tag]) == list: ret_dict[child.tag] = [ret_dict[child.tag]] - ret_dict[child.tag].append(child.text or "") + ret_dict[child.tag].append(content or "") else: - ret_dict[child.tag] = child.text or "" + ret_dict[child.tag] = content or "" return ret_dict __all__.append("getDictFromTree") This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-08 13:45:50
|
Revision: 473 http://s3tools.svn.sourceforge.net/s3tools/?rev=473&view=rev Author: ludvigm Date: 2011-04-08 13:45:43 +0000 (Fri, 08 Apr 2011) Log Message: ----------- Support for CloudFront invalidation using [sync --cf-invalidate] * s3cmd, S3/CloudFront.py, S3/Config.py: Support for CloudFront invalidation using [sync --cf-invalidate] command. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/CloudFront.py s3cmd/trunk/S3/Config.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-08 13:45:08 UTC (rev 472) +++ s3cmd/trunk/ChangeLog 2011-04-08 13:45:43 UTC (rev 473) @@ -1,5 +1,7 @@ 2011-04-10 Michal Ludvig <ml...@lo...> + * s3cmd, S3/CloudFront.py, S3/Config.py: Support for CloudFront + invalidation using [sync --cf-invalidate] command. * S3/Utils.py: getDictFromTree() now recurses into sub-trees. Modified: s3cmd/trunk/S3/CloudFront.py =================================================================== --- s3cmd/trunk/S3/CloudFront.py 2011-04-08 13:45:08 UTC (rev 472) +++ s3cmd/trunk/S3/CloudFront.py 2011-04-08 13:45:43 UTC (rev 473) @@ -6,6 +6,8 @@ import sys import time import httplib +import random +from datetime import datetime from logging import debug, info, warning, error try: @@ -17,7 +19,11 @@ from Exceptions import * from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, sign_string, getBucketFromHostname, getHostnameFromBucket from S3Uri import S3Uri, S3UriS3 +from FileLists import fetch_remote_list +cloudfront_api_version = "2010-11-01" +cloudfront_resource = "/%(api_ver)s/distribution" % { 'api_ver' : cloudfront_api_version } + def output(message): sys.stdout.write(message + "\n") @@ -34,7 +40,12 @@ ## <Status>Deployed</Status> ## <LastModifiedTime>2009-01-16T11:49:02.189Z</LastModifiedTime> ## <DomainName>blahblahblah.cloudfront.net</DomainName> - ## <Origin>example.bucket.s3.amazonaws.com</Origin> + ## <S3Origin> + ## <DNSName>example.bucket.s3.amazonaws.com</DNSName> + ## </S3Origin> + ## <CNAME>cdn.example.com</CNAME> + ## <CNAME>img.example.com</CNAME> + ## <Comment>What Ever</Comment> ## <Enabled>true</Enabled> ## </DistributionSummary> @@ -46,6 +57,8 @@ def parse(self, tree): self.info = getDictFromTree(tree) self.info['Enabled'] = (self.info['Enabled'].lower() == "true") + if self.info.has_key("CNAME") and type(self.info['CNAME']) != list: + self.info['CNAME'] = [self.info['CNAME']] def uri(self): return S3Uri("cf://%s" % self.info['Id']) @@ -121,7 +134,7 @@ ## </DistributionConfig> EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>" - xmlns = "http://cloudfront.amazonaws.com/doc/2010-07-15/" + xmlns = "http://cloudfront.amazonaws.com/doc/%(api_ver)s/" % { 'api_ver' : cloudfront_api_version } def __init__(self, xml = None, tree = None): if xml is None: xml = DistributionConfig.EMPTY_CONFIG @@ -178,6 +191,45 @@ tree.append(logging_el) return ET.tostring(tree) +class InvalidationBatch(object): + ## Example: + ## + ## <InvalidationBatch> + ## <Path>/image1.jpg</Path> + ## <Path>/image2.jpg</Path> + ## <Path>/videos/movie.flv</Path> + ## <Path>/sound%20track.mp3</Path> + ## <CallerReference>my-batch</CallerReference> + ## </InvalidationBatch> + + def __init__(self, reference = None, distribution = None, paths = []): + if reference: + self.reference = reference + else: + if not distribution: + distribution="0" + self.reference = "%s.%s.%s" % (distribution, + datetime.strftime(datetime.now(),"%Y%m%d%H%M%S"), + random.randint(1000,9999)) + self.paths = [] + self.add_objects(paths) + + def add_objects(self, paths): + self.paths.extend(paths) + + def get_reference(self): + return self.reference + + def __str__(self): + tree = ET.Element("InvalidationBatch") + + for path in self.paths: + if path[0] != "/": + path = "/" + path + appendXmlTextNode("Path", path, tree) + appendXmlTextNode("CallerReference", self.reference, tree) + return ET.tostring(tree) + class CloudFront(object): operations = { "CreateDist" : { 'method' : "POST", 'resource' : "" }, @@ -186,6 +238,9 @@ "GetDistInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s" }, "GetDistConfig" : { 'method' : "GET", 'resource' : "/%(dist_id)s/config" }, "SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" }, + "Invalidate" : { 'method' : "POST", 'resource' : "/%(dist_id)s/invalidation" }, + "GetInvalList" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation" }, + "GetInvalStatus" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(invalidation_id)s" }, } ## Maximum attempts of re-issuing failed requests @@ -311,6 +366,27 @@ body = request_body, headers = headers) return response + def InvalidateObjects(self, uri, paths): + # uri could be either cf:// or s3:// uri + cfuri = self.get_dist_name_for_bucket(uri) + if len(paths) > 999: + try: + tmp_filename = Utils.mktmpfile() + f = open(tmp_filename, "w") + f.write("\n".join(paths)+"\n") + f.close() + warning("Request to invalidate %d paths (max 999 supported)" % len(paths)) + warning("All the paths are now saved in: %s" % tmp_filename) + except: + pass + raise ParameterError("Too many paths to invalidate") + invalbatch = InvalidationBatch(distribution = cfuri.dist_id(), paths = paths) + debug("InvalidateObjects(): request_body: %s" % invalbatch) + response = self.send_request("Invalidate", dist_id = cfuri.dist_id(), + body = str(invalbatch)) + debug("InvalidateObjects(): response: %s" % response) + return response, invalbatch.get_reference() + ## -------------------------------------------------- ## Low-level methods for handling CloudFront requests ## -------------------------------------------------- @@ -350,7 +426,7 @@ return response def create_request(self, operation, dist_id = None, headers = None): - resource = self.config.cloudfront_resource + ( + resource = cloudfront_resource + ( operation['resource'] % { 'dist_id' : dist_id }) if not headers: @@ -400,9 +476,13 @@ response = self.GetList() CloudFront.dist_list = {} for d in response['dist_list'].dist_summs: - CloudFront.dist_list[getBucketFromHostname(d.info['Origin'])[0]] = d.uri() + CloudFront.dist_list[getBucketFromHostname(d.info['S3Origin']['DNSName'])[0]] = d.uri() debug("dist_list: %s" % CloudFront.dist_list) - return CloudFront.dist_list[uri.bucket()] + try: + return CloudFront.dist_list[uri.bucket()] + except Exception, e: + debug(e) + raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % arg) class Cmd(object): """ @@ -430,11 +510,7 @@ cf = CloudFront(Config()) cfuris = [] for arg in args: - try: - uri = cf.get_dist_name_for_bucket(S3Uri(arg)) - except Exception, e: - debug(e) - raise ParameterError("Unable to translate S3 URI to CloudFront distribution name: %s" % arg) + uri = cf.get_dist_name_for_bucket(S3Uri(arg)) cfuris.append(uri) return cfuris @@ -444,9 +520,17 @@ if not args: response = cf.GetList() for d in response['dist_list'].dist_summs: - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(d.info['Origin'])) + if d.info.has_key("S3Origin"): + origin = S3UriS3.httpurl_to_s3uri(d.info['S3Origin']['DNSName']) + elif d.info.has_key("CustomOrigin"): + origin = "http://%s/" % d.info['CustomOrigin']['DNSName'] + else: + origin = "<unknown>" + pretty_output("Origin", origin) pretty_output("DistId", d.uri()) pretty_output("DomainName", d.info['DomainName']) + if d.info.has_key("CNAME"): + pretty_output("CNAMEs", ", ".join(d.info['CNAME'])) pretty_output("Status", d.info['Status']) pretty_output("Enabled", d.info['Enabled']) output("") @@ -456,11 +540,18 @@ response = cf.GetDistInfo(cfuri) d = response['distribution'] dc = d.info['DistributionConfig'] - pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin'])) + if dc.info.has_key("S3Origin"): + origin = S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName']) + elif dc.info.has_key("CustomOrigin"): + origin = "http://%s/" % dc.info['CustomOrigin']['DNSName'] + else: + origin = "<unknown>" + pretty_output("Origin", origin) pretty_output("DistId", d.uri()) pretty_output("DomainName", d.info['DomainName']) + if dc.info.has_key("CNAME"): + pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) pretty_output("Status", d.info['Status']) - pretty_output("CNAMEs", ", ".join(dc.info['CNAME'])) pretty_output("Comment", dc.info['Comment']) pretty_output("Enabled", dc.info['Enabled']) pretty_output("DfltRootObject", dc.info['DefaultRootObject']) @@ -542,3 +633,7 @@ pretty_output("Enabled", dc.info['Enabled']) pretty_output("DefaultRootObject", dc.info['DefaultRootObject']) pretty_output("Etag", response['headers']['etag']) + + @staticmethod + def invalidate(args): + cf = CloudFront(Config()) Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2011-04-08 13:45:08 UTC (rev 472) +++ s3cmd/trunk/S3/Config.py 2011-04-08 13:45:43 UTC (rev 473) @@ -19,7 +19,6 @@ host_bucket = "%(bucket)s.s3.amazonaws.com" simpledb_host = "sdb.amazonaws.com" cloudfront_host = "cloudfront.amazonaws.com" - cloudfront_resource = "/2010-07-15/distribution" verbosity = logging.WARNING progress_meter = True progress_class = Progress.ProgressCR @@ -76,6 +75,7 @@ reduced_redundancy = False follow_symlinks = False socket_timeout = 10 + invalidate_on_cf = False ## Creating a singleton def __new__(self, configfile = None): Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2011-04-08 13:45:08 UTC (rev 472) +++ s3cmd/trunk/s3cmd 2011-04-08 13:45:43 UTC (rev 473) @@ -841,6 +841,7 @@ s3.object_delete(uri) output(u"deleted: '%s'" % uri) + uploaded_objects_list = [] total_size = 0 total_elapsed = 0.0 timestamp_start = time.time() @@ -872,6 +873,7 @@ (item['full_name_unicode'], uri, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1], seq_label)) total_size += response["size"] + uploaded_objects_list.append(uri.object()) total_elapsed = time.time() - timestamp_start total_speed = total_elapsed and total_size/total_elapsed or 0.0 @@ -885,6 +887,16 @@ else: info(outstr) + if cfg.invalidate_on_cf: + if len(uploaded_objects_list) == 0: + info("Nothing to invalidate in CloudFront") + else: + # 'uri' from the last iteration is still valid at this point + cf = CloudFront(cfg) + result, inval_id = cf.InvalidateObjects(uri, uploaded_objects_list) + print result + output("Created invalidation request: %s" % inval_id) + def cmd_sync(args): if (len(args) < 2): raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param']) @@ -1438,6 +1450,7 @@ optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).") optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)") optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)") + optparser.add_option( "--cf-invalidate", dest="invalidate_on_cf", action="store_true", help="Invalidate the uploaded filed in CloudFront. Also see [cfinval] command.") optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)") optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") @@ -1684,6 +1697,7 @@ from S3.Utils import * from S3.Progress import Progress from S3.CloudFront import Cmd as CfCmd + from S3.CloudFront import CloudFront from S3.FileLists import * main() This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-08 15:05:37
|
Revision: 474 http://s3tools.svn.sourceforge.net/s3tools/?rev=474&view=rev Author: ludvigm Date: 2011-04-08 15:05:30 +0000 (Fri, 08 Apr 2011) Log Message: ----------- Support for checking status of CF Invalidation Requests [cfinvalinfo]. * s3cmd, S3/CloudFront.py, S3/S3Uri.py: Support for checking status of CF Invalidation Requests [cfinvalinfo]. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/CloudFront.py s3cmd/trunk/S3/S3Uri.py s3cmd/trunk/s3cmd Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-08 13:45:43 UTC (rev 473) +++ s3cmd/trunk/ChangeLog 2011-04-08 15:05:30 UTC (rev 474) @@ -1,5 +1,7 @@ 2011-04-10 Michal Ludvig <ml...@lo...> + * s3cmd, S3/CloudFront.py, S3/S3Uri.py: Support for checking + status of CF Invalidation Requests [cfinvalinfo]. * s3cmd, S3/CloudFront.py, S3/Config.py: Support for CloudFront invalidation using [sync --cf-invalidate] command. * S3/Utils.py: getDictFromTree() now recurses into Modified: s3cmd/trunk/S3/CloudFront.py =================================================================== --- s3cmd/trunk/S3/CloudFront.py 2011-04-08 13:45:43 UTC (rev 473) +++ s3cmd/trunk/S3/CloudFront.py 2011-04-08 15:05:30 UTC (rev 474) @@ -191,6 +191,63 @@ tree.append(logging_el) return ET.tostring(tree) +class Invalidation(object): + ## Example: + ## + ## <Invalidation xmlns="http://cloudfront.amazonaws.com/doc/2010-11-01/"> + ## <Id>id</Id> + ## <Status>status</Status> + ## <CreateTime>date</CreateTime> + ## <InvalidationBatch> + ## <Path>/image1.jpg</Path> + ## <Path>/image2.jpg</Path> + ## <Path>/videos/movie.flv</Path> + ## <CallerReference>my-batch</CallerReference> + ## </InvalidationBatch> + ## </Invalidation> + + def __init__(self, xml): + tree = getTreeFromXml(xml) + if tree.tag != "Invalidation": + raise ValueError("Expected <Invalidation /> xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + + def __str__(self): + return str(self.info) + +class InvalidationList(object): + ## Example: + ## + ## <InvalidationList> + ## <Marker/> + ## <NextMarker>Invalidation ID</NextMarker> + ## <MaxItems>2</MaxItems> + ## <IsTruncated>true</IsTruncated> + ## <InvalidationSummary> + ## <Id>[Second Invalidation ID]</Id> + ## <Status>Completed</Status> + ## </InvalidationSummary> + ## <InvalidationSummary> + ## <Id>[First Invalidation ID]</Id> + ## <Status>Completed</Status> + ## </InvalidationSummary> + ## </InvalidationList> + + def __init__(self, xml): + tree = getTreeFromXml(xml) + if tree.tag != "InvalidationList": + raise ValueError("Expected <InvalidationList /> xml, got: <%s />" % tree.tag) + self.parse(tree) + + def parse(self, tree): + self.info = getDictFromTree(tree) + + def __str__(self): + return str(self.info) + class InvalidationBatch(object): ## Example: ## @@ -240,7 +297,7 @@ "SetDistConfig" : { 'method' : "PUT", 'resource' : "/%(dist_id)s/config" }, "Invalidate" : { 'method' : "POST", 'resource' : "/%(dist_id)s/invalidation" }, "GetInvalList" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation" }, - "GetInvalStatus" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(invalidation_id)s" }, + "GetInvalInfo" : { 'method' : "GET", 'resource' : "/%(dist_id)s/invalidation/%(request_id)s" }, } ## Maximum attempts of re-issuing failed requests @@ -384,18 +441,38 @@ debug("InvalidateObjects(): request_body: %s" % invalbatch) response = self.send_request("Invalidate", dist_id = cfuri.dist_id(), body = str(invalbatch)) + response['dist_id'] = cfuri.dist_id() + if response['status'] == 201: + inval_info = Invalidation(response['data']).info + response['request_id'] = inval_info['Id'] debug("InvalidateObjects(): response: %s" % response) - return response, invalbatch.get_reference() + return response + def GetInvalList(self, cfuri): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + response = self.send_request("GetInvalList", dist_id = cfuri.dist_id()) + response['inval_list'] = InvalidationList(response['data']) + return response + + def GetInvalInfo(self, cfuri): + if cfuri.type != "cf": + raise ValueError("Expected CFUri instead of: %s" % cfuri) + if cfuri.request_id() is None: + raise ValueError("Expected CFUri with Request ID") + response = self.send_request("GetInvalInfo", dist_id = cfuri.dist_id(), request_id = cfuri.request_id()) + response['inval_status'] = Invalidation(response['data']) + return response + ## -------------------------------------------------- ## Low-level methods for handling CloudFront requests ## -------------------------------------------------- - def send_request(self, op_name, dist_id = None, body = None, headers = {}, retries = _max_retries): + def send_request(self, op_name, dist_id = None, request_id = None, body = None, headers = {}, retries = _max_retries): operation = self.operations[op_name] if body: headers['content-type'] = 'text/plain' - request = self.create_request(operation, dist_id, headers) + request = self.create_request(operation, dist_id, request_id, headers) conn = self.get_connection() debug("send_request(): %s %s" % (request['method'], request['resource'])) conn.request(request['method'], request['resource'], body, request['headers']) @@ -425,9 +502,9 @@ return response - def create_request(self, operation, dist_id = None, headers = None): + def create_request(self, operation, dist_id = None, request_id = None, headers = None): resource = cloudfront_resource + ( - operation['resource'] % { 'dist_id' : dist_id }) + operation['resource'] % { 'dist_id' : dist_id, 'request_id' : request_id }) if not headers: headers = {} @@ -635,5 +712,24 @@ pretty_output("Etag", response['headers']['etag']) @staticmethod - def invalidate(args): + def invalinfo(args): cf = CloudFront(Config()) + cfuris = Cmd._parse_args(args) + requests = [] + for cfuri in cfuris: + if cfuri.request_id(): + requests.append(str(cfuri)) + else: + inval_list = cf.GetInvalList(cfuri) + for i in inval_list['inval_list'].info['InvalidationSummary']: + requests.append("/".join(["cf:/", cfuri.dist_id(), i["Id"]])) + for req in requests: + cfuri = S3Uri(req) + inval_info = cf.GetInvalInfo(cfuri) + st = inval_info['inval_status'].info + pretty_output("URI", str(cfuri)) + pretty_output("Status", st['Status']) + pretty_output("Created", st['CreateTime']) + pretty_output("Nr of paths", len(st['InvalidationBatch']['Path'])) + pretty_output("Reference", st['InvalidationBatch']['CallerReference']) + output("") Modified: s3cmd/trunk/S3/S3Uri.py =================================================================== --- s3cmd/trunk/S3/S3Uri.py 2011-04-08 13:45:43 UTC (rev 473) +++ s3cmd/trunk/S3/S3Uri.py 2011-04-08 15:05:30 UTC (rev 474) @@ -158,19 +158,26 @@ class S3UriCloudFront(S3Uri): type = "cf" - _re = re.compile("^cf://([^/]*)/?", re.IGNORECASE) + _re = re.compile("^cf://([^/]*)(/.*)?", re.IGNORECASE) def __init__(self, string): match = self._re.match(string) if not match: raise ValueError("%s: not a CloudFront URI" % string) groups = match.groups() self._dist_id = groups[0] + self._request_id = groups[1] != "/" and groups[1] or None def dist_id(self): return self._dist_id + def request_id(self): + return self._request_id + def uri(self): - return "/".join(["cf:/", self.dist_id()]) + uri = "cf://" + self.dist_id() + if self.request_id(): + uri += "/" + self.request_id() + return uri if __name__ == "__main__": uri = S3Uri("s3://bucket/object") Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2011-04-08 13:45:43 UTC (rev 473) +++ s3cmd/trunk/s3cmd 2011-04-08 15:05:30 UTC (rev 474) @@ -893,9 +893,10 @@ else: # 'uri' from the last iteration is still valid at this point cf = CloudFront(cfg) - result, inval_id = cf.InvalidateObjects(uri, uploaded_objects_list) - print result - output("Created invalidation request: %s" % inval_id) + result = cf.InvalidateObjects(uri, uploaded_objects_list) + if result['status'] == 201: + output("Created invalidation request for %d paths" % len(uploaded_objects_list)) + output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id'])) def cmd_sync(args): if (len(args) < 2): @@ -1328,6 +1329,8 @@ {"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1}, {"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1}, {"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1}, + #{"cmd":"cfinval", "label":"Invalidate CloudFront objects", "param":"s3://BUCKET/OBJECT [s3://BUCKET/OBJECT ...]", "func":CfCmd.invalidate, "argc":1}, + {"cmd":"cfinvalinfo", "label":"Display CloudFront invalidation request(s) status", "param":"cf://DIST_ID[/INVAL_ID]", "func":CfCmd.invalinfo, "argc":1}, ] def format_commands(progname, commands_list): This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-11 02:01:14
|
Revision: 475 http://s3tools.svn.sourceforge.net/s3tools/?rev=475&view=rev Author: ludvigm Date: 2011-04-11 02:01:08 +0000 (Mon, 11 Apr 2011) Log Message: ----------- * S3/Config.py: Increase socket_timeout from 10 secs to 5 mins. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/Config.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-08 15:05:30 UTC (rev 474) +++ s3cmd/trunk/ChangeLog 2011-04-11 02:01:08 UTC (rev 475) @@ -1,3 +1,7 @@ +2011-04-11 Michal Ludvig <ml...@lo...> + + * S3/Config.py: Increase socket_timeout from 10 secs to 5 mins. + 2011-04-10 Michal Ludvig <ml...@lo...> * s3cmd, S3/CloudFront.py, S3/S3Uri.py: Support for checking Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2011-04-08 15:05:30 UTC (rev 474) +++ s3cmd/trunk/S3/Config.py 2011-04-11 02:01:08 UTC (rev 475) @@ -74,7 +74,7 @@ log_target_prefix = "" reduced_redundancy = False follow_symlinks = False - socket_timeout = 10 + socket_timeout = 300 invalidate_on_cf = False ## Creating a singleton This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-11 02:02:02
|
Revision: 476 http://s3tools.svn.sourceforge.net/s3tools/?rev=476&view=rev Author: ludvigm Date: 2011-04-11 02:01:55 +0000 (Mon, 11 Apr 2011) Log Message: ----------- Released version 1.1.0-beta1 * S3/PkgInfo.py: Updated to 1.1.0-beta1 * NEWS: Updated. * s3cmd.1: Regenerated. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/PkgInfo.py s3cmd/trunk/s3cmd.1 Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-11 02:01:08 UTC (rev 475) +++ s3cmd/trunk/ChangeLog 2011-04-11 02:01:55 UTC (rev 476) @@ -1,5 +1,11 @@ 2011-04-11 Michal Ludvig <ml...@lo...> + * S3/PkgInfo.py: Updated to 1.1.0-beta1 + * NEWS: Updated. + * s3cmd.1: Regenerated. + +2011-04-11 Michal Ludvig <ml...@lo...> + * S3/Config.py: Increase socket_timeout from 10 secs to 5 mins. 2011-04-10 Michal Ludvig <ml...@lo...> Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2011-04-11 02:01:08 UTC (rev 475) +++ s3cmd/trunk/NEWS 2011-04-11 02:01:55 UTC (rev 476) @@ -1,3 +1,8 @@ +s3cmd 1.1.0 - ??? +=========== +* CloudFront invalidation via [sync --cf-invalidate] and [cfinvalinfo]. +* Increased socket_timeout from 10 secs to 5 mins. + s3cmd 1.0.0 - 2011-01-18 =========== * [sync] now supports --no-check-md5 Modified: s3cmd/trunk/S3/PkgInfo.py =================================================================== --- s3cmd/trunk/S3/PkgInfo.py 2011-04-11 02:01:08 UTC (rev 475) +++ s3cmd/trunk/S3/PkgInfo.py 2011-04-11 02:01:55 UTC (rev 476) @@ -1,5 +1,5 @@ package = "s3cmd" -version = "1.0.0" +version = "1.1.0-beta1" url = "http://s3tools.org" license = "GPL version 2" short_description = "Command line tool for managing Amazon S3 and CloudFront services" Modified: s3cmd/trunk/s3cmd.1 =================================================================== --- s3cmd/trunk/s3cmd.1 2011-04-11 02:01:08 UTC (rev 475) +++ s3cmd/trunk/s3cmd.1 2011-04-11 02:01:55 UTC (rev 476) @@ -84,6 +84,9 @@ .TP s3cmd \fBcfmodify\fR \fIcf://DIST_ID\fR Change CloudFront distribution point parameters +.TP +s3cmd \fBcfinvalinfo\fR \fIcf://DIST_ID[/INVAL_ID]\fR +Display CloudFront invalidation request(s) status .SH OPTIONS @@ -265,6 +268,10 @@ Enable given CloudFront distribution (only for [cfmodify] command) .TP +\fB\-\-cf\-invalidate\fR +Invalidate the uploaded filed in CloudFront. Also see +[cfinval] command. +.TP \fB\-\-cf\-add\-cname\fR=CNAME Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands) @@ -291,7 +298,7 @@ Enable debug output. .TP \fB\-\-version\fR -Show s3cmd version (1.0.0) and exit. +Show s3cmd version (1.1.0-beta1) and exit. .TP \fB\-F\fR, \fB\-\-follow\-symlinks\fR Follow symbolic links as if they are regular files This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-11 04:02:23
|
Revision: 477 http://s3tools.svn.sourceforge.net/s3tools/?rev=477&view=rev Author: ludvigm Date: 2011-04-11 04:02:17 +0000 (Mon, 11 Apr 2011) Log Message: ----------- * S3/CloudFront.py: Don't fail if there are no cfinval requests. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/CloudFront.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-11 02:01:55 UTC (rev 476) +++ s3cmd/trunk/ChangeLog 2011-04-11 04:02:17 UTC (rev 477) @@ -1,5 +1,10 @@ 2011-04-11 Michal Ludvig <ml...@lo...> + * S3/CloudFront.py: Don't fail if there are no cfinval + requests. + +2011-04-11 Michal Ludvig <ml...@lo...> + * S3/PkgInfo.py: Updated to 1.1.0-beta1 * NEWS: Updated. * s3cmd.1: Regenerated. Modified: s3cmd/trunk/S3/CloudFront.py =================================================================== --- s3cmd/trunk/S3/CloudFront.py 2011-04-11 02:01:55 UTC (rev 476) +++ s3cmd/trunk/S3/CloudFront.py 2011-04-11 04:02:17 UTC (rev 477) @@ -721,8 +721,11 @@ requests.append(str(cfuri)) else: inval_list = cf.GetInvalList(cfuri) - for i in inval_list['inval_list'].info['InvalidationSummary']: - requests.append("/".join(["cf:/", cfuri.dist_id(), i["Id"]])) + try: + for i in inval_list['inval_list'].info['InvalidationSummary']: + requests.append("/".join(["cf:/", cfuri.dist_id(), i["Id"]])) + except: + continue for req in requests: cfuri = S3Uri(req) inval_info = cf.GetInvalInfo(cfuri) This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |
From: <lu...@us...> - 2011-04-11 04:03:54
|
Revision: 478 http://s3tools.svn.sourceforge.net/s3tools/?rev=478&view=rev Author: ludvigm Date: 2011-04-11 04:03:47 +0000 (Mon, 11 Apr 2011) Log Message: ----------- * S3/S3Uri.py: Fixed cf:// uri parsing. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/S3/S3Uri.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2011-04-11 04:02:17 UTC (rev 477) +++ s3cmd/trunk/ChangeLog 2011-04-11 04:03:47 UTC (rev 478) @@ -1,5 +1,6 @@ 2011-04-11 Michal Ludvig <ml...@lo...> + * S3/S3Uri.py: Fixed cf:// uri parsing. * S3/CloudFront.py: Don't fail if there are no cfinval requests. Modified: s3cmd/trunk/S3/S3Uri.py =================================================================== --- s3cmd/trunk/S3/S3Uri.py 2011-04-11 04:02:17 UTC (rev 477) +++ s3cmd/trunk/S3/S3Uri.py 2011-04-11 04:03:47 UTC (rev 478) @@ -158,7 +158,7 @@ class S3UriCloudFront(S3Uri): type = "cf" - _re = re.compile("^cf://([^/]*)(/.*)?", re.IGNORECASE) + _re = re.compile("^cf://([^/]*)/*(.*)", re.IGNORECASE) def __init__(self, string): match = self._re.match(string) if not match: This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |