From: <lu...@us...> - 2010-03-19 03:18:28
|
Revision: 402 http://s3tools.svn.sourceforge.net/s3tools/?rev=402&view=rev Author: ludvigm Date: 2010-03-19 03:18:18 +0000 (Fri, 19 Mar 2010) Log Message: ----------- * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command. Modified Paths: -------------- s3cmd/trunk/ChangeLog s3cmd/trunk/NEWS s3cmd/trunk/S3/ACL.py s3cmd/trunk/S3/Config.py s3cmd/trunk/S3/S3.py s3cmd/trunk/S3/S3Uri.py s3cmd/trunk/S3/Utils.py s3cmd/trunk/s3cmd Added Paths: ----------- s3cmd/trunk/S3/AccessLog.py Modified: s3cmd/trunk/ChangeLog =================================================================== --- s3cmd/trunk/ChangeLog 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/ChangeLog 2010-03-19 03:18:18 UTC (rev 402) @@ -1,3 +1,7 @@ +2010-03-19 Michal Ludvig <ml...@lo...> + + * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command. + 2009-12-10 Michal Ludvig <ml...@lo...> * s3cmd: Path separator conversion on Windows hosts. Modified: s3cmd/trunk/NEWS =================================================================== --- s3cmd/trunk/NEWS 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/NEWS 2010-03-19 03:18:18 UTC (rev 402) @@ -1,3 +1,7 @@ +s3cmd 0.9.9.92 - ??? +============== +* Added [accesslog] command. (needs manpage!) + s3cmd 0.9.9.91 - 2009-10-08 ============== * Fixed invalid reference to a variable in failed upload handling. Modified: s3cmd/trunk/S3/ACL.py =================================================================== --- s3cmd/trunk/S3/ACL.py 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/S3/ACL.py 2010-03-19 03:18:18 UTC (rev 402) @@ -3,7 +3,7 @@ ## http://www.logix.cz/michal ## License: GPL Version 2 -from Utils import * +from Utils import getTreeFromXml try: import xml.etree.ElementTree as ET @@ -12,6 +12,7 @@ class Grantee(object): ALL_USERS_URI = "http://acs.amazonaws.com/groups/global/AllUsers" + LOG_DELIVERY_URI = "http://acs.amazonaws.com/groups/s3/LogDelivery" def __init__(self): self.xsi_type = None @@ -53,6 +54,17 @@ self.name = Grantee.ALL_USERS_URI self.permission = "READ" +class GranteeLogDelivery(Grantee): + def __init__(self, permission): + """ + permission must be either READ_ACP or WRITE + """ + Grantee.__init__(self) + self.xsi_type = "Group" + self.tag = "URI" + self.name = Grantee.LOG_DELIVERY_URI + self.permission = permission + class ACL(object): EMPTY_ACL = "<AccessControlPolicy><Owner><ID></ID></Owner><AccessControlList></AccessControlList></AccessControlPolicy>" @@ -109,11 +121,14 @@ def grantAnonRead(self): if not self.isAnonRead(): - self.grantees.append(GranteeAnonRead()) + self.appendGrantee(GranteeAnonRead()) def revokeAnonRead(self): self.grantees = [g for g in self.grantees if not g.isAnonRead()] + def appendGrantee(self, grantee): + self.grantees.append(grantee) + def __str__(self): tree = getTreeFromXml(ACL.EMPTY_ACL) tree.attrib['xmlns'] = "http://s3.amazonaws.com/doc/2006-03-01/" Added: s3cmd/trunk/S3/AccessLog.py =================================================================== --- s3cmd/trunk/S3/AccessLog.py (rev 0) +++ s3cmd/trunk/S3/AccessLog.py 2010-03-19 03:18:18 UTC (rev 402) @@ -0,0 +1,90 @@ +## Amazon S3 - Access Control List representation +## Author: Michal Ludvig <mi...@lo...> +## http://www.logix.cz/michal +## License: GPL Version 2 + +import S3Uri +from Exceptions import ParameterError +from Utils import getTreeFromXml +from ACL import GranteeAnonRead + +try: + import xml.etree.ElementTree as ET +except ImportError: + import elementtree.ElementTree as ET + +__all__ = [] +class AccessLog(object): + LOG_DISABLED = "<BucketLoggingStatus></BucketLoggingStatus>" + LOG_TEMPLATE = "<LoggingEnabled><TargetBucket></TargetBucket><TargetPrefix></TargetPrefix></LoggingEnabled>" + + def __init__(self, xml = None): + if not xml: + xml = self.LOG_DISABLED + self.tree = getTreeFromXml(xml) + self.tree.attrib['xmlns'] = "http://doc.s3.amazonaws.com/2006-03-01" + + def isLoggingEnabled(self): + return bool(self.tree.find(".//LoggingEnabled")) + + def disableLogging(self): + el = self.tree.find(".//LoggingEnabled") + if el: + self.tree.remove(el) + + def enableLogging(self, target_prefix_uri): + el = self.tree.find(".//LoggingEnabled") + if not el: + el = getTreeFromXml(self.LOG_TEMPLATE) + self.tree.append(el) + el.find(".//TargetBucket").text = target_prefix_uri.bucket() + el.find(".//TargetPrefix").text = target_prefix_uri.object() + + def targetPrefix(self): + if self.isLoggingEnabled(): + el = self.tree.find(".//LoggingEnabled") + target_prefix = "s3://%s/%s" % ( + self.tree.find(".//LoggingEnabled//TargetBucket").text, + self.tree.find(".//LoggingEnabled//TargetPrefix").text) + return S3Uri.S3Uri(target_prefix) + else: + return "" + + def setAclPublic(self, acl_public): + le = self.tree.find(".//LoggingEnabled") + if not le: + raise ParameterError("Logging not enabled, can't set default ACL for logs") + tg = le.find(".//TargetGrants") + if not acl_public: + if not tg: + ## All good, it's not been there + return + else: + le.remove(tg) + else: # acl_public == True + anon_read = GranteeAnonRead().getElement() + if not tg: + tg = ET.SubElement(le, "TargetGrants") + ## What if TargetGrants already exists? We should check if + ## AnonRead is there before appending a new one. Later... + tg.append(anon_read) + + def isAclPublic(self): + raise NotImplementedError() + + def __str__(self): + return ET.tostring(self.tree) +__all__.append("AccessLog") + +if __name__ == "__main__": + from S3Uri import S3Uri + log = AccessLog() + print log + log.enableLogging(S3Uri("s3://targetbucket/prefix/log-")) + print log + log.setAclPublic(True) + print log + log.setAclPublic(False) + print log + log.disableLogging() + print log Modified: s3cmd/trunk/S3/Config.py =================================================================== --- s3cmd/trunk/S3/Config.py 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/S3/Config.py 2010-03-19 03:18:18 UTC (rev 402) @@ -29,6 +29,7 @@ human_readable_sizes = False extra_headers = SortedDict(ignore_case = True) force = False + enable = None get_continue = False skip_existing = False recursive = False @@ -69,6 +70,7 @@ debug_include = {} encoding = "utf-8" urlencoding_mode = "normal" + log_target_prefix = "" ## Creating a singleton def __new__(self, configfile = None): Modified: s3cmd/trunk/S3/S3.py =================================================================== --- s3cmd/trunk/S3/S3.py 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/S3/S3.py 2010-03-19 03:18:18 UTC (rev 402) @@ -9,6 +9,7 @@ import httplib import logging import mimetypes +import re from logging import debug, info, warning, error from stat import ST_SIZE @@ -22,8 +23,11 @@ from BidirMap import BidirMap from Config import Config from Exceptions import * -from ACL import ACL +from ACL import ACL, GranteeLogDelivery +from AccessLog import AccessLog +from S3Uri import S3Uri +__all__ = [] class S3Request(object): def __init__(self, s3, method_string, resource, headers, params = {}): self.s3 = s3 @@ -322,6 +326,41 @@ response = self.send_request(request, body) return response + def get_accesslog(self, uri): + request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging") + response = self.send_request(request) + accesslog = AccessLog(response['data']) + return accesslog + + def set_accesslog_acl(self, uri): + acl = self.get_acl(uri) + debug("Current ACL(%s): %s" % (uri.uri(), str(acl))) + acl.appendGrantee(GranteeLogDelivery("READ_ACP")) + acl.appendGrantee(GranteeLogDelivery("WRITE")) + debug("Updated ACL(%s): %s" % (uri.uri(), str(acl))) + self.set_acl(uri, acl) + + def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False): + request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging") + accesslog = AccessLog() + if enable: + accesslog.enableLogging(log_target_prefix_uri) + accesslog.setAclPublic(acl_public) + else: + accesslog.disableLogging() + body = str(accesslog) + debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body)) + try: + response = self.send_request(request, body) + except S3Error, e: + if e.info['Code'] == "InvalidTargetBucketForLogging": + info("Setting up log-delivery ACL for target bucket.") + self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket())) + response = self.send_request(request, body) + else: + raise + return accesslog, response + ## Low level methods def urlencode_string(self, string, urlencoding_mode = None): if type(string) == unicode: @@ -720,3 +759,4 @@ return S3.check_bucket_name(bucket, dns_strict = True) except ParameterError: return False +__all__.append("S3") Modified: s3cmd/trunk/S3/S3Uri.py =================================================================== --- s3cmd/trunk/S3/S3Uri.py 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/S3/S3Uri.py 2010-03-19 03:18:18 UTC (rev 402) @@ -8,7 +8,7 @@ import sys from BidirMap import BidirMap from logging import debug -from S3 import S3 +import S3 from Utils import unicodise class S3Uri(object): @@ -73,7 +73,7 @@ return "/".join(["s3:/", self._bucket, self._object]) def is_dns_compatible(self): - return S3.check_bucket_name_dns_conformity(self._bucket) + return S3.S3.check_bucket_name_dns_conformity(self._bucket) def public_url(self): if self.is_dns_compatible(): Modified: s3cmd/trunk/S3/Utils.py =================================================================== --- s3cmd/trunk/S3/Utils.py 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/S3/Utils.py 2010-03-19 03:18:18 UTC (rev 402) @@ -29,6 +29,7 @@ import elementtree.ElementTree as ET from xml.parsers.expat import ExpatError +__all__ = [] def parseNodes(nodes): ## WARNING: Ignores text nodes from mixed xml/text. ## For instance <tag1>some text<tag2>other text</tag2></tag1> @@ -44,6 +45,7 @@ retval_item[name] = node.findtext(".//%s" % child.tag) retval.append(retval_item) return retval +__all__.append("parseNodes") def stripNameSpace(xml): """ @@ -56,6 +58,7 @@ else: xmlns = None return xml, xmlns +__all__.append("stripNameSpace") def getTreeFromXml(xml): xml, xmlns = stripNameSpace(xml) @@ -67,11 +70,13 @@ except ExpatError, e: error(e) raise Exceptions.ParameterError("Bucket contains invalid filenames. Please run: s3cmd fixbucket s3://your-bucket/") +__all__.append("getTreeFromXml") def getListFromXml(xml, node): tree = getTreeFromXml(xml) nodes = tree.findall('.//%s' % (node)) return parseNodes(nodes) +__all__.append("getListFromXml") def getDictFromTree(tree): ret_dict = {} @@ -86,6 +91,7 @@ else: ret_dict[child.tag] = child.text or "" return ret_dict +__all__.append("getDictFromTree") def getTextFromXml(xml, xpath): tree = getTreeFromXml(xml) @@ -93,15 +99,18 @@ return tree.text else: return tree.findtext(xpath) +__all__.append("getTextFromXml") def getRootTagName(xml): tree = getTreeFromXml(xml) return tree.tag +__all__.append("getRootTagName") def xmlTextNode(tag_name, text): el = ET.Element(tag_name) el.text = unicode(text) return el +__all__.append("xmlTextNode") def appendXmlTextNode(tag_name, text, parent): """ @@ -111,22 +120,27 @@ Returns the newly created Node. """ parent.append(xmlTextNode(tag_name, text)) +__all__.append("appendXmlTextNode") def dateS3toPython(date): date = re.compile("(\.\d*)?Z").sub(".000Z", date) return time.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") +__all__.append("dateS3toPython") def dateS3toUnix(date): ## FIXME: This should be timezone-aware. ## Currently the argument to strptime() is GMT but mktime() ## treats it as "localtime". Anyway... return time.mktime(dateS3toPython(date)) +__all__.append("dateS3toUnix") def dateRFC822toPython(date): return rfc822.parsedate(date) +__all__.append("dateRFC822toPython") def dateRFC822toUnix(date): return time.mktime(dateRFC822toPython(date)) +__all__.append("dateRFC822toUnix") def formatSize(size, human_readable = False, floating_point = False): size = floating_point and float(size) or int(size) @@ -139,17 +153,19 @@ return (size, coeff) else: return (size, "") +__all__.append("formatSize") def formatDateTime(s3timestamp): return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp)) +__all__.append("formatDateTime") def convertTupleListToDict(list): retval = {} for tuple in list: retval[tuple[0]] = tuple[1] return retval +__all__.append("convertTupleListToDict") - _rnd_chars = string.ascii_letters+string.digits _rnd_chars_len = len(_rnd_chars) def rndstr(len): @@ -158,6 +174,7 @@ retval += _rnd_chars[random.randint(0, _rnd_chars_len-1)] len -= 1 return retval +__all__.append("rndstr") def mktmpsomething(prefix, randchars, createfunc): old_umask = os.umask(0077) @@ -175,13 +192,16 @@ os.umask(old_umask) return dirname +__all__.append("mktmpsomething") def mktmpdir(prefix = "/tmp/tmpdir-", randchars = 10): return mktmpsomething(prefix, randchars, os.mkdir) +__all__.append("mktmpdir") def mktmpfile(prefix = "/tmp/tmpfile-", randchars = 20): createfunc = lambda filename : os.close(os.open(filename, os.O_CREAT | os.O_EXCL)) return mktmpsomething(prefix, randchars, createfunc) +__all__.append("mktmpfile") def hash_file_md5(filename): h = md5() @@ -194,6 +214,7 @@ h.update(data) f.close() return h.hexdigest() +__all__.append("hash_file_md5") def mkdir_with_parents(dir_name): """ @@ -220,6 +241,7 @@ warning("%s: %s" % (cur_dir, e)) return False return True +__all__.append("mkdir_with_parents") def unicodise(string, encoding = None, errors = "replace"): """ @@ -236,6 +258,7 @@ return string.decode(encoding, errors) except UnicodeDecodeError: raise UnicodeDecodeError("Conversion to unicode failed: %r" % string) +__all__.append("unicodise") def deunicodise(string, encoding = None, errors = "replace"): """ @@ -253,6 +276,7 @@ return string.encode(encoding, errors) except UnicodeEncodeError: raise UnicodeEncodeError("Conversion from unicode failed: %r" % string) +__all__.append("deunicodise") def unicodise_safe(string, encoding = None): """ @@ -261,6 +285,7 @@ """ return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?') +__all__.append("unicodise_safe") def replace_nonprintables(string): """ @@ -284,9 +309,11 @@ if modified and Config.Config().urlencoding_mode != "fixbucket": warning("%d non-printable characters replaced in: %s" % (modified, new_string)) return new_string +__all__.append("replace_nonprintables") def sign_string(string_to_sign): #debug("string_to_sign: %s" % string_to_sign) signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip() #debug("signature: %s" % signature) return signature +__all__.append("sign_string") Modified: s3cmd/trunk/s3cmd =================================================================== --- s3cmd/trunk/s3cmd 2009-12-09 11:11:08 UTC (rev 401) +++ s3cmd/trunk/s3cmd 2010-03-19 03:18:18 UTC (rev 402) @@ -1122,6 +1122,27 @@ if retsponse['status'] == 200: output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label)) +def cmd_accesslog(args): + s3 = S3(cfg) + bucket_uri = S3Uri(args.pop()) + if bucket_uri.object(): + raise ParameterError("Only bucket name is required for [accesslog] command") + if cfg.enable == True: + log_target_prefix_uri = S3Uri(cfg.log_target_prefix) + if log_target_prefix_uri.type != "s3": + raise ParameterError("--log-target-prefix must be a S3 URI") + accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public) + elif cfg.enable == False: + accesslog, response = s3.set_accesslog(bucket_uri, enable = False) + else: # cfg.enable == None + accesslog = s3.get_accesslog(bucket_uri) + + output(u"Access logging for: %s" % bucket_uri.uri()) + output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled()) + if accesslog.isLoggingEnabled(): + output(u" Target prefix: %s" % accesslog.targetPrefix().uri()) + #output(u" Public Access: %s" % accesslog.isAclPublic()) + def cmd_sign(args): string_to_sign = args.pop() debug("string-to-sign: %r" % string_to_sign) @@ -1426,6 +1447,7 @@ {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2}, {"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2}, {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1}, + {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1}, {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1}, {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1}, @@ -1516,6 +1538,8 @@ optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. Either EU or US (default)") + optparser.add_option( "--log-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI)") + optparser.add_option("-m", "--mime-type", dest="default_mime_type", type="mimetype", metavar="MIME/TYPE", help="Default MIME-type to be set for objects stored.") optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension. Falls back to default MIME-Type as specified by --mime-type option") @@ -1529,8 +1553,8 @@ optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).") optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).") - optparser.add_option( "--enable", dest="cf_enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)") - optparser.add_option( "--disable", dest="cf_enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)") + optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (for [cfmodify] command) or access logging (for [accesslog] command)") + optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command) or access logging (for [accesslog] command)") optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)") optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)") @@ -1617,7 +1641,13 @@ except AttributeError: ## Some Config() options are not settable from command line pass - + + ## Special handling for tri-state options (True, False, None) + cfg.update_option("enable", options.enable) + + ## CloudFront's cf_enable and Config's enable share the same --enable switch + options.cf_enable = options.enable + ## Update CloudFront options if some were set for option in CfCmd.options.option_list(): try: @@ -1735,9 +1765,9 @@ ## detect any syntax errors in there from S3.Exceptions import * from S3 import PkgInfo - from S3.S3 import * + from S3.S3 import S3 from S3.Config import Config - from S3.S3Uri import * + from S3.S3Uri import S3Uri from S3 import Utils from S3.Utils import unicodise from S3.Progress import Progress This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |