[cedar-backup-svn] SF.net SVN: cedar-backup:[1018] cedar-backup2/trunk
Brought to you by:
pronovic
Menu
▾
▴
[cedar-backup-svn] SF.net SVN: cedar-backup:[1018] cedar-backup2/trunk
|
From: <pro...@us...> - 2011-10-10 03:27:41
|
Revision: 1018
http://cedar-backup.svn.sourceforge.net/cedar-backup/?rev=1018&view=rev
Author: pronovic
Date: 2011-10-10 03:27:34 +0000 (Mon, 10 Oct 2011)
Log Message:
-----------
Implement recursive collect (original patch from Zoran Bosnjak)
Modified Paths:
--------------
cedar-backup2/trunk/CREDITS
cedar-backup2/trunk/CedarBackup2/actions/collect.py
cedar-backup2/trunk/CedarBackup2/config.py
cedar-backup2/trunk/CedarBackup2/release.py
cedar-backup2/trunk/Changelog
cedar-backup2/trunk/manual/src/config.xml
cedar-backup2/trunk/testcase/configtests.py
cedar-backup2/trunk/testcase/data/cback.conf.8
Modified: cedar-backup2/trunk/CREDITS
===================================================================
--- cedar-backup2/trunk/CREDITS 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/CREDITS 2011-10-10 03:27:34 UTC (rev 1018)
@@ -23,7 +23,7 @@
software, as indicated in the source code itself.
Unless otherwise indicated, all Cedar Backup source code is Copyright
-(c) 2004-2010 Kenneth J. Pronovici and is released under the GNU General
+(c) 2004-2011 Kenneth J. Pronovici and is released under the GNU General
Public License. The contents of the GNU General Public License can be
found in the LICENSE file, or can be downloaded from http://www.gnu.org/.
@@ -32,6 +32,9 @@
the optimized media blanking strategy as well as improvements to the DVD
writer implementation.
+Zoran Bosnjak contributed changes to collect.py to implement recursive
+collect behavior based on recursion level.
+
The PostgreSQL extension was contributed by Antoine Beaupre, based on
the existing MySQL extension.
Modified: cedar-backup2/trunk/CedarBackup2/actions/collect.py
===================================================================
--- cedar-backup2/trunk/CedarBackup2/actions/collect.py 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/CedarBackup2/actions/collect.py 2011-10-10 03:27:34 UTC (rev 1018)
@@ -8,7 +8,7 @@
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
-# Copyright (c) 2004-2008,2010 Kenneth J. Pronovici.
+# Copyright (c) 2004-2008,2011 Kenneth J. Pronovici.
# All rights reserved.
#
# This program is free software; you can redistribute it and/or
@@ -53,7 +53,7 @@
import pickle
# Cedar Backup modules
-from CedarBackup2.filesystem import BackupFileList
+from CedarBackup2.filesystem import BackupFileList, FilesystemList
from CedarBackup2.util import isStartOfWeek, changeOwnership, displayBytes, buildNormalizedPath
from CedarBackup2.actions.constants import DIGEST_EXTENSION, COLLECT_INDICATOR
from CedarBackup2.actions.util import writeIndicatorFile
@@ -111,8 +111,8 @@
logger.debug("Working with collect file [%s]" % collectFile.absolutePath)
collectMode = _getCollectMode(config, collectFile)
archiveMode = _getArchiveMode(config, collectFile)
- digestPath = _getDigestPath(config, collectFile)
- tarfilePath = _getTarfilePath(config, collectFile, archiveMode)
+ digestPath = _getDigestPath(config, collectFile.absolutePath)
+ tarfilePath = _getTarfilePath(config, collectFile.absolutePath, archiveMode)
if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
logger.debug("File meets criteria to be backed up today.")
_collectFile(config, collectFile.absolutePath, tarfilePath,
@@ -128,14 +128,13 @@
ignoreFile = _getIgnoreFile(config, collectDir)
linkDepth = _getLinkDepth(collectDir)
dereference = _getDereference(collectDir)
- digestPath = _getDigestPath(config, collectDir)
- tarfilePath = _getTarfilePath(config, collectDir, archiveMode)
+ recursionLevel = _getRecursionLevel(collectDir)
(excludePaths, excludePatterns) = _getExclusions(config, collectDir)
if fullBackup or (collectMode in ['daily', 'incr', ]) or (collectMode == 'weekly' and todayIsStart):
logger.debug("Directory meets criteria to be backed up today.")
- _collectDirectory(config, collectDir.absolutePath, tarfilePath,
+ _collectDirectory(config, collectDir.absolutePath,
collectMode, archiveMode, ignoreFile, linkDepth, dereference,
- resetDigest, digestPath, excludePaths, excludePatterns)
+ resetDigest, excludePaths, excludePatterns, recursionLevel)
else:
logger.debug("Directory will not be backed up, per collect mode.")
logger.info("Completed collecting directory [%s]" % collectDir.absolutePath)
@@ -182,9 +181,9 @@
# _collectDirectory() function
###############################
-def _collectDirectory(config, absolutePath, tarfilePath, collectMode, archiveMode,
- ignoreFile, linkDepth, dereference, resetDigest, digestPath,
- excludePaths, excludePatterns):
+def _collectDirectory(config, absolutePath, collectMode, archiveMode,
+ ignoreFile, linkDepth, dereference, resetDigest,
+ excludePaths, excludePatterns, recursionLevel):
"""
Collects a configured collect directory.
@@ -199,25 +198,51 @@
@param config: Config object.
@param absolutePath: Absolute path of directory to collect.
- @param tarfilePath: Path to tarfile that should be created.
@param collectMode: Collect mode to use.
@param archiveMode: Archive mode to use.
@param ignoreFile: Ignore file to use.
@param linkDepth: Link depth value to use.
@param dereference: Dereference flag to use.
@param resetDigest: Reset digest flag.
- @param digestPath: Path to digest file on disk, if needed.
@param excludePaths: List of absolute paths to exclude.
@param excludePatterns: List of patterns to exclude.
+ @param recursionLevel: Recursion level (zero for no recursion)
"""
- backupList = BackupFileList()
- backupList.ignoreFile = ignoreFile
- backupList.excludePaths = excludePaths
- backupList.excludePatterns = excludePatterns
- backupList.addDirContents(absolutePath, linkDepth=linkDepth, dereference=dereference)
- _executeBackup(config, backupList, absolutePath, tarfilePath, collectMode, archiveMode, resetDigest, digestPath)
+ if recursionLevel == 0:
+ # Collect the actual directory because we're at recursion level 0
+ logger.info("Collecting directory [%s]" % absolutePath)
+ tarfilePath = _getTarfilePath(config, absolutePath, archiveMode)
+ digestPath = _getDigestPath(config, absolutePath)
+ backupList = BackupFileList()
+ backupList.ignoreFile = ignoreFile
+ backupList.excludePaths = excludePaths
+ backupList.excludePatterns = excludePatterns
+ backupList.addDirContents(absolutePath, linkDepth=linkDepth, dereference=dereference)
+ _executeBackup(config, backupList, absolutePath, tarfilePath, collectMode, archiveMode, resetDigest, digestPath)
+ else:
+ # Find all of the immediate subdirectories
+ subdirs = FilesystemList()
+ subdirs.excludeFiles = True
+ subdirs.excludeLinks = True
+ subdirs.excludePaths = excludePaths
+ subdirs.excludePatterns = excludePatterns
+ subdirs.addDirContents(path=absolutePath, recursive=False, addSelf=False)
+
+ # Back up the subdirectories separately
+ for subdir in subdirs:
+ _collectDirectory(config, subdir, collectMode, archiveMode,
+ ignoreFile, linkDepth, dereference, resetDigest,
+ excludePaths, excludePatterns, recursionLevel-1)
+ excludePaths.append(subdir) # this directory is already backed up, so exclude it
+
+ # Back up everything that hasn't previously been backed up
+ _collectDirectory(config, absolutePath, collectMode, archiveMode,
+ ignoreFile, linkDepth, dereference, resetDigest,
+ excludePaths, excludePatterns, 0)
+
+
############################
# _executeBackup() function
############################
@@ -398,7 +423,7 @@
Gets the link depth that should be used for a collect directory.
If possible, use the one on the directory, otherwise set a value of 0 (zero).
@param item: C{CollectDir} object
- @return: Ignore file to use.
+ @return: Link depth to use.
"""
if item.linkDepth is None:
linkDepth = 0
@@ -417,7 +442,7 @@
Gets the dereference flag that should be used for a collect directory.
If possible, use the one on the directory, otherwise set a value of False.
@param item: C{CollectDir} object
- @return: Ignore file to use.
+ @return: Dereference flag to use.
"""
if item.dereference is None:
dereference = False
@@ -427,18 +452,37 @@
return dereference
+################################
+# _getRecursionLevel() function
+################################
+
+def _getRecursionLevel(item):
+ """
+ Gets the recursion level that should be used for a collect directory.
+ If possible, use the one on the directory, otherwise set a value of 0 (zero).
+ @param item: C{CollectDir} object
+ @return: Recursion level to use.
+ """
+ if item.recursionLevel is None:
+ recursionLevel = 0
+ else:
+ recursionLevel = item.recursionLevel
+ logger.debug("Recursion level is [%d]" % recursionLevel)
+ return recursionLevel
+
+
############################
# _getDigestPath() function
############################
-def _getDigestPath(config, item):
+def _getDigestPath(config, absolutePath):
"""
Gets the digest path associated with a collect directory or file.
@param config: Config object.
- @param item: C{CollectFile} or C{CollectDir} object
+ @param absolutePath: Absolute path to generate digest for
@return: Absolute path to the digest associated with the collect directory or file.
"""
- normalized = buildNormalizedPath(item.absolutePath)
+ normalized = buildNormalizedPath(absolutePath)
filename = "%s.%s" % (normalized, DIGEST_EXTENSION)
digestPath = os.path.join(config.options.workingDir, filename)
logger.debug("Digest path is [%s]" % digestPath)
@@ -449,11 +493,11 @@
# _getTarfilePath() function
#############################
-def _getTarfilePath(config, item, archiveMode):
+def _getTarfilePath(config, absolutePath, archiveMode):
"""
Gets the tarfile path (including correct extension) associated with a collect directory.
@param config: Config object.
- @param item: C{CollectFile} or C{CollectDir} object
+ @param absolutePath: Absolute path to generate tarfile for
@param archiveMode: Archive mode to use for this tarfile.
@return: Absolute path to the tarfile associated with the collect directory.
"""
@@ -463,7 +507,7 @@
extension = "tar.gz"
elif archiveMode == 'tarbz2':
extension = "tar.bz2"
- normalized = buildNormalizedPath(item.absolutePath)
+ normalized = buildNormalizedPath(absolutePath)
filename = "%s.%s" % (normalized, extension)
tarfilePath = os.path.join(config.collect.targetDir, filename)
logger.debug("Tarfile path is [%s]" % tarfilePath)
Modified: cedar-backup2/trunk/CedarBackup2/config.py
===================================================================
--- cedar-backup2/trunk/CedarBackup2/config.py 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/CedarBackup2/config.py 2011-10-10 03:27:34 UTC (rev 1018)
@@ -2934,11 +2934,12 @@
@sort: __init__, __repr__, __str__, __cmp__, targetDir,
collectMode, archiveMode, ignoreFile, absoluteExcludePaths,
- excludePatterns, collectFiles, collectDirs
+ excludePatterns, collectFiles, collectDirs, recursionLevel
"""
def __init__(self, targetDir=None, collectMode=None, archiveMode=None, ignoreFile=None,
- absoluteExcludePaths=None, excludePatterns=None, collectFiles=None, collectDirs=None):
+ absoluteExcludePaths=None, excludePatterns=None, collectFiles=None,
+ collectDirs=None, recursionLevel=None):
"""
Constructor for the C{CollectConfig} class.
@@ -2950,6 +2951,7 @@
@param excludePatterns: List of regular expression patterns to exclude.
@param collectFiles: List of collect files.
@param collectDirs: List of collect directories.
+ @param recursionLevel: Recursion level to use for recursive directory collection
@raise ValueError: If one of the values is invalid.
"""
@@ -2961,6 +2963,7 @@
self._excludePatterns = None
self._collectFiles = None
self._collectDirs = None
+ self._recursionLevel = None
self.targetDir = targetDir
self.collectMode = collectMode
self.archiveMode = archiveMode
@@ -2969,15 +2972,17 @@
self.excludePatterns = excludePatterns
self.collectFiles = collectFiles
self.collectDirs = collectDirs
+ self.recursionLevel = recursionLevel
def __repr__(self):
"""
Official string representation for class instance.
"""
- return "CollectConfig(%s, %s, %s, %s, %s, %s, %s, %s)" % (self.targetDir, self.collectMode, self.archiveMode,
- self.ignoreFile, self.absoluteExcludePaths,
- self.excludePatterns, self.collectFiles, self.collectDirs)
-
+ return "CollectConfig(%s, %s, %s, %s, %s, %s, %s, %s, %s)" % (self.targetDir, self.collectMode, self.archiveMode,
+ self.ignoreFile, self.absoluteExcludePaths,
+ self.excludePatterns, self.collectFiles,
+ self.collectDirs, self.recursionLevel)
+
def __str__(self):
"""
Informal string representation for class instance.
@@ -3033,6 +3038,11 @@
return -1
else:
return 1
+ if self.recursionLevel != other.recursionLevel:
+ if self.recursionLevel < other.recursionLevel:
+ return -1
+ else:
+ return 1
return 0
def _setTargetDir(self, value):
@@ -3197,6 +3207,27 @@
"""
return self._collectDirs
+ def _setRecursionLevel(self, value):
+ """
+ Property target used to set the recursionLevel.
+ The value must be an integer.
+ @raise ValueError: If the value is not valid.
+ """
+ if value is None:
+ self._recursionLevel = None
+ else:
+ try:
+ value = int(value)
+ except TypeError:
+ raise ValueError("Recusion level value must be an integer.")
+ self._recursionLevel = value
+
+ def _getRecursionLevel(self):
+ """
+ Property target used to get the action recursionLevel.
+ """
+ return self._recursionLevel
+
targetDir = property(_getTargetDir, _setTargetDir, None, "Directory to collect files into.")
collectMode = property(_getCollectMode, _setCollectMode, None, "Default collect mode.")
archiveMode = property(_getArchiveMode, _setArchiveMode, None, "Default archive mode for collect files.")
@@ -3205,6 +3236,7 @@
excludePatterns = property(_getExcludePatterns, _setExcludePatterns, None, "List of regular expressions patterns to exclude.")
collectFiles = property(_getCollectFiles, _setCollectFiles, None, "List of collect files.")
collectDirs = property(_getCollectDirs, _setCollectDirs, None, "List of collect directories.")
+ recursionLevel = property(_getRecursionLevel, _setRecursionLevel, None, "Recursion level to use for recursive directory collection")
########################################################################
@@ -4453,6 +4485,7 @@
collectMode //cb_config/collect/collect_mode
archiveMode //cb_config/collect/archive_mode
ignoreFile //cb_config/collect/ignore_file
+ recursionLevel //cb_config/collect/recursion_level
We also read groups of the following items, one list element per
item::
@@ -4479,6 +4512,7 @@
collect.collectMode = readString(sectionNode, "collect_mode")
collect.archiveMode = readString(sectionNode, "archive_mode")
collect.ignoreFile = readString(sectionNode, "ignore_file")
+ collect.recursionLevel = readInteger(sectionNode, "recursion_level")
(collect.absoluteExcludePaths, unused, collect.excludePatterns) = Config._parseExclusions(sectionNode)
collect.collectFiles = Config._parseCollectFiles(sectionNode)
collect.collectDirs = Config._parseCollectDirs(sectionNode)
@@ -5110,6 +5144,7 @@
collectMode //cb_config/collect/collect_mode
archiveMode //cb_config/collect/archive_mode
ignoreFile //cb_config/collect/ignore_file
+ recursionLevel //cb_config/collect/recursion_level
We also add groups of the following items, one list element per
item::
@@ -5134,6 +5169,7 @@
addStringNode(xmlDom, sectionNode, "collect_mode", collectConfig.collectMode)
addStringNode(xmlDom, sectionNode, "archive_mode", collectConfig.archiveMode)
addStringNode(xmlDom, sectionNode, "ignore_file", collectConfig.ignoreFile)
+ addStringNode(xmlDom, sectionNode, "recursion_level", collectConfig.recursionLevel)
if ((collectConfig.absoluteExcludePaths is not None and collectConfig.absoluteExcludePaths != []) or
(collectConfig.excludePatterns is not None and collectConfig.excludePatterns != [])):
excludeNode = addContainerNode(xmlDom, sectionNode, "exclude")
@@ -5694,10 +5730,10 @@
"""
Validates collect configuration.
- The target directory must be filled in. The collect mode, archive mode
- and ignore file are all optional. The list of absolute paths to exclude
- and patterns to exclude may be either C{None} or an empty list C{[]} if
- desired.
+ The target directory must be filled in. The collect mode, archive mode,
+ ignore file, and recursion level are all optional. The list of absolute
+ paths to exclude and patterns to exclude may be either C{None} or an
+ empty list C{[]} if desired.
Each collect directory entry must contain an absolute path to collect,
and then must either be able to take collect mode, archive mode and
Modified: cedar-backup2/trunk/CedarBackup2/release.py
===================================================================
--- cedar-backup2/trunk/CedarBackup2/release.py 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/CedarBackup2/release.py 2011-10-10 03:27:34 UTC (rev 1018)
@@ -33,8 +33,8 @@
AUTHOR = "Kenneth J. Pronovici"
EMAIL = "pro...@ie..."
-COPYRIGHT = "2004-2010"
-VERSION = "2.20.1"
-DATE = "19 Oct 2010"
+COPYRIGHT = "2004-2011"
+VERSION = "2.21.0"
+DATE = "unreleased"
URL = "http://cedar-backup.sourceforge.net/"
Modified: cedar-backup2/trunk/Changelog
===================================================================
--- cedar-backup2/trunk/Changelog 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/Changelog 2011-10-10 03:27:34 UTC (rev 1018)
@@ -1,3 +1,10 @@
+Version 2.21.0 unreleased
+
+ * Implement configurable recursion for collect action.
+ - Update collect.py to handle recursion (patch by Zoran Bosnjak)
+ - Add new configuration item CollectDir.recursionLevel
+ - Update user manual, CREDITS, etc. for new functionality
+
Version 2.20.1 19 Oct 2010
* Fix minor formatting issues in manpages, pointed out by Debian lintian.
Modified: cedar-backup2/trunk/manual/src/config.xml
===================================================================
--- cedar-backup2/trunk/manual/src/config.xml 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/manual/src/config.xml 2011-10-10 03:27:34 UTC (rev 1018)
@@ -1366,6 +1366,53 @@
</varlistentry>
<varlistentry>
+ <term><literal>recursion_level</literal></term>
+ <listitem>
+ <para>Recursion level to use when collecting directories.</para>
+ <para>
+ This is an integer value that Cedar Backup will consider
+ when generating archive files for a configured collect
+ directory.
+ </para>
+ <para>
+ Normally, Cedar Backup generates one archive file per
+ collect directory. So, if you collect
+ <literal>/etc</literal> you get
+ <literal>etc.tar.gz</literal>. Most of the time, this is
+ what you want. However, you may sometimes wish to
+ generate multiple archive files for a single collect
+ directory.
+ </para>
+ <para>
+ The most obvious example is for <literal>/home</literal>.
+ By default, Cedar Backup will generate
+ <literal>home.tar.gz</literal>. If instead, you want one
+ archive file per home directory you can set a recursion
+ level of <literal>1</literal>. Cedar Backup will generate
+ <literal>home-user1.tar.gz</literal>,
+ <literal>home-user2.tar.gz</literal>, etc.
+ </para>
+ <para>
+ Higher recursion levels (<literal>2</literal>,
+ <literal>3</literal>, etc.) are legal, and it doesn't
+ matter if the configured recursion level is deeper than
+ the directory tree that is being collected. You can use a
+ negative recursion level (like <literal>-1</literal>) to
+ specify an infinite level of recursion. This will exhaust
+ the tree in the same way as if the recursion level is set
+ too high.
+ </para>
+ <para>
+ This field is optional. if it doesn't exist, the backup
+ will use the default list recursion level of zero.
+ </para>
+ <para>
+ <emphasis>Restrictions:</emphasis> Must be an integer.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><literal>exclude</literal></term>
<listitem>
<para>List of paths or patterns to exclude from the backup.</para>
Modified: cedar-backup2/trunk/testcase/configtests.py
===================================================================
--- cedar-backup2/trunk/testcase/configtests.py 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/testcase/configtests.py 2011-10-10 03:27:34 UTC (rev 1018)
@@ -11256,6 +11256,7 @@
config = Config(xmlPath=path, validate=False)
expected = Config()
expected.collect = CollectConfig("/opt/backup/collect", "daily", "targz", ".cbignore")
+ expected.collect.recursionLevel = 1
expected.collect.absoluteExcludePaths = ["/etc/cback.conf", "/etc/X11", ]
expected.collect.excludePatterns = [".*tmp.*", ".*\.netscape\/.*", ]
expected.collect.collectFiles = []
Modified: cedar-backup2/trunk/testcase/data/cback.conf.8
===================================================================
--- cedar-backup2/trunk/testcase/data/cback.conf.8 2011-03-02 17:35:31 UTC (rev 1017)
+++ cedar-backup2/trunk/testcase/data/cback.conf.8 2011-10-10 03:27:34 UTC (rev 1018)
@@ -6,6 +6,7 @@
<collect_mode>daily</collect_mode>
<archive_mode>targz</archive_mode>
<ignore_file>.cbignore</ignore_file>
+ <recursion_level>1</recursion_level>
<exclude>
<abs_path>/etc/cback.conf</abs_path>
<abs_path>/etc/X11</abs_path>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
