Menu
▾
▴
[Zxsync-commits] zXSync/zxsync engine.py,1.10,1.11
|
From: <luc...@us...> - 2003-07-13 09:21:05
|
Update of /cvsroot/zxsync/zXSync/zxsync
In directory sc8-pr-cvs1:/tmp/cvs-serv31011
Modified Files:
engine.py
Log Message:
- Added automatic backups
- Added automatic sync mode assignment based on the backups
- Added (very simple) logging via the context
- Output a lot of log messages to better track the progress while syncing
- A lot of bug fixes for dry-runs
- Skip a lot of sync stuff if an application cannot be found on a device (improves performance and necessary for backups)
- Various small bug fixes
Index: engine.py
===================================================================
RCS file: /cvsroot/zxsync/zXSync/zxsync/engine.py,v
retrieving revision 1.10
retrieving revision 1.11
diff -C2 -d -r1.10 -r1.11
*** engine.py 23 Jun 2003 21:10:52 -0000 1.10
--- engine.py 13 Jul 2003 09:21:02 -0000 1.11
***************
*** 27,33 ****
--- 27,35 ----
import vcard
import report
+ import backup
i18n = {
+ "AUTO_SYNC_FAILED": "Automatic sync mode assignment for a device provided by plugin %1 failed",
"BEGIN_SYNC_FAILED": "Beginning sync for a device provided by plugin %1 failed",
"END_SYNC_FAILED": "Ending sync for a device provided by plugin %1 failed",
***************
*** 41,44 ****
--- 43,47 ----
"INIT_DEVICE_FAILED": "Initializing a device provided by plugin %1 failed",
"LOADING_PLUGIN_FAILED": "Loading plugin %1 failed",
+ "POST_SYNC_BACKUP_FAILED": "Creating backups after syncing failed",
"PROBE_FAILED": "Probing plugin %1 for devices failed",
"READ_ALL_FAILED": "Reading entries from a device provided by plugin %1 failed",
***************
*** 107,110 ****
--- 110,121 ----
raise
+
+ def log(self, message):
+ """
+ Logs a message. This implementation simply prints the message to the console.
+ """
+ print "[zXSync]", message
+
+
globalContext = SyncContext()
***************
*** 149,155 ****
self.removeRejectedList = [] # removals vetoed by the device
self.changeRejectedList = [] # changes rejected by the device
- self.addListBackup = []
- self.removeListBackup = []
- self.changeListBackup = []
self.apps = []
# set to an empty list in case the next call raises an exception
--- 160,163 ----
***************
*** 251,254 ****
--- 259,263 ----
pass
try:
+ globalContext.log("Probing '" + plugin.__name__ + "' ...")
moduleDevices = plugin.__dict__["probe"](pluginSettings, globalContext)
except:
***************
*** 277,280 ****
--- 286,290 ----
Begins a a sync operation for the specified application.
"""
+ appToSyncFound = 0
for deviceInfo in self.devices:
device = deviceInfo.device
***************
*** 289,292 ****
--- 299,303 ----
deviceApp = appInfo.app
if deviceApp.getId() == appToSync:
+ appToSyncFound = 1
if appInfo.syncStarted:
appInfo.syncStarted = 0
***************
*** 296,299 ****
--- 307,313 ----
self.handleError(0, i18n["END_SYNC_FAILED"], deviceInfo.plugin.__name__);
try:
+ globalContext.log("Calling beginSync() for '" +
+ deviceInfo.plugin.__name__ + "/" +
+ device.getId() + "' ...")
writeMode = deviceApp.beginSync()
appInfo.syncStarted = 1
***************
*** 304,312 ****
globalContext.fatalDefault = READ_ERRORS_ARE_FATAL
try:
deviceEntries = deviceApp.readAllEntries()
removeList = []
for deviceEntry in deviceEntries:
if not deviceEntry.isValid():
! print "Invalid entry:", deviceEntry
removeList.append(deviceEntry)
for removeEntry in removeList:
--- 318,329 ----
globalContext.fatalDefault = READ_ERRORS_ARE_FATAL
try:
+ globalContext.log("Calling readAllEntries() for '" +
+ deviceInfo.plugin.__name__ + "/" +
+ device.getId() + "' ...")
deviceEntries = deviceApp.readAllEntries()
removeList = []
for deviceEntry in deviceEntries:
if not deviceEntry.isValid():
! globalContext.log("Invalid entry: " + str(deviceEntry))
removeList.append(deviceEntry)
for removeEntry in removeList:
***************
*** 319,322 ****
--- 336,340 ----
deviceInfo.data.append(newApp)
deviceInfo.entriesRead += len(deviceEntries)
+ globalContext.fatalDefault = oldFatalDefault
except:
globalContext.fatalDefault = oldFatalDefault
***************
*** 325,329 ****
# (2) the plugin called handleError() with fatal == true
# or (3) the default for read errors is fatal
! globalContext.fatalDefault = oldFatalDefault
--- 343,543 ----
# (2) the plugin called handleError() with fatal == true
# or (3) the default for read errors is fatal
! if appToSyncFound:
! try:
! self.determineStatus(appToSync)
! self.setAutoSyncModes(appToSync)
! except:
! self.handleError(0, i18n["AUTO_SYNC_FAILED"], deviceInfo.plugin.__name__);
!
!
! def determineStatus(self, appToSync):
! """
! Determines the sync modes according to the latest backups for the
! specified app.
!
! Note that entries and fields that have been added since the last
! backup will have a status of STATUS_UNKNOWN instead of STATUS_ADDED.
! This is OK since the default sync mode of SYNC_BOTH will still get
! them distributed. Probably STATUS_ADDED will be removed altogether
! some day (since it's additinal unnecessary work to find out what has
! been addded and assign that status).
! """
! backups = backup.loadBackups(self.devices, appToSync, 1)
! # "1" means load post-sync backups
! if len(backups) > 0:
! globalContext.log("Using previous '" + appToSync +
! "' backups to automatically assign sync modes ...")
! else:
! globalContext.log("No previous '" + appToSync +
! "' backups found - no sync modes will be assigned automatically")
!
! # determine the status of all entries and fields
! temporaries = []
! #print backups.keys()
! for backupKey in backups.keys():
! oldVersion = backups[backupKey]
! for deviceInfo in self.devices:
! if deviceInfo.plugin.__name__ == backupKey[0] and \
! deviceInfo.device.getId() == backupKey[1]:
! #print "found", backupKey
! globalContext.log("Examining " + str(backupKey) + " ...")
! break
! else:
! continue
! newVersion = deviceInfo.data
! for oldApp in oldVersion:
! if oldApp.getId() != appToSync:
! continue
! for newApp in newVersion:
! if newApp.getId() == oldApp.getId():
! #print "found", newApp.getId();
! globalContext.log("Examining " + str(newApp.getId()) + " ...")
! break
! else:
! continue
! for oldEntry in oldApp.getChildren():
! #globalContext.log("Examining " + str(oldEntry) + " ...")
! matches = oldEntry.matchesInSequence(newApp.getChildren())
! if len(matches) > 1:
! # not much we can do
! continue
! elif len(matches) == 1:
! # examine the entry anyway, regardless of ids
! # (which might be bogus)
! newEntry = matches[0]
! else:
! deleted = 1
! for newEntry in newApp.getChildren():
! if oldEntry.getId() == newEntry.getId():
! # we have no match but equal ids:
! # check if key fields (involved in matching)
! # have changed
! matches = newEntry.matchesInSequence(oldApp.getChildren())
! if len(matches) == 0:
! # we have no match in either direction,
! # but ids are equal -> assume key fields
! # have changed
! deleted = 0
! break
! if deleted:
! globalContext.log("Setting status to deleted: " + str(oldEntry))
! temporaryEntry = oldEntry.deepClone()
! temporaryEntry.setStatus(syncbase.STATUS_DELETED)
! temporaryEntry.setParent(newApp)
! temporaries.append(temporaryEntry)
! continue
! # if we're here, we either have a matching entry or two
! # entries with equal ids (and no match at all)
! if oldEntry.isValid():
! newEntry.setPreviousEntry(oldEntry)
! # The validity check is necessary because the
! # isValid() implementation might have changed after
! # the backup was made. Since the previous entry is
! # used for matching, it is essential that it is
! # valid.
! changed = 0
! for oldField in oldEntry.getChildren():
! matches = oldField.matchesInSequence(newEntry.getChildren())
! if len(matches) == 0:
! #print "found", oldField
! globalContext.log("Setting status to deleted: " + str(oldField))
! temporaryField = oldField.deepClone()
! temporaryField.setStatus(syncbase.STATUS_DELETED)
! temporaryField.setParent(newEntry)
! temporaries.append(temporaryField)
! changed = 1
! elif len(matches) == 1:
! newField = matches[0]
! if not oldField.equals(newField):
! #print "found", oldField
! globalContext.log("Setting status to changed: " + str(oldField) + " to " + str(newField))
! newField.setStatus(syncbase.STATUS_CHANGED)
! changed = 1
! else:
! newField.setStatus(syncbase.STATUS_UNCHANGED)
! else:
! for newField in matches:
! if not oldField.equals(newField):
! globalContext.log("Strange match for '" +
! deviceInfo.plugin.__name__ + "/" +
! deviceInfo.device.getId() + "/" +
! oldApp.getId() + "/" +
! oldField.getId() + "'!");
! # this shouldn't normally happen
! # since the only case where we have
! # several matches here should be when
! # the exact same field is present
! # more than once
! globalContext.log("Setting status to changed")
! newField.setStatus(syncbase.STATUS_CHANGED)
! changed = 1
! else:
! pass
! # leave the status as STATUS_UNKNOWN since
! # this might be a newly added duplicate
! # (and there's no sense flagging that as
! # UNCHANGED)
! if changed:
! newEntry.setStatus(syncbase.STATUS_CHANGED, 0)
! else:
! pass
! # leave the status as STATUS_UNKNOWN since we don't
! # currently check for added entries
! # for oldEntry in oldApp
! # for oldApp in oldVersion
! # for backup in backups.keys()
!
! for temporary in temporaries:
! temporary.getParent().addChild(temporary)
!
!
! def setAutoSyncModes(self, appToSync):
! """
! Automatically sets sync modes according to the status of each
! entry/field.
! """
!
! temporaries = []
! #print len(self.devices), "devices present"
! for deviceInfo in self.devices:
! for app in deviceInfo.data:
! if app.getId() != appToSync:
! continue
! matchingApps = []
! for checkDeviceInfo in self.devices:
! for checkApp in checkDeviceInfo.data:
! if checkApp.getId() == appToSync:
! matchingApps.append(checkApp)
! for entry in app.getChildren():
! status = entry.getStatus()
! if status == syncbase.STATUS_DELETED:
! matchingEntries = entry.matchesInParentSequence(matchingApps)
! for matchingEntry in matchingEntries:
! matchingEntryStatus = matchingEntry.getStatus()
! if matchingEntryStatus == syncbase.STATUS_UNKNOWN or \
! matchingEntryStatus == syncbase.STATUS_UNCHANGED:
! matchingEntry.setSyncMode(syncbase.SYNC_TO)
! temporaries.append(entry)
! elif status == syncbase.STATUS_CHANGED:
! matchingEntries = entry.matchesInParentSequence(matchingApps)
! #print "changed entry found with", len(matchingEntries), "matching entries"
! for field in entry.getChildren():
! status = field.getStatus()
! if status == syncbase.STATUS_DELETED or \
! status == syncbase.STATUS_CHANGED:
! matchingFields = field.matchesInParentSequence(matchingEntries)
! #print len(matchingFields), "matching fields found"
! for matchingField in matchingFields:
! matchingFieldStatus = matchingField.getStatus()
! #print "status:", matchingFieldStatus
! if matchingFieldStatus == syncbase.STATUS_UNKNOWN or \
! matchingFieldStatus == syncbase.STATUS_UNCHANGED:
! #print "setting field mode to SYNC_TO"
! matchingField.setSyncMode(syncbase.SYNC_TO)
! if status == syncbase.STATUS_DELETED:
! temporaries.append(field)
!
! for temporary in temporaries:
! temporary.getParent().removeChild(temporary)
***************
*** 427,430 ****
--- 641,645 ----
# information ...)
deviceInfo.changeList.append(newElement)
+ #print "check:", newElement.getParent().isLeaf()
else:
# check if we have to add data
***************
*** 466,469 ****
--- 681,685 ----
while information to that entry has been added on two other devices.
"""
+ appFound = 0
self.aggregatedData = syncbase.HierarchyElement()
self.conflictList = []
***************
*** 471,474 ****
--- 687,695 ----
for syncApp in deviceInfo.data:
if syncApp.getId() == appToSync:
+ appFound = 1
+ globalContext.log("Aggregating data from '" +
+ deviceInfo.plugin.__name__ + "/" +
+ deviceInfo.device.getId() + "/" +
+ appToSync + "' into the global pool ...")
# if reading is not allowed, back out here
if not syncApp.allowsReading():
***************
*** 476,479 ****
--- 697,701 ----
self.doAggregateData(syncApp, self.aggregatedData,
self.findMatchingApps(syncApp))
+ return appFound
***************
*** 486,489 ****
--- 708,715 ----
for syncApp in deviceInfo.data:
if syncApp.getId() == appToSync:
+ globalContext.log("Distributing changes to '" +
+ deviceInfo.plugin.__name__ + "/" +
+ deviceInfo.device.getId() + "/" +
+ appToSync + "' ...")
self.doDistributeData(deviceInfo, syncApp,
syncApp.matchesInSequence(self.aggregatedData.getChildren()))
***************
*** 492,498 ****
def saveInternalData(self):
"""
! Clones all internal data and the add/remove/change lists of the
! devices. Then, the clones are put into the place of the originals
! which are stored in backup attributes.
"""
for deviceInfo in self.devices:
--- 718,725 ----
def saveInternalData(self):
"""
! Clones all internal data. Then, the clones are put into the place of
! the originals which are stored in a backup.
!
! TODO: restrict this to the current app that is synced
"""
for deviceInfo in self.devices:
***************
*** 502,526 ****
deviceInfo.dataBackup = deviceInfo.data
deviceInfo.data = newData
- newAddList = []
for element in deviceInfo.addList:
! newElement = element.deepClone()
! newElement.parent = element.getParent().getClone()
! newAddList.append(newElement)
! deviceInfo.addListBackup = deviceInfo.addList
! deviceInfo.addList = newAddList
! newChangeList = []
for element in deviceInfo.changeList:
! newElement = element.deepClone()
! newElement.parent = element.getParent().getClone()
! newChangeList.append(newElement)
! deviceInfo.changeListBackup = deviceInfo.changeList
! deviceInfo.changeList = newChangeList
! newRemoveList = []
! for element in deviceInfo.removeList:
! newElement = element.deepClone()
! newElement.parent = element.getParent().getClone()
! newRemoveList.append(newElement)
! deviceInfo.removeListBackup = deviceInfo.removeList
! deviceInfo.removeList = newRemoveList
--- 729,744 ----
deviceInfo.dataBackup = deviceInfo.data
deviceInfo.data = newData
for element in deviceInfo.addList:
! element.parent = element.getParent().getClone()
for element in deviceInfo.changeList:
! #print "check 2:", element.getParent().isLeaf()
! element.parent = element.getParent().getClone()
! #print "check 3:", element.getParent().isLeaf()
! # special handling for remove list:
! # the removed entries are the exact same ones
! # present in their parents (and this property must be
! # preserved for the remove the work)
! for index in range(len(deviceInfo.removeList)):
! deviceInfo.removeList[index] = deviceInfo.removeList[index].getClone()
***************
*** 530,545 ****
the same time, the rejected lists are changed so that they point to
the original data instead of the cloned version.
"""
for deviceInfo in self.devices:
deviceInfo.data = deviceInfo.dataBackup
! deviceInfo.addList = deviceInfo.addListBackup
! deviceInfo.changeList = deviceInfo.changeListBackup
! deviceInfo.removeList = deviceInfo.removeListBackup
for element in deviceInfo.addRejectedList:
element.parent = element.getParent().getOriginal()
for element in deviceInfo.changeRejectedList:
element.parent = element.getParent().getOriginal()
! for element in deviceInfo.removeRejectedList:
! element.parent = element.getParent().getOriginal()
--- 748,770 ----
the same time, the rejected lists are changed so that they point to
the original data instead of the cloned version.
+
+ TODO: restrict this to the current app that is synced
"""
for deviceInfo in self.devices:
deviceInfo.data = deviceInfo.dataBackup
! for element in deviceInfo.addList:
! element.parent = element.getParent().getOriginal()
! for element in deviceInfo.changeList:
! #print "check 4:", element.getParent().isLeaf()
! element.parent = element.getParent().getOriginal()
! #print "check 5:", element.getParent().isLeaf()
! for index in range(len(deviceInfo.removeList)):
! deviceInfo.removeList[index] = deviceInfo.removeList[index].getOriginal()
for element in deviceInfo.addRejectedList:
element.parent = element.getParent().getOriginal()
for element in deviceInfo.changeRejectedList:
element.parent = element.getParent().getOriginal()
! for index in range(len(deviceInfo.removeRejectedList)):
! deviceInfo.removeRejectedList[index] = deviceInfo.removeRejectedList[index].getOriginal()
***************
*** 550,568 ****
"""
! self.aggregateData(appToSync)
# aggregates the data from all devices and checks for conflicts
if len(self.conflictList) > 0:
return
self.distributeData(appToSync)
! # adds, changes, and deletes entries
-
# backup the data so changes do not really take effect
# (however, they have to be made anyway or we won't get
# sensible rejected lists)
! #if checkConflictsOnly:
! # self.saveInternalData()
# change the (maybe saved before) internal data
--- 775,798 ----
"""
! appFound = self.aggregateData(appToSync)
# aggregates the data from all devices and checks for conflicts
if len(self.conflictList) > 0:
+ globalContext.log("Conflicts found!")
+ return
+ if not appFound:
return
self.distributeData(appToSync)
! # builds add, change, and remove lists
# backup the data so changes do not really take effect
# (however, they have to be made anyway or we won't get
# sensible rejected lists)
! if checkConflictsOnly:
! self.saveInternalData()
! else:
! # make a pre-sync backup
! backup.createBackups(self.devices, appToSync, 0)
# change the (maybe saved before) internal data
***************
*** 572,575 ****
--- 802,814 ----
for deviceInfo in self.devices:
+ if checkConflictsOnly:
+ globalContext.log("Simulating writing data to '" +
+ deviceInfo.plugin.__name__ + "/" +
+ deviceInfo.device.getId() + "' ...")
+ else:
+ globalContext.log("Writing data to '" +
+ deviceInfo.plugin.__name__ + "/" +
+ deviceInfo.device.getId() + "' ...")
+
# actually put the info into the data tree (necessary for
# WRITE_ALL mode) and additionally translate everything
***************
*** 584,591 ****
parent = element.getParent()
if element.isLeaf():
! retVal = parent.getParent.getSource().nofityRemoveField(
element, parent)
if retVal:
parent.removeChild(element)
if not (parent in changedEntries):
changedEntries.append(parent)
--- 823,832 ----
parent = element.getParent()
if element.isLeaf():
! retVal = parent.getParent().getSource().notifyDeleteField(
element, parent)
if retVal:
parent.removeChild(element)
+ element.parent = parent
+ # restore parent (has been set to None by removeChild())
if not (parent in changedEntries):
changedEntries.append(parent)
***************
*** 597,601 ****
# this must be an entry since whole applications
# are never removed and fields are leafs
! retVal = parent.getSource().notifyRemoveEntry(
element, parent)
if retVal:
--- 838,842 ----
# this must be an entry since whole applications
# are never removed and fields are leafs
! retVal = parent.getSource().notifyDeleteEntry(
element, parent)
if retVal:
***************
*** 611,615 ****
while index < len(deviceInfo.changeList):
# changed elements must be fields
! element = deviceInfo.changeList[index]
oldElement = element.getParent()
parent = oldElement.getParent()
--- 852,859 ----
while index < len(deviceInfo.changeList):
# changed elements must be fields
! temp = deviceInfo.changeList[index]
! element = temp.deepClone()
! # clone in order to preserve the element in the list exactly
! element.parent = temp.getParent()
oldElement = element.getParent()
parent = oldElement.getParent()
***************
*** 629,633 ****
index = 0
while index < len(deviceInfo.addList):
! element = deviceInfo.addList[index]
parent = element.getParent()
if element.isLeaf():
--- 873,880 ----
index = 0
while index < len(deviceInfo.addList):
! temp = deviceInfo.addList[index]
! element = temp.deepClone()
! # clone in order to preserve the element in the list exactly
! element.parent = temp.getParent()
parent = element.getParent()
if element.isLeaf():
***************
*** 659,663 ****
if not checkConflictsOnly:
for syncApp in deviceInfo.data:
! if syncApp.getSource().getId() == appToSync:
if syncApp.getWriteMode() == syncbase.WRITE_SINGLE:
for entry in addedEntries:
--- 906,910 ----
if not checkConflictsOnly:
for syncApp in deviceInfo.data:
! if syncApp.getId() == appToSync:
if syncApp.getWriteMode() == syncbase.WRITE_SINGLE:
for entry in addedEntries:
***************
*** 669,673 ****
--- 916,958 ----
else:
syncApp.getSource().writeAllEntries(syncApp.getChildren())
+
+ # if this was no dry-run, make post-sync backups
+ if not checkConflictsOnly:
+ try:
+ # we have to re-read everything since what we tell
+ # a plugin to write is not necessarily what it actually
+ # writes ...
+ for deviceInfo in self.devices:
+ for app in deviceInfo.data:
+ if app.getId() == appToSync:
+ deviceApp = app.getSource()
+ deviceEntries = deviceApp.readAllEntries()
+ removeList = []
+ for deviceEntry in deviceEntries:
+ if not deviceEntry.isValid():
+ globalContext.log("Invalid entry: " + str(deviceEntry))
+ removeList.append(deviceEntry)
+ for removeEntry in removeList:
+ deviceEntries.remove(removeEntry)
+ newApp = syncbase.Application(deviceApp.getId())
+ newApp.setSource(deviceApp)
+ newApp.setDeviceInfo(deviceInfo)
+ newApp.addChildren(deviceEntries)
+ backup.createBackup(deviceInfo.plugin.__name__,
+ deviceInfo.device.getId(),
+ newApp, 1)
+ except:
+ self.handleError(0, i18n["POST_SYNC_BACKUP_FAILED"]);
+ # not considered fatal
+ # TODO: better error message (e.g. which device failed)
+
+ # if this was only a dry-run, restore the internal data
+ if checkConflictsOnly:
+ self.restoreInternalData()
+ globalContext.fatalDefault = oldFatalDefault
+
except:
+ if checkConflictsOnly:
+ self.restoreInternalData()
globalContext.fatalDefault = oldFatalDefault
self.handleError(1, i18n["WRITE_FAILED"], deviceInfo.plugin.__name__);
***************
*** 675,686 ****
# (2) the plugin called handleError() with fatal == true
# or (3) the default for write errors is fatal
!
! globalContext.fatalDefault = oldFatalDefault
!
! # if this was only a conflict check, revert the internal data to
! # the saved state
! #if checkConflictsOnly:
! # self.restoreInternalData()
!
def cancelSync(self):
--- 960,966 ----
# (2) the plugin called handleError() with fatal == true
# or (3) the default for write errors is fatal
! # note: handleError might throw an exception
!
!
def cancelSync(self):
|