Menu
▾
▴
SF.net SVN: mailmanager: [3330] MailManager/tags
|
From: <ke...@us...> - 2006-07-27 23:32:26
|
Revision: 3330 Author: kevca Date: 2006-07-27 11:31:57 -0700 (Thu, 27 Jul 2006) ViewCVS: http://svn.sourceforge.net/mailmanager/?rev=3330&view=rev Log Message: ----------- Tagging release for internal testing Modified Paths: -------------- MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/ruleset.xml MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/testing.xml MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/unicode.xml Added Paths: ----------- MailManager/tags/RELENG_2_1_RC6_PRE7/ MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py MailManager/tags/RELENG_2_1_RC6_PRE7/sql/v2_1/getFirstHistoryDate.zsql MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt Removed Paths: ------------- MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt Copied: MailManager/tags/RELENG_2_1_RC6_PRE7 (from rev 3328, MailManager/branches/RELENG_2_1) Modified: MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/ruleset.xml =================================================================== --- MailManager/branches/RELENG_2_1/docs/development/source/sections/ruleset.xml 2006-07-27 17:29:56 UTC (rev 3328) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/ruleset.xml 2006-07-27 18:31:57 UTC (rev 3330) @@ -1,4 +1,4 @@ -<!-- $Id:$ --> +<!-- $Id$ --> <?xml version="1.0" encoding="ISO-8859-1"?> <!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN" Modified: MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/testing.xml =================================================================== --- MailManager/branches/RELENG_2_1/docs/development/source/sections/testing.xml 2006-07-27 17:29:56 UTC (rev 3328) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/testing.xml 2006-07-27 18:31:57 UTC (rev 3330) @@ -1,4 +1,4 @@ -<!-- $Id:$ --> +<!-- $Id$ --> <?xml version="1.0" encoding="ISO-8859-1"?> <!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN" Modified: MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/unicode.xml =================================================================== --- MailManager/branches/RELENG_2_1/docs/development/source/sections/unicode.xml 2006-07-27 17:29:56 UTC (rev 3328) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/docs/development/source/sections/unicode.xml 2006-07-27 18:31:57 UTC (rev 3330) @@ -1,4 +1,4 @@ -<!-- $Id:$ --> +<!-- $Id$ --> <?xml version="1.0" encoding="ISO-8859-1"?> <!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN" Deleted: MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py =================================================================== --- MailManager/branches/RELENG_2_1/migrations/v2_1/__init__.py 2006-07-27 17:29:56 UTC (rev 3328) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py 2006-07-27 18:31:57 UTC (rev 3330) @@ -1,530 +0,0 @@ -# (c) Copyright Logicalware Ltd 2002-2006 -# Logicalware MailManager comes with ABSOLUTELY NO WARRANTY. Further details -# including conditions of redistribution are contained in LICENSE.txt -# http://www.logicalware.org/ -# $Id$ - -__version__ = '$Revision$' - -from tempfile import NamedTemporaryFile -import copy - -import logging -from Products.MailManager.support.logger import log -from DateTime.DateTime import DateTime -import mx.DateTime - - -def migrate_v2_1_0_v2_1_1(self, migrationParams): - """ Add on the TID flag for accounts """ - - schema = migrationParams.get('schema', '') - create_tables = migrationParams.get('create_tables', False) - dbplatform = migrationParams.get('dbplatform', 'postgres') - - self.sql.migrateAccountTIDFlag() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_1' - -def migrate_v2_1_1_v2_1_2(self, migrationParams): - """ Move to multiple ruleset descriptions in the database - - The previous release only had a single ruleset, entitled queuesystem. - We now need to move to using multiple rulesets and migrate existing - tickets between rulesets. Normally end users are only going to have - a single ruleset in use, which should be the latest revision of - queuesystem. - - Multiple rulesets in the same system would require some fixes still. - The account table would need a rsname attribute, and the ruleset - engine would need to load the ruleset from a ticket each time. Also, - difficulties arise when dealing with supporters over multiple - rulesets. - """ - - schema = migrationParams.get('schema', '') - create_tables = migrationParams.get('create_tables', False) - dbplatform = migrationParams.get('dbplatform', 'postgres') - - self.sql.migrateRulesetStates() - - # Load in additional ruleset and update engine - self.engine.setup(rsname = 'queuesystem_2_1_1') - - # Convert any tickets - for id in [t.id for t in self.sql.listTickets(sqv_state = 'New')]: - self.sql.editTicket(sqv_id = id, sqv_state = 'Open') - for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem')]: - self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_1') - - # Update the mversion to show we are complete - self.mversion = 'v2_1_2' - -def migrate_v2_1_2_v2_1_3(self, migrationParams): - """ Add on default templates and close action - - We add on an attribute to an account entitled default_template, which - automatically adds a template on ticket replies. We also update the - ruleset to add on a new Close action. - """ - - schema = migrationParams.get('schema', '') - create_tables = migrationParams.get('create_tables', False) - dbplatform = migrationParams.get('dbplatform', 'postgres') - - self.sql.migrateDefaultTemplate() - - # Load in additional ruleset and update engine - self.engine.setup(rsname = 'queuesystem_2_1_2') - - # All tickets should now be using this ruleset - for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem_2_1_1')]: - self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_2') - - # Update the mversion to show we are complete - self.mversion = 'v2_1_3' - -def migrate_v2_1_3_v2_1_4(self, migrationParams): - """ Fix missing user abilities for any migrated users - - Late divergence fix for users coming from 2.0, and for users with - early 2.1 datasets. Users migrated from earlier releases, or created - or modified under 2.1 may lack a black ability in some category. This - prevents the SQL operations from working corretly. - - Some tickets migrated from 2.0 also lack rsname in the database. - - Ticket migrated from 2.0 lack the Overdue state, which was previously - calculated in the SQL method. This migration explicitly sets that - state in the database. The API methods are not used as these would - cause actions to be triggered, potentially mailing users. - - This fix should be convergent and is not needed during the previous - migration process. - - Fixes Overdue status is lost on migration (#1493442) - Fixes No Send buttons on Tickets screen (#1493452) - Fixes Incorrect abilities for new accounts (#1492592) - """ - - for user in self.sql.listUsers(): - username = user.username - - # Ensure that an empty ability exists for all categories - for catno in [0,1,2]: - self.sql.deleteUserAbility( - sqv_catno = catno, - sqv_username = username, - sqv_ability = '' - ) - - self.sql.addUserAbility( - sqv_catno = catno, - sqv_username = username, - sqv_ability = '' - ) - - # Convert any tickets to the Overdue state - # Fixes Overdue status is lost on migration (#1493442) - self.sql.migrateSetOverdueState() - - # Set an rsname for any tickets which lack one - # Fixes No Send buttons on Tickets screen (#1493452) - self.sql.migrateAddMissingRsname( - sqv_default_rsname='queuesystem_2_1_2' - ) - - # Replace the ruleset with one with fixes for AddNote, which does not - # deal with the Overdue state. - # Fixes Add Note missing from overdue tickets (#1494173) - # Load in additional ruleset and update engine - # Also adds in transitions for filters in order to fix - # Filters do not allow setting of state (#1495442) - self.engine.setup(rsname = 'queuesystem_2_1_3') - - # All tickets should now be using this ruleset - for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem_2_1_2')]: - self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_3') - - # Update the mversion to show we are complete - self.mversion = 'v2_1_4' - - -def migrate_v2_1_4_v2_1_5(self, migrationParams): - """ Fix overdue and viewed status on tickets - - Another late divergence fix for any datasets pre 2_1_4. Tickets - in the database will not have either the Responded or Viewed - attributes set. These can be inferred from the existing data in - the database. - - Fixes Replied tickets are being marked as overdue (#1497428) - Fixes Ruleset permissions are too strict (#1499441) - Fixes Adding a queue fails (#1499449) - Fixes Replying to a ticket puts it on hold (#1502192) - Fixes On Hold tickets lack actions (#1502194) - Fixes Queues list shows role user (#1498085) - """ - - # Add the Responded and Viewed attributes for each ticket - for t in self.sql.listTickets(): - if t.date_responded: - self.sql.setTicketAttribute( - sqv_ticket_id = t.id, - sqv_attribute = 'Responded', - sqv_setting = self.sql_true - ) - - if not t.unread: - self.sql.setTicketAttribute( - sqv_ticket_id = t.id, - sqv_attribute = 'Viewed', - sqv_setting = self.sql_true - ) - - # Update the ruleset to remove the ACL restrictions - # Fixes Replying to a ticket puts it on hold (#1502192) - # Fixes On Hold tickets lack actions (#1502194) - self.engine.setup(rsname = 'queuesystem_2_1_5') - self.sql.modifyTicketRulesets( - sqv_new_ruleset = 'queuesystem_2_1_5' - ) - - # Fixes Adding a queue fails (#1499449) - # Fixes Username limitation is too short (#1501616) - self.sql.migrateUserGroupNameSizes() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_5' - - -def migrate_v2_1_5_v2_1_6(self, migrationParams): - """ Adds on settings to allow autologout to be configured. - - Fixes Expired logouts cause issues (#1502271) - """ - - self.sql.migrateCreateLoginSettings() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_6' - - -def migrate_v2_1_6_v2_1_7(self, migrationParams): - """ Conversions to datatypes and indexes - - Fixes Missing index on categories (#1510557) - Fixes Replies to tickets are not processed via ruleset engine (#1511436) - """ - # Fixes Missing index on categories (#1510557) - self.sql.migrateAddCategoryIndexes() - - # Fixes Replies to tickets are not processed via ruleset engine (#1511436) - self.engine.setup(rsname = 'queuesystem_2_1_6') - self.sql.modifyTicketRulesets( - sqv_new_ruleset = 'queuesystem_2_1_6' - ) - - # Update the mversion to show we are complete - self.mversion = 'v2_1_7' - -def migrate_v2_1_7_v2_1_8(self, migrationParams): - """ Conversions to datatypes and indexes - - Fixes unicode issue in getmail (#1493483) - - Converts raw_headers field into binary type in db - Fixes No index on mm_ticket.unread (#1498049) - """ - - # Fixes Unicode issue in getmail (#1493483) - if self.dbplatform == 'postgres': - import psycopg - - # Need to convert all existing data in the database. This needs done in - # a couple of stages. First, add a new field of type bytea called - # raw_headers_new - self.sql.migrateConvertRawHeaders_a() - - # Now load existing data from the database, and convert. This may - # require disabling unicode output from the database temporarily. - # We don't do this using an SQL method as converting from text to - # bytea requires some undocumented features of postgres (see bug - # report) - ticket_ids = [t.id for t in self.sql.listTickets(sqv_id_only=self.sql_true)] - for tid in ticket_ids: - # Find all of the messages in this ticket - # Read the data from the database - for msg in self.sql.listMessages(sqv_ticket_id = tid): - - # Insert the bytea data - self.sql.migrateSetMessageRawHeaders( - sqv_id = msg.id, - sqv_raw_headers = psycopg.Binary(msg.raw_headers) - ) - - # Finally, drop the old row and replace it with the newly created one - self.sql.migrateConvertRawHeaders_b() - - elif self.dbplatform == 'mysql': - # MySQL only needs 1 step - self.sql.migrateConvertRawHeaders_a() - - # Fixes No index on mm_ticket.unread (#1498049) - self.sql.migrateAddUnreadIndex() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_8' - - -def migrate_v2_1_8_v2_1_9(self, migrationParams): - """ Improvements to reporting and cron scheduling - - Fixes Cannot control getMail frequency effectively (#1521234) - Fixes Snapshot reports unusable at scale (#1521221) - Fixes Cannot set a filter to a group/user of certain length (#1526419) - """ - - # Fixes Cannot control getMail frequency effectively (#1521234) - self.sql.migrateCreateCronSettings() - - # Fixes Cannot set a filter to a group/user of certain length (#1526419) - self.sql.migrateFilterGroupNameSizes() - - # Set the initial cron settings - notify_home = False - self.sql.setCronSettings( - sqv_modify_notify_home = self.sql_true, - sqv_notify_home = notify_home and self.sql_truevar or self.sql_falsevar, - sqv_notify_home_freq = 86400, - sqv_modify_gen_report = self.sql_true, - sqv_gen_report = self.sql_falsevar, - sqv_gen_report_freq = 3600, - sqv_modify_get_mail = self.sql_true, - sqv_get_mail = self.sql_truevar, - sqv_get_mail_freq = 300, - ) - - # Create the snapshot indexes - # Fixes Snapshot reports unusable at scale (#1521221) - self.sql.migrateCreateSnapshotIndexes() - self.rebuildSnapshotIndexes() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_9' - -def migrate_v2_1_9_v2_1_10(self, migrationParams): - """ Fixes incorrect overdue tickets and add missing indexes - - Tickets incorrectly set as Overdue (#1516996) - Categories and date_opened lack indexes (#1527686) - Fixes Snapshot reports unusable at scale (#1521221) - Fixes Reporting performance (#1521220) - """ - - # Add on a cron entry to compress the snapshot caches - # Fixes Snapshot reports unusable at scale (#1521221) - self.sql.migrateCronAddSnapshot() - - # Sort out the overdue tickets - # Fixes Tickets incorrectly set as Overdue (#1516996) - for tick in self.sql.listTickets(sqv_state='Overdue'): - ticket = self.ticket(id=tick.id)[0] - respond_by = ticket.calculateOverdueTime() - if respond_by < mx.DateTime.utc(): - # Ticket should not be Overdue - ticket.save(changed_by='', state='Open') - if respond_by is not None: - self.engine.queueTimeEvent(eventname='FixOverdue', - ticket=ticket, - time=DateTime(respond_by.ticks(), 'UTC')) - - # Add the missing indexes - # Fixes Categories and date_opened lack indexes (#1527686) - self.sql.migrateAddDateOpenedIndex() - self.sql.migrateAddTicketCategoryIndexes() - - # Add in a reports cache - # Fixes Reporting performance (#1521220) - self.sql.migrateCreateReportCache() - - # Update the mversion to show we are complete - self.mversion = 'v2_1_10' - -def migrate_convert_postgres_unicode(self, migrationParams): - """ Convert postgres database to using unicode """ - - # Update the database object - self.mailmanager_db.encoding = 'UTF-8' - self.mailmanager_db.ustrings = True - - # Find out the location of pg_dump - if sys.platform.startswith("win"): - if not os.path.isfile('C:\Program\ Files/PostgreSQL/8.0/bin/pg_dump.exe'): - raise MigrationError("""Cannot run migration. Your postgres database -needs converted to UTF-8, but the conversion program pg_dump cannot be found -on the system. Please report this error, or ask on the mailmanager-users -mailing list on sourceforge for a solution""") - if not os.path.isfile('C:\Program\ Files/PostgreSQL/8.0/bin/pg_restore.exe'): - raise MigrationError("""Cannot run migration. Your postgres database -needs converted to UTF-8, but the conversion program pg_restore cannot be found -on the system. Please report this error, or ask on the mailmanager-users -mailing list on sourceforge for a solution""") - pg_dump = 'C:\Program\ Files/PostgreSQL/8.0/bin/pg_dump.exe' - pg_restore = 'C:\Program\ Files/PostgreSQL/8.0/bin/pg_restore.exe' - else: - # Attempt to find pg_dump in $PATH - pg_dump = None - for prefix in os.environ['PATH'].split(':'): - if os.path.isfile(os.path.join(prefix, 'pg_dump')): - pg_dump = os.path.join(prefix, 'pg_dump') - continue - if not pg_dump: - raise MigrationError("""Cannot run migration. Your postgres database -needs converted to UTF-8, but the conversion program pg_dump cannot be found -on the system. Please report this error, or ask on the mailmanager-users -mailing list on sourceforge for a solution""") - pg_restore = None - for prefix in os.environ['PATH'].split(':'): - if os.path.isfile(os.path.join(prefix, 'pg_restore')): - pg_restore = os.path.join(prefix, 'pg_restore') - continue - if not pg_restore: - raise MigrationError("""Cannot run migration. Your postgres database -needs converted to UTF-8, but the conversion program pg_restore cannot be found -on the system. Please report this error, or ask on the mailmanager-users -mailing list on sourceforge for a solution""") - - # Convert the mailmanager_db connection string into variables to - # supply to pg_dump and pg_restore. The formation of the connection - # string looks like 'dbname=... user=... password=... host=...' - params = {} - kvassign = self.mailmanager_db.connection_string.strip().split() - for (key,value) in [kv.split('=') for kv in kvassign]: - params[key] = value - - # Create a tempfile, and export the database - tmpfile = NamedTemporaryFile() - tmpfile.flush() - tmpfile.close() - - # Store the connection details into the environment - environ = copy.copy(os.environ()) - if params.has_key('password'): - environ['PGPASSWORD'] = params['password'] - if params.has_key('user'): - environ['PGUSER'] = params['user'] - if params.has_key('host'): - environ['PGHOST'] = params['host'] - - # Dump the data to the tempfile - args = [pg_dump, '-F', 'c'] - if self.schema: - args.extend(['-n', self.schema.replace('.','')]) - args.extend(['-f', tmpfile.name, params['dbname']]) - - res = os.spawnvpe(os.P_WAIT, pg_dump, args, environ) - if not (res == 0): - raise MigrationError('Could not dump database - error code %i' % res) - - # Run the utf-8 conversion for a pg dumpfile - outfile = NamedTemporaryFile() - infile = open(tmpfile.name, 'r') - from Products.MailManager.support.convertdb import convert - convert(infile, outfile) - outfile.flush() - outfile.close() - - # Now drop the database and reload using the converted sql data - self.mailmanager_db.query('DROP SCHEMA %s CASCADE;') - self.mailmanager_db.get_transaction.commit() - - args = [pg_restore, '-d', params['dbname'], outfile.name] - res = os.spawnvpe(os.P_WAIT, pg_restore, args, environ) - if not (res == 0): - raise MigrationError('Could not restore database - error code %i' % res) - - # Update the mversion to show we are complete - self.mversion = migrationParams['target'] - - -migrations = { - 'v2_1_0' : { - 'targetstate' : 'v2_1_1', - 'overview' : 'Add on TID flag to accounts', - 'method' : migrate_v2_1_0_v2_1_1, - }, - 'v2_1_1' : { - 'targetstate' : 'v2_1_2', - 'overview' : 'Move to multiple ruleset descriptions in the database', - 'method' : migrate_v2_1_1_v2_1_2, - }, - 'v2_1_2' : { - 'targetstate' : 'v2_1_3', - 'overview' : 'Add on default template for accounts (#1464963)', - 'method' : migrate_v2_1_2_v2_1_3, - }, - 'v2_1_3' : { - 'targetstate' : 'v2_1_4', - 'overview' : 'Fix for incorrect abilities for new accounts (#1492592)\\n' + \ - 'Adds an rsname for any tickets which lack one - fixes #1493452\\n' + \ - 'Fixes Overdue status is lost on migration (#1493442)\\n' + \ - 'Fixes No Send buttons on Tickets screen (#1493452)\\n' + \ - 'Fixes Incorrect abilities for new accounts (#1492592)\\n', - 'method' : migrate_v2_1_3_v2_1_4, - }, - 'v2_1_4' : { - 'targetstate' : 'v2_1_5', - 'overview' : 'Fixes Replied tickets are being marked as overdue (#1497428)\\n' + \ - 'Fixes Ruleset permissions are too strict (#1499441)\\n' + \ - 'Fixes Adding a queue fails (#1499449)\\n' + \ - 'Fixes Replying to a ticket puts it on hold (#1502192)\\n' + \ - 'Fixes On Hold tickets lack actions (#1502194)\\n' + \ - 'Fixes Queues list shows role user (#1498085)', - 'method' : migrate_v2_1_4_v2_1_5, - }, - 'v2_1_5' : { - 'targetstate' : 'v2_1_6', - 'overview' : 'Fixes Expired logouts cause issues (#1502271)', - 'method' : migrate_v2_1_5_v2_1_6, - }, - 'v2_1_6' : { - 'targetstate' : 'v2_1_7', - 'overview' : 'Fixes Missing index on categories (#1510557)\n' + \ - 'Fixes Replies to tickets are not processed via ruleset engine (#1511436)\n', - 'method' : migrate_v2_1_6_v2_1_7, - }, - 'v2_1_7' : { - 'targetstate' : 'v2_1_8', - 'overview' : 'Fixes unicode issue in getmail (#1493483)\n' + \ - 'Converts raw_headers field into binary type in db\n' + \ - 'Fixes No index on mm_ticket.unread (#1498049)\n', - 'method' : migrate_v2_1_7_v2_1_8, - }, - 'v2_1_8' : { - 'targetstate' : 'v2_1_9', - 'overview' : 'Fixes Cannot control getMail frequency effectively (#1521234)', - 'method' : migrate_v2_1_8_v2_1_9, - }, - 'v2_1_9' : { - 'targetstate' : 'v2_1_10', - 'overview': 'Fixed Tickets incorrectly set as Overdue (#1516996)\n' + \ - 'Categories and date_opened lack indexes (#1527686)\n' + \ - 'Fixes Reporting performance (#1521220)', - 'method' : migrate_v2_1_9_v2_1_10, - }, -} - -# Add on mangling for unicode conversion for postgres users -# Known affected versions are v2_1_0 - v2_1_3 inclusive. Later -# dataset revisions will have been created with the correct -# encoding - -for src_ver in migrations.keys(): - if src_ver in ['v2_1_0', 'v2_1_1', 'v2_1_2', 'v2_1_3']: - migrations['%s_no_u' % src_ver] = { - 'targetstate' : src_ver, - 'overview' : 'Convert database to unicode', - 'method' : migrate_convert_postgres_unicode, - } - Copied: MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py (from rev 3329, MailManager/branches/RELENG_2_1/migrations/v2_1/__init__.py) =================================================================== --- MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py (rev 0) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/migrations/v2_1/__init__.py 2006-07-27 18:31:57 UTC (rev 3330) @@ -0,0 +1,540 @@ +# (c) Copyright Logicalware Ltd 2002-2006 +# Logicalware MailManager comes with ABSOLUTELY NO WARRANTY. Further details +# including conditions of redistribution are contained in LICENSE.txt +# http://www.logicalware.org/ +# $Id$ + +__version__ = '$Revision$' + +from tempfile import NamedTemporaryFile +import copy + +import logging +from Products.MailManager.support.logger import log +from DateTime.DateTime import DateTime +import mx.DateTime + + +def migrate_v2_1_0_v2_1_1(self, migrationParams): + """ Add on the TID flag for accounts """ + + schema = migrationParams.get('schema', '') + create_tables = migrationParams.get('create_tables', False) + dbplatform = migrationParams.get('dbplatform', 'postgres') + + self.sql.migrateAccountTIDFlag() + + # Update the mversion to show we are complete + self.mversion = 'v2_1_1' + +def migrate_v2_1_1_v2_1_2(self, migrationParams): + """ Move to multiple ruleset descriptions in the database + + The previous release only had a single ruleset, entitled queuesystem. + We now need to move to using multiple rulesets and migrate existing + tickets between rulesets. Normally end users are only going to have + a single ruleset in use, which should be the latest revision of + queuesystem. + + Multiple rulesets in the same system would require some fixes still. + The account table would need a rsname attribute, and the ruleset + engine would need to load the ruleset from a ticket each time. Also, + difficulties arise when dealing with supporters over multiple + rulesets. + """ + + schema = migrationParams.get('schema', '') + create_tables = migrationParams.get('create_tables', False) + dbplatform = migrationParams.get('dbplatform', 'postgres') + + self.sql.migrateRulesetStates() + + # Load in additional ruleset and update engine + self.engine.setup(rsname = 'queuesystem_2_1_1') + + # Convert any tickets + for id in [t.id for t in self.sql.listTickets(sqv_state = 'New')]: + self.sql.editTicket(sqv_id = id, sqv_state = 'Open') + for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem')]: + self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_1') + + # Update the mversion to show we are complete + self.mversion = 'v2_1_2' + +def migrate_v2_1_2_v2_1_3(self, migrationParams): + """ Add on default templates and close action + + We add on an attribute to an account entitled default_template, which + automatically adds a template on ticket replies. We also update the + ruleset to add on a new Close action. + """ + + schema = migrationParams.get('schema', '') + create_tables = migrationParams.get('create_tables', False) + dbplatform = migrationParams.get('dbplatform', 'postgres') + + self.sql.migrateDefaultTemplate() + + # Load in additional ruleset and update engine + self.engine.setup(rsname = 'queuesystem_2_1_2') + + # All tickets should now be using this ruleset + for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem_2_1_1')]: + self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_2') + + # Update the mversion to show we are complete + self.mversion = 'v2_1_3' + +def migrate_v2_1_3_v2_1_4(self, migrationParams): + """ Fix missing user abilities for any migrated users + + Late divergence fix for users coming from 2.0, and for users with + early 2.1 datasets. Users migrated from earlier releases, or created + or modified under 2.1 may lack a black ability in some category. This + prevents the SQL operations from working corretly. + + Some tickets migrated from 2.0 also lack rsname in the database. + + Ticket migrated from 2.0 lack the Overdue state, which was previously + calculated in the SQL method. This migration explicitly sets that + state in the database. The API methods are not used as these would + cause actions to be triggered, potentially mailing users. + + This fix should be convergent and is not needed during the previous + migration process. + + Fixes Overdue status is lost on migration (#1493442) + Fixes No Send buttons on Tickets screen (#1493452) + Fixes Incorrect abilities for new accounts (#1492592) + """ + + for user in self.sql.listUsers(): + username = user.username + + # Ensure that an empty ability exists for all categories + for catno in [0,1,2]: + self.sql.deleteUserAbility( + sqv_catno = catno, + sqv_username = username, + sqv_ability = '' + ) + + self.sql.addUserAbility( + sqv_catno = catno, + sqv_username = username, + sqv_ability = '' + ) + + # Convert any tickets to the Overdue state + # Fixes Overdue status is lost on migration (#1493442) + self.sql.migrateSetOverdueState() + + # Set an rsname for any tickets which lack one + # Fixes No Send buttons on Tickets screen (#1493452) + self.sql.migrateAddMissingRsname( + sqv_default_rsname='queuesystem_2_1_2' + ) + + # Replace the ruleset with one with fixes for AddNote, which does not + # deal with the Overdue state. + # Fixes Add Note missing from overdue tickets (#1494173) + # Load in additional ruleset and update engine + # Also adds in transitions for filters in order to fix + # Filters do not allow setting of state (#1495442) + self.engine.setup(rsname = 'queuesystem_2_1_3') + + # All tickets should now be using this ruleset + for id in [t.id for t in self.sql.listTickets(sqv_rsname = 'queuesystem_2_1_2')]: + self.sql.editTicket(sqv_id = id, sqv_rsname = 'queuesystem_2_1_3') + + # Update the mversion to show we are complete + self.mversion = 'v2_1_4' + + +def migrate_v2_1_4_v2_1_5(self, migrationParams): + """ Fix overdue and viewed status on tickets + + Another late divergence fix for any datasets pre 2_1_4. Tickets + in the database will not have either the Responded or Viewed + attributes set. These can be inferred from the existing data in + the database. + + Fixes Replied tickets are being marked as overdue (#1497428) + Fixes Ruleset permissions are too strict (#1499441) + Fixes Adding a queue fails (#1499449) + Fixes Replying to a ticket puts it on hold (#1502192) + Fixes On Hold tickets lack actions (#1502194) + Fixes Queues list shows role user (#1498085) + """ + + # Add the Responded and Viewed attributes for each ticket + for t in self.sql.listTickets(): + if t.date_responded: + self.sql.setTicketAttribute( + sqv_ticket_id = t.id, + sqv_attribute = 'Responded', + sqv_setting = self.sql_true + ) + + if not t.unread: + self.sql.setTicketAttribute( + sqv_ticket_id = t.id, + sqv_attribute = 'Viewed', + sqv_setting = self.sql_true + ) + + # Update the ruleset to remove the ACL restrictions + # Fixes Replying to a ticket puts it on hold (#1502192) + # Fixes On Hold tickets lack actions (#1502194) + self.engine.setup(rsname = 'queuesystem_2_1_5') + self.sql.modifyTicketRulesets( + sqv_new_ruleset = 'queuesystem_2_1_5' + ) + + # Fixes Adding a queue fails (#1499449) + # Fixes Username limitation is too short (#1501616) + self.sql.migrateUserGroupNameSizes() + + # Update the mversion to show we are complete + self.mversion = 'v2_1_5' + + +def migrate_v2_1_5_v2_1_6(self, migrationParams): + """ Adds on settings to allow autologout to be configured. + + Fixes Expired logouts cause issues (#1502271) + """ + + self.sql.migrateCreateLoginSettings() + + # Update the mversion to show we are complete + self.mversion = 'v2_1_6' + + +def migrate_v2_1_6_v2_1_7(self, migrationParams): + """ Conversions to datatypes and indexes + + Fixes Missing index on categories (#1510557) + Fixes Replies to tickets are not processed via ruleset engine (#1511436) + """ + # Fixes Missing index on categories (#1510557) + self.sql.migrateAddCategoryIndexes() + + # Fixes Replies to tickets are not processed via ruleset engine (#1511436) + self.engine.setup(rsname = 'queuesystem_2_1_6') + self.sql.modifyTicketRulesets( + sqv_new_ruleset = 'queuesystem_2_1_6' + ) + + # Update the mversion to show we are complete + self.mversion = 'v2_1_7' + +def migrate_v2_1_7_v2_1_8(self, migrationParams): + """ Conversions to datatypes and indexes + + Fixes unicode issue in getmail (#1493483) + - Converts raw_headers field into binary type in db + Fixes No index on mm_ticket.unread (#1498049) + """ + + # Fixes Unicode issue in getmail (#1493483) + if self.dbplatform == 'postgres': + import psycopg + + # Need to convert all existing data in the database. This needs done in + # a couple of stages. First, add a new field of type bytea called + # raw_headers_new + self.sql.migrateConvertRawHeaders_a() + + # Now load existing data from the database, and convert. This may + # require disabling unicode output from the database temporarily. + # We don't do this using an SQL method as converting from text to + # bytea requires some undocumented features of postgres (see bug + # report) + ticket_ids = [t.id for t in self.sql.listTickets(sqv_id_only=self.sql_true)] + for tid in ticket_ids: + # Find all of the messages in this ticket + # Read the data from the database + for msg in self.sql.listMessages(sqv_ticket_id = tid): + + # Insert the bytea data + self.sql.migrateSetMessageRawHeaders( + sqv_id = msg.id, + sqv_raw_headers = psycopg.Binary(msg.raw_headers) + ) + + # Finally, drop the old row and replace it with the newly created one + self.sql.migrateConvertRawHeaders_b() + + elif self.dbplatform == 'mysql': + # MySQL only needs 1 step + self.sql.migrateConvertRawHeaders_a() + + # Fixes No index on mm_ticket.unread (#1498049) + self.sql.migrateAddUnreadIndex() + + # Update the mversion to show we are complete + self.mversion = 'v2_1_8' + + +def migrate_v2_1_8_v2_1_9(self, migrationParams): + """ Improvements to reporting and cron scheduling + + Fixes Cannot control getMail frequency effectively (#1521234) + Fixes Snapshot reports unusable at scale (#1521221) + Fixes Cannot set a filter to a group/user of certain length (#1526419) + """ + + # Fixes Cannot control getMail frequency effectively (#1521234) + self.sql.migrateCreateCronSettings() + + # Fixes Cannot set a filter to a group/user of certain length (#1526419) + self.sql.migrateFilterGroupNameSizes() + + # Set the initial cron settings + notify_home = False + self.sql.setCronSettings( + sqv_modify_notify_home = self.sql_true, + sqv_notify_home = notify_home and self.sql_truevar or self.sql_falsevar, + sqv_notify_home_freq = 86400, + sqv_modify_gen_report = self.sql_true, + sqv_gen_report = self.sql_falsevar, + sqv_gen_report_freq = 3600, + sqv_modify_get_mail = self.sql_true, + sqv_get_mail = self.sql_truevar, + sqv_get_mail_freq = 300, + ) + + # Create the snapshot indexes + # Fixes Snapshot reports unusable at scale (#1521221) + self.sql.migrateCreateSnapshotIndexes() + self.rebuildSnapshotIndexes() + + # Update the mversion to show we are complete + self.mversion = 'v2_1_9' + +def migrate_v2_1_9_v2_1_10(self, migrationParams): + """ Fixes incorrect overdue tickets and add missing indexes + + Tickets incorrectly set as Overdue (#1516996) + Categories and date_opened lack indexes (#1527686) + Fixes Snapshot reports unusable at scale (#1521221) + Fixes Reporting performance (#1521220) + """ + + # Add on a cron entry to compress the snapshot caches + # Fixes Snapshot reports unusable at scale (#1521221) + self.sql.migrateCronAddSnapshot() + + # Sort out the overdue tickets + # Fixes Tickets incorrectly set as Overdue (#1516996) + for tick in self.sql.listTickets(sqv_state='Overdue'): + ticket = self.ticket(id=tick.id)[0] + respond_by = ticket.calculateOverdueTime() + if respond_by < mx.DateTime.utc(): + # Ticket should not be Overdue + ticket.save(changed_by='', state='Open') + if respond_by is not None: + self.engine.queueTimeEvent(eventname='FixOverdue', + ticket=ticket, + time=DateTime(respond_by.ticks(), 'UTC')) + + # Add the missing indexes + # Fixes Categories and date_opened lack indexes (#1527686) + self.sql.migrateAddDateOpenedIndex() + self.sql.migrateAddTicketCategoryIndexes() + + # Add in a reports cache + # Fixes Reporting performance (#1521220) + self.sql.migrateCreateReportCache() + start_date = self.sql.getFirstHistoryDate()[0].change_date + rep_date = DateTime(start_date.strftime('%Y-%m-%d 00:00')) + now_date = DateTime(self.sql.getCurrentTime()[0].now.strftime('%Y-%m-%d 00:00')) + while rep_date < now_date: + # Generate a report for this date + self.generateReportCache(rep_date) + log('%sGenerating a report cache entry for date %s' % ( + self.getLogName(), rep_date), + logging.INFO, 'migration') + rep_date = rep_date + 1 + + # Update the mversion to show we are complete + self.mversion = 'v2_1_10' + +def migrate_convert_postgres_unicode(self, migrationParams): + """ Convert postgres database to using unicode """ + + # Update the database object + self.mailmanager_db.encoding = 'UTF-8' + self.mailmanager_db.ustrings = True + + # Find out the location of pg_dump + if sys.platform.startswith("win"): + if not os.path.isfile('C:\Program\ Files/PostgreSQL/8.0/bin/pg_dump.exe'): + raise MigrationError("""Cannot run migration. Your postgres database +needs converted to UTF-8, but the conversion program pg_dump cannot be found +on the system. Please report this error, or ask on the mailmanager-users +mailing list on sourceforge for a solution""") + if not os.path.isfile('C:\Program\ Files/PostgreSQL/8.0/bin/pg_restore.exe'): + raise MigrationError("""Cannot run migration. Your postgres database +needs converted to UTF-8, but the conversion program pg_restore cannot be found +on the system. Please report this error, or ask on the mailmanager-users +mailing list on sourceforge for a solution""") + pg_dump = 'C:\Program\ Files/PostgreSQL/8.0/bin/pg_dump.exe' + pg_restore = 'C:\Program\ Files/PostgreSQL/8.0/bin/pg_restore.exe' + else: + # Attempt to find pg_dump in $PATH + pg_dump = None + for prefix in os.environ['PATH'].split(':'): + if os.path.isfile(os.path.join(prefix, 'pg_dump')): + pg_dump = os.path.join(prefix, 'pg_dump') + continue + if not pg_dump: + raise MigrationError("""Cannot run migration. Your postgres database +needs converted to UTF-8, but the conversion program pg_dump cannot be found +on the system. Please report this error, or ask on the mailmanager-users +mailing list on sourceforge for a solution""") + pg_restore = None + for prefix in os.environ['PATH'].split(':'): + if os.path.isfile(os.path.join(prefix, 'pg_restore')): + pg_restore = os.path.join(prefix, 'pg_restore') + continue + if not pg_restore: + raise MigrationError("""Cannot run migration. Your postgres database +needs converted to UTF-8, but the conversion program pg_restore cannot be found +on the system. Please report this error, or ask on the mailmanager-users +mailing list on sourceforge for a solution""") + + # Convert the mailmanager_db connection string into variables to + # supply to pg_dump and pg_restore. The formation of the connection + # string looks like 'dbname=... user=... password=... host=...' + params = {} + kvassign = self.mailmanager_db.connection_string.strip().split() + for (key,value) in [kv.split('=') for kv in kvassign]: + params[key] = value + + # Create a tempfile, and export the database + tmpfile = NamedTemporaryFile() + tmpfile.flush() + tmpfile.close() + + # Store the connection details into the environment + environ = copy.copy(os.environ()) + if params.has_key('password'): + environ['PGPASSWORD'] = params['password'] + if params.has_key('user'): + environ['PGUSER'] = params['user'] + if params.has_key('host'): + environ['PGHOST'] = params['host'] + + # Dump the data to the tempfile + args = [pg_dump, '-F', 'c'] + if self.schema: + args.extend(['-n', self.schema.replace('.','')]) + args.extend(['-f', tmpfile.name, params['dbname']]) + + res = os.spawnvpe(os.P_WAIT, pg_dump, args, environ) + if not (res == 0): + raise MigrationError('Could not dump database - error code %i' % res) + + # Run the utf-8 conversion for a pg dumpfile + outfile = NamedTemporaryFile() + infile = open(tmpfile.name, 'r') + from Products.MailManager.support.convertdb import convert + convert(infile, outfile) + outfile.flush() + outfile.close() + + # Now drop the database and reload using the converted sql data + self.mailmanager_db.query('DROP SCHEMA %s CASCADE;') + self.mailmanager_db.get_transaction.commit() + + args = [pg_restore, '-d', params['dbname'], outfile.name] + res = os.spawnvpe(os.P_WAIT, pg_restore, args, environ) + if not (res == 0): + raise MigrationError('Could not restore database - error code %i' % res) + + # Update the mversion to show we are complete + self.mversion = migrationParams['target'] + + +migrations = { + 'v2_1_0' : { + 'targetstate' : 'v2_1_1', + 'overview' : 'Add on TID flag to accounts', + 'method' : migrate_v2_1_0_v2_1_1, + }, + 'v2_1_1' : { + 'targetstate' : 'v2_1_2', + 'overview' : 'Move to multiple ruleset descriptions in the database', + 'method' : migrate_v2_1_1_v2_1_2, + }, + 'v2_1_2' : { + 'targetstate' : 'v2_1_3', + 'overview' : 'Add on default template for accounts (#1464963)', + 'method' : migrate_v2_1_2_v2_1_3, + }, + 'v2_1_3' : { + 'targetstate' : 'v2_1_4', + 'overview' : 'Fix for incorrect abilities for new accounts (#1492592)\\n' + \ + 'Adds an rsname for any tickets which lack one - fixes #1493452\\n' + \ + 'Fixes Overdue status is lost on migration (#1493442)\\n' + \ + 'Fixes No Send buttons on Tickets screen (#1493452)\\n' + \ + 'Fixes Incorrect abilities for new accounts (#1492592)\\n', + 'method' : migrate_v2_1_3_v2_1_4, + }, + 'v2_1_4' : { + 'targetstate' : 'v2_1_5', + 'overview' : 'Fixes Replied tickets are being marked as overdue (#1497428)\\n' + \ + 'Fixes Ruleset permissions are too strict (#1499441)\\n' + \ + 'Fixes Adding a queue fails (#1499449)\\n' + \ + 'Fixes Replying to a ticket puts it on hold (#1502192)\\n' + \ + 'Fixes On Hold tickets lack actions (#1502194)\\n' + \ + 'Fixes Queues list shows role user (#1498085)', + 'method' : migrate_v2_1_4_v2_1_5, + }, + 'v2_1_5' : { + 'targetstate' : 'v2_1_6', + 'overview' : 'Fixes Expired logouts cause issues (#1502271)', + 'method' : migrate_v2_1_5_v2_1_6, + }, + 'v2_1_6' : { + 'targetstate' : 'v2_1_7', + 'overview' : 'Fixes Missing index on categories (#1510557)\n' + \ + 'Fixes Replies to tickets are not processed via ruleset engine (#1511436)\n', + 'method' : migrate_v2_1_6_v2_1_7, + }, + 'v2_1_7' : { + 'targetstate' : 'v2_1_8', + 'overview' : 'Fixes unicode issue in getmail (#1493483)\n' + \ + 'Converts raw_headers field into binary type in db\n' + \ + 'Fixes No index on mm_ticket.unread (#1498049)\n', + 'method' : migrate_v2_1_7_v2_1_8, + }, + 'v2_1_8' : { + 'targetstate' : 'v2_1_9', + 'overview' : 'Fixes Cannot control getMail frequency effectively (#1521234)', + 'method' : migrate_v2_1_8_v2_1_9, + }, + 'v2_1_9' : { + 'targetstate' : 'v2_1_10', + 'overview': 'Fixed Tickets incorrectly set as Overdue (#1516996)\n' + \ + 'Categories and date_opened lack indexes (#1527686)\n' + \ + 'Fixes Reporting performance (#1521220)', + 'method' : migrate_v2_1_9_v2_1_10, + }, +} + +# Add on mangling for unicode conversion for postgres users +# Known affected versions are v2_1_0 - v2_1_3 inclusive. Later +# dataset revisions will have been created with the correct +# encoding + +for src_ver in migrations.keys(): + if src_ver in ['v2_1_0', 'v2_1_1', 'v2_1_2', 'v2_1_3']: + migrations['%s_no_u' % src_ver] = { + 'targetstate' : src_ver, + 'overview' : 'Convert database to unicode', + 'method' : migrate_convert_postgres_unicode, + } + Copied: MailManager/tags/RELENG_2_1_RC6_PRE7/sql/v2_1/getFirstHistoryDate.zsql (from rev 3329, MailManager/branches/RELENG_2_1/sql/v2_1/getFirstHistoryDate.zsql) =================================================================== --- MailManager/tags/RELENG_2_1_RC6_PRE7/sql/v2_1/getFirstHistoryDate.zsql (rev 0) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/sql/v2_1/getFirstHistoryDate.zsql 2006-07-27 18:31:57 UTC (rev 3330) @@ -0,0 +1,13 @@ +<dtml-comment> +title:Get first history item date (PostgeSQL, MySQL) +connection_id: mailmanager_db +max_rows:0 +arguments: + +Get the date of the first history item. This should generally show when +MailManager was first installed. + +</dtml-comment> + +SELECT COALESCE(MIN(change_date), <dtml-var sql_now>) AS change_date FROM <dtml-var schema>mm_history + Deleted: MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt =================================================================== --- MailManager/branches/RELENG_2_1/version.txt 2006-07-27 17:29:56 UTC (rev 3328) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt 2006-07-27 18:31:57 UTC (rev 3330) @@ -1 +0,0 @@ -2.1-rc6-pre6 Copied: MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt (from rev 3329, MailManager/branches/RELENG_2_1/version.txt) =================================================================== --- MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt (rev 0) +++ MailManager/tags/RELENG_2_1_RC6_PRE7/version.txt 2006-07-27 18:31:57 UTC (rev 3330) @@ -0,0 +1 @@ +2.1-rc6-pre7 This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site. |