Mercurial > bitten > bitten-test
diff bitten/upgrades.py @ 708:0d7d6552477e
Database upgrade cleanup:
* Adds database server specific sequence updating and
index dropping support.
* Fixes numerous upgrade scripts that relied on
bitten.model being in the same state it was when the upgrade scripts were
written.
* Avoids temporary table name clashes by appending _vX to temporary table
names where X is the upgrade version number.
* Adds sequence fixing upgrade function to version 10 upgrade (Fixes #452).
* Corrects dropping and creating of indexes in
add_config_platform_rev_index_to_build (Fixes #461).
* Commits database at each step of the upgrade process (alleviates some
issues with upgrades that fail part way, see #462).
Adds basic unit tests for upgrade functions. New upgrades tested against MySQL,
PostGreSQL and SQLite.
author | hodgestar |
---|---|
date | Tue, 27 Oct 2009 23:20:00 +0000 |
parents | 0c4fec90c8e2 |
children | 508636ffbf11 dc51831e6120 |
line wrap: on
line diff
--- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -21,26 +21,85 @@ __docformat__ = 'restructuredtext en' +# database abstraction functions + +def _parse_scheme(env): + """Retrieve the environment database scheme.""" + connection_uri = DatabaseManager(env).connection_uri + parts = connection_uri.split(':', 1) + scheme = parts[0].lower() + return scheme + +def update_sequence(env, db, tbl, col): + """Update a sequence associated with an autoincrement column.""" + # Hopefully Trac will eventually implement its own version + # of this function. + scheme = _parse_scheme(env) + if scheme == "postgres": + seq = '%s_%s_seq' % (tbl, col) + cursor = db.cursor() + cursor.execute("SELECT setval('%s', (SELECT MAX(%s) FROM %s))" + % (seq, col, tbl)) + +def drop_index(env, db, tbl, idx): + """Drop an index associated with a table.""" + # Hopefully Trac will eventually implement its own version + # of this function. + scheme = _parse_scheme(env) + cursor = db.cursor() + if scheme == "mysql": + cursor.execute("DROP INDEX %s ON %s" % (idx, tbl)) + else: + cursor.execute("DROP INDEX %s" % (idx,)) + +# upgrade scripts + def add_log_table(env, db): """Add a table for storing the builds logs.""" - from bitten.model import BuildLog, BuildStep + from trac.db import Table, Column + INFO_LEVEL = 'I' + cursor = db.cursor() + build_log_schema_v3 = [ + Table('bitten_log', key='id')[ + Column('id', auto_increment=True), Column('build', type='int'), + Column('step'), Column('type') + ], + Table('bitten_log_message', key=('log', 'line'))[ + Column('log', type='int'), Column('line', type='int'), + Column('level', size=1), Column('message') + ] + ] + + build_step_schema_v3 = [ + Table('bitten_step', key=('build', 'name'))[ + Column('build', type='int'), Column('name'), Column('description'), + Column('status', size=1), Column('started', type='int'), + Column('stopped', type='int') + ] + ] + connector, _ = DatabaseManager(env)._get_connector() - for table in BuildLog._schema: + for table in build_log_schema_v3: for stmt in connector.to_sql(table): cursor.execute(stmt) + update_cursor = db.cursor() cursor.execute("SELECT build,name,log FROM bitten_step " "WHERE log IS NOT NULL") for build, step, log in cursor: - build_log = BuildLog(env, build, step) - build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()] - build_log.insert(db) + update_cursor.execute("INSERT INTO bitten_log (build, step) " + "VALUES (%s,%s)", (build, step)) + log_id = db.get_last_id(update_cursor, 'bitten_log') + messages = [(log_id, line, INFO_LEVEL, msg) + for line, msg in enumerate(log.splitlines())] + update_cursor.executemany("INSERT INTO bitten_log_message (log, line, level, message) " + "VALUES (%s, %s, %s, %s)", messages) cursor.execute("CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step") cursor.execute("DROP TABLE bitten_step") - for table in BuildStep._schema: + for table in build_step_schema_v3: for stmt in connector.to_sql(table): cursor.execute(stmt) cursor.execute("INSERT INTO bitten_step (build,name,description,status," @@ -117,7 +176,7 @@ from bitten.model import BuildLog cursor = db.cursor() - cursor.execute("CREATE TEMPORARY TABLE old_log AS " + cursor.execute("CREATE TEMPORARY TABLE old_log_v5 AS " "SELECT * FROM bitten_log") cursor.execute("DROP TABLE bitten_log") @@ -126,7 +185,7 @@ cursor.execute(stmt) cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno) " - "SELECT id,build,step,type,0 FROM old_log") + "SELECT id,build,step,type,0 FROM old_log_v5") def add_report_tables(env, db): """Add database tables for report storage.""" @@ -289,7 +348,7 @@ from bitten.model import BuildLog cursor = db.cursor() - cursor.execute("CREATE TEMPORARY TABLE old_log AS " + cursor.execute("CREATE TEMPORARY TABLE old_log_v8 AS " "SELECT * FROM bitten_log") cursor.execute("DROP TABLE bitten_log") @@ -298,7 +357,7 @@ cursor.execute(stmt) cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno,filename) " - "SELECT id,build,step,generator,orderno,'' FROM old_log") + "SELECT id,build,step,generator,orderno,'' FROM old_log_v8") def migrate_logs_to_files(env, db): """Migrates logs that are stored in the bitten_log_messages table into files.""" @@ -309,7 +368,7 @@ if not os.path.isabs(logs_dir): logs_dir = os.path.join(env.path, logs_dir) if not os.path.exists(logs_dir): - os.mkdir(logs_dir) + os.makedirs(logs_dir) message_table = Table('bitten_log_message', key=('log', 'line'))[ Column('log', type='int'), Column('line', type='int'), @@ -345,7 +404,6 @@ def recreate_rule_with_int_id(env, db): """Recreates the bitten_rule table with an integer id column rather than a text one.""" - from trac.db import Table, Column from bitten.model import TargetPlatform cursor = db.cursor() connector, _ = DatabaseManager(env)._get_connector() @@ -393,11 +451,27 @@ if not duplicates_exist: cursor = db.cursor() - cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config,rev,platform)") - cursor.execute("DROP INDEX bitten_build_config_rev_slave_idx") + scheme = _parse_scheme(env) + if scheme == "mysql": + # 111 = 333 / len(columns in index) -- this is the Trac default + cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config(111), rev(111), platform)") + else: + cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config,rev,platform)") + drop_index(env, db, 'bitten_build', 'bitten_build_config_rev_slave_idx') else: raise TracError('') +def fix_sequences(env, db): + """Fixes any auto increment sequences that might have been left in an inconsistent state. + + Upgrade scripts for schema versions > 10 should handle sequence updates correctly themselves. + """ + update_sequence(env, db, 'bitten_build', 'id') + update_sequence(env, db, 'bitten_log', 'id') + update_sequence(env, db, 'bitten_platform', 'id') + update_sequence(env, db, 'bitten_report', 'id') + + map = { 2: [add_log_table], 3: [add_recipe_to_config], @@ -407,5 +481,5 @@ 7: [add_error_table], 8: [add_filename_to_logs,migrate_logs_to_files], 9: [recreate_rule_with_int_id], - 10: [add_config_platform_rev_index_to_build], + 10: [add_config_platform_rev_index_to_build, fix_sequences], }