cmlenz@379: # -*- coding: utf-8 -*-
cmlenz@112: #
osimons@832: # Copyright (C) 2007-2010 Edgewall Software
cmlenz@408: # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
cmlenz@163: # All rights reserved.
cmlenz@112: #
cmlenz@163: # This software is licensed as described in the file COPYING, which
cmlenz@163: # you should have received as part of this distribution. The terms
cmlenz@408: # are also available at http://bitten.edgewall.org/wiki/License.
cmlenz@112: 
cmlenz@316: """Automated upgrades for the Bitten database tables, and other data stored
osimons@775: in the Trac environment.
osimons@775: 
osimons@775: **Do not import and call directly!**"""
cmlenz@316: 
cmlenz@174: import os
cmlenz@203: import sys
cmlenz@174: 
osimons@656: from trac.core import TracError
hodgestar@841: from trac.db import DatabaseManager, Table, Column, Index
dfraser@519: from trac.util.text import to_unicode
dfraser@521: import codecs
cmlenz@320: 
cmlenz@411: __docformat__ = 'restructuredtext en'
cmlenz@411: 
hodgestar@708: # database abstraction functions
hodgestar@708: 
hodgestar@846: def parse_scheme(env):
hodgestar@708:     """Retrieve the environment database scheme."""
hodgestar@708:     connection_uri = DatabaseManager(env).connection_uri
hodgestar@708:     parts = connection_uri.split(':', 1)
hodgestar@708:     scheme = parts[0].lower()
hodgestar@708:     return scheme
hodgestar@708: 
hodgestar@708: def update_sequence(env, db, tbl, col):
hodgestar@708:     """Update a sequence associated with an autoincrement column."""
hodgestar@708:     # Hopefully Trac will eventually implement its own version
hodgestar@708:     # of this function.
hodgestar@846:     scheme = parse_scheme(env)
hodgestar@708:     if scheme == "postgres":
hodgestar@708:         seq = '%s_%s_seq' % (tbl, col)
hodgestar@708:         cursor = db.cursor()
hodgestar@708:         cursor.execute("SELECT setval('%s', (SELECT MAX(%s) FROM %s))"
hodgestar@708:             % (seq, col, tbl))
hodgestar@708: 
hodgestar@708: def drop_index(env, db, tbl, idx):
hodgestar@708:     """Drop an index associated with a table."""
hodgestar@708:     # Hopefully Trac will eventually implement its own version
hodgestar@708:     # of this function.
hodgestar@846:     scheme = parse_scheme(env)
hodgestar@708:     cursor = db.cursor()
hodgestar@708:     if scheme == "mysql":
hodgestar@708:         cursor.execute("DROP INDEX %s ON %s" % (idx, tbl))
hodgestar@708:     else:
hodgestar@708:         cursor.execute("DROP INDEX %s" % (idx,))
hodgestar@708: 
hodgestar@708: # upgrade scripts
hodgestar@708: 
cmlenz@112: def add_log_table(env, db):
cmlenz@203:     """Add a table for storing the builds logs."""
hodgestar@708:     INFO_LEVEL = 'I'
hodgestar@708: 
cmlenz@112:     cursor = db.cursor()
cmlenz@112: 
hodgestar@708:     build_log_schema_v3 = [
hodgestar@708:         Table('bitten_log', key='id')[
hodgestar@708:             Column('id', auto_increment=True), Column('build', type='int'),
hodgestar@708:             Column('step'), Column('type')
hodgestar@708:         ],
hodgestar@708:         Table('bitten_log_message', key=('log', 'line'))[
hodgestar@708:             Column('log', type='int'), Column('line', type='int'),
hodgestar@708:             Column('level', size=1), Column('message')
hodgestar@708:         ]
hodgestar@708:     ]
hodgestar@708: 
hodgestar@708:     build_step_schema_v3 = [
hodgestar@708:         Table('bitten_step', key=('build', 'name'))[
hodgestar@708:             Column('build', type='int'), Column('name'), Column('description'),
hodgestar@708:             Column('status', size=1), Column('started', type='int'),
hodgestar@708:             Column('stopped', type='int')
hodgestar@708:         ]
hodgestar@708:     ]
hodgestar@708: 
cmlenz@410:     connector, _ = DatabaseManager(env)._get_connector()
hodgestar@708:     for table in build_log_schema_v3:
cmlenz@410:         for stmt in connector.to_sql(table):
cmlenz@112:             cursor.execute(stmt)
cmlenz@112: 
hodgestar@708:     update_cursor = db.cursor()
cmlenz@112:     cursor.execute("SELECT build,name,log FROM bitten_step "
cmlenz@112:                    "WHERE log IS NOT NULL")
cmlenz@112:     for build, step, log in cursor:
hodgestar@708:         update_cursor.execute("INSERT INTO bitten_log (build, step) "
hodgestar@708:                 "VALUES (%s,%s)", (build, step))
hodgestar@708:         log_id = db.get_last_id(update_cursor, 'bitten_log')
hodgestar@708:         messages = [(log_id, line, INFO_LEVEL, msg)
hodgestar@708:             for line, msg in enumerate(log.splitlines())]
hodgestar@708:         update_cursor.executemany("INSERT INTO bitten_log_message (log, line, level, message) "
hodgestar@708:             "VALUES (%s, %s, %s, %s)", messages)
cmlenz@112: 
dfraser@524:     cursor.execute("CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step")
cmlenz@112:     cursor.execute("DROP TABLE bitten_step")
hodgestar@708:     for table in build_step_schema_v3:
cmlenz@410:         for stmt in connector.to_sql(table):
cmlenz@112:             cursor.execute(stmt)
cmlenz@112:     cursor.execute("INSERT INTO bitten_step (build,name,description,status,"
cmlenz@112:                    "started,stopped) SELECT build,name,description,status,"
cmlenz@112:                    "started,stopped FROM old_step")
cmlenz@112: 
cmlenz@147: def add_recipe_to_config(env, db):
cmlenz@203:     """Add a column for storing the build recipe to the build configuration
cmlenz@203:     table."""
cmlenz@147:     cursor = db.cursor()
cmlenz@147: 
hodgestar@841:     build_config_schema_v3 = Table('bitten_config', key='name')[
hodgestar@841:             Column('name'), Column('path'), Column('active', type='int'),
hodgestar@841:             Column('recipe'), Column('min_rev'), Column('max_rev'),
hodgestar@841:             Column('label'), Column('description')
hodgestar@841:         ]
hodgestar@841: 
hodgestar@841:     cursor.execute("CREATE TEMPORARY TABLE old_config_v2 AS "
cmlenz@147:                    "SELECT * FROM bitten_config")
cmlenz@147:     cursor.execute("DROP TABLE bitten_config")
cmlenz@410: 
cmlenz@410:     connector, _ = DatabaseManager(env)._get_connector()
hodgestar@841:     for stmt in connector.to_sql(build_config_schema_v3):
hodgestar@841:         cursor.execute(stmt)
cmlenz@410: 
cmlenz@147:     cursor.execute("INSERT INTO bitten_config (name,path,active,recipe,min_rev,"
cmlenz@147:                    "max_rev,label,description) SELECT name,path,0,'',NULL,"
hodgestar@841:                    "NULL,label,description FROM old_config_v2")
cmlenz@147: 
wbell@762: def add_last_activity_to_build(env, db):
wbell@762:     """Add a column for storing the last activity to the build table."""
wbell@762:     cursor = db.cursor()
wbell@762: 
wbell@762:     build_table_schema_v12 = Table('bitten_build', key='id')[
wbell@762:             Column('id', auto_increment=True), Column('config'), Column('rev'),
wbell@762:             Column('rev_time', type='int'), Column('platform', type='int'),
wbell@762:             Column('slave'), Column('started', type='int'),
wbell@762:             Column('stopped', type='int'), Column('status', size=1),
wbell@762:             Column('last_activity', type='int'),
wbell@762:             Index(['config', 'rev', 'platform'], unique=True)
wbell@762:         ]
wbell@762: 
wbell@762:     cursor.execute("CREATE TEMPORARY TABLE old_build_v11 AS "
wbell@762:                    "SELECT * FROM bitten_build")
wbell@762:     cursor.execute("DROP TABLE bitten_build")
wbell@762: 
wbell@762:     connector, _ = DatabaseManager(env)._get_connector()
wbell@762:     for stmt in connector.to_sql(build_table_schema_v12):
wbell@762:         cursor.execute(stmt)
wbell@762: 
wbell@762:     # it's safe to make the last activity the stop time of the build
hodgestar@788:     cursor.execute("INSERT INTO bitten_build (id,config,rev,rev_time,platform,"
wbell@762:                    "slave,started,stopped,last_activity,status) "
hodgestar@788:                    "SELECT id,config,rev,rev_time,platform,"
wbell@762:                    "slave,started,stopped,stopped,status FROM old_build_v11")
wbell@762: 
hodgestar@847:     update_sequence(env, db, 'bitten_build', 'id')
hodgestar@847: 
cmlenz@174: def add_config_to_reports(env, db):
cmlenz@203:     """Add the name of the build configuration as metadata to report documents
cmlenz@203:     stored in the BDB XML database."""
cmlenz@174:     try:
cmlenz@203:         from bsddb3 import db as bdb
cmlenz@174:         import dbxml
cmlenz@219:     except ImportError:
cmlenz@174:         return
cmlenz@174: 
cmlenz@174:     dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
cmlenz@174:     if not os.path.isfile(dbfile):
cmlenz@174:         return
cmlenz@174: 
cmlenz@203:     dbenv = bdb.DBEnv()
cmlenz@200:     dbenv.open(os.path.dirname(dbfile),
cmlenz@203:                bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
cmlenz@203:                bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
cmlenz@200: 
cmlenz@200:     mgr = dbxml.XmlManager(dbenv, 0)
cmlenz@200:     xtn = mgr.createTransaction()
cmlenz@203:     container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
cmlenz@174:     uc = mgr.createUpdateContext()
cmlenz@174: 
cmlenz@200:     container.addIndex(xtn, '', 'config', 'node-metadata-equality-string', uc)
cmlenz@174: 
cmlenz@174:     qc = mgr.createQueryContext()
cmlenz@200:     for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc):
cmlenz@174:         doc = value.asDocument()
cmlenz@174:         metaval = dbxml.XmlValue()
cmlenz@174:         if doc.getMetaData('', 'build', metaval):
cmlenz@174:             build_id = int(metaval.asNumber())
hodgestar@841: 
hodgestar@841:             cursor = db.cursor()
hodgestar@841:             cursor.execute("SELECT config FROM bitten_build WHERE id=%s", (build_id,))
hodgestar@841:             row = cursor.fetchone()
hodgestar@841: 
hodgestar@841:             if row:
hodgestar@841:                 doc.setMetaData('', 'config', dbxml.XmlValue(row[0]))
cmlenz@200:                 container.updateDocument(xtn, doc, uc)
cmlenz@175:             else:
cmlenz@175:                 # an orphaned report, for whatever reason... just remove it
cmlenz@200:                 container.deleteDocument(xtn, doc, uc)
cmlenz@200: 
cmlenz@200:     xtn.commit()
cmlenz@200:     container.close()
cmlenz@200:     dbenv.close(0)
cmlenz@174: 
cmlenz@203: def add_order_to_log(env, db):
cmlenz@203:     """Add order column to log table to make sure that build logs are displayed
cmlenz@203:     in the order they were generated."""
cmlenz@203:     cursor = db.cursor()
cmlenz@203: 
hodgestar@841:     log_table_schema_v6 = Table('bitten_log', key='id')[
hodgestar@841:             Column('id', auto_increment=True), Column('build', type='int'),
hodgestar@841:             Column('step'), Column('generator'), Column('orderno', type='int'),
hodgestar@841:             Index(['build', 'step'])
hodgestar@841:         ]
hodgestar@841: 
hodgestar@708:     cursor.execute("CREATE TEMPORARY TABLE old_log_v5 AS "
cmlenz@203:                    "SELECT * FROM bitten_log")
cmlenz@203:     cursor.execute("DROP TABLE bitten_log")
cmlenz@410: 
cmlenz@410:     connector, _ = DatabaseManager(env)._get_connector()
hodgestar@841:     for stmt in connector.to_sql(log_table_schema_v6):
cmlenz@203:         cursor.execute(stmt)
cmlenz@410: 
cmlenz@203:     cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno) "
hodgestar@708:                    "SELECT id,build,step,type,0 FROM old_log_v5")
cmlenz@203: 
cmlenz@203: def add_report_tables(env, db):
cmlenz@203:     """Add database tables for report storage."""
cmlenz@203:     cursor = db.cursor()
cmlenz@203: 
hodgestar@841:     report_schema_v6 = Table('bitten_report', key='id')[
hodgestar@841:             Column('id', auto_increment=True), Column('build', type='int'),
hodgestar@841:             Column('step'), Column('category'), Column('generator'),
hodgestar@841:             Index(['build', 'step', 'category'])
hodgestar@841:         ]
hodgestar@841:     report_item_schema_v6 = Table('bitten_report_item', key=('report', 'item', 'name'))[
hodgestar@841:             Column('report', type='int'), Column('item', type='int'),
hodgestar@841:             Column('name'), Column('value')
hodgestar@841:         ]
hodgestar@841: 
cmlenz@410:     connector, _ = DatabaseManager(env)._get_connector()
hodgestar@841:     for table in [report_schema_v6, report_item_schema_v6]:
cmlenz@410:         for stmt in connector.to_sql(table):
cmlenz@203:             cursor.execute(stmt)
cmlenz@203: 
cmlenz@203: def xmldb_to_db(env, db):
cmlenz@203:     """Migrate report data from Berkeley DB XML to SQL database.
wbell@762: 
cmlenz@203:     Depending on the number of reports stored, this might take rather long.
cmlenz@203:     After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
cmlenz@203:     may be deleted. BDB XML is no longer used by Bitten.
cmlenz@203:     """
cmlenz@203:     from bitten.util import xmlio
cmlenz@203:     try:
cmlenz@203:         from bsddb3 import db as bdb
cmlenz@203:         import dbxml
cmlenz@219:     except ImportError:
cmlenz@203:         return
cmlenz@203: 
cmlenz@203:     dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
cmlenz@203:     if not os.path.isfile(dbfile):
cmlenz@203:         return
cmlenz@203: 
cmlenz@203:     dbenv = bdb.DBEnv()
cmlenz@203:     dbenv.open(os.path.dirname(dbfile),
cmlenz@203:                bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
cmlenz@203:                bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)
cmlenz@203: 
cmlenz@203:     mgr = dbxml.XmlManager(dbenv, 0)
cmlenz@203:     xtn = mgr.createTransaction()
cmlenz@203:     container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)
cmlenz@203: 
cmlenz@203:     def get_pylint_items(xml):
cmlenz@203:         for problems_elem in xml.children('problems'):
cmlenz@203:             for problem_elem in problems_elem.children('problem'):
cmlenz@203:                 item = {'type': 'problem'}
cmlenz@203:                 item.update(problem_elem.attr)
cmlenz@203:                 yield item
cmlenz@203: 
cmlenz@203:     def get_trace_items(xml):
cmlenz@203:         for cov_elem in xml.children('coverage'):
cmlenz@203:             item = {'type': 'coverage', 'name': cov_elem.attr['module'],
cmlenz@203:                     'file': cov_elem.attr['file'],
cmlenz@203:                     'percentage': cov_elem.attr['percentage']}
cmlenz@203:             lines = 0
cmlenz@203:             line_hits = []
cmlenz@203:             for line_elem in cov_elem.children('line'):
cmlenz@203:                 lines += 1
cmlenz@203:                 line_hits.append(line_elem.attr['hits'])
cmlenz@203:             item['lines'] = lines
cmlenz@203:             item['line_hits'] = ' '.join(line_hits)
cmlenz@203:             yield item
cmlenz@203: 
cmlenz@203:     def get_unittest_items(xml):
cmlenz@203:         for test_elem in xml.children('test'):
cmlenz@203:             item = {'type': 'test'}
cmlenz@203:             item.update(test_elem.attr)
cmlenz@203:             for child_elem in test_elem.children():
cmlenz@203:                 item[child_elem.name] = child_elem.gettext()
cmlenz@203:             yield item
cmlenz@203: 
cmlenz@203:     qc = mgr.createQueryContext()
cmlenz@203:     for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
cmlenz@203:         doc = value.asDocument()
cmlenz@203:         metaval = dbxml.XmlValue()
cmlenz@203:         build, step = None, None
cmlenz@203:         if doc.getMetaData('', 'build', metaval):
cmlenz@203:             build = metaval.asNumber()
cmlenz@203:         if doc.getMetaData('', 'step', metaval):
cmlenz@203:             step = metaval.asString()
cmlenz@203: 
cmlenz@203:         report_types = {'pylint':   ('lint', get_pylint_items),
cmlenz@203:                         'trace':    ('coverage', get_trace_items),
cmlenz@203:                         'unittest': ('test', get_unittest_items)}
cmlenz@203:         xml = xmlio.parse(value.asString())
cmlenz@203:         report_type = xml.attr['type']
cmlenz@203:         category, get_items = report_types[report_type]
cmlenz@203:         sys.stderr.write('.')
cmlenz@203:         sys.stderr.flush()
hodgestar@841: 
hodgestar@841:         items = list(get_items(xml))
hodgestar@841: 
hodgestar@841:         cursor = db.cursor()
hodgestar@841:         cursor.execute("SELECT bitten_report.id FROM bitten_report "
hodgestar@841:                        "WHERE build=%s AND step=%s AND category=%s",
hodgestar@841:                        (build, step, category))
hodgestar@841:         rows = cursor.fetchall()
hodgestar@841:         if rows:
cmlenz@203:             # Duplicate report, skip
hodgestar@841:             continue
hodgestar@841: 
hodgestar@841:         cursor.execute("INSERT INTO bitten_report "
hodgestar@841:                        "(build,step,category,generator) VALUES (%s,%s,%s,%s)",
hodgestar@841:                        (build, step, category, report_type))
hodgestar@841:         id = db.get_last_id(cursor, 'bitten_report')
hodgestar@841: 
hodgestar@841:         for idx, item in enumerate(items):
hodgestar@841:             cursor.executemany("INSERT INTO bitten_report_item "
hodgestar@841:                                "(report,item,name,value) VALUES (%s,%s,%s,%s)",
hodgestar@841:                                [(id, idx, key, value) for key, value
hodgestar@841:                                 in item.items()])
hodgestar@841: 
cmlenz@203:     sys.stderr.write('\n')
cmlenz@203:     sys.stderr.flush()
cmlenz@203: 
cmlenz@203:     xtn.abort()
cmlenz@203:     container.close()
cmlenz@203:     dbenv.close(0)
cmlenz@203: 
cmlenz@213: def normalize_file_paths(env, db):
cmlenz@213:     """Normalize the file separator in file names in reports."""
cmlenz@213:     cursor = db.cursor()
cmlenz@213:     cursor.execute("SELECT report,item,value FROM bitten_report_item "
cmlenz@213:                    "WHERE name='file'")
cmlenz@223:     rows = cursor.fetchall() or []
cmlenz@213:     for report, item, value in rows:
cmlenz@213:         if '\\' in value:
cmlenz@213:             cursor.execute("UPDATE bitten_report_item SET value=%s "
cmlenz@213:                            "WHERE report=%s AND item=%s AND name='file'",
cmlenz@213:                            (value.replace('\\', '/'), report, item))
cmlenz@213: 
cmlenz@213: def fixup_generators(env, db):
cmlenz@213:     """Upgrade the identifiers for the recipe commands that generated log
cmlenz@213:     messages and report data."""
cmlenz@213: 
cmlenz@213:     mapping = {
osimons@683:         'pipe': 'http://bitten.edgewall.org/tools/sh#pipe',
osimons@683:         'make': 'http://bitten.edgewall.org/tools/c#make',
osimons@683:         'distutils': 'http://bitten.edgewall.org/tools/python#distutils',
osimons@683:         'exec_': 'http://bitten.edgewall.org/tools/python#exec' # Ambigious
cmlenz@213:     }
cmlenz@213:     cursor = db.cursor()
cmlenz@213:     cursor.execute("SELECT id,generator FROM bitten_log "
cmlenz@213:                    "WHERE generator IN (%s)"
cmlenz@213:                    % ','.join([repr(key) for key in mapping.keys()]))
cmlenz@219:     for log_id, generator in cursor:
cmlenz@213:         cursor.execute("UPDATE bitten_log SET generator=%s "
cmlenz@219:                        "WHERE id=%s", (mapping[generator], log_id))
cmlenz@213: 
cmlenz@213:     mapping = {
osimons@683:         'unittest': 'http://bitten.edgewall.org/tools/python#unittest',
osimons@683:         'trace': 'http://bitten.edgewall.org/tools/python#trace',
osimons@683:         'pylint': 'http://bitten.edgewall.org/tools/python#pylint'
cmlenz@213:     }
cmlenz@213:     cursor.execute("SELECT id,generator FROM bitten_report "
cmlenz@213:                    "WHERE generator IN (%s)"
cmlenz@213:                    % ','.join([repr(key) for key in mapping.keys()]))
cmlenz@219:     for report_id, generator in cursor:
cmlenz@213:         cursor.execute("UPDATE bitten_report SET generator=%s "
cmlenz@219:                        "WHERE id=%s", (mapping[generator], report_id))
cmlenz@213: 
cmlenz@245: def add_error_table(env, db):
cmlenz@245:     """Add the bitten_error table for recording step failure reasons."""
cmlenz@245:     table = Table('bitten_error', key=('build', 'step', 'orderno'))[
cmlenz@245:                 Column('build', type='int'), Column('step'), Column('message'),
cmlenz@245:                 Column('orderno', type='int')
cmlenz@245:             ]
cmlenz@245:     cursor = db.cursor()
cmlenz@410: 
cmlenz@410:     connector, _ = DatabaseManager(env)._get_connector()
cmlenz@410:     for stmt in connector.to_sql(table):
cmlenz@245:         cursor.execute(stmt)
cmlenz@245: 
dfraser@516: def add_filename_to_logs(env, db):
dfraser@516:     """Add filename column to log table to save where log files are stored."""
dfraser@516:     cursor = db.cursor()
dfraser@516: 
hodgestar@841:     build_log_schema_v9 = Table('bitten_log', key='id')[
hodgestar@841:             Column('id', auto_increment=True), Column('build', type='int'),
hodgestar@841:             Column('step'), Column('generator'), Column('orderno', type='int'),
hodgestar@841:             Column('filename'),
hodgestar@841:             Index(['build', 'step'])
hodgestar@841:         ]
hodgestar@841: 
hodgestar@708:     cursor.execute("CREATE TEMPORARY TABLE old_log_v8 AS "
dfraser@516:                    "SELECT * FROM bitten_log")
dfraser@516:     cursor.execute("DROP TABLE bitten_log")
dfraser@516: 
dfraser@516:     connector, _ = DatabaseManager(env)._get_connector()
hodgestar@841:     for stmt in connector.to_sql(build_log_schema_v9):
dfraser@516:         cursor.execute(stmt)
dfraser@516: 
dfraser@516:     cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno,filename) "
hodgestar@708:                    "SELECT id,build,step,generator,orderno,'' FROM old_log_v8")
dfraser@516: 
dfraser@516: def migrate_logs_to_files(env, db):
dfraser@516:     """Migrates logs that are stored in the bitten_log_messages table into files."""
dfraser@516:     logs_dir = env.config.get("bitten", "logs_dir", "log/bitten")
dfraser@516:     if not os.path.isabs(logs_dir):
dfraser@516:         logs_dir = os.path.join(env.path, logs_dir)
hodgestar@860: 
hodgestar@860:     if os.path.exists(logs_dir):
hodgestar@860:         print "Bitten log folder %r already exists" % (logs_dir,)
hodgestar@860:         print "Upgrade cannot be performed until the existing folder is moved."
hodgestar@860:         print "The upgrade script will now exit with an error:\n"
hodgestar@860:         raise TracError("")
hodgestar@860: 
hodgestar@860:     os.makedirs(logs_dir)
dfraser@516: 
dfraser@516:     cursor = db.cursor()
dfraser@516:     message_cursor = db.cursor()
dfraser@516:     update_cursor = db.cursor()
dfraser@516:     cursor.execute("SELECT id FROM bitten_log")
dfraser@516:     for log_id, in cursor.fetchall():
dfraser@516:         filename = "%s.log" % (log_id,)
dfraser@516:         message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,))
dfraser@516:         full_filename = os.path.join(logs_dir, filename)
dfraser@521:         message_file = codecs.open(full_filename, "wb", "UTF-8")
dfraser@727:         # Note: the original version of this code erroneously wrote to filename + ".level" instead of ".levels", producing unused level files
hodgestar@735:         level_file = codecs.open(full_filename + '.levels', "wb", "UTF-8")
dfraser@516:         for message, level in message_cursor.fetchall() or []:
dfraser@521:             message_file.write(to_unicode(message) + "\n")
dfraser@521:             level_file.write(to_unicode(level) + "\n")
dfraser@516:         message_file.close()
dfraser@516:         level_file.close()
dfraser@516:         update_cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (filename, log_id))
dfraser@516:         env.log.info("Migrated log %s", log_id)
dfraser@518:     env.log.warning("Logs have been migrated from the database to files in %s. "
dfraser@516:         "Ensure permissions are set correctly on this file. "
dfraser@619:         "Since we presume that the migration worked correctly, "
dfraser@619:         "we are now dropping the bitten_log_message table in the database (aren't you glad you backed up)", logs_dir)
dfraser@619:     cursor.close()
dfraser@619:     cursor = db.cursor()
dfraser@619:     cursor.execute("DROP TABLE bitten_log_message")
dfraser@619:     cursor.close()
dfraser@619:     env.log.warning("We have dropped the bitten_log_message table - you may want to vaccuum/compress your database to save space")
dfraser@516: 
hodgestar@735: def fix_log_levels_misnaming(env, db):
osimons@775:     """Renames or removes \*.log.level files created by older versions of migrate_logs_to_files."""
hodgestar@735:     logs_dir = env.config.get("bitten", "logs_dir", "log/bitten")
hodgestar@735:     if not os.path.isabs(logs_dir):
hodgestar@735:         logs_dir = os.path.join(env.path, logs_dir)
hodgestar@735:     if not os.path.isdir(logs_dir):
hodgestar@735:         return
hodgestar@735: 
hodgestar@735:     rename_count = 0
hodgestar@735:     rename_error_count = 0
hodgestar@735:     delete_count = 0
hodgestar@735:     delete_error_count = 0
hodgestar@735: 
hodgestar@735:     for wrong_filename in os.listdir(logs_dir):
hodgestar@735:         if not wrong_filename.endswith('.log.level'):
hodgestar@735:             continue
hodgestar@735: 
hodgestar@735:         log_filename = os.path.splitext(wrong_filename)[0]
hodgestar@735:         right_filename = log_filename + '.levels'
hodgestar@735:         full_log_filename = os.path.join(logs_dir, log_filename)
hodgestar@735:         full_wrong_filename = os.path.join(logs_dir, wrong_filename)
hodgestar@735:         full_right_filename = os.path.join(logs_dir, right_filename)
hodgestar@735: 
hodgestar@735:         if not os.path.exists(full_log_filename):
hodgestar@735:             try:
hodgestar@735:                 os.remove(full_wrong_filename)
hodgestar@735:                 delete_count += 1
hodgestar@735:                 env.log.info("Deleted stray log level file %s", wrong_filename)
hodgestar@735:             except Exception, e:
hodgestar@735:                 delete_error_count += 1
hodgestar@735:                 env.log.warning("Error removing stray log level file %s: %s", wrong_filename, e)
hodgestar@735:         else:
hodgestar@735:             if os.path.exists(full_right_filename):
hodgestar@735:                 env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: new filename already exists",
hodgestar@735:                     full_wrong_filename, full_right_filename)
hodgestar@735:                 rename_error_count += 1
hodgestar@735:                 continue
hodgestar@735:             try:
hodgestar@735:                 os.rename(full_wrong_filename, full_right_filename)
hodgestar@735:                 rename_count += 1
hodgestar@735:                 env.log.info("Renamed incorrectly named log level file %s to %s", wrong_filename, right_filename)
hodgestar@735:             except Exception, e:
hodgestar@735:                 env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: %s", full_wrong_filename, full_right_filename, e)
hodgestar@735:                 rename_error_count += 1
hodgestar@735: 
hodgestar@735:     env.log.info("Renamed %d incorrectly named log level files from previous migrate (%d errors)", rename_count, rename_error_count)
hodgestar@735:     env.log.info("Deleted %d stray log level (%d errors)", delete_count, delete_error_count)
hodgestar@735: 
hodgestar@737: def remove_stray_log_levels_files(env, db):
osimons@775:     """Remove \*.log.levels files without a matching \*.log file (old Bitten
osimons@775:     versions did not delete .log.levels files when builds were deleted)"""
hodgestar@737:     logs_dir = env.config.get("bitten", "logs_dir", "log/bitten")
hodgestar@737:     if not os.path.isabs(logs_dir):
hodgestar@737:         logs_dir = os.path.join(env.path, logs_dir)
hodgestar@737:     if not os.path.isdir(logs_dir):
hodgestar@737:         return
hodgestar@737: 
hodgestar@737:     delete_count = 0
hodgestar@737:     delete_error_count = 0
hodgestar@737: 
hodgestar@737:     for filename in os.listdir(logs_dir):
hodgestar@737:         if not filename.endswith('.log.levels'):
hodgestar@737:             continue
hodgestar@737: 
hodgestar@737:         log_filename = os.path.splitext(filename)[0]
hodgestar@737:         full_log_filename = os.path.join(logs_dir, log_filename)
hodgestar@737:         full_filename = os.path.join(logs_dir, filename)
hodgestar@737: 
hodgestar@737:         if not os.path.exists(full_log_filename):
hodgestar@737:             try:
hodgestar@737:                 os.remove(full_filename)
hodgestar@737:                 delete_count += 1
hodgestar@737:                 env.log.info("Deleted stray log levels file %s", filename)
hodgestar@737:             except Exception, e:
hodgestar@737:                 delete_error_count += 1
hodgestar@737:                 env.log.warning("Error removing stray log levels file %s: %s", filename, e)
hodgestar@737: 
hodgestar@737:     env.log.info("Deleted %d stray log levels (%d errors)", delete_count, delete_error_count)
hodgestar@737: 
wbell@565: def recreate_rule_with_int_id(env, db):
wbell@565:         """Recreates the bitten_rule table with an integer id column rather than a text one."""
wbell@565:         cursor = db.cursor()
hodgestar@841: 
hodgestar@841:         rule_schema_v9 = Table('bitten_rule', key=('id', 'propname'))[
hodgestar@841:             Column('id', type='int'), Column('propname'), Column('pattern'),
hodgestar@841:             Column('orderno', type='int')
hodgestar@841:         ]
hodgestar@841: 
hodgestar@841:         env.log.info("Migrating bitten_rule table to integer ids")
wbell@565:         connector, _ = DatabaseManager(env)._get_connector()
wbell@565: 
hodgestar@841:         cursor.execute("CREATE TEMPORARY TABLE old_rule_v9 AS SELECT * FROM bitten_rule")
hodgestar@841:         cursor.execute("DROP TABLE bitten_rule")
hodgestar@841:         for stmt in connector.to_sql(rule_schema_v9):
hodgestar@841:             cursor.execute(stmt)
hodgestar@841:         cursor.execute("INSERT INTO bitten_rule (id,propname,pattern,orderno) "
hodgestar@841:             "SELECT %s,propname,pattern,orderno FROM old_rule_v9" % db.cast('id', 'int'))
wbell@565: 
osimons@650: def add_config_platform_rev_index_to_build(env, db):
osimons@650:     """Adds a unique index on (config, platform, rev) to the bitten_build table.
osimons@650:        Also drops the old index on bitten_build that serves no real purpose anymore."""
osimons@656:     # check for existing duplicates
osimons@656:     duplicates_cursor = db.cursor()
osimons@656:     build_cursor = db.cursor()
osimons@656: 
osimons@656:     duplicates_cursor.execute("SELECT config, rev, platform FROM bitten_build GROUP BY config, rev, platform HAVING COUNT(config) > 1")
osimons@656:     duplicates_exist = False
osimons@656:     for config, rev, platform in duplicates_cursor.fetchall():
osimons@656:         if not duplicates_exist:
osimons@656:             duplicates_exist = True
osimons@656:             print "\nConfig Name, Revision, Platform :: [<list of build ids>]"
osimons@656:             print "--------------------------------------------------------"
osimons@656: 
osimons@656:         build_cursor.execute("SELECT id FROM bitten_build WHERE config='%s' AND rev='%s' AND platform='%s'" % (config, rev, platform))
osimons@656:         build_ids = [row[0] for row in build_cursor.fetchall()]
osimons@656:         print "%s, %s, %s :: %s" % (config, rev, platform, build_ids)
osimons@656: 
osimons@656:     if duplicates_exist:
osimons@656:         print "--------------------------------------------------------\n"
osimons@656:         print "Duplicate builds found. You can remove the builds you don't want to"
osimons@656:         print "keep by using this one-line command:\n"
osimons@656:         print "$  python -c \"from bitten.model import Build; from trac.env import Environment; " \
osimons@656:                         "Build.fetch(Environment('%s'), <buildid>).delete()\"" % env.path
osimons@656:         print "\n...where <buildid> is the id of the build to remove.\n"
osimons@656:         print "Upgrades cannot be performed until conflicts are resolved."
osimons@656:         print "The upgrade script will now exit with an error:\n"
osimons@656: 
osimons@656:     duplicates_cursor.close()
osimons@656:     build_cursor.close()
osimons@656: 
osimons@656:     if not duplicates_exist:
osimons@656:         cursor = db.cursor()
hodgestar@846:         scheme = parse_scheme(env)
hodgestar@708:         if scheme == "mysql":
hodgestar@708:             # 111 = 333 / len(columns in index) -- this is the Trac default
hodgestar@708:             cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config(111), rev(111), platform)")
hodgestar@708:         else:
hodgestar@708:             cursor.execute("CREATE UNIQUE INDEX bitten_build_config_rev_platform_idx ON bitten_build (config,rev,platform)")
hodgestar@708:         drop_index(env, db, 'bitten_build', 'bitten_build_config_rev_slave_idx')
osimons@656:     else:
osimons@656:         raise TracError('')
wbell@565: 
hodgestar@708: def fix_sequences(env, db):
hodgestar@708:     """Fixes any auto increment sequences that might have been left in an inconsistent state.
hodgestar@708: 
hodgestar@708:        Upgrade scripts for schema versions > 10 should handle sequence updates correctly themselves.
hodgestar@708:        """
hodgestar@708:     update_sequence(env, db, 'bitten_build', 'id')
hodgestar@708:     update_sequence(env, db, 'bitten_log', 'id')
hodgestar@708:     update_sequence(env, db, 'bitten_platform', 'id')
hodgestar@708:     update_sequence(env, db, 'bitten_report', 'id')
hodgestar@708: 
hodgestar@708: 
cmlenz@112: map = {
cmlenz@147:     2: [add_log_table],
cmlenz@174:     3: [add_recipe_to_config],
cmlenz@203:     4: [add_config_to_reports],
cmlenz@213:     5: [add_order_to_log, add_report_tables, xmldb_to_db],
cmlenz@245:     6: [normalize_file_paths, fixup_generators],
dfraser@516:     7: [add_error_table],
dfraser@516:     8: [add_filename_to_logs,migrate_logs_to_files],
wbell@565:     9: [recreate_rule_with_int_id],
hodgestar@708:    10: [add_config_platform_rev_index_to_build, fix_sequences],
hodgestar@737:    11: [fix_log_levels_misnaming, remove_stray_log_levels_files],
wbell@762:    12: [add_last_activity_to_build],
cmlenz@112: }