Mercurial > bitten > bitten-test
changeset 112:a38eabd4b6e1
* Store build logs in a structured way, for example to highlight messages on the error stream.
* Add basic infrastructure for database upgrades.
| author | cmlenz |
|---|---|
| date | Thu, 04 Aug 2005 20:15:39 +0000 |
| parents | 8d76fd3918a5 |
| children | 142d95b9e8b0 |
| files | bitten/master.py bitten/model.py bitten/tests/model.py bitten/trac_ext/__init__.py bitten/trac_ext/htdocs/build.css bitten/trac_ext/main.py bitten/trac_ext/templates/build.cs bitten/trac_ext/web_ui.py bitten/upgrades.py |
| diffstat | 9 files changed, 374 insertions(+), 56 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/master.py +++ b/bitten/master.py @@ -30,7 +30,7 @@ import time from trac.env import Environment -from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep +from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep, BuildLog from bitten.util import archive, beep, xmlio log = logging.getLogger('bitten.master') @@ -70,17 +70,19 @@ try: repos.sync() - for config in BuildConfig.select(self.env): + db = self.env.get_db_cnx() + for config in BuildConfig.select(self.env, db=db): log.debug('Checking for changes to "%s" at %s', config.label, config.path) node = repos.get_node(config.path) for path, rev, chg in node.get_history(): enqueued = False - for platform in TargetPlatform.select(self.env, config.name): + for platform in TargetPlatform.select(self.env, + config.name, db=db): # Check whether the latest revision of the configuration # has already been built on this platform builds = Build.select(self.env, config.name, rev, - platform.id) + platform.id, db=db) if not list(builds): log.info('Enqueuing build of configuration "%s" at ' 'revision [%s] on %s', config.name, rev, @@ -90,9 +92,10 @@ build.rev = str(rev) build.rev_time = repos.get_changeset(rev).date build.platform = platform.id - build.insert() + build.insert(db) enqueued = True if enqueued: + db.commit() break finally: repos.close() @@ -119,7 +122,13 @@ build.status = Build.PENDING build.update(db=db) for build in Build.select(self.env, status=Build.PENDING, db=db): + for step in BuildStep.select(self.env, build=build.id, db=db): + for log in BuildLog.select(self.env, build=build.id, + step=step.name, db=db): + log.delete(db=db) + step.delete(db=db) build.delete(db=db) + db.commit() def _cleanup_snapshots(self, when): log.debug('Checking for unused snapshot archives...') @@ -332,16 +341,19 @@ step.status = BuildStep.FAILURE else: step.status = BuildStep.SUCCESS + step.insert(db=db) - # TODO: Insert log messages into separate table, and also store reports - log_lines = [] + # TODO: Store reports, too + level_map = {'debug': BuildLog.DEBUG, 'info': BuildLog.INFO, + 'warning': BuildLog.WARNING, 'error': BuildLog.ERROR} for log_elem in elem.children('log'): + build_log = BuildLog(self.env, build=build.id, step=step.name, + type=log_elem.attr.get('type')) for messages_elem in log_elem.children('messages'): for message_elem in messages_elem.children('message'): - log_lines.append(message_elem.gettext()) - step.log = '\n'.join(log_lines) - - step.insert(db=db) + build_log.messages.append((message_elem.attr['level'], + message_elem.gettext())) + build_log.insert(db=db) def _build_completed(self, db, build, elem): log.info('Slave %s completed build %d ("%s" as of [%s])', self.name, @@ -378,11 +390,11 @@ except ValueError, e: raise ValueError, 'Invalid ISO date/time %s (%s)' % (string, e) - def main(): from bitten import __version__ as VERSION from optparse import OptionParser + # Parse command-line arguments parser = OptionParser(usage='usage: %prog [options] env-path', version='%%prog %s' % VERSION) parser.add_option('-p', '--port', action='store', type='int', dest='port',
--- a/bitten/model.py +++ b/bitten/model.py @@ -359,7 +359,7 @@ "stopped=%s,status=%s WHERE id=%s", (self.slave or '', self.started or 0, self.stopped or 0, self.status, self.id)) - cursor.execute("DELETE FROM bitten_slave WHERE build=%s", (self.id)) + cursor.execute("DELETE FROM bitten_slave WHERE build=%s", (self.id,)) cursor.executemany("INSERT INTO bitten_slave VALUES (%s,%s,%s)", [(self.id, name, value) for name, value in self.slave_info.items()]) @@ -425,8 +425,8 @@ _schema = [ Table('bitten_step', key=('build', 'name'))[ Column('build', type='int'), Column('name'), Column('description'), - Column('status', size=1), Column('log'), - Column('started', type='int'), Column('stopped', type='int') + Column('status', size=1), Column('started', type='int'), + Column('stopped', type='int') ] ] @@ -435,13 +435,12 @@ FAILURE = 'F' def __init__(self, env, build=None, name=None, description=None, - status=None, log=None, started=None, stopped=None): + status=None, started=None, stopped=None): self.env = env self.build = build self.name = name self.description = description self.status = status - self.log = log self.started = started self.stopped = stopped @@ -473,10 +472,9 @@ cursor = db.cursor() cursor.execute("INSERT INTO bitten_step (build,name,description,status," - "log,started,stopped) VALUES (%s,%s,%s,%s,%s,%s,%s)", + "started,stopped) VALUES (%s,%s,%s,%s,%s,%s)", (self.build, self.name, self.description or '', - self.status, self.log or '', self.started or 0, - self.stopped or 0)) + self.status, self.started or 0, self.stopped or 0)) if handle_ta: db.commit() @@ -485,15 +483,15 @@ db = env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT description,status,log,started,stopped " + cursor.execute("SELECT description,status,started,stopped " "FROM bitten_step WHERE build=%s AND name=%s", (build, name)) row = cursor.fetchone() if not row: return None - return BuildStep(env, build, name, row[0] or '', row[1], row[2] or '', - row[3] and int(row[3]), row[4] and int(row[4])) + return BuildStep(env, build, name, row[0] or '', row[1], + row[2] and int(row[2]), row[3] and int(row[3])) fetch = classmethod(fetch) def select(cls, env, build=None, name=None, db=None): @@ -511,16 +509,129 @@ where = "" cursor = db.cursor() - cursor.execute("SELECT build,name,description,status,log,started," - "stopped FROM bitten_step %s ORDER BY stopped" + cursor.execute("SELECT build,name,description,status,started,stopped " + "FROM bitten_step %s ORDER BY stopped" % where, [wc[1] for wc in where_clauses]) - for build, name, description, status, log, started, stopped in cursor: + for build, name, description, status, started, stopped in cursor: yield BuildStep(env, build, name, description or '', status, - log or '', started and int(started), - stopped and int(stopped)) + started and int(started), stopped and int(stopped)) + select = classmethod(select) + + +class BuildLog(object): + """Represents a build log.""" + + _schema = [ + Table('bitten_log', key='id')[ + Column('id', auto_increment=True), Column('build', type='int'), + Column('step'), Column('type') + ], + Table('bitten_log_message', key=('log', 'line'))[ + Column('log', type='int'), Column('line', type='int'), + Column('level', size=1), Column('message') + ] + ] + + # Message levels + DEBUG = 'D' + INFO = 'I' + WARNING = 'W' + ERROR = 'E' + + def __init__(self, env, build=None, step=None, type=None): + self.env = env + self.id = None + self.build = build + self.step = step + self.type = type + self.messages = [] + + exists = property(fget=lambda self: self.id is not None) + + def delete(self, db=None): + assert self.exists, 'Cannot delete a non-existing build log' + if not db: + db = self.env.get_db_cnx() + handle_ta = True + else: + handle_ta = False + + cursor = db.cursor() + cursor.execute("DELETE FROM bitten_log WHERE id=%s", (self.id,)) + cursor.execute("DELETE FROM bitten_log_message WHERE log=%s", + (self.id,)) + + if handle_ta: + db.commit() + self.id = None + + def insert(self, db=None): + if not db: + db = self.env.get_db_cnx() + handle_ta = True + else: + handle_ta = False + + assert self.build and self.step + + cursor = db.cursor() + cursor.execute("INSERT INTO bitten_log (build,step,type) " + "VALUES (%s,%s,%s)", (self.build, self.step, self.type)) + id = db.get_last_id(cursor, 'bitten_log') + cursor.executemany("INSERT INTO bitten_log_message " + "(log,line,level,message) VALUES (%s,%s,%s,%s)", + [(id, idx, message[0], message[1]) for idx, message + in enumerate(self.messages)]) + + if handle_ta: + db.commit() + self.id = id + + def fetch(cls, env, id, db=None): + if not db: + db = env.get_db_cnx() + + cursor = db.cursor() + cursor.execute("SELECT build,step,type FROM bitten_log " + "WHERE id=%s", (id,)) + row = cursor.fetchone() + if not row: + return None + log = BuildLog(env, int(row[0]), row[1], row[2] or '') + log.id = id + cursor.execute("SELECT level,message FROM bitten_log_message " + "WHERE log=%s ORDER BY line", (id,)) + log.messages = cursor.fetchall() + + return log + + fetch = classmethod(fetch) + + def select(cls, env, build=None, step=None, type=None, db=None): + if not db: + db = env.get_db_cnx() + + where_clauses = [] + if build is not None: + where_clauses.append(("build=%s", build)) + if step is not None: + where_clauses.append(("step=%s", step)) + if type is not None: + where_clauses.append(("type=%s", type)) + if where_clauses: + where = "WHERE " + " AND ".join([wc[0] for wc in where_clauses]) + else: + where = "" + + cursor = db.cursor() + cursor.execute("SELECT id FROM bitten_log %s ORDER BY type" + % where, [wc[1] for wc in where_clauses]) + for (id, ) in cursor: + yield BuildLog.fetch(env, id, db=db) + select = classmethod(select) schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \ - BuildStep._schema -schema_version = 1 + BuildStep._schema + BuildLog._schema +schema_version = 2
--- a/bitten/tests/model.py +++ b/bitten/tests/model.py @@ -21,7 +21,7 @@ import unittest from trac.test import EnvironmentStub -from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep +from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep, BuildLog class BuildConfigTestCase(unittest.TestCase): @@ -268,6 +268,23 @@ self.assertEquals('127.0.0.1', build.slave_info[Build.IP_ADDRESS]) self.assertEquals('joe@example.org', build.slave_info[Build.MAINTAINER]) + def test_update(self): + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("INSERT INTO bitten_build (config,rev,rev_time,platform," + "slave,started,stopped,status) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", + ('test', '42', 12039, 1, 'tehbox', 15006, 16007, + Build.SUCCESS)) + build_id = db.get_last_id(cursor, 'bitten_build') + cursor.executemany("INSERT INTO bitten_slave VALUES (%s,%s,%s)", + [(build_id, Build.IP_ADDRESS, '127.0.0.1'), + (build_id, Build.MAINTAINER, 'joe@example.org')]) + + build = Build.fetch(self.env, build_id) + build.status = Build.FAILURE + build.update() + class BuildStepTestCase(unittest.TestCase): @@ -292,9 +309,9 @@ db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT build,name,description,status,log,started" - ",stopped FROM bitten_step") - self.assertEqual((1, 'test', 'Foo bar', BuildStep.SUCCESS, '', 0, 0), + cursor.execute("SELECT build,name,description,status,started,stopped " + "FROM bitten_step") + self.assertEqual((1, 'test', 'Foo bar', BuildStep.SUCCESS, 0, 0), cursor.fetchone()) def test_insert_no_build_or_name(self): @@ -307,8 +324,8 @@ def test_fetch(self): db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("INSERT INTO bitten_step VALUES (%s,%s,%s,%s,%s,%s,%s)", - (1, 'test', 'Foo bar', BuildStep.SUCCESS, '', 0, 0)) + cursor.execute("INSERT INTO bitten_step VALUES (%s,%s,%s,%s,%s,%s)", + (1, 'test', 'Foo bar', BuildStep.SUCCESS, 0, 0)) step = BuildStep.fetch(self.env, build=1, name='test') self.assertEqual(1, step.build) @@ -317,12 +334,127 @@ self.assertEqual(BuildStep.SUCCESS, step.status) +class BuildLogTestCase(unittest.TestCase): + + def setUp(self): + self.env = EnvironmentStub() + db = self.env.get_db_cnx() + cursor = db.cursor() + for table in BuildLog._schema: + for stmt in db.to_sql(table): + cursor.execute(stmt) + db.commit() + + def test_new(self): + log = BuildLog(self.env) + self.assertEqual(False, log.exists) + self.assertEqual(None, log.id) + self.assertEqual(None, log.build) + self.assertEqual(None, log.step) + self.assertEqual(None, log.type) + self.assertEqual([], log.messages) + + def test_insert(self): + log = BuildLog(self.env, build=1, step='test', type='distutils') + log.messages = [ + (BuildLog.INFO, 'running tests'), + (BuildLog.ERROR, 'tests failed') + ] + log.insert() + self.assertNotEqual(None, log.id) + + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("SELECT build,step,type FROM bitten_log " + "WHERE id=%s", (log.id,)) + self.assertEqual((1, 'test', 'distutils'), cursor.fetchone()) + cursor.execute("SELECT level,message FROM bitten_log_message " + "WHERE log=%s ORDER BY line", (log.id,)) + self.assertEqual((BuildLog.INFO, 'running tests'), cursor.fetchone()) + self.assertEqual((BuildLog.ERROR, 'tests failed'), cursor.fetchone()) + + def test_insert_no_build_or_step(self): + log = BuildLog(self.env, step='test') + self.assertRaises(AssertionError, log.insert) # No build + + step = BuildStep(self.env, build=1) + self.assertRaises(AssertionError, log.insert) # No step + + def test_delete(self): + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("INSERT INTO bitten_log (build,step,type) " + "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + id = db.get_last_id(cursor, 'bitten_log') + cursor.executemany("INSERT INTO bitten_log_message " + "VALUES (%s,%s,%s,%s)", + [(id, 1, BuildLog.INFO, 'running tests'), + (id, 2, BuildLog.ERROR, 'tests failed')]) + + log = BuildLog.fetch(self.env, id=id, db=db) + self.assertEqual(True, log.exists) + log.delete() + self.assertEqual(False, log.exists) + + cursor.execute("SELECT * FROM bitten_log WHERE id=%s", (id,)) + self.assertEqual(True, not cursor.fetchall()) + cursor.execute("SELECT * FROM bitten_log_message WHERE log=%s", (id,)) + self.assertEqual(True, not cursor.fetchall()) + + def test_delete_new(self): + log = BuildLog(self.env, build=1, step='test', type='foo') + self.assertRaises(AssertionError, log.delete) + + def test_fetch(self): + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("INSERT INTO bitten_log (build,step,type) " + "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + id = db.get_last_id(cursor, 'bitten_log') + cursor.executemany("INSERT INTO bitten_log_message " + "VALUES (%s,%s,%s,%s)", + [(id, 1, BuildLog.INFO, 'running tests'), + (id, 2, BuildLog.ERROR, 'tests failed')]) + + log = BuildLog.fetch(self.env, id=id, db=db) + self.assertEqual(True, log.exists) + self.assertEqual(id, log.id) + self.assertEqual(1, log.build) + self.assertEqual('test', log.step) + self.assertEqual('distutils', log.type) + self.assertEqual((BuildLog.INFO, 'running tests'), log.messages[0]) + self.assertEqual((BuildLog.ERROR, 'tests failed'), log.messages[1]) + + def test_select(self): + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("INSERT INTO bitten_log (build,step,type) " + "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + id = db.get_last_id(cursor, 'bitten_log') + cursor.executemany("INSERT INTO bitten_log_message " + "VALUES (%s,%s,%s,%s)", + [(id, 1, BuildLog.INFO, 'running tests'), + (id, 2, BuildLog.ERROR, 'tests failed')]) + + logs = BuildLog.select(self.env, build=1, step='test', db=db) + log = logs.next() + self.assertEqual(True, log.exists) + self.assertEqual(id, log.id) + self.assertEqual(1, log.build) + self.assertEqual('test', log.step) + self.assertEqual('distutils', log.type) + self.assertEqual((BuildLog.INFO, 'running tests'), log.messages[0]) + self.assertEqual((BuildLog.ERROR, 'tests failed'), log.messages[1]) + self.assertRaises(StopIteration, logs.next) + + def suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(BuildConfigTestCase, 'test')) suite.addTest(unittest.makeSuite(TargetPlatformTestCase, 'test')) suite.addTest(unittest.makeSuite(BuildTestCase, 'test')) suite.addTest(unittest.makeSuite(BuildStepTestCase, 'test')) + suite.addTest(unittest.makeSuite(BuildLogTestCase, 'test')) return suite if __name__ == '__main__':
--- a/bitten/trac_ext/__init__.py +++ b/bitten/trac_ext/__init__.py @@ -17,3 +17,6 @@ # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # Author: Christopher Lenz <cmlenz@gmx.de> + +import bitten.trac_ext.main +import bitten.trac_ext.web_ui
--- a/bitten/trac_ext/htdocs/build.css +++ b/bitten/trac_ext/htdocs/build.css @@ -16,4 +16,6 @@ #content.build #builds td.failed { background: #d99; } #content.build #builds td.in-progress { background: #ff9; } -#content.build pre.log { overflow: auto; white-space: pre; } +#content.build .log { overflow: auto; white-space: pre; } +#content.build .log .warning { color: #660; font-weight: bold; } +#content.build .log .error { color: #900; font-weight: bold; }
--- a/bitten/trac_ext/main.py +++ b/bitten/trac_ext/main.py @@ -55,6 +55,8 @@ cursor = db.cursor() cursor.execute("SELECT value FROM system WHERE name='bitten_version'") row = cursor.fetchone() + self.log.debug("Current DB version is %s, we need %s", + row and int(row[0]) or None, schema_version) if not row or int(row[0]) < schema_version: return True @@ -65,19 +67,15 @@ if not row: self.environment_created() else: - current_version = int(row.fetchone()[0]) - for i in range(current_version + 1, schema_version + 1): - name = 'db%i' % i - try: - upgrades = __import__('upgrades', globals(), locals(), - [name]) - script = getattr(upgrades, name) - except AttributeError: - err = 'No upgrade module for version %i (%s.py)' % (i, name) - raise TracError, err - script.do_upgrade(self.env, i, cursor) + current_version = int(row[0]) + from bitten import upgrades + for version in range(current_version + 1, schema_version + 1): + self.log.debug('Updating to schema version %s', version) + for function in upgrades.map.get(version): + self.log.debug('Executing upgrade function %s', function) + function(self.env, db) cursor.execute("UPDATE system SET value=%s WHERE " - "name='bitten_version'", (schema_version)) + "name='bitten_version'", (schema_version,)) self.log.info('Upgraded Bitten tables from version %d to %d', current_version, schema_version)
--- a/bitten/trac_ext/templates/build.cs +++ b/bitten/trac_ext/templates/build.cs @@ -160,7 +160,10 @@ each:step = build.steps ?> <h2><?cs var:step.name ?> (<?cs var:step.duration ?>)</h2> <p><?cs var:step.description ?></p> - <pre class="log"><?cs var:step.log ?></pre><?cs + <div class="log"><?cs + each:item = step.log ?><code class="<?cs var:item.level ?>"><?cs + var:item.message ?></code><br /><?cs + /each ?></div><?cs /each ?><?cs /if ?>
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -31,7 +31,7 @@ add_link, add_stylesheet from trac.web.main import IRequestHandler from trac.wiki import wiki_to_html -from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep +from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep, BuildLog class BuildModule(Component): @@ -43,6 +43,10 @@ _status_label = {Build.IN_PROGRESS: 'in progress', Build.SUCCESS: 'completed', Build.FAILURE: 'failed'} + _level_label = {BuildLog.DEBUG: 'debug', + BuildLog.INFO: 'info', + BuildLog.WARNING: 'warning', + BuildLog.ERROR: 'error'} # INavigationContributor methods @@ -380,7 +384,7 @@ Build.IN_PROGRESS: 'In Progress'} req.hdf['title'] = 'Build %s - %s' % (build_id, status2title[build.status]) - req.hdf['build'] = self._build_to_hdf(build) + req.hdf['build'] = self._build_to_hdf(build, include_output=True) req.hdf['build.mode'] = 'view_build' config = BuildConfig.fetch(self.env, build.config) @@ -389,7 +393,7 @@ 'href': self.env.href.build(config.name) } - def _build_to_hdf(self, build): + def _build_to_hdf(self, build, include_output=False): hdf = {'id': build.id, 'name': build.slave, 'rev': build.rev, 'status': self._status_label[build.status], 'cls': self._status_label[build.status].replace(' ', '-'), @@ -411,13 +415,19 @@ 'machine': build.slave_info.get(Build.MACHINE), 'processor': build.slave_info.get(Build.PROCESSOR) } + db = self.env.get_db_cnx() steps = [] - for step in BuildStep.select(self.env, build=build.id): + for step in BuildStep.select(self.env, build=build.id, db=db): steps.append({ 'name': step.name, 'description': step.description, - 'duration': pretty_timedelta(step.started, step.stopped), - 'log': step.log + 'duration': pretty_timedelta(step.started, step.stopped) }) + if include_output: + for log in BuildLog.select(self.env, build=build.id, + step=step.name, db=db): + steps[-1]['log'] = [{'level': level, + 'message': message} + for level, message in log.messages] hdf['steps'] = steps return hdf
new file mode 100644 --- /dev/null +++ b/bitten/upgrades.py @@ -0,0 +1,47 @@ +# -*- coding: iso8859-1 -*- +# +# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de> +# +# Bitten is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Trac is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# Author: Christopher Lenz <cmlenz@gmx.de> + +def add_log_table(env, db): + from bitten.model import BuildLog, BuildStep + cursor = db.cursor() + + for table in BuildLog._schema: + for stmt in db.to_sql(table): + cursor.execute(stmt) + + cursor.execute("SELECT build,name,log FROM bitten_step " + "WHERE log IS NOT NULL") + for build, step, log in cursor: + build_log = BuildLog(env, build, step) + build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()] + build_log.insert(db) + + cursor.execute("CREATE TEMP TABLE old_step AS SELECT * FROM bitten_step") + cursor.execute("DROP TABLE bitten_step") + for table in BuildStep._schema: + for stmt in db.to_sql(table): + cursor.execute(stmt) + cursor.execute("INSERT INTO bitten_step (build,name,description,status," + "started,stopped) SELECT build,name,description,status," + "started,stopped FROM old_step") + +map = { + 2: [add_log_table] +}