Mercurial > bitten > bitten-test
changeset 47:083e848088ee
* Improvements to the model classes, and a couple of unit tests.
* The build master now stores information about ongoing builds in the Trac database.
* The web interface displays the status of ongoing builds.
| author | cmlenz |
|---|---|
| date | Fri, 24 Jun 2005 15:35:23 +0000 |
| parents | f7bbf9d2bbe7 |
| children | 757aa3bf9594 |
| files | bitten/master.py bitten/model.py bitten/slave.py bitten/tests/__init__.py bitten/tests/model.py bitten/trac_ext/web_ui.py bitten/util/tests/beep.py |
| diffstat | 7 files changed, 335 insertions(+), 85 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/master.py +++ b/bitten/master.py @@ -20,6 +20,7 @@ import logging import os.path +import time from trac.env import Environment from bitten import __version__ as VERSION @@ -34,16 +35,23 @@ def __init__(self, env_path, ip, port): beep.Listener.__init__(self, ip, port) self.profiles[OrchestrationProfileHandler.URI] = OrchestrationProfileHandler - self.env = Environment(env_path) + self.slaves = {} + + # path to generated snapshot archives, key is (config name, revision) + self.snapshots = {} + self.schedule(self.TRIGGER_INTERVAL, self._check_build_triggers) - self.build_queue = {} + + def close(self): + # Remove all pending builds + for build in Build.select(self.env, status=Build.PENDING): + build.delete() + beep.Listener.close(self) def _check_build_triggers(self, master, when): self.schedule(self.TRIGGER_INTERVAL, self._check_build_triggers) - if not self.slaves: - return logging.debug('Checking for build triggers...') repos = self.env.get_repository() @@ -52,36 +60,41 @@ for config in BuildConfig.select(self.env): node = repos.get_node(config.path) - if (node.path, node.rev) in self.build_queue: - # Builds already pending - continue # Check whether the latest revision of that configuration has # already been built - builds = list(Build.select(self.env, node.path, node.rev)) - if not builds: - logging.info('Enqueuing build of configuration "%s" as of revision [%s]', - config.name, node.rev) + builds = Build.select(self.env, config.name, node.rev) + if not list(builds): snapshot = archive.make_archive(self.env, repos, node.path, node.rev, config.name) logging.info('Created snapshot archive at %s' % snapshot) - self.build_queue[(node.path, node.rev)] = (config, snapshot) + self.snapshots[(config.name, str(node.rev))] = snapshot + + logging.info('Enqueuing build of configuration "%s" as of revision [%s]', + config.name, node.rev) + build = Build(self.env) + build.config = config.name + build.rev = node.rev + build.insert() finally: repos.close() - if self.build_queue: - self.schedule(5, self._check_build_queue) + self.schedule(5, self._check_build_queue) def _check_build_queue(self, master, when): - if self.build_queue: - for path, rev in self.build_queue.keys(): - config, snapshot = self.build_queue[(path, rev)] - logging.info('Building configuration "%s" as of revision [%s]', - config.name, rev) - for slave in self.slaves.values(): - if not slave.building: - slave.send_build(snapshot) - break + if not self.slaves: + return + logging.info('Checking for pending builds...') + for build in Build.select(self.env, status=Build.PENDING): + logging.info('Building configuration "%s" as of revision [%s]', + build.config, build.rev) + snapshot = self.snapshots[(build.config, build.rev)] + for slave in self.slaves.values(): + active_builds = Build.select(self.env, slave=slave.name, + status=Build.IN_PROGRESS) + if not list(active_builds): + slave.send_build(build, snapshot) + break class OrchestrationProfileHandler(beep.ProfileHandler): @@ -94,11 +107,21 @@ self.master = self.session.listener assert self.master self.building = False - self.slave_name = None + self.name = None def handle_disconnect(self): - del self.master.slaves[self.slave_name] - logging.info('Unregistered slave "%s"', self.slave_name) + del self.master.slaves[self.name] + logging.info('Unregistered slave "%s"', self.name) + if self.building: + for build in Build.select(self.master.env, slave=self.name, + status=Build.IN_PROGRESS): + logging.info('Build [%s] of "%s" by %s cancelled', build.rev, + build.config, self.name) + build.slave = None + build.status = Build.PENDING + build.time = None + build.update() + break def handle_msg(self, msgno, msg): assert msg.get_content_type() == beep.BEEP_XML @@ -114,16 +137,16 @@ os_family = child.family os_version = child.version - self.slave_name = elem.name - self.master.slaves[self.slave_name] = self + self.name = elem.name + self.master.slaves[self.name] = self xml = xmlio.Element('ok') self.channel.send_rpy(msgno, beep.MIMEMessage(xml)) logging.info('Registered slave "%s" (%s running %s %s [%s])', - self.slave_name, platform, os, os_version, os_family) + self.name, platform, os, os_version, os_family) - def send_build(self, archive_path, handle_reply=None): - logging.info('Initiating build on slave %s', self.slave_name) + def send_build(self, build, snapshot_path, handle_reply=None): + logging.info('Initiating build on slave %s', self.name) self.building = True def handle_reply(cmd, msgno, msg): @@ -133,12 +156,18 @@ if elem.tagname == 'error': logging.warning('Slave refused build request: %s (%d)', elem.gettext(), int(elem.code)) + build.slave = self.name + build.time = int(time.time()) + build.status = Build.IN_PROGRESS + build.update() logging.info('Build started') - archive_name = os.path.basename(archive_path) - message = beep.MIMEMessage(file(archive_path).read(), + # TODO: should not block while reading the file; rather stream it using + # asyncore push_with_producer() + snapshot_name = os.path.basename(snapshot_path) + message = beep.MIMEMessage(file(snapshot_path).read(), content_type='application/tar', - content_disposition=archive_name, + content_disposition=snapshot_name, content_encoding='gzip') self.channel.send_msg(message, handle_reply=handle_reply)
--- a/bitten/model.py +++ b/bitten/model.py @@ -69,8 +69,8 @@ cursor.execute("INSERT INTO bitten_config " "(name,path,label,active,description) " "VALUES (%s,%s,%s,%s,%s)", - (self.name, self.path, self.label, int(self.active or 0), - self.description)) + (self.name, self.path, self.label or '', + int(self.active or 0), self.description or '')) if handle_ta: db.commit() @@ -119,47 +119,63 @@ class Build(object): """Representation of a build.""" - _table = Table('bitten_build', key=('path', 'rev', 'slave'))[ - Column('rev'), Column('path'), Column('slave'), + _table = Table('bitten_build', key=('config', 'rev', 'slave'))[ + Column('config'), Column('rev'), Column('slave'), Column('time', type='int'), Column('duration', type='int'), - Column('status', type='int') + Column('status', size='1') ] - FAILURE = 'failure' - IN_PROGRESS = 'in-progress' - SUCCESS = 'success' + PENDING = 'P' + IN_PROGRESS = 'I' + SUCCESS = 'S' + FAILURE = 'F' - def __init__(self, env, path=None, rev=None, slave=None, db=None): + def __init__(self, env, config=None, rev=None, slave=None, db=None): self.env = env - self.rev = self.path = self.slave = None - self.time = self.duration = self.status = None - if rev: - self._fetch(rev, path, slave, db) + self.config = self.rev = self.slave = self.time = self.duration = None + if config and rev and slave: + self._fetch(config, rev, slave, db) + else: + self.time = self.duration = 0 + self.status = self.PENDING - def _fetch(self, rev, path, slave, db=None): + def _fetch(self, config, rev, slave, db=None): if not db: db = self.env.get_db_cnx() cursor = db.cursor() cursor.execute("SELECT time,duration,status FROM bitten_build " - "WHERE rev=%s AND path=%s AND slave=%s", - (rev, path, slave)) + "WHERE config=%s AND rev=%s AND slave=%s", + (config, rev, slave)) row = cursor.fetchone() if not row: raise Exception, "Build not found" + self.config = config self.rev = rev - self.path = path self.slave = slave self.time = row[0] and int(row[0]) self.duration = row[1] and int(row[1]) - if row[2] is not None: - self.status = row[2] and Build.SUCCESS or Build.FAILURE - else: - self.status = Build.IN_PROGRESS + self.status = row[2] completed = property(fget=lambda self: self.status != Build.IN_PROGRESS) successful = property(fget=lambda self: self.status == Build.SUCCESS) + def delete(self, db=None): + if not db: + db = self.env.get_db_cnx() + handle_ta = True + else: + handle_ta = False + + assert self.status == self.PENDING, 'Only pending builds can be deleted' + + cursor = db.cursor() + cursor.execute("DELETE FROM bitten_build WHERE config=%s AND rev=%s " + "AND slave=%s", (self.config, self.rev, + self.slave or '')) + if handle_ta: + db.commit() + def insert(self, db=None): if not db: db = self.env.get_db_cnx() @@ -167,41 +183,70 @@ else: handle_ta = False + assert self.config and self.rev + assert self.status in (self.PENDING, self.IN_PROGRESS, self.SUCCESS, + self.FAILURE) + if not self.slave: + assert self.status == self.PENDING + cursor = db.cursor() cursor.execute("INSERT INTO bitten_build VALUES (%s,%s,%s,%s,%s,%s)", - (self.rev, self.path, self.slave, self.time, - self.duration, self.status or Build.IN_PROGRESS)) + (self.config, self.rev, self.slave or '', self.time or 0, + self.duration or 0, self.status)) + if handle_ta: + db.commit() - def select(cls, env, path=None, rev=None, slave=None, db=None): + def update(self, db=None): + if not db: + db = self.env.get_db_cnx() + handle_ta = True + else: + handle_ta = False + + assert self.config and self.rev + assert self.status in (self.PENDING, self.IN_PROGRESS, self.SUCCESS, + self.FAILURE) + if not self.slave: + assert self.status == self.PENDING + + cursor = db.cursor() + cursor.execute("UPDATE bitten_build SET slave=%s,time=%s,duration=%s," + "status=%s WHERE config=%s AND rev=%s", + (self.slave or '', self.time or 0, self.duration or 0, + self.status, self.config, self.rev)) + if handle_ta: + db.commit() + + def select(cls, env, config=None, rev=None, slave=None, status=None, + db=None): if not db: db = env.get_db_cnx() where_clauses = [] + if config is not None: + where_clauses.append(("config=%s", config)) if rev is not None: where_clauses.append(("rev=%s", rev)) - if path is not None: - where_clauses.append(("path=%s", path)) if slave is not None: - where_clauses.append(("slave=%s", path)) + where_clauses.append(("slave=%s", slave)) + if status is not None: + where_clauses.append(("status=%s", status)) if where_clauses: where = "WHERE " + " AND ".join([wc[0] for wc in where_clauses]) else: where = "" cursor = db.cursor() - cursor.execute("SELECT rev,path,slave,time,duration,status " - "FROM bitten_build " + where, + cursor.execute("SELECT config,rev,slave,time,duration,status " + "FROM bitten_build %s ORDER BY config,rev,slave" % where, [wc[1] for wc in where_clauses]) - for rev, path, slave, time, duration, status in cursor: + for config, rev, slave, time, duration, status in cursor: build = Build(env) + build.config = config build.rev = rev - build.path = path build.slave = slave - build.time = time and int(time) - build.duration = duration and int(duration) - if status is not None: - build.status = status and Build.SUCCESS or Build.FAILURE - else: - build.status = Build.FAILURE + build.time = time and int(time) or 0 + build.duration = duration and int(duration) or 0 + build.status = status yield build select = classmethod(select)
--- a/bitten/slave.py +++ b/bitten/slave.py @@ -32,8 +32,9 @@ def greeting_received(self, profiles): if OrchestrationProfileHandler.URI not in profiles: - logging.error('Peer does not support Bitten profile') - raise beep.TerminateSession, 'Peer does not support Bitten profile' + err = 'Peer does not support the Bitten orchestration profile' + logging.error(err) + raise beep.TerminateSession, err self.channels[0].profile.send_start([OrchestrationProfileHandler]) @@ -65,18 +66,16 @@ def handle_msg(self, msgno, msg): if msg.get_content_type() == 'application/tar': - logging.info('Received snapshot') workdir = tempfile.mkdtemp(prefix='bitten') archive_name = msg.get('Content-Disposition', 'snapshot.tar.gz') archive_path = os.path.join(workdir, archive_name) file(archive_path, 'wb').write(msg.get_payload()) - logging.info('Stored snapshot archive at %s', archive_path) + logging.info('Received snapshot archive: %s', archive_path) # TODO: Spawn the build process xml = xmlio.Element('ok') self.channel.send_rpy(msgno, beep.MIMEMessage(xml)) - logging.info('Sent <ok/> in reply to build request') else: xml = xmlio.Element('error', code=500)['Sorry, what?'] @@ -104,7 +103,7 @@ try: port = int(args[1]) assert (1 <= port <= 65535), 'port number out of range' - except AssertionError, ValueError: + except (AssertionError, ValueError): parser.error('port must be an integer in the range 1-65535') else: port = 7633
--- a/bitten/tests/__init__.py +++ b/bitten/tests/__init__.py @@ -20,11 +20,12 @@ import unittest -from bitten.tests import recipe +from bitten.tests import model, recipe from bitten.util import tests as util def suite(): suite = unittest.TestSuite() + suite.addTest(model.suite()) suite.addTest(recipe.suite()) suite.addTest(util.suite()) return suite
new file mode 100644 --- /dev/null +++ b/bitten/tests/model.py @@ -0,0 +1,125 @@ +# -*- coding: iso8859-1 -*- +# +# Copyright (C) 2005 Christopher Lenz <cmlenz@gmx.de> +# +# Bitten is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation; either version 2 of the +# License, or (at your option) any later version. +# +# Trac is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +# +# Author: Christopher Lenz <cmlenz@gmx.de> + +import unittest + +from trac.test import EnvironmentStub +from bitten.model import Build, BuildConfig + + +class BuildConfigTestCase(unittest.TestCase): + + def setUp(self): + self.env = EnvironmentStub() + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute(db.to_sql(BuildConfig._table)) + db.commit() + + def test_new_config(self): + config = BuildConfig(self.env) + assert not config.exists + + def test_insert_config(self): + config = BuildConfig(self.env) + config.name = 'test' + config.label = 'Test' + config.path = 'trunk' + config.insert() + + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("SELECT name,path,label,active,description " + "FROM bitten_config") + self.assertEqual(('test', 'trunk', 'Test', 0, ''), cursor.fetchone()) + + def test_insert_config_no_name(self): + config = BuildConfig(self.env) + self.assertRaises(AssertionError, config.insert) + + +class BuildTestCase(unittest.TestCase): + + def setUp(self): + self.env = EnvironmentStub() + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute(db.to_sql(Build._table)) + db.commit() + + def test_new_build(self): + build = Build(self.env) + self.assertEqual(Build.PENDING, build.status) + self.assertEqual(0, build.time) + self.assertEqual(0, build.duration) + + def test_insert_build(self): + build = Build(self.env) + build.config = 'test' + build.rev = '42' + build.insert() + + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("SELECT config,rev,slave,time,duration,status " + "FROM bitten_build") + self.assertEqual(('test', '42', '', 0, 0, 'P'), cursor.fetchone()) + + def test_insert_build_no_config_or_rev(self): + build = Build(self.env) + self.assertRaises(AssertionError, build.insert) + + build = Build(self.env) + build.config = 'test' + self.assertRaises(AssertionError, build.insert) + + build = Build(self.env) + build.rev = '42' + self.assertRaises(AssertionError, build.insert) + + def test_insert_build_no_slave(self): + build = Build(self.env) + build.config = 'test' + build.rev = '42' + build.status = Build.SUCCESS + self.assertRaises(AssertionError, build.insert) + build.status = Build.FAILURE + self.assertRaises(AssertionError, build.insert) + build.status = Build.IN_PROGRESS + self.assertRaises(AssertionError, build.insert) + build.status = Build.PENDING + build.insert() + + def test_insert_invalid_status(self): + build = Build(self.env) + build.config = 'test' + build.rev = '42' + build.status = 'DUNNO' + self.assertRaises(AssertionError, build.insert) + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(BuildConfigTestCase, 'test')) + suite.addTest(unittest.makeSuite(BuildTestCase, 'test')) + return suite + +if __name__ == '__main__': + unittest.main(defaultTest='suite')
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -19,13 +19,14 @@ # Author: Christopher Lenz <cmlenz@gmx.de> import re +import time from trac.core import * -from trac.util import escape +from trac.util import escape, pretty_timedelta from trac.web.chrome import INavigationContributor from trac.web.main import IRequestHandler from trac.wiki import wiki_to_html -from bitten.model import BuildConfig +from bitten.model import Build, BuildConfig class BuildModule(Component): @@ -88,9 +89,30 @@ <li>Active: <?cs if:build.config.active ?>yes<?cs else ?>no<?cs /if ?></li> <li>Path: <?cs if:build.config.path ?><a href="<?cs var:build.config.browser_href ?>"><?cs - var:build.config.path ?></a></li><?cs /if ?></ul> - <?cs if:build.config.description ?><div class="description"><?cs - var:build.config.description ?></div><?cs /if ?> + var:build.config.path ?></a></li><?cs /if ?></ul><?cs + if:build.config.description ?><div class="description"><?cs + var:build.config.description ?></div><?cs /if ?> + <div id="builds"><h3>Builds</h3><?cs + if:len(build.config.builds) ?><ul><?cs + each:b = build.config.builds ?><li><a href="<?cs + var:b.href ?>">[<?cs var:b.rev ?>]</a> built by <?cs + var:len(b.slaves) ?> slave(s)<?cs + if:len(b.slaves) ?>:<ul><?cs + each:slave = b.slaves ?><li><strong><?cs var:slave.name ?></strong>: <?cs + var:slave.status ?> (started <?cs + var:slave.started_delta ?> ago<?cs + if:slave.stopped ?>, stopped <?cs + var:slave.stopped_delta ?> ago, took <?cs + var:slave.duration ?><?cs + /if ?>)</li><?cs + /each ?> + </ul><?cs + /if ?> + </li><?cs + /each ?> + </ul><?cs + else ?><p>None</p><?cs + /if ?></div> <div class="buttons"> <form method="get" action=""><div> <input type="hidden" name="action" value="edit" /> @@ -116,7 +138,7 @@ # IRequestHandler methods def match_request(self, req): - match = re.match(r'/build(?:/(\w+))?(?:/(\w+))?', req.path_info) + match = re.match(r'/build(?:/([\w.-]+))?(?:/([\w+.-]))?', req.path_info) if match: if match.group(1): req.args['config'] = match.group(1) @@ -206,6 +228,36 @@ 'active': config.active, 'description': description, 'browser_href': self.env.href.browser(config.path) } + + builds = Build.select(self.env, config=config.name) + curr_rev = None + slave_idx = 0 + for idx, build in enumerate(builds): + if build.rev != curr_rev: + slave_idx = 0 + curr_rev = build.rev + req.hdf['build.config.builds.%d' % idx] = { + 'rev': build.rev, + 'href': self.env.href.changeset(build.rev) + } + if not build.slave: + continue + status_label = {Build.PENDING: 'pending', + Build.IN_PROGRESS: 'in progress', + Build.SUCCESS: 'success', Build.FAILURE: 'failed'} + prefix = 'build.config.builds.%d.slaves.%d' % (idx, slave_idx) + req.hdf[prefix] = {'name': build.slave, + 'status': status_label[build.status]} + if build.time: + started = build.time + req.hdf[prefix + '.started'] = time.strftime('%x %X', time.localtime(started)) + req.hdf[prefix + '.started_delta'] = pretty_timedelta(started) + if build.duration: + stopped = build.time + build.duration + req.hdf[prefix + '.duration'] = pretty_timedelta(stopped, build.time) + req.hdf[prefix + '.stopped'] = time.strftime('%x %X', time.localtime(stopped)) + req.hdf[prefix + '.stopped_delta'] = pretty_timedelta(stopped) + req.hdf['build.mode'] = 'view_config' def _render_config_form(self, req, config_name=None):