Mercurial > bitten > bitten-test
changeset 69:b92d7c7d70fd
Record build slave properties in database.
| author | cmlenz |
|---|---|
| date | Thu, 30 Jun 2005 18:27:12 +0000 |
| parents | 234600bf0d49 |
| children | ccd03b6f04ef |
| files | bitten/master.py bitten/model.py bitten/slave.py bitten/tests/model.py bitten/trac_ext/main.py bitten/trac_ext/web_ui.py |
| diffstat | 6 files changed, 237 insertions(+), 133 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/master.py +++ b/bitten/master.py @@ -23,7 +23,7 @@ import time from trac.env import Environment -from bitten.model import Build, BuildConfig +from bitten.model import Build, BuildConfig, SlaveInfo from bitten.util import archive, beep, xmlio @@ -60,19 +60,16 @@ for config in BuildConfig.select(self.env): node = repos.get_node(config.path) for path, rev, chg in node.get_history(): - # Check whether the latest revision of that configuration has - # already been built + # Check whether the latest revision of that configuration + # has already been built builds = Build.select(self.env, config.name, rev) if not list(builds): logging.info('Enqueuing build of configuration "%s" as ' 'of revision [%s]', config.name, rev) build = Build(self.env) build.config = config.name - - chgset = repos.get_changeset(rev) build.rev = rev - build.rev_time = chgset.date - + build.rev_time = repos.get_changeset(rev).date build.insert() break finally: @@ -118,22 +115,26 @@ def handle_connect(self): self.master = self.session.listener assert self.master + self.env = self.master.env + assert self.env self.name = None + self.props = {} def handle_disconnect(self): if self.name is None: - # Slave didn't successfully register before disconnecting + # Slave didn't successfully register before disconnecting, so + # there's nothing to clean up return del self.master.slaves[self.name] - for build in Build.select(self.master.env, slave=self.name, + for build in Build.select(self.env, slave=self.name, status=Build.IN_PROGRESS): logging.info('Build [%s] of "%s" by %s cancelled', build.rev, build.config, self.name) build.slave = None build.status = Build.PENDING - build.time = None + build.started = 0 build.update() break logging.info('Unregistered slave "%s"', self.name) @@ -143,23 +144,22 @@ elem = xmlio.parse(msg.get_payload()) if elem.name == 'register': - platform, os, os_family, os_version = None, None, None, None + self.name = elem.attr['name'] for child in elem.children(): if child.name == 'platform': - platform = child.gettext() - processor = child.attr.get('processor') + self.props[SlaveInfo.MACHINE] = child.gettext() + self.props[SlaveInfo.PROCESSOR] = child.attr.get('processor') elif child.name == 'os': - os = child.gettext() - os_family = child.attr.get('family') - os_version = child.attr.get('version') + self.props[SlaveInfo.OS_NAME] = child.gettext() + self.props[SlaveInfo.OS_FAMILY] = child.attr.get('family') + self.props[SlaveInfo.OS_VERSION] = child.attr.get('version') self.name = elem.attr['name'] self.master.slaves[self.name] = self xml = xmlio.Element('ok') self.channel.send_rpy(msgno, beep.MIMEMessage(xml)) - logging.info('Registered slave "%s" (%s running %s %s [%s])', - self.name, platform, os, os_version, os_family) + logging.info('Registered slave "%s"', self.name) def send_initiation(self, build): logging.debug('Initiating build of "%s" on slave %s', build.config, @@ -211,7 +211,7 @@ if elem.name == 'started': self.steps = [] build.slave = self.name - build.time = int(time.time()) + build.started = int(time.time()) build.status = Build.IN_PROGRESS build.update() logging.info('Slave %s started build of "%s" as of [%s]', @@ -226,7 +226,7 @@ elif elem.name == 'abort': logging.info('Slave "%s" aborted build', self.name) build.slave = None - build.time = 0 + build.started = 0 build.status = Build.PENDING elif elem.name == 'error': build.status = Build.FAILURE @@ -234,7 +234,7 @@ if build.status != Build.PENDING: # Completed logging.info('Slave %s completed build of "%s" as of [%s]', self.name, build.config, build.rev) - build.duration = int(time.time()) - build.time + build.stopped = int(time.time()) if build.status is Build.IN_PROGRESS: # Find out whether the build failed or succeeded if [st for st in self.steps if st[1] == 'failure']: @@ -243,9 +243,16 @@ build.status = Build.SUCCESS else: # Aborted build.slave = None - build.time = 0 + build.started = 0 build.update() + # Insert slave info + slave_info = SlaveInfo(self.env) + slave_info.build = build.id + for name, value in self.props.items(): + slave_info[name] = value + slave_info.insert() + # TODO: should not block while reading the file; rather stream it using # asyncore push_with_producer() snapshot_path = self.master.get_snapshot(build, type, encoding)
--- a/bitten/model.py +++ b/bitten/model.py @@ -18,7 +18,7 @@ # # Author: Christopher Lenz <cmlenz@gmx.de> -from trac.db_default import Table, Column +from trac.db_default import Table, Column, Index schema_version = 1 @@ -119,10 +119,11 @@ class Build(object): """Representation of a build.""" - _table = Table('bitten_build', key=('config', 'rev', 'slave'))[ - Column('config'), Column('rev'), Column('slave'), - Column('time', type='int'), Column('duration', type='int'), - Column('status', size='1'), Column('rev_time', type='int') + _table = Table('bitten_build', key='id')[ + Column('id', auto_increment=True), Column('config'), Column('rev'), + Column('rev_time', type='int'), Column('slave'), + Column('started', type='int'), Column('stopped', type='int'), + Column('status', size='1'), Index(['config', 'rev', 'slave']) ] PENDING = 'P' @@ -130,35 +131,35 @@ SUCCESS = 'S' FAILURE = 'F' - def __init__(self, env, config=None, rev=None, slave=None, db=None): + def __init__(self, env, id=None, db=None): self.env = env - self.config = self.rev = self.slave = None - self.time = self.duration = self.rev_time = None - if config and rev and slave: - self._fetch(config, rev, slave, db) + if id is not None: + self._fetch(id, db) else: - self.time = self.duration = self.rev_time = 0 + self.id = self.config = self.rev = self.slave = None + self.started = self.stopped = self.rev_time = 0 self.status = self.PENDING - def _fetch(self, config, rev, slave, db=None): + def _fetch(self, id, db=None): if not db: db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT time,duration,status,rev_time FROM bitten_build " - "WHERE config=%s AND rev=%s AND slave=%s", - (config, rev, slave)) + cursor.execute("SELECT config,rev,rev_time,slave,started,stopped," + "status FROM bitten_build WHERE id=%s", (id,)) row = cursor.fetchone() if not row: - raise Exception, "Build not found" - self.config = config - self.rev = rev - self.slave = slave - self.time = row[0] and int(row[0]) - self.duration = row[1] and int(row[1]) - self.status = row[2] - self.rev_time = int(row[3]) + raise Exception, "Build %s not found" % id + self.id = id + self.config = row[0] + self.rev = row[1] + self.rev_time = int(row[2]) + self.slave = row[3] + self.started = row[4] and int(row[4]) + self.stopped = row[5] and int(row[5]) + self.status = row[6] + exists = property(fget=lambda self: self.id is not None) completed = property(fget=lambda self: self.status != Build.IN_PROGRESS) successful = property(fget=lambda self: self.status == Build.SUCCESS) @@ -172,9 +173,7 @@ assert self.status == self.PENDING, 'Only pending builds can be deleted' cursor = db.cursor() - cursor.execute("DELETE FROM bitten_build WHERE config=%s AND rev=%s " - "AND slave=%s", (self.config, self.rev, - self.slave or '')) + cursor.execute("DELETE FROM bitten_build WHERE id=%s", (self.id,)) if handle_ta: db.commit() @@ -192,9 +191,12 @@ assert self.status == self.PENDING cursor = db.cursor() - cursor.execute("INSERT INTO bitten_build VALUES (%s,%s,%s,%s,%s,%s,%s)", - (self.config, self.rev, self.slave or '', self.time or 0, - self.duration or 0, self.status, self.rev_time)) + cursor.execute("INSERT INTO bitten_build (config,rev,rev_time,slave," + "started,stopped,status) " + "VALUES (%s,%s,%s,%s,%s,%s,%s)", + (self.config, self.rev, self.rev_time, self.slave or '', + self.started or 0, self.stopped or 0, self.status)) + self.id = db.get_last_id('bitten_build') if handle_ta: db.commit() @@ -212,10 +214,10 @@ assert self.status == self.PENDING cursor = db.cursor() - cursor.execute("UPDATE bitten_build SET slave=%s,time=%s,duration=%s," - "status=%s WHERE config=%s AND rev=%s", - (self.slave or '', self.time or 0, self.duration or 0, - self.status, self.config, self.rev)) + cursor.execute("UPDATE bitten_build SET slave=%s,started=%s," + "stopped=%s,status=%s WHERE id=%s", + (self.slave or '', self.started or 0, + self.stopped or 0, self.status, self.id)) if handle_ta: db.commit() @@ -239,17 +241,78 @@ where = "" cursor = db.cursor() - cursor.execute("SELECT config,rev,slave,time,duration,status,rev_time " - "FROM bitten_build %s ORDER BY config,rev_time DESC," - "slave" % where, [wc[1] for wc in where_clauses]) - for config, rev, slave, time, duration, status, rev_time in cursor: + cursor.execute("SELECT id,config,rev,slave,started,stopped,status," + "rev_time FROM bitten_build %s " + "ORDER BY config,rev_time DESC,slave" + % where, [wc[1] for wc in where_clauses]) + for id, config, rev, slave, started, stopped, status, rev_time \ + in cursor: build = Build(env) + build.id = id build.config = config build.rev = rev build.slave = slave - build.time = time and int(time) or 0 - build.duration = duration and int(duration) or 0 + build.started = started and int(started) or 0 + build.stopped = stopped and int(stopped) or 0 build.status = status build.rev_time = int(rev_time) yield build select = classmethod(select) + + +class SlaveInfo(object): + _table = Table('bitten_slave', key=('build', 'propname'))[ + Column('build', type='int'), Column('propname'), Column('propvalue') + ] + + # Standard properties + IP_ADDRESS = 'ipnr' + MAINTAINER = 'owner' + OS_NAME = 'os' + OS_FAMILY = 'family' + OS_VERSION = 'version' + MACHINE = 'machine' + PROCESSOR = 'processor' + + def __init__(self, env, build=None, db=None): + self.env = env + self.properties = {} + if build: + self._fetch(build, db) + else: + self.build = None + + def _fetch(self, build, db=None): + if not db: + db = self.env.get_db_cnx() + + cursor = db.cursor() + cursor.execute("SELECT propname,propvalue FROM bitten_slave " + "WHERE build=%s", (build,)) + for propname, propvalue in cursor: + self.properties[propname] = propvalue + if not self.properties: + raise Exception, "Slave info for '%s' not found" % name + self.build = build + + def __getitem__(self, name): + return self.properties[name] + + def __setitem__(self, name, value): + self.properties[name] = value + + def insert(self, db=None): + if not db: + db = self.env.get_db_cnx() + handle_ta = True + else: + handle_ta = False + + assert self.build + + cursor = db.cursor() + cursor.executemany("INSERT INTO bitten_slave VALUES (%s,%s,%s)", + [(self.build, name, value) for name, value + in self.properties.items()]) + if handle_ta: + db.commit()
--- a/bitten/slave.py +++ b/bitten/slave.py @@ -131,7 +131,6 @@ def execute_build(self, msgno, basedir, recipe_path): logging.info('Building in directory %s using recipe %s', basedir, recipe_path) - try: recipe = Recipe(recipe_path, basedir)
--- a/bitten/tests/model.py +++ b/bitten/tests/model.py @@ -21,7 +21,7 @@ import unittest from trac.test import EnvironmentStub -from bitten.model import Build, BuildConfig +from bitten.model import Build, BuildConfig, SlaveInfo class BuildConfigTestCase(unittest.TestCase): @@ -66,9 +66,10 @@ def test_new_build(self): build = Build(self.env) + self.assertEqual(None, build.id) self.assertEqual(Build.PENDING, build.status) - self.assertEqual(0, build.time) - self.assertEqual(0, build.duration) + self.assertEqual(0, build.stopped) + self.assertEqual(0, build.started) def test_insert_build(self): build = Build(self.env) @@ -79,8 +80,8 @@ db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT config,rev,slave,time,duration,status " - "FROM bitten_build") + cursor.execute("SELECT config,rev,slave,started,stopped,status " + "FROM bitten_build WHERE id=%s" % build.id) self.assertEqual(('test', '42', '', 0, 0, 'P'), cursor.fetchone()) def test_insert_build_no_config_or_rev_or_rev_time(self): @@ -125,10 +126,48 @@ self.assertRaises(AssertionError, build.insert) +class SlaveInfoTestCase(unittest.TestCase): + + def setUp(self): + self.env = EnvironmentStub() + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute(db.to_sql(SlaveInfo._table)) + db.commit() + + def test_insert_slave_info(self): + slave_info = SlaveInfo(self.env) + slave_info.build = 42 + slave_info[SlaveInfo.IP_ADDRESS] = '127.0.0.1' + slave_info[SlaveInfo.MAINTAINER] = 'joe@example.org' + slave_info.insert() + + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.execute("SELECT propname,propvalue FROM bitten_slave") + expected = {SlaveInfo.IP_ADDRESS: '127.0.0.1', + SlaveInfo.MAINTAINER: 'joe@example.org'} + for propname, propvalue in cursor: + self.assertEqual(expected[propname], propvalue) + + def test_fetch_slave_info(self): + db = self.env.get_db_cnx() + cursor = db.cursor() + cursor.executemany("INSERT INTO bitten_slave VALUES (42,%s,%s)", + [(SlaveInfo.IP_ADDRESS, '127.0.0.1'), + (SlaveInfo.MAINTAINER, 'joe@example.org')]) + + slave_info = SlaveInfo(self.env, 42) + self.assertEquals(42, slave_info.build) + self.assertEquals('127.0.0.1', slave_info[SlaveInfo.IP_ADDRESS]) + self.assertEquals('joe@example.org', slave_info[SlaveInfo.MAINTAINER]) + + def suite(): suite = unittest.TestSuite() suite.addTest(unittest.makeSuite(BuildConfigTestCase, 'test')) suite.addTest(unittest.makeSuite(BuildTestCase, 'test')) + suite.addTest(unittest.makeSuite(SlaveInfoTestCase, 'test')) return suite if __name__ == '__main__':
--- a/bitten/trac_ext/main.py +++ b/bitten/trac_ext/main.py @@ -23,7 +23,7 @@ from trac.core import * from trac.env import IEnvironmentSetupParticipant from trac.perm import IPermissionRequestor -from bitten.model import Build, BuildConfig, schema_version +from bitten.model import Build, BuildConfig, SlaveInfo, schema_version from bitten.trac_ext import web_ui class BuildSystem(Component): @@ -36,7 +36,7 @@ # Create the required tables db = self.env.get_db_cnx() cursor = db.cursor() - for table in [Build._table, BuildConfig._table]: + for table in [Build._table, BuildConfig._table, SlaveInfo._table]: cursor.execute(db.to_sql(table)) tarballs_dir = os.path.join(self.env.path, 'snapshots')
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -27,7 +27,7 @@ from trac.web.chrome import INavigationContributor from trac.web.main import IRequestHandler from trac.wiki import wiki_to_html -from bitten.model import Build, BuildConfig +from bitten.model import Build, BuildConfig, SlaveInfo class BuildModule(Component): @@ -93,34 +93,23 @@ var:build.config.browser_href ?>"><?cs var:build.config.path ?></a></li><?cs /if ?></ul><?cs if:build.config.description ?><div class="description"><?cs - var:build.config.description ?></div><?cs /if ?> - <div id="builds"><h3>Builds</h3><?cs - if:len(build.config.builds) ?><ul><?cs - each:b = build.config.builds ?><li><a href="<?cs - var:b.href ?>">[<?cs var:b.rev ?>]</a> built by <?cs - var:len(b.slaves) ?> slave(s)<?cs - if:len(b.slaves) ?>:<ul><?cs - each:slave = b.slaves ?><li><strong><?cs var:slave.name ?></strong>: <?cs - var:slave.status ?> (started <?cs - var:slave.started_delta ?> ago<?cs - if:slave.stopped ?>, stopped <?cs - var:slave.stopped_delta ?> ago, took <?cs - var:slave.duration ?><?cs - /if ?>)</li><?cs - /each ?> - </ul><?cs - /if ?> - </li><?cs - /each ?> - </ul><?cs - else ?><p>(None)</p><?cs - /if ?></div><?cs + var:build.config.description ?></div><?cs /if ?><?cs if:build.can_modify ?><div class="buttons"> <form method="get" action=""><div> <input type="hidden" name="action" value="edit" /> <input type="submit" value="Edit configuration" /> </div></form><?cs /if ?></div><?cs + + elif:build.mode == 'view_build' ?> + <p class="trigger">Triggered by: Changeset <a href="<?cs + var:build.chgset_href ?>">[<?cs var:build.rev ?>]</a> of <a href="<?cs + var:build.config.href ?>"><?cs var:build.config.name ?></a></p> + <p class="slave">Built by: <strong><?cs + var:build.slave.name ?></strong> (<?cs var:build.slave.os ?> <?cs + var:build.slave.os.version ?> on <?cs var:build.slave.machine ?>)</p> + <p class="time">Completed: <?cs var:build.started ?> (<?cs + var:build.started_delta ?> ago)<br />Took: <?cs var:build.duration ?></p><?cs /if ?> </div> @@ -128,7 +117,7 @@ """ _status_label = {Build.IN_PROGRESS: 'in progress', - Build.SUCCESS: 'success', + Build.SUCCESS: 'completed', Build.FAILURE: 'failed'} # INavigationContributor methods @@ -146,7 +135,7 @@ # IRequestHandler methods def match_request(self, req): - match = re.match(r'/build(?:/([\w.-]+))?(?:/([\w+.-]))?', req.path_info) + match = re.match(r'/build(?:/([\w.-]+))?(?:/([\w.-]+))?', req.path_info) if match: if match.group(1): req.args['config'] = match.group(1) @@ -192,20 +181,20 @@ if 'build' in filters: db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT name,label,rev,slave,time,status " + cursor.execute("SELECT id,config,label,rev,slave,stopped,status " "FROM bitten_build " " INNER JOIN bitten_config ON (name=config) " - "WHERE time>=%s AND time<=%s " - "AND status IN (%s, %s) ORDER BY time", + "WHERE stopped>=%s AND stopped<=%s " + "AND status IN (%s, %s) ORDER BY stopped", (start, stop, Build.SUCCESS, Build.FAILURE)) event_kinds = {Build.SUCCESS: 'successbuild', Build.FAILURE: 'failedbuild'} - for name, label, rev, slave, time, status in cursor: - title = '<em>%s</em> [%s] built by %s' \ - % (escape(label), escape(rev), escape(slave)) - href = self.env.href.build(name) - message = self._status_label[status] - yield event_kinds[status], href, title, time, None, message + for id, config, label, rev, slave, stopped, status in cursor: + title = 'Build <em title="[%s] of %s">%s</em> by %s %s' \ + % (escape(rev), escape(label), escape(id), + escape(slave), self._status_label[status]) + href = self.env.href.build(config, id) + yield event_kinds[status], href, title, stopped, None, '' # Internal methods @@ -271,35 +260,6 @@ 'browser_href': self.env.href.browser(config.path) } - builds = Build.select(self.env, config=config.name) - curr_rev = None - slave_idx = 0 - for idx, build in enumerate(builds): - if build.rev != curr_rev: - slave_idx = 0 - curr_rev = build.rev - req.hdf['build.config.builds.%d' % idx] = { - 'rev': build.rev, - 'href': self.env.href.changeset(build.rev) - } - if not build.slave: - continue - prefix = 'build.config.builds.%d.slaves.%d' % (idx, slave_idx) - req.hdf[prefix] = {'name': build.slave, - 'status': self._status_label[build.status]} - if build.time: - started = build.time - req.hdf[prefix + '.started'] = strftime('%x %X', - localtime(started)) - req.hdf[prefix + '.started_delta'] = pretty_timedelta(started) - if build.duration: - stopped = build.time + build.duration - req.hdf[prefix + '.duration'] = pretty_timedelta(stopped, - build.time) - req.hdf[prefix + '.stopped'] = strftime('%x %X', - localtime(stopped)) - req.hdf[prefix + '.stopped_delta'] = pretty_timedelta(stopped) - req.hdf['build.mode'] = 'view_config' req.hdf['build.can_modify'] = req.perm.has_permission('BUILD_MODIFY') @@ -320,4 +280,40 @@ req.hdf['build.mode'] = 'edit_config' def _render_build(self, req, config_name, build_id): - raise NotImplementedError, 'Not implemented yet' + build = Build(self.env, build_id) + assert build.exists + status2title = {Build.SUCCESS: 'Success', Build.FAILURE: 'Failure'} + req.hdf['title'] = 'Build %s - %s' % (build_id, + status2title[build.status]) + req.hdf['build'] = self._build_to_hdf(build) + req.hdf['build.mode'] = 'view_build' + + config = BuildConfig(self.env, build.config) + req.hdf['build.config'] = { + 'name': config.label, + 'href': self.env.href.build(config.name) + } + + slave_info = SlaveInfo(self.env, build.id) + req.hdf['build.slave'] = { + 'name': build.slave, + 'ip_address': slave_info.properties.get(SlaveInfo.IP_ADDRESS), + 'os': slave_info.properties.get(SlaveInfo.OS_NAME), + 'os.family': slave_info.properties.get(SlaveInfo.OS_FAMILY), + 'os.version': slave_info.properties.get(SlaveInfo.OS_VERSION), + 'machine': slave_info.properties.get(SlaveInfo.MACHINE), + 'processor': slave_info.properties.get(SlaveInfo.PROCESSOR) + } + + def _build_to_hdf(self, build): + hdf = {'name': build.slave, 'status': self._status_label[build.status], + 'rev': build.rev, + 'chgset_href': self.env.href.changeset(build.rev)} + if build.started: + hdf['started'] = strftime('%x %X', localtime(build.started)) + hdf['started_delta'] = pretty_timedelta(build.started) + if build.stopped: + hdf['stopped'] = strftime('%x %X', localtime(build.stopped)) + hdf['stopped_delta'] = pretty_timedelta(build.stopped) + hdf['duration'] = pretty_timedelta(build.stopped, build.started) + return hdf