cmlenz@379: # -*- coding: utf-8 -*-
cmlenz@13: #
cmlenz@408: # Copyright (C) 2007 Edgewall Software
cmlenz@408: # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
cmlenz@163: # All rights reserved.
cmlenz@13: #
cmlenz@163: # This software is licensed as described in the file COPYING, which
cmlenz@163: # you should have received as part of this distribution. The terms
cmlenz@408: # are also available at http://bitten.edgewall.org/wiki/License.
cmlenz@13: 
cmlenz@392: """Build master implementation."""
cmlenz@313: 
cmlenz@312: import calendar
cmlenz@392: import re
cmlenz@47: import time
cmlenz@18: 
osimons@577: from trac.config import BoolOption, IntOption, Option
cmlenz@392: from trac.core import *
cmlenz@392: from trac.web import IRequestHandler, HTTPBadRequest, HTTPConflict, \
cmlenz@392:                      HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, \
cmlenz@392:                      RequestDone
cmlenz@392: 
dfraser@554: from bitten.model import BuildConfig, Build, BuildStep, BuildLog, Report, \
dfraser@554:                      TargetPlatform
dfraser@554: 
cmlenz@410: from bitten.main import BuildSystem
cmlenz@227: from bitten.queue import BuildQueue
cmlenz@392: from bitten.recipe import Recipe
cmlenz@392: from bitten.util import xmlio
cmlenz@83: 
cmlenz@411: __all__ = ['BuildMaster']
cmlenz@411: __docformat__ = 'restructuredtext en'
cmlenz@411: 
cmlenz@56: 
cmlenz@392: class BuildMaster(Component):
wbell@542:     """Trac request handler implementation for the build master."""
cmlenz@13: 
cmlenz@392:     implements(IRequestHandler)
cmlenz@51: 
cmlenz@392:     # Configuration options
cmlenz@51: 
cmlenz@392:     adjust_timestamps = BoolOption('bitten', 'adjust_timestamps', False, doc=
cmlenz@432:         """Whether the timestamps of builds should be adjusted to be close
cmlenz@392:         to the timestamps of the corresponding changesets.""")
cmlenz@392: 
cmlenz@392:     build_all = BoolOption('bitten', 'build_all', False, doc=
cmlenz@392:         """Whether to request builds of older revisions even if a younger
cmlenz@392:         revision has already been built.""")
cmlenz@468:     
cmlenz@468:     stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc=
cmlenz@468:         """The time in seconds to wait for the repository to stabilize before
cmlenz@468:         queuing up a new build.  This allows time for developers to check in
cmlenz@468:         a group of related changes back to back without spawning multiple
cmlenz@468:         builds.""")
cmlenz@392: 
cmlenz@392:     slave_timeout = IntOption('bitten', 'slave_timeout', 3600, doc=
cmlenz@392:         """The time in seconds after which a build is cancelled if the slave
cmlenz@392:         does not report progress.""")
cmlenz@392: 
osimons@577:     logs_dir = Option('bitten', 'logs_dir', "log/bitten", doc=
dfraser@516:          """The directory on the server in which client log files will be stored.""")
dfraser@516: 
dfraser@557:     quick_status = BoolOption('bitten', 'quick_status', False, doc=
dfraser@557:          """Whether to show the current build status withing the Trac main 
dfraser@557:             navigation bar""")
dfraser@557: 
osimons@568:     def __init__(self):
osimons@568:         self.env.systeminfo.append(('Bitten',
osimons@568:                 __import__('bitten', ['__version__']).__version__))
osimons@568: 
cmlenz@392:     # IRequestHandler methods
cmlenz@392: 
cmlenz@392:     def match_request(self, req):
cmlenz@401:         match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$',
cmlenz@392:                          req.path_info)
cmlenz@392:         if match:
cmlenz@392:             if match.group(1):
cmlenz@392:                 req.args['id'] = match.group(1)
cmlenz@392:                 req.args['collection'] = match.group(2)
cmlenz@392:                 req.args['member'] = match.group(3)
cmlenz@392:             return True
cmlenz@392: 
cmlenz@392:     def process_request(self, req):
cmlenz@392:         req.perm.assert_permission('BUILD_EXEC')
cmlenz@392: 
cmlenz@392:         if 'id' not in req.args:
cmlenz@392:             if req.method != 'POST':
cmlenz@392:                 raise HTTPMethodNotAllowed('Method not allowed')
cmlenz@392:             return self._process_build_creation(req)
cmlenz@392: 
cmlenz@392:         build = Build.fetch(self.env, req.args['id'])
cmlenz@392:         if not build:
cmlenz@392:             raise HTTPNotFound('No such build')
cmlenz@392:         config = BuildConfig.fetch(self.env, build.config)
cmlenz@392: 
cmlenz@392:         if not req.args['collection']:
cmlenz@420:             if req.method == 'DELETE':
cmlenz@420:                 return self._process_build_cancellation(req, config, build)
cmlenz@420:             else:
cmlenz@420:                 return self._process_build_initiation(req, config, build)
cmlenz@392: 
cmlenz@401:         if req.method != 'POST':
cmlenz@392:             raise HTTPMethodNotAllowed('Method not allowed')
cmlenz@392: 
cmlenz@392:         if req.args['collection'] == 'steps':
cmlenz@401:             return self._process_build_step(req, config, build)
cmlenz@392:         else:
cmlenz@392:             raise HTTPNotFound('No such collection')
cmlenz@392: 
cmlenz@392:     def _process_build_creation(self, req):
cmlenz@468:         queue = BuildQueue(self.env, build_all=self.build_all, 
cmlenz@468:                            stabilize_wait=self.stabilize_wait,
cmlenz@419:                            timeout=self.slave_timeout)
cmlenz@392:         queue.populate()
cmlenz@392: 
cmlenz@392:         try:
cmlenz@392:             elem = xmlio.parse(req.read())
cmlenz@392:         except xmlio.ParseError, e:
cmlenz@492:             self.log.error('Error parsing build initialization request: %s', e,
cmlenz@492:                            exc_info=True)
cmlenz@392:             raise HTTPBadRequest('XML parser error')
cmlenz@392: 
cmlenz@426:         slavename = elem.attr['name']
cmlenz@463:         properties = {'name': slavename, Build.IP_ADDRESS: req.remote_addr}
cmlenz@426:         self.log.info('Build slave %r connected from %s', slavename,
cmlenz@426:                       req.remote_addr)
cmlenz@392: 
cmlenz@392:         for child in elem.children():
cmlenz@392:             if child.name == 'platform':
cmlenz@392:                 properties[Build.MACHINE] = child.gettext()
cmlenz@392:                 properties[Build.PROCESSOR] = child.attr.get('processor')
cmlenz@392:             elif child.name == 'os':
cmlenz@392:                 properties[Build.OS_NAME] = child.gettext()
cmlenz@392:                 properties[Build.OS_FAMILY] = child.attr.get('family')
cmlenz@392:                 properties[Build.OS_VERSION] = child.attr.get('version')
cmlenz@392:             elif child.name == 'package':
cmlenz@392:                 for name, value in child.attr.items():
cmlenz@392:                     if name == 'name':
cmlenz@392:                         continue
cmlenz@392:                     properties[child.attr['name'] + '.' + name] = value
cmlenz@392: 
cmlenz@444:         self.log.debug('Build slave configuration: %r', properties)
cmlenz@444: 
cmlenz@426:         build = queue.get_build_for_slave(slavename, properties)
cmlenz@392:         if not build:
cmlenz@392:             req.send_response(204)
cmlenz@420:             req.write('')
cmlenz@392:             raise RequestDone
cmlenz@392: 
cmlenz@392:         req.send_response(201)
cmlenz@392:         req.send_header('Content-Type', 'text/plain')
cmlenz@392:         req.send_header('Location', req.abs_href.builds(build.id))
cmlenz@392:         req.write('Build pending')
cmlenz@392:         raise RequestDone
cmlenz@392: 
cmlenz@420:     def _process_build_cancellation(self, req, config, build):
cmlenz@420:         self.log.info('Build slave %r cancelled build %d', build.slave,
cmlenz@420:                       build.id)
cmlenz@420:         build.status = Build.PENDING
cmlenz@420:         build.slave = None
cmlenz@420:         build.slave_info = {}
cmlenz@420:         build.started = 0
cmlenz@420:         db = self.env.get_db_cnx()
cmlenz@420:         for step in list(BuildStep.select(self.env, build=build.id, db=db)):
cmlenz@420:             step.delete(db=db)
cmlenz@420:         build.update(db=db)
cmlenz@420:         db.commit()
cmlenz@420: 
cmlenz@458:         for listener in BuildSystem(self.env).listeners:
cmlenz@458:             listener.build_aborted(build)
cmlenz@458: 
cmlenz@420:         req.send_response(204)
cmlenz@420:         req.write('')
cmlenz@420:         raise RequestDone
cmlenz@420: 
cmlenz@392:     def _process_build_initiation(self, req, config, build):
cmlenz@420:         self.log.info('Build slave %r initiated build %d', build.slave,
cmlenz@420:                       build.id)
cmlenz@392:         build.started = int(time.time())
cmlenz@392:         build.update()
cmlenz@277: 
cmlenz@458:         for listener in BuildSystem(self.env).listeners:
cmlenz@458:             listener.build_started(build)
cmlenz@458: 
cmlenz@287:         xml = xmlio.parse(config.recipe)
cmlenz@392:         xml.attr['path'] = config.path
cmlenz@392:         xml.attr['revision'] = build.rev
cmlenz@466:         xml.attr['config'] = config.name
cmlenz@466:         xml.attr['build'] = str(build.id)
dfraser@554:         target_platform = TargetPlatform.fetch(self.env, build.platform)
dfraser@554:         xml.attr['platform'] = target_platform.name
cmlenz@392:         body = str(xml)
wbell@374: 
wbell@473:         self.log.info('Build slave %r initiated build %d', build.slave,
wbell@473:                       build.id)
wbell@473: 
cmlenz@392:         req.send_response(200)
cmlenz@392:         req.send_header('Content-Type', 'application/x-bitten+xml')
cmlenz@392:         req.send_header('Content-Length', str(len(body)))
cmlenz@392:         req.send_header('Content-Disposition',
cmlenz@392:                         'attachment; filename=recipe_%s_r%s.xml' %
cmlenz@392:                         (config.name, build.rev))
cmlenz@392:         req.write(body)
cmlenz@392:         raise RequestDone
cmlenz@227: 
cmlenz@401:     def _process_build_step(self, req, config, build):
cmlenz@401:         try:
cmlenz@401:             elem = xmlio.parse(req.read())
cmlenz@401:         except xmlio.ParseError, e:
cmlenz@492:             self.log.error('Error parsing build step result: %s', e,
cmlenz@492:                            exc_info=True)
cmlenz@401:             raise HTTPBadRequest('XML parser error')
cmlenz@401:         stepname = elem.attr['step']
wbell@473: 	
wbell@479:         # make sure it's the right slave.
cmlenz@494:         if build.status != Build.IN_PROGRESS or \
cmlenz@494:                 build.slave_info.get(Build.IP_ADDRESS) != req.remote_addr:
cmlenz@494:             raise HTTPForbidden('Build %s has been invalidated for host %s.'
cmlenz@494:                                 % (build.id, req.remote_addr))
cmlenz@401: 
cmlenz@392:         step = BuildStep.fetch(self.env, build=build.id, name=stepname)
cmlenz@392:         if step:
cmlenz@392:             raise HTTPConflict('Build step already exists')
cmlenz@253: 
cmlenz@392:         recipe = Recipe(xmlio.parse(config.recipe))
cmlenz@392:         index = None
cmlenz@392:         current_step = None
cmlenz@392:         for num, recipe_step in enumerate(recipe):
cmlenz@392:             if recipe_step.id == stepname:
cmlenz@392:                 index = num
cmlenz@392:                 current_step = recipe_step
cmlenz@392:         if index is None:
cmlenz@392:             raise HTTPForbidden('No such build step')
cmlenz@392:         last_step = index == num
cmlenz@200: 
cmlenz@494:         self.log.debug('Slave %s (build %d) completed step %d (%s) with '
cmlenz@494:                        'status %s', build.slave, build.id, index, stepname,
cmlenz@494:                        elem.attr['status'])
cmlenz@392: 
cmlenz@392:         db = self.env.get_db_cnx()
cmlenz@392: 
cmlenz@392:         step = BuildStep(self.env, build=build.id, name=stepname)
cmlenz@392:         try:
cmlenz@392:             step.started = int(_parse_iso_datetime(elem.attr['time']))
cmlenz@392:             step.stopped = step.started + float(elem.attr['duration'])
cmlenz@392:         except ValueError, e:
cmlenz@492:             self.log.error('Error parsing build step timestamp: %s', e,
cmlenz@492:                            exc_info=True)
cmlenz@392:             raise HTTPBadRequest(e.args[0])
cmlenz@392:         if elem.attr['status'] == 'failure':
cmlenz@392:             self.log.warning('Build %s step %s failed', build.id, stepname)
cmlenz@109:             step.status = BuildStep.FAILURE
cmlenz@459:             if current_step.onerror == 'fail':
cmlenz@459:                 last_step = True
cmlenz@109:         else:
cmlenz@109:             step.status = BuildStep.SUCCESS
cmlenz@277:         step.errors += [error.gettext() for error in elem.children('error')]
cmlenz@112:         step.insert(db=db)
cmlenz@109: 
cmlenz@392:         # Collect log messages from the request body
cmlenz@203:         for idx, log_elem in enumerate(elem.children('log')):
cmlenz@401:             build_log = BuildLog(self.env, build=build.id, step=stepname,
cmlenz@203:                                  generator=log_elem.attr.get('generator'),
cmlenz@203:                                  orderno=idx)
cmlenz@115:             for message_elem in log_elem.children('message'):
cmlenz@115:                 build_log.messages.append((message_elem.attr['level'],
cmlenz@115:                                            message_elem.gettext()))
cmlenz@112:             build_log.insert(db=db)
cmlenz@109: 
cmlenz@392:         # Collect report data from the request body
cmlenz@203:         for report_elem in elem.children('report'):
cmlenz@401:             report = Report(self.env, build=build.id, step=stepname,
cmlenz@213:                             category=report_elem.attr.get('category'),
cmlenz@213:                             generator=report_elem.attr.get('generator'))
cmlenz@203:             for item_elem in report_elem.children():
cmlenz@203:                 item = {'type': item_elem.name}
cmlenz@203:                 item.update(item_elem.attr)
cmlenz@203:                 for child_elem in item_elem.children():
cmlenz@203:                     item[child_elem.name] = child_elem.gettext()
cmlenz@203:                 report.items.append(item)
cmlenz@203:             report.insert(db=db)
cmlenz@116: 
cmlenz@392:         # If this was the last step in the recipe we mark the build as
cmlenz@392:         # completed
cmlenz@459:         if last_step:
cmlenz@392:             self.log.info('Slave %s completed build %d ("%s" as of [%s])',
cmlenz@392:                           build.slave, build.id, build.config, build.rev)
cmlenz@392:             build.stopped = step.stopped
cmlenz@392: 
cmlenz@392:             # Determine overall outcome of the build by checking the outcome
cmlenz@392:             # of the individual steps against the "onerror" specification of
cmlenz@392:             # each step in the recipe
cmlenz@392:             for num, recipe_step in enumerate(recipe):
cmlenz@392:                 step = BuildStep.fetch(self.env, build.id, recipe_step.id)
cmlenz@392:                 if step.status == BuildStep.FAILURE:
cmlenz@392:                     if recipe_step.onerror != 'ignore':
cmlenz@392:                         build.status = Build.FAILURE
cmlenz@392:                         break
cmlenz@392:             else:
cmlenz@392:                 build.status = Build.SUCCESS
cmlenz@392: 
cmlenz@392:             build.update(db=db)
cmlenz@392: 
cmlenz@200:         db.commit()
cmlenz@200: 
cmlenz@459:         if last_step:
cmlenz@392:             for listener in BuildSystem(self.env).listeners:
cmlenz@392:                 listener.build_completed(build)
cmlenz@253: 
cmlenz@392:         body = 'Build step processed'
cmlenz@402:         req.send_response(201)
cmlenz@392:         req.send_header('Content-Type', 'text/plain')
cmlenz@392:         req.send_header('Content-Length', str(len(body)))
cmlenz@402:         req.send_header('Location', req.abs_href.builds(build.id, 'steps',
cmlenz@402:                         stepname))
cmlenz@392:         req.write(body)
cmlenz@392:         raise RequestDone
cmlenz@109: 
cmlenz@13: 
cmlenz@82: def _parse_iso_datetime(string):
cmlenz@82:     """Minimal parser for ISO date-time strings.
cmlenz@82:     
cmlenz@82:     Return the time as floating point number. Only handles UTC timestamps
cmlenz@82:     without time zone information."""
cmlenz@82:     try:
cmlenz@82:         string = string.split('.', 1)[0] # strip out microseconds
cmlenz@312:         return calendar.timegm(time.strptime(string, '%Y-%m-%dT%H:%M:%S'))
cmlenz@82:     except ValueError, e:
cmlenz@392:         raise ValueError('Invalid ISO date/time %r' % string)