cmlenz@379: # -*- coding: utf-8 -*-
cmlenz@13: #
osimons@832: # Copyright (C) 2007-2010 Edgewall Software
cmlenz@408: # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
cmlenz@163: # All rights reserved.
cmlenz@13: #
cmlenz@163: # This software is licensed as described in the file COPYING, which
cmlenz@163: # you should have received as part of this distribution. The terms
cmlenz@408: # are also available at http://bitten.edgewall.org/wiki/License.
cmlenz@13: 
cmlenz@313: """Implementation of the build slave."""
cmlenz@313: 
cmlenz@82: from datetime import datetime
jonas@457: import errno
osimons@648: import urllib
cmlenz@402: import urllib2
cmlenz@15: import logging
cmlenz@13: import os
cmlenz@66: import platform
cmlenz@90: import shutil
jonas@457: import socket
osimons@654: import sys
cmlenz@42: import tempfile
cmlenz@392: import time
osimons@648: import re
osimons@648: import cookielib
wbell@785: import threading
wbell@785: import os
osimons@835: import mimetools
osimons@675: from ConfigParser import MissingSectionHeaderError
cmlenz@13: 
osimons@654: from bitten import PROTOCOL_VERSION
cmlenz@61: from bitten.build import BuildError
osimons@675: from bitten.build.config import Configuration, ConfigFileNotFound
cmlenz@436: from bitten.recipe import Recipe
cmlenz@392: from bitten.util import xmlio
hodgestar@895: from bitten.util.compat import HTTPBasicAuthHandler
cmlenz@13: 
dfraser@495: EX_OK = getattr(os, "EX_OK", 0)
dfraser@495: EX_UNAVAILABLE = getattr(os, "EX_UNAVAILABLE", 69)
osimons@675: EX_IOERR = getattr(os, "EX_IOERR", 74)
dfraser@495: EX_PROTOCOL = getattr(os, "EX_PROTOCOL", 76)
osimons@648: EX_NOPERM = getattr(os, "EX_NOPERM", 77)
osimons@648: 
osimons@648: FORM_TOKEN_RE = re.compile('__FORM_TOKEN\" value=\"(.+)\"')
dfraser@495: 
cmlenz@411: __all__ = ['BuildSlave', 'ExitSlave']
cmlenz@411: __docformat__ = 'restructuredtext en'
cmlenz@411: 
cmlenz@93: log = logging.getLogger('bitten.slave')
cmlenz@93: 
jonas@457: # List of network errors which are usually temporary and non critical.
jonas@457: temp_net_errors = [errno.ENETUNREACH, errno.ENETDOWN, errno.ETIMEDOUT,
jonas@457:                    errno.ECONNREFUSED]
jonas@457: 
wbell@474: def _rmtree(root):
dfraser@549:     """Catch shutil.rmtree failures on Windows when files are read-only, and only remove if root exists."""
wbell@474:     def _handle_error(fn, path, excinfo):
wbell@474:        os.chmod(path, 0666)
wbell@474:        fn(path)
dfraser@549:     if os.path.exists(root):
dfraser@549:         return shutil.rmtree(root, onerror=_handle_error) 
dfraser@549:     else:
dfraser@549:         return False
cmlenz@13: 
cmlenz@402: 
cmlenz@420: class SaneHTTPRequest(urllib2.Request):
cmlenz@420: 
cmlenz@420:     def __init__(self, method, url, data=None, headers={}):
cmlenz@420:         urllib2.Request.__init__(self, url, data, headers)
cmlenz@420:         self.method = method
cmlenz@420: 
cmlenz@420:     def get_method(self):
cmlenz@420:         if self.method is None:
cmlenz@420:             self.method = self.has_data() and 'POST' or 'GET'
cmlenz@420:         return self.method
cmlenz@420: 
osimons@835: 
osimons@835: def encode_multipart_formdata(fields):
osimons@835:     """
osimons@835:     Given a dictionary field parameters, returns the HTTP request body and the
osimons@835:     content_type (which includes the boundary string), to be used with an
osimons@835:     httplib-like call.
osimons@835: 
osimons@835:     Normal key/value items are treated as regular parameters, but key/tuple
osimons@835:     items are treated as files, where a value tuple is a (filename, data) tuple.
osimons@835: 
hodgestar@870:     For example::
hodgestar@870: 
osimons@835:         fields = {
osimons@835:             'foo': 'bar',
osimons@835:             'foofile': ('foofile.txt', 'contents of foofile'),
osimons@835:         }
osimons@835:         body, content_type = encode_multipart_formdata(fields)
osimons@835:     
osimons@835:     Note: Adapted from http://code.google.com/p/urllib3/ (MIT license)
osimons@835:     """
osimons@835: 
osimons@835:     BOUNDARY = mimetools.choose_boundary()
osimons@835:     ENCODE_TEMPLATE= "--%(boundary)s\r\n" \
osimons@835:         "Content-Disposition: form-data; name=\"%(name)s\"\r\n" \
osimons@835:         "\r\n%(value)s\r\n"
osimons@835:     ENCODE_TEMPLATE_FILE = "--%(boundary)s\r\n" \
osimons@835:         "Content-Disposition: form-data; name=\"%(name)s\"; " \
osimons@835:                 "filename=\"%(filename)s\"\r\n" \
osimons@835:         "Content-Type: %(contenttype)s\r\n" \
osimons@835:         "\r\n%(value)s\r\n"
osimons@835: 
osimons@835:     body = ""
osimons@835:     for key, value in fields.iteritems():
osimons@835:         if isinstance(value, tuple):
osimons@835:             filename, value = value
osimons@835:             body += ENCODE_TEMPLATE_FILE % {
osimons@835:                         'boundary': BOUNDARY,
osimons@835:                         'name': str(key),
osimons@835:                         'value': str(value),
osimons@835:                         'filename': str(filename),
osimons@835:                         'contenttype': 'application/octet-stream'
osimons@835:                     }
osimons@835:         else:
osimons@835:             body += ENCODE_TEMPLATE % {
osimons@835:                         'boundary': BOUNDARY,
osimons@835:                         'name': str(key),
osimons@835:                         'value': str(value)
osimons@835:                     }
osimons@835:     body += '--%s--\r\n' % BOUNDARY
osimons@835:     content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
osimons@835:     return body, content_type
osimons@835: 
osimons@835: 
wbell@785: class KeepAliveThread(threading.Thread):
wbell@785:     "A thread to periodically send keep-alive messages to the master"
wbell@785:     
wbell@785:     def __init__(self, opener, build_url, single_build, keepalive_interval):
wbell@785:         threading.Thread.__init__(self, None, None, "KeepaliveThread")
wbell@785:         self.build_url = build_url
wbell@785:         self.keepalive_interval = keepalive_interval
wbell@785:         self.single_build = single_build
wbell@785:         self.last_keepalive = int(time.time())
wbell@785:         self.kill = False
wbell@785:         self.opener = opener
wbell@785: 
wbell@785:     def keepalive(self):
wbell@785:         log.debug('Sending keepalive')
wbell@785:         method = 'POST'
wbell@785:         url = self.build_url + '/keepalive/'
wbell@785:         body = None
wbell@785:         shutdown = False
wbell@785:         headers = {
osimons@891:             'Content-Type': 'application/x-bitten+xml',
osimons@891:             'Content-Length': '0'
wbell@785:             }
wbell@785:         log.debug('Sending %s request to %r', method, url)
wbell@785:         req = SaneHTTPRequest(method, url, body, headers or {})
wbell@785:         try:
wbell@785:             return self.opener.open(req)
wbell@785:         except urllib2.HTTPError, e:
wbell@785:             # a conflict error lets us know that we've been
wbell@785:             # invalidated. Ideally, we'd engineer something to stop any
wbell@785:             # running steps in progress, but killing threads is tricky
wbell@785:             # stuff. For now, we'll wait for whatever's going
wbell@785:             # on to stop, and the main thread'll figure out that we've
wbell@785:             # been invalidated.
wbell@785:             log.warning('Server returned keepalive error %d: %s', e.code, e.msg)
wbell@785:         except:
wbell@785:             log.warning('Server returned unknown keepalive error')
wbell@785: 
wbell@785:     def run(self):
wbell@785:         log.debug('Keepalive thread starting.')
wbell@785:         while (not self.kill):
wbell@785:             now = int(time.time())
wbell@785:             if (self.last_keepalive + self.keepalive_interval) < now:
wbell@785:                 self.keepalive()
wbell@785:                 self.last_keepalive = now
wbell@785:             
wbell@785:             time.sleep(1)
wbell@785:         log.debug('Keepalive thread exiting.')
wbell@785: 
wbell@785:     def stop(self):
wbell@785:         log.debug('Stopping keepalive thread')
wbell@785:         self.kill = True
wbell@785:         self.join(30)
wbell@785:         log.debug('Keepalive thread stopped')
wbell@785:         
cmlenz@420: 
cmlenz@392: class BuildSlave(object):
wbell@542:     """HTTP client implementation for the build slave."""
cmlenz@13: 
cmlenz@494:     def __init__(self, urls, name=None, config=None, dry_run=False,
cmlenz@466:                  work_dir=None, build_dir="build_${build}",
cmlenz@466:                  keep_files=False, single_build=False,
wbell@785:                  poll_interval=300, keepalive_interval = 60,
wbell@785:                  username=None, password=None,
osimons@648:                  dump_reports=False, no_loop=False, form_auth=False):
cmlenz@313:         """Create the build slave instance.
cmlenz@313:         
cmlenz@494:         :param urls: a list of URLs of the build masters to connect to, or a
cmlenz@494:                      single-element list containing the path to a build recipe
cmlenz@494:                      file
cmlenz@414:         :param name: the name with which this slave should identify itself
cmlenz@414:         :param config: the path to the slave configuration file
cmlenz@414:         :param dry_run: wether the build outcome should not be reported back
cmlenz@411:                         to the master
cmlenz@414:         :param work_dir: the working directory to use for build execution
cmlenz@466:         :param build_dir: the pattern to use for naming the build subdir
cmlenz@414:         :param keep_files: whether files and directories created for build
cmlenz@411:                            execution should be kept when done
cmlenz@414:         :param single_build: whether this slave should exit after completing a 
cmlenz@411:                              single build, or continue processing builds forever
cmlenz@449:         :param poll_interval: the time in seconds to wait between requesting
cmlenz@451:                               builds from the build master (default is five
cmlenz@451:                               minutes)
hodgestar@870:         :param keepalive_interval: the time in seconds to wait between sending
wbell@785:                                     keepalive heartbeats (default is 30 seconds)
cmlenz@411:         :param username: the username to use when authentication against the
cmlenz@411:                          build master is requested
cmlenz@411:         :param password: the password to use when authentication is needed
cmlenz@442:         :param dump_reports: whether report data should be written to the
cmlenz@442:                              standard output, in addition to being transmitted
cmlenz@442:                              to the build master
dfraser@525:         :param no_loop: for this slave to just perform a single check, regardless
dfraser@525:                         of whether a build is done or not
osimons@648:         :param form_auth: login using AccountManager HTML form instead of
osimons@648:                                 HTTP authentication for all urls
cmlenz@313:         """
cmlenz@494:         self.urls = urls
cmlenz@494:         self.local = len(urls) == 1 and not urls[0].startswith('http://') \
cmlenz@494:                                     and not urls[0].startswith('https://')
cmlenz@392:         if name is None:
cmlenz@392:             name = platform.node().split('.', 1)[0].lower()
cmlenz@63:         self.name = name
cmlenz@392:         self.config = Configuration(config)
cmlenz@185:         self.dry_run = dry_run
cmlenz@208:         if not work_dir:
cmlenz@208:             work_dir = tempfile.mkdtemp(prefix='bitten')
cmlenz@208:         elif not os.path.exists(work_dir):
cmlenz@208:             os.makedirs(work_dir)
cmlenz@208:         self.work_dir = work_dir
cmlenz@466:         self.build_dir = build_dir
cmlenz@241:         self.keep_files = keep_files
cmlenz@377:         self.single_build = single_build
dfraser@525:         self.no_loop = no_loop
cmlenz@449:         self.poll_interval = poll_interval
wbell@785:         self.keepalive_interval = keepalive_interval
cmlenz@442:         self.dump_reports = dump_reports
osimons@648:         self.cookiejar = cookielib.CookieJar()
osimons@648:         self.username = username \
osimons@648:                         or self.config['authentication.username'] or ''
cmlenz@63: 
cmlenz@414:         if not self.local:
osimons@569:             self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
osimons@648:             if self.username:
osimons@648:                 log.debug('Enabling authentication with username %r',
osimons@648:                                 self.username)
osimons@648:                 self.form_auth = form_auth
osimons@648:                 password = password \
osimons@648:                            or self.config['authentication.password'] or ''
osimons@648:                 self.config.packages.pop('authentication', None)
osimons@648:                 urls = [url[:-7] for url in urls]
osimons@648:                 self.password_mgr.add_password(
osimons@648:                                 None, urls, self.username, password)
osimons@648:                 self.auth_map = dict(map(lambda x: (x, False), urls))
osimons@569: 
osimons@569:     def _get_opener(self):
osimons@652:         opener = urllib2.build_opener(urllib2.HTTPErrorProcessor())
hodgestar@895:         opener.add_handler(HTTPBasicAuthHandler(self.password_mgr))
osimons@569:         opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr))
osimons@648:         opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar))
osimons@569:         return opener
osimons@569:     opener = property(_get_opener)
cmlenz@13: 
cmlenz@402:     def request(self, method, url, body=None, headers=None):
cmlenz@420:         log.debug('Sending %s request to %r', method, url)
cmlenz@420:         req = SaneHTTPRequest(method, url, body, headers or {})
cmlenz@402:         try:
dfraser@559:             resp = self.opener.open(req)
dfraser@559:             if not hasattr(resp, 'code'):
dfraser@559:                 resp.code = 200
dfraser@559:             return resp
cmlenz@402:         except urllib2.HTTPError, e:
cmlenz@402:             if e.code >= 300:
osimons@647:                 if hasattr(e, 'headers') and \
osimons@647:                             e.headers.getheader('Content-Type', ''
osimons@647:                                         ).startswith('text/plain'):
osimons@645:                     content = e.read()
osimons@645:                 else:
wbell@785:                     content = 'no message available'
wbell@785:                 log.debug('Server returned error %d: %s (%s)',
wbell@785:                                         e.code, e.msg, content)
cmlenz@402:                 raise
cmlenz@402:             return e
cmlenz@29: 
cmlenz@392:     def run(self):
cmlenz@414:         if self.local:
cmlenz@494:             fileobj = open(self.urls[0])
cmlenz@414:             try:
cmlenz@414:                 self._execute_build(None, fileobj)
cmlenz@414:             finally:
cmlenz@414:                 fileobj.close()
dfraser@495:             return EX_OK
cmlenz@414: 
cmlenz@494:         urls = []
cmlenz@392:         while True:
cmlenz@494:             if not urls:
cmlenz@494:                 urls[:] = self.urls
cmlenz@494:             url = urls.pop(0)
cmlenz@392:             try:
cmlenz@392:                 try:
osimons@648:                     if self.username and not self.auth_map.get(url):
osimons@715:                         login_url = '%s/login?referer=%s' % (url[:-7],
osimons@715:                                                         urllib.quote_plus(url))
osimons@648:                         # First request to url, authentication needed
osimons@648:                         if self.form_auth:
osimons@648:                             log.debug('Performing http form authentication')
osimons@715:                             resp = self.request('POST', login_url)
osimons@648:                             match = FORM_TOKEN_RE.search(resp.read())
osimons@648:                             if not match:
osimons@648:                                 log.error("Project %s does not support form "
osimons@648:                                           "authentication" % url[:-7])
osimons@648:                                 raise ExitSlave(EX_NOPERM)
osimons@648:                             values = {'user': self.username, 
osimons@648:                                       'password':
osimons@648:                                           self.password_mgr.find_user_password(
osimons@648:                                                                 None, url)[1],
osimons@648:                                       'referer': '',
osimons@648:                                       '__FORM_TOKEN': match.group(1)} 
osimons@715:                             self.request('POST', login_url,
osimons@648:                                          body=urllib.urlencode(values))
osimons@648:                         else:
osimons@648:                             log.debug('Performing basic/digest authentication')
osimons@715:                             self.request('HEAD', login_url)
osimons@648:                         self.auth_map[url] = True
osimons@648:                     elif self.username:
osimons@648:                         log.debug('Reusing authentication information.')
osimons@648:                     else:
osimons@648:                         log.debug('Authentication not provided. Attempting to '
osimons@648:                                   'execute build anonymously.')
cmlenz@494:                     job_done = self._create_build(url)
wbell@480:                     if job_done:
wbell@480:                         continue
mgood@465:                 except urllib2.HTTPError, e:
mgood@465:                     # HTTPError doesn't have the "reason" attribute of URLError
mgood@465:                     log.error(e)
dfraser@495:                     raise ExitSlave(EX_UNAVAILABLE)
cmlenz@402:                 except urllib2.URLError, e:
jonas@457:                     # Is this a temporary network glitch or something a bit
jonas@457:                     # more severe?
jonas@457:                     if isinstance(e.reason, socket.error) and \
jonas@457:                         e.reason.args[0] in temp_net_errors:
jonas@457:                         log.warning(e)
jonas@457:                     else:
jonas@457:                         log.error(e)
dfraser@495:                         raise ExitSlave(EX_UNAVAILABLE)
mgood@490:             except ExitSlave, e:
mgood@490:                 return e.exit_code
dfraser@525:             if self.no_loop:
dfraser@525:                 break
cmlenz@449:             time.sleep(self.poll_interval)
cmlenz@127: 
cmlenz@392:     def quit(self):
cmlenz@392:         log.info('Shutting down')
dfraser@495:         raise ExitSlave(EX_OK)
cmlenz@392: 
cmlenz@494:     def _create_build(self, url):
osimons@649:         xml = xmlio.Element('slave', name=self.name, version=PROTOCOL_VERSION)[
cmlenz@233:             xmlio.Element('platform', processor=self.config['processor'])[
cmlenz@233:                 self.config['machine']
cmlenz@233:             ],
cmlenz@233:             xmlio.Element('os', family=self.config['family'],
cmlenz@244:                                 version=self.config['version'])[
cmlenz@233:                 self.config['os']
cmlenz@233:             ],
cmlenz@29:         ]
cmlenz@392: 
cmlenz@392:         log.debug('Configured packages: %s', self.config.packages)
cmlenz@233:         for package, properties in self.config.packages.items():
cmlenz@233:             xml.append(xmlio.Element('package', name=package, **properties))
cmlenz@233: 
cmlenz@392:         body = str(xml)
cmlenz@392:         log.debug('Sending slave configuration: %s', body)
cmlenz@494:         resp = self.request('POST', url, body, {
dfraser@559:             'Content-Length': str(len(body)),
cmlenz@392:             'Content-Type': 'application/x-bitten+xml'
cmlenz@392:         })
cmlenz@277: 
cmlenz@402:         if resp.code == 201:
cmlenz@402:             self._initiate_build(resp.info().get('location'))
wbell@480:             return True
cmlenz@402:         elif resp.code == 204:
cmlenz@420:             log.info('No pending builds')
wbell@480:             return False
cmlenz@392:         else:
cmlenz@402:             log.error('Unexpected response (%d %s)', resp.code, resp.msg)
dfraser@495:             raise ExitSlave(EX_PROTOCOL)
cmlenz@56: 
cmlenz@392:     def _initiate_build(self, build_url):
cmlenz@418:         log.info('Build pending at %s', build_url)
cmlenz@420:         try:
cmlenz@420:             resp = self.request('GET', build_url)
cmlenz@420:             if resp.code == 200:
cmlenz@420:                 self._execute_build(build_url, resp)
cmlenz@420:             else:
cmlenz@420:                 log.error('Unexpected response (%d): %s', resp.code, resp.msg)
dfraser@495:                 self._cancel_build(build_url, exit_code=EX_PROTOCOL)
cmlenz@420:         except KeyboardInterrupt:
cmlenz@420:             log.warning('Build interrupted')
cmlenz@420:             self._cancel_build(build_url)
cmlenz@392: 
cmlenz@414:     def _execute_build(self, build_url, fileobj):
cmlenz@414:         build_id = build_url and int(build_url.split('/')[-1]) or 0
cmlenz@414:         xml = xmlio.parse(fileobj)
osimons@580:         basedir = ''
cmlenz@414:         try:
wbell@785:             if not self.local:
osimons@796:                 keepalive_thread = KeepAliveThread(self.opener, build_url,
osimons@796:                                     self.single_build, self.keepalive_interval)
wbell@785:                 keepalive_thread.start()
cmlenz@466:             recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), 
cmlenz@466:                             self.config)
cmlenz@466:             basedir = recipe.ctxt.basedir
mgood@490:             log.debug('Running build in directory %s' % basedir)
cmlenz@466:             if not os.path.exists(basedir):
cmlenz@466:                 os.mkdir(basedir)
cmlenz@466: 
cmlenz@414:             for step in recipe:
wbell@754:                 try:
wbell@754:                     log.info('Executing build step %r, onerror = %s', step.id, step.onerror)
wbell@754:                     if not self._execute_step(build_url, recipe, step):
wbell@754:                         log.warning('Stopping build due to failure')
wbell@754:                         break
wbell@754:                 except Exception, e:
wbell@754:                     log.error('Exception raised processing step %s. Reraising %s', step.id, e)
wbell@754:                     raise
cmlenz@414:             else:
cmlenz@417:                 log.info('Build completed')
cmlenz@461:             if self.dry_run:
cmlenz@461:                 self._cancel_build(build_url)
cmlenz@414:         finally:
osimons@796:             if not self.local:
osimons@796:                 keepalive_thread.stop()
osimons@580:             if not self.keep_files and os.path.isdir(basedir):
cmlenz@414:                 log.debug('Removing build directory %s' % basedir)
wbell@474:                 _rmtree(basedir)
cmlenz@414:             if self.single_build:
cmlenz@414:                 log.info('Exiting after single build completed.')
dfraser@495:                 raise ExitSlave(EX_OK)
cmlenz@392: 
cmlenz@392:     def _execute_step(self, build_url, recipe, step):
cmlenz@392:         failed = False
wbell@758:         started = int(time.time())
wbell@758:         xml = xmlio.Element('result', step=step.id)
cmlenz@392:         try:
cmlenz@392:             for type, category, generator, output in \
cmlenz@392:                     step.execute(recipe.ctxt):
cmlenz@392:                 if type == Recipe.ERROR:
cmlenz@82:                     failed = True
cmlenz@442:                 if type == Recipe.REPORT and self.dump_reports:
cmlenz@442:                     print output
osimons@835:                 if type == Recipe.ATTACH:
osimons@835:                     # Attachments are added out-of-band due to major
osimons@835:                     # performance issues with inlined base64 xml content
osimons@835:                     self._attach_file(build_url, recipe, output)
cmlenz@392:                 xml.append(xmlio.Element(type, category=category,
cmlenz@392:                                          generator=generator)[
cmlenz@392:                     output
cmlenz@392:                 ])
cmlenz@414:         except KeyboardInterrupt:
cmlenz@417:             log.warning('Build interrupted')
cmlenz@420:             self._cancel_build(build_url)
cmlenz@392:         except BuildError, e:
osimons@628:             log.error('Build step %r failed', step.id)
cmlenz@392:             failed = True
cmlenz@392:         except Exception, e:
cmlenz@392:             log.error('Internal error in build step %r', step.id, exc_info=True)
cmlenz@392:             failed = True
wbell@758:         xml.attr['duration'] = (time.time() - started)
cmlenz@392:         if failed:
cmlenz@392:             xml.attr['status'] = 'failure'
cmlenz@392:         else:
cmlenz@392:             xml.attr['status'] = 'success'
cmlenz@392:             log.info('Build step %s completed successfully', step.id)
cmlenz@63: 
cmlenz@424:         if not self.local and not self.dry_run:
cmlenz@420:             try:
cmlenz@420:                 resp = self.request('POST', build_url + '/steps/', str(xml), {
cmlenz@420:                     'Content-Type': 'application/x-bitten+xml'
cmlenz@420:                 })
cmlenz@420:                 if resp.code != 201:
cmlenz@420:                     log.error('Unexpected response (%d): %s', resp.code,
cmlenz@420:                               resp.msg)
cmlenz@420:             except KeyboardInterrupt:
cmlenz@420:                 log.warning('Build interrupted')
cmlenz@420:                 self._cancel_build(build_url)
cmlenz@392:         return not failed or step.onerror != 'fail'
cmlenz@63: 
dfraser@495:     def _cancel_build(self, build_url, exit_code=EX_OK):
cmlenz@420:         log.info('Cancelling build at %s', build_url)
cmlenz@420:         if not self.local:
cmlenz@420:             resp = self.request('DELETE', build_url)
cmlenz@420:             if resp.code not in (200, 204):
cmlenz@420:                 log.error('Unexpected response (%d): %s', resp.code, resp.msg)
mgood@490:         raise ExitSlave(exit_code)
cmlenz@420: 
osimons@835:     def _attach_file(self, build_url, recipe, attachment):
osimons@835:         form_token = recipe._root.attr.get('form_token', '')
osimons@835:         if self.local or self.dry_run or not form_token:
osimons@835:             log.info('Attachment %s not sent due to current slave options',
osimons@835:                         attachment.attr['file'])
osimons@835:             return
osimons@835:         resource_type = attachment.attr['resource']
osimons@835:         url = str(build_url + '/attach/' + resource_type)
osimons@835:         path = recipe.ctxt.resolve(attachment.attr['filename'])
osimons@835:         filename = os.path.basename(path)
osimons@835:         log.debug('Attaching file %s to %s...', attachment.attr['filename'],
osimons@835:                                                     resource_type)
osimons@889:         f = open(path, 'rb')
osimons@835:         try:
osimons@835:             data, content_type = encode_multipart_formdata({
osimons@835:                                 'file': (filename, f.read()),
osimons@835:                                 'description': attachment.attr['description'],
osimons@835:                                 '__FORM_TOKEN': form_token})
osimons@835:         finally:
osimons@835:             f.close()
osimons@835:         resp = self.request('POST', url , data, {
osimons@835:                                 'Content-Type': content_type})
osimons@835:         if not resp.code == 201:
osimons@835:             msg = 'Error attaching %s to %s'
osimons@835:             log.error(msg, filename, resource_type)
osimons@835:             raise BuildError(msg, filename, resource_type)
cmlenz@63: 
cmlenz@392: class ExitSlave(Exception):
cmlenz@392:     """Exception used internally by the slave to signal that the slave process
cmlenz@411:     should be stopped.
cmlenz@411:     """
mgood@490:     def __init__(self, exit_code):
mgood@490:         self.exit_code = exit_code
mgood@490:         Exception.__init__(self)
cmlenz@13: 
cmlenz@13: 
cmlenz@31: def main():
cmlenz@313:     """Main entry point for running the build slave."""
cmlenz@56:     from bitten import __version__ as VERSION
cmlenz@19:     from optparse import OptionParser
cmlenz@19: 
cmlenz@494:     parser = OptionParser(usage='usage: %prog [options] url1 [url2] ...',
cmlenz@19:                           version='%%prog %s' % VERSION)
cmlenz@185:     parser.add_option('--name', action='store', dest='name',
cmlenz@63:                       help='name of this slave (defaults to host name)')
cmlenz@208:     parser.add_option('-f', '--config', action='store', dest='config',
cmlenz@127:                       metavar='FILE', help='path to configuration file')
cmlenz@402:     parser.add_option('-u', '--user', dest='username',
cmlenz@402:                       help='the username to use for authentication')
cmlenz@402:     parser.add_option('-p', '--password', dest='password',
cmlenz@402:                       help='the password to use when authenticating')
osimons@587:     def _ask_password(option, opt_str, value, parser):
osimons@587:         from getpass import getpass
osimons@587:         parser.values.password = getpass('Passsword: ')
osimons@587:     parser.add_option('-P', '--ask-password', action='callback',
osimons@587:                       callback=_ask_password, help='Prompt for password')
osimons@648:     parser.add_option('--form-auth', action='store_true', 
osimons@648:                       dest='form_auth',
osimons@648:                       help='login using AccountManager HTML form instead of '
osimons@648:                            'HTTP authentication for all urls')
cmlenz@442: 
cmlenz@442:     group = parser.add_option_group('building')
cmlenz@442:     group.add_option('-d', '--work-dir', action='store', dest='work_dir',
cmlenz@442:                      metavar='DIR', help='working directory for builds')
cmlenz@466:     group.add_option('--build-dir', action='store', dest='build_dir',
cmlenz@466:                      default = 'build_${config}_${build}',
cmlenz@466:                      help='name pattern for the build dir to use inside the '
cmlenz@466:                           'working dir ["%default"]')
cmlenz@442:     group.add_option('-k', '--keep-files', action='store_true',
cmlenz@442:                      dest='keep_files', 
cmlenz@442:                      help='don\'t delete files after builds')
cmlenz@442:     group.add_option('-s', '--single', action='store_true',
cmlenz@442:                      dest='single_build',
cmlenz@442:                      help='exit after completing a single build')
dfraser@525:     group.add_option('', '--no-loop', action='store_true',
dfraser@525:                      dest='no_loop',
dfraser@525:                      help='exit after completing a single check and running '
dfraser@525:                           'the required builds')
cmlenz@442:     group.add_option('-n', '--dry-run', action='store_true', dest='dry_run',
cmlenz@442:                      help='don\'t report results back to master')
cmlenz@449:     group.add_option('-i', '--interval', dest='interval', metavar='SECONDS',
cmlenz@449:                      type='int', help='time to wait between requesting builds')
wbell@785:     group.add_option('-b', '--keepalive_interval', dest='keepalive_interval', metavar='SECONDS', type='int', help='time to wait between keepalive heartbeats')
cmlenz@442:     group = parser.add_option_group('logging')
cmlenz@442:     group.add_option('-l', '--log', dest='logfile', metavar='FILENAME',
cmlenz@442:                      help='write log messages to FILENAME')
cmlenz@442:     group.add_option('-v', '--verbose', action='store_const', dest='loglevel',
cmlenz@442:                      const=logging.DEBUG, help='print as much as possible')
cmlenz@442:     group.add_option('-q', '--quiet', action='store_const', dest='loglevel',
cmlenz@442:                      const=logging.WARN, help='print as little as possible')
cmlenz@442:     group.add_option('--dump-reports', action='store_true', dest='dump_reports',
cmlenz@442:                      help='whether report data should be printed')
cmlenz@442: 
cmlenz@242:     parser.set_defaults(dry_run=False, keep_files=False,
dfraser@525:                         loglevel=logging.INFO, single_build=False, no_loop=False,
wbell@785:                         dump_reports=False, interval=300, keepalive_interval=60,
wbell@785:                         form_auth=False)
cmlenz@19:     options, args = parser.parse_args()
cmlenz@19: 
cmlenz@13:     if len(args) < 1:
cmlenz@19:         parser.error('incorrect number of arguments')
cmlenz@494:     urls = args
cmlenz@13: 
cmlenz@157:     logger = logging.getLogger('bitten')
cmlenz@157:     logger.setLevel(options.loglevel)
cmlenz@93:     handler = logging.StreamHandler()
cmlenz@93:     handler.setLevel(options.loglevel)
cmlenz@93:     formatter = logging.Formatter('[%(levelname)-8s] %(message)s')
cmlenz@93:     handler.setFormatter(formatter)
cmlenz@157:     logger.addHandler(handler)
cmlenz@339:     if options.logfile:
cmlenz@339:         handler = logging.FileHandler(options.logfile)
cmlenz@339:         handler.setLevel(options.loglevel)
cmlenz@339:         formatter = logging.Formatter('%(asctime)s [%(name)s] %(levelname)s: '
cmlenz@339:                                       '%(message)s')
cmlenz@339:         handler.setFormatter(formatter)
cmlenz@339:         logger.addHandler(handler)
cmlenz@13: 
osimons@610:     log.info("Slave launched at %s" % \
osimons@610:                 datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
osimons@610: 
osimons@675:     slave = None
osimons@675:     try:
osimons@675:         slave = BuildSlave(urls, name=options.name, config=options.config,
cmlenz@392:                        dry_run=options.dry_run, work_dir=options.work_dir,
cmlenz@466:                        build_dir=options.build_dir,
cmlenz@392:                        keep_files=options.keep_files,
cmlenz@402:                        single_build=options.single_build,
dfraser@543:                        no_loop=options.no_loop,
cmlenz@449:                        poll_interval=options.interval,
wbell@785:                        keepalive_interval=options.keepalive_interval,
cmlenz@442:                        username=options.username, password=options.password,
osimons@648:                        dump_reports=options.dump_reports,
osimons@648:                        form_auth=options.form_auth)
cmlenz@392:         try:
mgood@490:             exit_code = slave.run()
cmlenz@392:         except KeyboardInterrupt:
cmlenz@392:             slave.quit()
osimons@675:     except ConfigFileNotFound, e:
osimons@675:         log.error(e)
osimons@675:         exit_code = EX_IOERR
osimons@675:     except MissingSectionHeaderError:
osimons@675:         log.error("Error parsing configuration file %r. Wrong format?" \
osimons@675:                             % options.config)
osimons@675:         exit_code = EX_IOERR
mgood@490:     except ExitSlave, e:
mgood@490:         exit_code = e.exit_code
cmlenz@31: 
osimons@675:     if slave and not (options.work_dir or options.keep_files):
osimons@576:         log.debug('Removing working directory %s' % slave.work_dir)
wbell@474:         _rmtree(slave.work_dir)
osimons@610: 
osimons@610:     log.info("Slave exited at %s" % \
osimons@610:                 datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
osimons@610: 
mgood@490:     return exit_code
cmlenz@208: 
cmlenz@31: if __name__ == '__main__':
mgood@490:     sys.exit(main())