# HG changeset patch # User cmlenz # Date 1187130157 0 # Node ID d6e1a05f32f7b205d2e6763627d7cbdd96370177 # Parent 53d99d3151882748bae543d13228ad412a013a48 Start webadmin integration. diff --git a/bitten/admin.py b/bitten/admin.py new file mode 100644 --- /dev/null +++ b/bitten/admin.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2007 Edgewall Software +# All rights reserved. +# +# This software is licensed as described in the file COPYING, which +# you should have received as part of this distribution. The terms +# are also available at http://bitten.edgewall.org/wiki/License. + +"""Implementation of the web administration interface.""" + +import re + +from trac.core import * +from webadmin.web_ui import IAdminPageProvider + +from bitten.model import BuildConfig, TargetPlatform +from bitten.recipe import Recipe +from bitten.util import xmlio + + +class BuildMasterAdminPageProvider(Component): + """Web administration panel for configuring the build master.""" + + implements(IAdminPageProvider) + + # IAdminPageProvider methods + + def get_admin_pages(self, req): + if req.perm.has_permission('BUILD_ADMIN'): + yield ('bitten', 'Bitten', 'master', 'Build Master') + + def process_admin_request(self, req, cat, page, path_info): + from bitten.master import BuildMaster + master = BuildMaster(self.env) + + if req.method == 'POST': + changed = False + build_all = 'build_all' in req.args + if build_all != master.build_all: + self.config['bitten'].set('build_all', + build_all and 'yes' or 'no') + changed = True + adjust_timestamps = 'adjust_timestamps' in req.args + if adjust_timestamps != master.adjust_timestamps: + self.config['bitten'].set('adjust_timestamps', + adjust_timestamps and 'yes' or 'no') + changed = True + slave_timeout = int(req.args.get('slave_timeout', 0)) + if slave_timeout != master.slave_timeout: + self.config['bitten'].set('slave_timeout', str(slave_timeout)) + changed = True + if changed: + self.config.save() + + req.hdf['admin.master'] = { + 'build_all': master.build_all, + 'adjust_timestamps': master.adjust_timestamps, + 'slave_timeout': master.slave_timeout, + } + return 'bitten_admin_master.cs', None + + +class BuildConfigurationsAdminPageProvider(Component): + """Web administration panel for configuring the build master.""" + + implements(IAdminPageProvider) + + # IAdminPageProvider methods + + def get_admin_pages(self, req): + if req.perm.has_permission('BUILD_MODIFY'): + yield ('bitten', 'Bitten', 'configs', 'Configurations') + + def process_admin_request(self, req, cat, page, config_name): + data = {} + + if config_name: + config = BuildConfig.fetch(self.env, config_name) + platforms = list(TargetPlatform.select(self.env, config=config.name)) + + if req.method == 'POST': + if 'save' in req.args: + name = req.args.get('name') + if not name: + raise TracError('Missing required field "name"', + 'Missing field') + if not re.match(r'^[\w.-]+$', name): + raise TracError('The field "name" may only contain ' + 'letters, digits, periods, or dashes.', + 'Invalid field') + + path = req.args.get('path', '') + repos = self.env.get_repository(req.authname) + max_rev = req.args.get('max_rev') or None + try: + node = repos.get_node(path, max_rev) + assert node.isdir, '%s is not a directory' % node.path + except (AssertionError, TracError), e: + raise TracError(e, 'Invalid repository path') + if req.args.get('min_rev'): + try: + repos.get_node(path, req.args.get('min_rev')) + except TracError, e: + raise TracError(e, + 'Invalid value for oldest revision') + + recipe_xml = req.args.get('recipe', '') + if recipe_xml: + try: + Recipe(xmlio.parse(recipe_xml)).validate() + except xmlio.ParseError, e: + raise TracError('Failure parsing recipe: %s' % e, + 'Invalid recipe') + except InvalidRecipeError, e: + raise TracError(e, 'Invalid recipe') + + config.name = name + config.path = repos.normalize_path(path) + config.recipe = recipe_xml + config.min_rev = req.args.get('min_rev') + config.max_rev = req.args.get('max_rev') + config.label = req.args.get('label', '') + config.description = req.args.get('description', '') + config.update() + req.redirect(self.env.href.admin(cat, page)) + + elif 'cancel' in req.args: + req.redirect(self.env.href.admin(cat, page)) + + data['config'] = { + 'name': config.name, 'label': config.label or config.name, + 'active': config.active, 'path': config.path, + 'min_rev': config.min_rev, 'max_rev': config.max_rev, + 'description': config.description, + 'recipe': config.recipe, + 'platforms': [{ + 'name': platform.name, + 'id': platform.id, + 'href': req.href.admin('bitten', 'configs', config.name, + platform.id) + } for platform in platforms] + } + + else: + if req.method == 'POST': + # Add configuration + if 'add' in req.args: + config = BuildConfig(self.env) + config.name = req.args.get('name') + config.label = req.args.get('label', config.name) + config.path = req.args.get('path') + config.insert() + req.redirect(self.env.href.admin(cat, page, config.name)) + + # Remove configurations + elif 'remove' in req.args: + sel = req.args.get('sel') + sel = isinstance(sel, list) and sel or [sel] + if not sel: + raise TracError('No configuration selected') + db = self.env.get_db_cnx() + for name in sel: + config = BuildConfig.fetch(self.env, name, db=db) + config.delete(db=db) + db.commit() + req.redirect(self.env.href.admin(cat, page)) + + # Set active state + elif 'apply' in req.args: + active = req.args.get('active') + active = isinstance(active, list) and active or [active] + db = self.env.get_db_cnx() + for config in BuildConfig.select(self.env, db=db, + include_inactive=True): + config.active = config.name in active + config.update(db=db) + db.commit() + req.redirect(self.env.href.admin(cat, page)) + + configs = [] + for config in BuildConfig.select(self.env, include_inactive=True): + configs.append({ + 'name': config.name, 'label': config.label or config.name, + 'active': config.active, 'path': config.path, + 'min_rev': config.min_rev, 'max_rev': config.max_rev, + 'href': req.href.admin('bitten', 'configs', config.name), + }) + data['configs'] = configs + + req.hdf['admin'] = data + return 'bitten_admin_configs.cs', None diff --git a/bitten/build/pythontools.py b/bitten/build/pythontools.py --- a/bitten/build/pythontools.py +++ b/bitten/build/pythontools.py @@ -163,6 +163,136 @@ except IOError, e: log.warning('Error opening pylint results file (%s)', e) +def coverage(ctxt, summary=None, coverdir=None, include=None, exclude=None): + """Extract data from a ``coverage.py`` run. + + :param ctxt: the build context + :type ctxt: `Context` + :param summary: path to the file containing the coverage summary + :param coverdir: name of the directory containing the per-module coverage + details + :param include: patterns of files or directories to include in the report + :param exclude: patterns of files or directories to exclude from the report + """ + assert summary, 'Missing required attribute "summary"' + assert coverdir, 'Missing required attribute "coverdir"' + + summary_line_re = re.compile(r'^\s*(?P\d+)\s+(?P\d+)%\s+' + r'(?P.*?)\s+\((?P.*?)\)') + coverage_line_re = re.compile(r'\s*(?:(?P\d+): )?(?P.*)') + + fileset = FileSet(ctxt.basedir, include, exclude) + missing_files = [] + for filename in fileset: + if os.path.splitext(filename)[1] != '.py': + continue + missing_files.append(filename) + covered_modules = set() + + def handle_file(elem, sourcefile, coverfile=None): + code_lines = set() + for lineno, linetype, line in loc.count(sourcefile): + if linetype == loc.CODE: + code_lines.add(lineno) + num_covered = 0 + lines = [] + + if coverfile: + prev_hits = '0' + for idx, coverline in enumerate(coverfile): + match = coverage_line_re.search(coverline) + if match: + hits = match.group(1) + if hits: # Line covered + if hits != '0': + num_covered += 1 + lines.append(hits) + prev_hits = hits + elif coverline.startswith('>'): # Line not covered + lines.append('0') + prev_hits = '0' + elif idx not in code_lines: # Not a code line + lines.append('-') + prev_hits = '0' + else: # A code line not flagged by trace.py + if prev_hits != '0': + num_covered += 1 + lines.append(prev_hits) + + elem.append(xmlio.Element('line_hits')[' '.join(lines)]) + + num_lines = len(code_lines) + if num_lines: + percentage = int(round(num_covered * 100 / num_lines)) + else: + percentage = 0 + elem.attr['percentage'] = percentage + elem.attr['lines'] = num_lines + + try: + summary_file = open(ctxt.resolve(summary), 'r') + try: + coverage = xmlio.Fragment() + for summary_line in summary_file: + match = summary_line_re.search(summary_line) + if match: + modname = match.group(3) + filename = match.group(4) + if not os.path.isabs(filename): + filename = os.path.normpath(os.path.join(ctxt.basedir, + filename)) + else: + filename = os.path.realpath(filename) + if not filename.startswith(ctxt.basedir): + continue + filename = filename[len(ctxt.basedir) + 1:] + if not filename in fileset: + continue + + missing_files.remove(filename) + covered_modules.add(modname) + module = xmlio.Element('coverage', name=modname, + file=filename.replace(os.sep, '/')) + sourcefile = file(ctxt.resolve(filename)) + try: + coverpath = ctxt.resolve(coverdir, modname + '.cover') + if os.path.isfile(coverpath): + coverfile = file(coverpath, 'r') + else: + log.warning('No coverage file for module %s at %s', + modname, coverpath) + coverfile = None + try: + handle_file(module, sourcefile, coverfile) + finally: + if coverfile: + coverfile.close() + finally: + sourcefile.close() + coverage.append(module) + + for filename in missing_files: + modname = os.path.splitext(filename.replace(os.sep, '.'))[0] + if modname in covered_modules: + continue + covered_modules.add(modname) + module = xmlio.Element('coverage', name=modname, + file=filename.replace(os.sep, '/'), + percentage=0) + filepath = ctxt.resolve(filename) + fileobj = file(filepath, 'r') + try: + handle_file(module, fileobj) + finally: + fileobj.close() + coverage.append(module) + + ctxt.report('coverage', coverage) + finally: + summary_file.close() + except IOError, e: + log.warning('Error opening coverage summary file (%s)', e) + def trace(ctxt, summary=None, coverdir=None, include=None, exclude=None): """Extract data from a ``trace.py`` run. diff --git a/bitten/templates/bitten_admin_configs.cs b/bitten/templates/bitten_admin_configs.cs new file mode 100644 --- /dev/null +++ b/bitten/templates/bitten_admin_configs.cs @@ -0,0 +1,100 @@ +

Manage Build Configurations

+
+ + + + + +
+ +

+ +
+
+ Build Recipe + +
+
+ Repository Mapping + + + + + + + + +
+
+
+ Target Platforms
    +
  • +
+
+
+ + +
+
+
+
+ Add Configuration: +
+ +
+
+ +
+
+ +
+
+
+ +
+ + + + + + + + + + + +
 NamePathActive
checked="checked" >
+
+ + +
+
diff --git a/bitten/templates/bitten_admin_master.cs b/bitten/templates/bitten_admin_master.cs new file mode 100644 --- /dev/null +++ b/bitten/templates/bitten_admin_master.cs @@ -0,0 +1,47 @@ +

Manage Build Master

+ +
+ +
+ Configuration Options +
+ +
+

+ Whether to build older revisions even when a more recent revision has + already been built. +

+
+ +
+

+ Whether the timestamps of builds should be adjusted to be close to the + timestamps of the corresponding changesets. +

+
+
+ +
+

+ The timeout in milliseconds after which a build started by a slave is + considered aborted, in case there has been no activity from that slave + in that time. +

+
+ +
+ +
+
diff --git a/bitten/util/testrunner.py b/bitten/util/testrunner.py --- a/bitten/util/testrunner.py +++ b/bitten/util/testrunner.py @@ -8,15 +8,15 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.edgewall.org/wiki/License. +from distutils import log +from distutils.errors import DistutilsOptionError import os import re -try: - from cStringIO import StringIO -except: - from StringIO import StringIO +from StringIO import StringIO import sys import time -from pkg_resources import * +from pkg_resources import Distribution, EntryPoint, PathMetadata, \ + normalize_path, require, working_set from setuptools.command.test import test from unittest import _TextTestResult, TextTestRunner @@ -120,7 +120,10 @@ ('coverage-dir=', None, "Directory where coverage files are to be stored"), ('coverage-summary=', None, - "Path to the file where the coverage summary should be stored") + "Path to the file where the coverage summary should be stored"), + ('coverage-method=', None, + "Whether to use trace.py or coverage.py to collect code coverage. " + "Valid options are 'trace' (the default) or 'coverage'.") ] def initialize_options(self): @@ -129,31 +132,50 @@ self.xml_output_file = None self.coverage_summary = None self.coverage_dir = None + self.coverage_method = 'trace' def finalize_options(self): test.finalize_options(self) + if self.xml_output is not None: if not os.path.exists(os.path.dirname(self.xml_output)): os.makedirs(os.path.dirname(self.xml_output)) self.xml_output_file = open(self.xml_output, 'w') + if self.coverage_method not in ('trace', 'coverage'): + raise DistutilsOptionError('Unknown coverage method %r' % + self.coverage_method) + def run_tests(self): if self.coverage_dir: - from trace import Trace - trace = Trace(ignoredirs=[sys.prefix, sys.exec_prefix], - trace=False, count=True) - try: - trace.runfunc(self._run_tests) - finally: - results = trace.results() - real_stdout = sys.stdout - sys.stdout = open(self.coverage_summary, 'w') + + if self.coverage_method == 'coverage': + import coverage + coverage.erase() + coverage.start() + log.info('running tests under coverage.py') try: - results.write_results(show_missing=True, summary=True, - coverdir=self.coverage_dir) + self._run_tests() finally: - sys.stdout.close() - sys.stdout = real_stdout + coverage.stop() + + else: + from trace import Trace + trace = Trace(ignoredirs=[sys.prefix, sys.exec_prefix], + trace=False, count=True) + try: + trace.runfunc(self._run_tests) + finally: + results = trace.results() + real_stdout = sys.stdout + sys.stdout = open(self.coverage_summary, 'w') + try: + results.write_results(show_missing=True, summary=True, + coverdir=self.coverage_dir) + finally: + sys.stdout.close() + sys.stdout = real_stdout + else: self._run_tests() diff --git a/scripts/build.py b/scripts/build.py deleted file mode 100755 --- a/scripts/build.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2005-2007 Christopher Lenz -# Copyright (C) 2007 Edgewall Software -# All rights reserved. -# -# This software is licensed as described in the file COPYING, which -# you should have received as part of this distribution. The terms -# are also available at http://bitten.edgewall.org/wiki/License. - -import itertools -import logging -import sys - -from bitten.build import BuildError -from bitten.recipe import Recipe -from bitten.util import xmlio - -def main(): - from bitten import __version__ as VERSION - from optparse import OptionParser - - parser = OptionParser(usage='usage: %prog [options] [step1] [step2] ...', - version='%%prog %s' % VERSION) - parser.add_option('-f', '--recipe-file', action='store', dest='recipe_file', - metavar='FILE', help='read build recipe from FILE') - parser.add_option('--print-logs', action='store_const', - dest='print_logs', const=True, - help='print build logs') - parser.add_option('--print-reports', action='store_const', - dest='print_reports', const=True, - help='print generated reports') - parser.add_option('-v', '--verbose', action='store_const', dest='loglevel', - const=logging.DEBUG, help='print as much as possible') - parser.add_option('-q', '--quiet', action='store_const', dest='loglevel', - const=logging.ERROR, help='print as little as possible') - parser.set_defaults(loglevel=logging.INFO, recipe_file='recipe.xml') - options, args = parser.parse_args() - - log = logging.getLogger('bitten') - log.setLevel(options.loglevel) - handler = logging.StreamHandler() - handler.setLevel(options.loglevel) - formatter = logging.Formatter('%(message)s') - handler.setFormatter(formatter) - log.addHandler(handler) - - steps_to_run = dict([(step, False) for step in args]) - - recipe_file = file(options.recipe_file, 'r') - try: - recipe = Recipe(xmlio.parse(recipe_file)) - for step in recipe: - if not steps_to_run or step.id in steps_to_run: - print - print '-->', step.id - for type, category, generator, output in step.execute(recipe.ctxt): - if type == Recipe.ERROR: - log.error(output) - elif type == Recipe.LOG and options.print_logs: - output.write(sys.stdout, newlines=True) - elif type == Recipe.REPORT and options.print_reports: - output.write(sys.stdout, newlines=True) - if step.id in steps_to_run: - steps_to_run[step.id] = True - finally: - recipe_file.close() - -if __name__ == '__main__': - try: - main() - except BuildError, e: - print - print>>sys.stderr, 'FAILED: %s' % e - sys.exit(-1) - print - print 'SUCCESS' diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ 'unittest = bitten.util.testrunner:unittest' ], 'trac.plugins': [ + 'bitten.admin = bitten.admin', 'bitten.main = bitten.main', 'bitten.master = bitten.master', 'bitten.web_ui = bitten.web_ui',