changeset 429:d6e1a05f32f7

Start webadmin integration.
author cmlenz
date Tue, 14 Aug 2007 22:22:37 +0000
parents 53d99d315188
children c85dcf71e28e
files bitten/admin.py bitten/build/pythontools.py bitten/templates/bitten_admin_configs.cs bitten/templates/bitten_admin_master.cs bitten/util/testrunner.py scripts/build.py setup.py
diffstat 7 files changed, 511 insertions(+), 97 deletions(-) [+]
line wrap: on
line diff
new file mode 100644
--- /dev/null
+++ b/bitten/admin.py
@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2007 Edgewall Software
+# All rights reserved.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at http://bitten.edgewall.org/wiki/License.
+
+"""Implementation of the web administration interface."""
+
+import re
+
+from trac.core import *
+from webadmin.web_ui import IAdminPageProvider
+
+from bitten.model import BuildConfig, TargetPlatform
+from bitten.recipe import Recipe
+from bitten.util import xmlio
+
+
+class BuildMasterAdminPageProvider(Component):
+    """Web administration panel for configuring the build master."""
+
+    implements(IAdminPageProvider)
+
+    # IAdminPageProvider methods
+
+    def get_admin_pages(self, req):
+        if req.perm.has_permission('BUILD_ADMIN'):
+            yield ('bitten', 'Bitten', 'master', 'Build Master')
+
+    def process_admin_request(self, req, cat, page, path_info):
+        from bitten.master import BuildMaster
+        master = BuildMaster(self.env)
+
+        if req.method == 'POST':
+            changed = False
+            build_all = 'build_all' in req.args
+            if build_all != master.build_all:
+                self.config['bitten'].set('build_all',
+                                          build_all and 'yes' or 'no')
+                changed = True
+            adjust_timestamps = 'adjust_timestamps' in req.args
+            if adjust_timestamps != master.adjust_timestamps:
+                self.config['bitten'].set('adjust_timestamps',
+                                          adjust_timestamps and 'yes' or 'no')
+                changed = True
+            slave_timeout = int(req.args.get('slave_timeout', 0))
+            if slave_timeout != master.slave_timeout:
+                self.config['bitten'].set('slave_timeout', str(slave_timeout))
+                changed = True
+            if changed:
+                self.config.save()
+
+        req.hdf['admin.master'] = {
+            'build_all': master.build_all,
+            'adjust_timestamps': master.adjust_timestamps,
+            'slave_timeout': master.slave_timeout,
+        }
+        return 'bitten_admin_master.cs', None
+
+
+class BuildConfigurationsAdminPageProvider(Component):
+    """Web administration panel for configuring the build master."""
+
+    implements(IAdminPageProvider)
+
+    # IAdminPageProvider methods
+
+    def get_admin_pages(self, req):
+        if req.perm.has_permission('BUILD_MODIFY'):
+            yield ('bitten', 'Bitten', 'configs', 'Configurations')
+
+    def process_admin_request(self, req, cat, page, config_name):
+        data = {}
+
+        if config_name:
+            config = BuildConfig.fetch(self.env, config_name)
+            platforms = list(TargetPlatform.select(self.env, config=config.name))
+
+            if req.method == 'POST':
+                if 'save' in req.args:
+                    name = req.args.get('name')
+                    if not name:
+                        raise TracError('Missing required field "name"',
+                                        'Missing field')
+                    if not re.match(r'^[\w.-]+$', name):
+                        raise TracError('The field "name" may only contain '
+                                        'letters, digits, periods, or dashes.',
+                                        'Invalid field')
+
+                    path = req.args.get('path', '')
+                    repos = self.env.get_repository(req.authname)
+                    max_rev = req.args.get('max_rev') or None
+                    try:
+                        node = repos.get_node(path, max_rev)
+                        assert node.isdir, '%s is not a directory' % node.path
+                    except (AssertionError, TracError), e:
+                        raise TracError(e, 'Invalid repository path')
+                    if req.args.get('min_rev'):
+                        try:
+                            repos.get_node(path, req.args.get('min_rev'))
+                        except TracError, e:
+                            raise TracError(e,
+                                            'Invalid value for oldest revision')
+
+                    recipe_xml = req.args.get('recipe', '')
+                    if recipe_xml:
+                        try:
+                            Recipe(xmlio.parse(recipe_xml)).validate()
+                        except xmlio.ParseError, e:
+                            raise TracError('Failure parsing recipe: %s' % e,
+                                            'Invalid recipe')
+                        except InvalidRecipeError, e:
+                            raise TracError(e, 'Invalid recipe')
+
+                    config.name = name
+                    config.path = repos.normalize_path(path)
+                    config.recipe = recipe_xml
+                    config.min_rev = req.args.get('min_rev')
+                    config.max_rev = req.args.get('max_rev')
+                    config.label = req.args.get('label', '')
+                    config.description = req.args.get('description', '')
+                    config.update()
+                    req.redirect(self.env.href.admin(cat, page))
+
+                elif 'cancel' in req.args:
+                    req.redirect(self.env.href.admin(cat, page))
+
+            data['config'] = {
+                'name': config.name, 'label': config.label or config.name,
+                'active': config.active, 'path': config.path,
+                'min_rev': config.min_rev, 'max_rev': config.max_rev,
+                'description': config.description,
+                'recipe': config.recipe,
+                'platforms': [{
+                    'name': platform.name,
+                    'id': platform.id,
+                    'href': req.href.admin('bitten', 'configs', config.name,
+                                           platform.id)
+                } for platform in platforms]
+            }
+
+        else:
+            if req.method == 'POST':
+                # Add configuration
+                if 'add' in req.args:
+                    config = BuildConfig(self.env)
+                    config.name = req.args.get('name')
+                    config.label = req.args.get('label', config.name)
+                    config.path = req.args.get('path')
+                    config.insert()
+                    req.redirect(self.env.href.admin(cat, page, config.name))
+
+                # Remove configurations
+                elif 'remove' in req.args:
+                    sel = req.args.get('sel')
+                    sel = isinstance(sel, list) and sel or [sel]
+                    if not sel:
+                        raise TracError('No configuration selected')
+                    db = self.env.get_db_cnx()
+                    for name in sel:
+                        config = BuildConfig.fetch(self.env, name, db=db)
+                        config.delete(db=db)
+                    db.commit()
+                    req.redirect(self.env.href.admin(cat, page))
+
+                # Set active state
+                elif 'apply' in req.args:
+                    active = req.args.get('active')
+                    active = isinstance(active, list) and active or [active]
+                    db = self.env.get_db_cnx()
+                    for config in BuildConfig.select(self.env, db=db,
+                                                     include_inactive=True):
+                        config.active = config.name in active
+                        config.update(db=db)
+                    db.commit()
+                    req.redirect(self.env.href.admin(cat, page))
+
+            configs = []
+            for config in BuildConfig.select(self.env, include_inactive=True):
+                configs.append({
+                    'name': config.name, 'label': config.label or config.name,
+                    'active': config.active, 'path': config.path,
+                    'min_rev': config.min_rev, 'max_rev': config.max_rev,
+                    'href': req.href.admin('bitten', 'configs', config.name),
+                })
+            data['configs'] = configs
+
+        req.hdf['admin'] = data
+        return 'bitten_admin_configs.cs', None
--- a/bitten/build/pythontools.py
+++ b/bitten/build/pythontools.py
@@ -163,6 +163,136 @@
     except IOError, e:
         log.warning('Error opening pylint results file (%s)', e)
 
+def coverage(ctxt, summary=None, coverdir=None, include=None, exclude=None):
+    """Extract data from a ``coverage.py`` run.
+    
+    :param ctxt: the build context
+    :type ctxt: `Context`
+    :param summary: path to the file containing the coverage summary
+    :param coverdir: name of the directory containing the per-module coverage
+                     details
+    :param include: patterns of files or directories to include in the report
+    :param exclude: patterns of files or directories to exclude from the report
+    """
+    assert summary, 'Missing required attribute "summary"'
+    assert coverdir, 'Missing required attribute "coverdir"'
+
+    summary_line_re = re.compile(r'^\s*(?P<lines>\d+)\s+(?P<cov>\d+)%\s+'
+                                 r'(?P<module>.*?)\s+\((?P<filename>.*?)\)')
+    coverage_line_re = re.compile(r'\s*(?:(?P<hits>\d+): )?(?P<line>.*)')
+
+    fileset = FileSet(ctxt.basedir, include, exclude)
+    missing_files = []
+    for filename in fileset:
+        if os.path.splitext(filename)[1] != '.py':
+            continue
+        missing_files.append(filename)
+    covered_modules = set()
+
+    def handle_file(elem, sourcefile, coverfile=None):
+        code_lines = set()
+        for lineno, linetype, line in loc.count(sourcefile):
+            if linetype == loc.CODE:
+                code_lines.add(lineno)
+        num_covered = 0
+        lines = []
+
+        if coverfile:
+            prev_hits = '0'
+            for idx, coverline in enumerate(coverfile):
+                match = coverage_line_re.search(coverline)
+                if match:
+                    hits = match.group(1)
+                    if hits: # Line covered
+                        if hits != '0':
+                            num_covered += 1
+                        lines.append(hits)
+                        prev_hits = hits
+                    elif coverline.startswith('>'): # Line not covered
+                        lines.append('0')
+                        prev_hits = '0'
+                    elif idx not in code_lines: # Not a code line
+                        lines.append('-')
+                        prev_hits = '0'
+                    else: # A code line not flagged by trace.py
+                        if prev_hits != '0':
+                            num_covered += 1
+                        lines.append(prev_hits)
+
+            elem.append(xmlio.Element('line_hits')[' '.join(lines)])
+
+        num_lines = len(code_lines)
+        if num_lines:
+            percentage = int(round(num_covered * 100 / num_lines))
+        else:
+            percentage = 0
+        elem.attr['percentage'] = percentage
+        elem.attr['lines'] = num_lines
+
+    try:
+        summary_file = open(ctxt.resolve(summary), 'r')
+        try:
+            coverage = xmlio.Fragment()
+            for summary_line in summary_file:
+                match = summary_line_re.search(summary_line)
+                if match:
+                    modname = match.group(3)
+                    filename = match.group(4)
+                    if not os.path.isabs(filename):
+                        filename = os.path.normpath(os.path.join(ctxt.basedir,
+                                                                 filename))
+                    else:
+                        filename = os.path.realpath(filename)
+                    if not filename.startswith(ctxt.basedir):
+                        continue
+                    filename = filename[len(ctxt.basedir) + 1:]
+                    if not filename in fileset:
+                        continue
+
+                    missing_files.remove(filename)
+                    covered_modules.add(modname)
+                    module = xmlio.Element('coverage', name=modname,
+                                           file=filename.replace(os.sep, '/'))
+                    sourcefile = file(ctxt.resolve(filename))
+                    try:
+                        coverpath = ctxt.resolve(coverdir, modname + '.cover')
+                        if os.path.isfile(coverpath):
+                            coverfile = file(coverpath, 'r')
+                        else:
+                            log.warning('No coverage file for module %s at %s',
+                                        modname, coverpath)
+                            coverfile = None
+                        try:
+                            handle_file(module, sourcefile, coverfile)
+                        finally:
+                            if coverfile:
+                                coverfile.close()
+                    finally:
+                        sourcefile.close()
+                    coverage.append(module)
+
+            for filename in missing_files:
+                modname = os.path.splitext(filename.replace(os.sep, '.'))[0]
+                if modname in covered_modules:
+                    continue
+                covered_modules.add(modname)
+                module = xmlio.Element('coverage', name=modname,
+                                       file=filename.replace(os.sep, '/'),
+                                       percentage=0)
+                filepath = ctxt.resolve(filename)
+                fileobj = file(filepath, 'r')
+                try:
+                    handle_file(module, fileobj)
+                finally:
+                    fileobj.close()
+                coverage.append(module)
+
+            ctxt.report('coverage', coverage)
+        finally:
+            summary_file.close()
+    except IOError, e:
+        log.warning('Error opening coverage summary file (%s)', e)
+
 def trace(ctxt, summary=None, coverdir=None, include=None, exclude=None):
     """Extract data from a ``trace.py`` run.
     
new file mode 100644
--- /dev/null
+++ b/bitten/templates/bitten_admin_configs.cs
@@ -0,0 +1,100 @@
+<h2>Manage Build Configurations</h2><?cs
+
+if admin.config.name ?>
+ <form class="mod" id="modconfig" method="post">
+  <table summary=""><tr>
+   <td class="name"><label>Name:<br />
+    <input type="text" name="name" value="<?cs var:admin.config.name ?>" />
+   </label></td>
+   <td class="label"><label>Label (for display):<br />
+    <input type="text" name="label" size="32" value="<?cs
+      var:admin.config.label ?>" />
+   </label></td>
+  </tr><tr>
+   <td colspan="2"><fieldset class="iefix">
+    <label for="description">Description (you may use <a tabindex="42" href="<?cs
+      var:trac.href.wiki ?>/WikiFormatting">WikiFormatting</a> here):</label>
+    <p><textarea id="description" name="description" class="wikitext" rows="5" cols="78"><?cs
+      var:admin.config.description ?></textarea></p>
+    <script type="text/javascript" src="<?cs
+      var:chrome.href ?>/common/js/wikitoolbar.js"></script>
+   </fieldset></td>
+  </tr></table>
+  <fieldset id="recipe">
+   <legend>Build Recipe</legend>
+   <textarea id="recipe" name="recipe" rows="8" cols="78"><?cs
+     var:admin.config.recipe ?></textarea>
+  </fieldset>
+  <fieldset id="repos">
+   <legend>Repository Mapping</legend>
+   <table summary=""><tr>
+    <th><label for="path">Path:</label></th>
+    <td colspan="3"><input type="text" name="path" size="48" value="<?cs
+      var:admin.config.path ?>" /></td>
+   </tr><tr>
+    <th><label for="min_rev">Oldest revision:</label></th>
+    <td><input type="text" name="min_rev" size="8" value="<?cs
+      var:admin.config.min_rev ?>" /></td>
+    <th><label for="min_rev">Youngest revision:</label></th>
+    <td><input type="text" name="max_rev" size="8" value="<?cs
+      var:admin.config.max_rev ?>" /></td>
+   </table>
+  </fieldset>
+  <fieldset>
+    <legend>Target Platforms</legend><?cs
+    if:len(admin.config.platforms) ?><ul><?cs
+     each:platform = admin.config.platforms ?>
+      <li><input type="checkbox" name="delete_platform" value="<?cs
+       var:platform.id ?>"> <a href="<?cs
+       var:platform.href ?>"><?cs var:platform.name ?></a>
+      </li><?cs
+     /each ?></ul><?cs
+    /if ?>
+  </fieldset>
+  <div class="buttons">
+   <input type="submit" name="cancel" value="Cancel" />
+   <input type="submit" name="save" value="Save" />
+  </div>
+ </form><?cs
+
+else ?>
+ <form class="addnew" id="addcomp" method="post">
+  <fieldset>
+   <legend>Add Configuration:</legend>
+   <div class="field">
+    <label>Name:<br /><input type="text" name="name" /></label>
+   </div>
+   <div class="field">
+    <label>Label:<br /><input type="text" name="label" /></label>
+   </div>
+   <div class="buttons">
+    <input type="submit" name="add" value="Add">
+   </div>
+  </fieldset>
+ </form>
+
+ <form method="POST">
+  <table class="listing" id="configlist">
+   <thead>
+    <tr><th class="sel">&nbsp;</th><th>Name</th>
+    <th>Path</th><th>Active</th></tr>
+   </thead><?cs each:config = admin.configs ?>
+    <tr>
+     <td class="sel"><input type="checkbox" name="sel" value="<?cs
+       var:config.name ?>" /></td>
+     <td class="name"><a href="<?cs var:config.href?>"><?cs
+       var:config.label ?></a></td>
+     <td class="path"><code><?cs var:config.path ?></code></td>
+     <td class="active"><input type="checkbox" name="active" value="<?cs
+       var:config.name ?>"<?cs
+       if:config.active ?> checked="checked" <?cs /if ?>></td>
+    </tr><?cs
+   /each ?>
+  </table>
+  <div class="buttons">
+   <input type="submit" name="remove" value="Remove selected items" />
+   <input type="submit" name="apply" value="Apply changes" />
+  </div>
+ </form><?cs
+
+/if ?>
new file mode 100644
--- /dev/null
+++ b/bitten/templates/bitten_admin_master.cs
@@ -0,0 +1,47 @@
+<h2>Manage Build Master</h2>
+
+<form class="mod" id="bitten" method="post">
+
+  <fieldset id="config">
+    <legend>Configuration Options</legend>
+    <div class="field">
+      <label>
+        <input type="checkbox" id="build_all" name="build_all"
+               <?cs if:admin.master.build_all ?> checked="checked"<?cs /if ?> />
+        Build all revisions
+      </label>
+    </div>
+    <p class="hint">
+      Whether to build older revisions even when a more recent revision has
+      already been built.
+    </p>
+    <div class="field">
+      <label>
+        <input type="checkbox" id="adjust_timestamps" name="adjust_timestamps"
+               <?cs if:admin.master.adjust_timestamps ?> checked="checked"<?cs /if ?> />
+        Adjust build timestamps
+      </label>
+    </div>
+    <p class="hint">
+      Whether the timestamps of builds should be adjusted to be close to the
+      timestamps of the corresponding changesets.
+    </p>
+    <hr />
+    <div class="field">
+      <label>
+        Connection timeout for build slaves:
+        <input type="text" id="slave_timeout" name="slave_timeout"
+               value="<?cs var:admin.master.slave_timeout ?>" size="5" />
+      </label>
+    </div>
+    <p class="hint">
+      The timeout in milliseconds after which a build started by a slave is
+      considered aborted, in case there has been no activity from that slave
+      in that time.
+    </p>
+  </fieldset>
+
+  <div class="buttons">
+    <input type="submit" value="Apply changes"/>
+  </div>
+</form>
--- a/bitten/util/testrunner.py
+++ b/bitten/util/testrunner.py
@@ -8,15 +8,15 @@
 # you should have received as part of this distribution. The terms
 # are also available at http://bitten.edgewall.org/wiki/License.
 
+from distutils import log
+from distutils.errors import DistutilsOptionError
 import os
 import re
-try:
-    from cStringIO import StringIO
-except:
-    from StringIO import StringIO
+from StringIO import StringIO
 import sys
 import time
-from pkg_resources import *
+from pkg_resources import Distribution, EntryPoint, PathMetadata, \
+                          normalize_path, require, working_set
 from setuptools.command.test import test
 from unittest import _TextTestResult, TextTestRunner
 
@@ -120,7 +120,10 @@
         ('coverage-dir=', None,
             "Directory where coverage files are to be stored"),
         ('coverage-summary=', None,
-            "Path to the file where the coverage summary should be stored")
+            "Path to the file where the coverage summary should be stored"),
+        ('coverage-method=', None,
+            "Whether to use trace.py or coverage.py to collect code coverage. "
+            "Valid options are 'trace' (the default) or 'coverage'.")
     ]
 
     def initialize_options(self):
@@ -129,31 +132,50 @@
         self.xml_output_file = None
         self.coverage_summary = None
         self.coverage_dir = None
+        self.coverage_method = 'trace'
 
     def finalize_options(self):
         test.finalize_options(self)
+
         if self.xml_output is not None:
             if not os.path.exists(os.path.dirname(self.xml_output)):
                 os.makedirs(os.path.dirname(self.xml_output))
             self.xml_output_file = open(self.xml_output, 'w')
 
+        if self.coverage_method not in ('trace', 'coverage'):
+            raise DistutilsOptionError('Unknown coverage method %r' %
+                                       self.coverage_method)
+
     def run_tests(self):
         if self.coverage_dir:
-            from trace import Trace
-            trace = Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
-                          trace=False, count=True)
-            try:
-                trace.runfunc(self._run_tests)
-            finally:
-                results = trace.results()
-                real_stdout = sys.stdout
-                sys.stdout = open(self.coverage_summary, 'w')
+
+            if self.coverage_method == 'coverage':
+                import coverage
+                coverage.erase()
+                coverage.start()
+                log.info('running tests under coverage.py')
                 try:
-                    results.write_results(show_missing=True, summary=True,
-                                          coverdir=self.coverage_dir)
+                    self._run_tests()
                 finally:
-                    sys.stdout.close()
-                    sys.stdout = real_stdout
+                    coverage.stop()
+
+            else:
+                from trace import Trace
+                trace = Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
+                              trace=False, count=True)
+                try:
+                    trace.runfunc(self._run_tests)
+                finally:
+                    results = trace.results()
+                    real_stdout = sys.stdout
+                    sys.stdout = open(self.coverage_summary, 'w')
+                    try:
+                        results.write_results(show_missing=True, summary=True,
+                                              coverdir=self.coverage_dir)
+                    finally:
+                        sys.stdout.close()
+                        sys.stdout = real_stdout
+
         else:
             self._run_tests()
 
deleted file mode 100755
--- a/scripts/build.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de>
-# Copyright (C) 2007 Edgewall Software
-# All rights reserved.
-#
-# This software is licensed as described in the file COPYING, which
-# you should have received as part of this distribution. The terms
-# are also available at http://bitten.edgewall.org/wiki/License.
-
-import itertools
-import logging
-import sys
-
-from bitten.build import BuildError
-from bitten.recipe import Recipe
-from bitten.util import xmlio
-
-def main():
-    from bitten import __version__ as VERSION
-    from optparse import OptionParser
-
-    parser = OptionParser(usage='usage: %prog [options] [step1] [step2] ...',
-                          version='%%prog %s' % VERSION)
-    parser.add_option('-f', '--recipe-file', action='store', dest='recipe_file',
-                      metavar='FILE', help='read build recipe from FILE')
-    parser.add_option('--print-logs', action='store_const',
-                      dest='print_logs', const=True,
-                      help='print build logs')
-    parser.add_option('--print-reports', action='store_const',
-                      dest='print_reports', const=True,
-                      help='print generated reports')
-    parser.add_option('-v', '--verbose', action='store_const', dest='loglevel',
-                      const=logging.DEBUG, help='print as much as possible')
-    parser.add_option('-q', '--quiet', action='store_const', dest='loglevel',
-                      const=logging.ERROR, help='print as little as possible')
-    parser.set_defaults(loglevel=logging.INFO, recipe_file='recipe.xml')
-    options, args = parser.parse_args()
-
-    log = logging.getLogger('bitten')
-    log.setLevel(options.loglevel)
-    handler = logging.StreamHandler()
-    handler.setLevel(options.loglevel)
-    formatter = logging.Formatter('%(message)s')
-    handler.setFormatter(formatter)
-    log.addHandler(handler)
-
-    steps_to_run = dict([(step, False) for step in args])
-
-    recipe_file = file(options.recipe_file, 'r')
-    try:
-        recipe = Recipe(xmlio.parse(recipe_file))
-        for step in recipe:
-            if not steps_to_run or step.id in steps_to_run:
-                print
-                print '-->', step.id
-                for type, category, generator, output in step.execute(recipe.ctxt):
-                    if type == Recipe.ERROR:
-                        log.error(output)
-                    elif type == Recipe.LOG and options.print_logs:
-                        output.write(sys.stdout, newlines=True)
-                    elif type == Recipe.REPORT and options.print_reports:
-                        output.write(sys.stdout, newlines=True)
-                if step.id in steps_to_run:
-                    steps_to_run[step.id] = True
-    finally:
-        recipe_file.close()
-
-if __name__ == '__main__':
-    try:
-        main()
-    except BuildError, e:
-        print
-        print>>sys.stderr, 'FAILED: %s' % e
-        sys.exit(-1)
-    print
-    print 'SUCCESS'
--- a/setup.py
+++ b/setup.py
@@ -49,6 +49,7 @@
             'unittest = bitten.util.testrunner:unittest'
         ],
         'trac.plugins': [
+            'bitten.admin = bitten.admin',
             'bitten.main = bitten.main',
             'bitten.master = bitten.master',
             'bitten.web_ui = bitten.web_ui',
Copyright (C) 2012-2017 Edgewall Software