Mercurial > bitten > bitten-test
changeset 213:25f84dd9f159
* Refactoring of build recipes, the file format has changed slightly:
* The namespace URIs of recipe command collections are now abstract, implementations are registered using setuptools entry points.
* Commands for report generation are no longer nested in a `<reports>` sub-element, but are at the same level as normal commands.
* Fixed linking to files from the test results and code coverage summarizers.
* Windows file separators are normalized to a forward slash by recipe commands (thereby also fixing linking to the repository browser from report summaries).
* Paths using backslashes as file separators are now recognized in build logs in the web interface, and linked to the repository browser.
* The `generator` column in the build log and report tables now has the qualified name of the recipe command that generated the log messages or report data.
* There's a database upgrade script to fix file separator normalization and generator values for existing reports and build logs.
| author | cmlenz |
|---|---|
| date | Tue, 20 Sep 2005 22:16:41 +0000 |
| parents | 62b668fc713d |
| children | f0e37bee64c5 |
| files | bitten/build/pythontools.py bitten/master.py bitten/model.py bitten/recipe.py bitten/tests/recipe.py bitten/trac_ext/api.py bitten/trac_ext/summarizers.py bitten/trac_ext/templates/bitten_summary_coverage.cs bitten/trac_ext/templates/bitten_summary_tests.cs bitten/trac_ext/web_ui.py bitten/upgrades.py setup.py |
| diffstat | 12 files changed, 145 insertions(+), 83 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/build/pythontools.py +++ b/bitten/build/pythontools.py @@ -86,8 +86,10 @@ lineno = int(match.group('line')) tag = match.group('tag') xmlio.SubElement(problems, 'problem', category=category, - type=msg_type, tag=tag, file=filename, - line=lineno)[match.group('msg') or ''] + type=msg_type, tag=tag, line=lineno, + file=filename.replace(os.sep, '/'))[ + match.group('msg') or '' + ] ctxt.report('lint', problems) finally: fd.close() @@ -128,7 +130,8 @@ missing_files.remove(filename) covered_modules.add(modname) module = xmlio.Element('coverage', name=modname, - file=filename, percentage=cov) + file=filename.replace(os.sep, '/'), + percentage=cov) coverage_path = ctxt.resolve(coverdir, modname + '.cover') if not os.path.exists(coverage_path): @@ -162,7 +165,8 @@ continue covered_modules.add(modname) module = xmlio.Element('coverage', name=modname, - file=filename, percentage=0) + file=filename.replace(os.sep, '/'), + percentage=0) filepath = ctxt.resolve(filename) fileobj = file(filepath, 'r') try: @@ -196,6 +200,7 @@ value = os.path.realpath(value) if value.startswith(ctxt.basedir): value = value[len(ctxt.basedir) + 1:] + value = value.replace(os.sep, '/') else: continue test.attr[name] = value
--- a/bitten/master.py +++ b/bitten/master.py @@ -385,13 +385,10 @@ message_elem.gettext())) build_log.insert(db=db) - report_types = {'unittest': 'test', 'trace': 'coverage', - 'pylint': 'lint'} for report_elem in elem.children('report'): - generator = report_elem.attr.get('generator') report = Report(self.env, build=build.id, step=step.name, - category=report_types[generator], - generator=generator) + category=report_elem.attr.get('category'), + generator=report_elem.attr.get('generator')) for item_elem in report_elem.children(): item = {'type': item_elem.name} item.update(item_elem.attr)
--- a/bitten/model.py +++ b/bitten/model.py @@ -882,4 +882,4 @@ schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \ BuildStep._schema + BuildLog._schema + Report._schema -schema_version = 5 +schema_version = 6
--- a/bitten/recipe.py +++ b/bitten/recipe.py @@ -11,6 +11,7 @@ import logging import os +from pkg_resources import WorkingSet from bitten.build import BuildError from bitten.util import xmlio @@ -26,22 +27,49 @@ class Context(object): """The context in which a recipe command or report is run.""" - current_step = None - current_function = None + step = None # The current step + generator = None # The current generator (namespace#name) def __init__(self, basedir): self.basedir = os.path.realpath(basedir) self.output = [] + def run(self, step, namespace, name, attr): + if not namespace: + log.warn('Ignoring element <%s> without namespace', name) + return + + group = 'bitten.recipe_commands' + qname = namespace + '#' + name + function = None + for entry_point in WorkingSet().iter_entry_points(group, qname): + function = entry_point.load() + break + else: + raise InvalidRecipeError, 'Unknown recipe command %s' % qname + + def escape(name): + name = name.replace('-', '_') + if keyword.iskeyword(name) or name in __builtins__: + name = name + '_' + return name + args = dict([(escape(name), attr[name]) for name in attr]) + + self.step = step + self.generator = qname + log.debug('Executing %s with arguments: %s', function, args) + function(self, **args) + self.generator = None + self.step = None + def error(self, message): - self.output.append((Recipe.ERROR, None, self.current_function, message)) + self.output.append((Recipe.ERROR, None, self.generator, message)) def log(self, xml_elem): - self.output.append((Recipe.LOG, None, self.current_function, xml_elem)) + self.output.append((Recipe.LOG, None, self.generator, xml_elem)) def report(self, category, xml_elem): - self.output.append((Recipe.REPORT, category, self.current_function, - xml_elem)) + self.output.append((Recipe.REPORT, category, self.generator, xml_elem)) def resolve(self, *path): return os.path.normpath(os.path.join(self.basedir, *path)) @@ -60,25 +88,10 @@ self.description = elem.attr.get('description') self.onerror = elem.attr.get('onerror', 'fail') - def __iter__(self): + def execute(self, ctxt): for child in self._elem: - if child.namespace: # Commands - yield self._function(child), self._args(child) - elif child.name == 'reports': # Reports - for grandchild in child: - yield self._function(grandchild), self._args(grandchild) - else: - raise InvalidRecipeError, "Unknown element <%s>" % child.name + ctxt.run(self, child.namespace, child.name, child.attr) - def execute(self, ctxt): - ctxt.current_step = self - try: - for function, args in self: - ctxt.current_function = function.__name__ - function(ctxt, **args) - ctxt.current_function = None - finally: - ctxt.current_step = None errors = [] while ctxt.output: type, category, generator, output = ctxt.output.pop(0) @@ -91,29 +104,6 @@ log.warning('Ignoring errors in step %s (%s)', self.id, ', '.join([error[1] for error in errors])) - def _args(self, elem): - return dict([(self._translate_name(name), value) for name, value - in elem.attr.items()]) - - def _function(self, elem): - if not elem.namespace.startswith('bitten:'): - # Ignore elements in foreign namespaces - return None - func_name = self._translate_name(elem.name) - try: - module = __import__(elem.namespace[7:], globals(), locals(), - [func_name]) - func = getattr(module, func_name) - return func - except (ImportError, AttributeError), e: - raise InvalidRecipeError, 'Cannot load "%s" (%s)' % (elem.name, e) - - def _translate_name(self, name): - name = name.replace('-', '_') - if keyword.iskeyword(name) or name in __builtins__: - name = name + '_' - return name - class Recipe(object): """A build recipe.
--- a/bitten/tests/recipe.py +++ b/bitten/tests/recipe.py @@ -32,7 +32,7 @@ steps = list(recipe) self.assertEqual(0, len(steps)) - def test_single_step(self): + def test_empty_step(self): xml = xmlio.parse('<build>' ' <step id="foo" description="Bar"></step>' '</build>')
--- a/bitten/trac_ext/api.py +++ b/bitten/trac_ext/api.py @@ -30,7 +30,7 @@ """Return a list of strings identifying the types of reports this component supports.""" - def render_summary(req, build, step, category): + def render_summary(req, config, build, step, category): """Render a summary for the given report and return the results HTML as a string."""
--- a/bitten/trac_ext/summarizers.py +++ b/bitten/trac_ext/summarizers.py @@ -21,10 +21,9 @@ def get_supported_categories(self): return ['test'] - def render_summary(self, req, build, step, category): + def render_summary(self, req, config, build, step, category): assert category == 'test' - hdf = HDFWrapper(loadpaths=Chrome(self.env).get_all_templates_dirs()) db = self.env.get_db_cnx() cursor = db.cursor() cursor.execute(""" @@ -52,10 +51,12 @@ data = [] for fixture, file, num_success, num_failure, num_error in cursor: - data.append({'name': fixture, 'href': self.env.href.browser(file), - 'num_success': num_success, 'num_error': num_error, - 'num_failure': num_failure}) + data.append({'name': fixture, 'num_success': num_success, + 'num_error': num_error, 'num_failure': num_failure}) + if file: + data[-1]['href'] = self.env.href.browser(config.path, file) + hdf = HDFWrapper(loadpaths=Chrome(self.env).get_all_templates_dirs()) hdf['data'] = data return hdf.render('bitten_summary_tests.cs') @@ -66,10 +67,9 @@ def get_supported_categories(self): return ['coverage'] - def render_summary(self, req, build, step, category): + def render_summary(self, req, config, build, step, category): assert category == 'coverage' - hdf = HDFWrapper(loadpaths=Chrome(self.env).get_all_templates_dirs()) db = self.env.get_db_cnx() cursor = db.cursor() cursor.execute(""" @@ -93,8 +93,10 @@ data = [] for unit, file, loc, cov in cursor: - data.append({'name': unit, 'href': self.env.href.browser(file), - 'loc': loc, 'cov': cov}) + data.append({'name': unit, 'loc': loc, 'cov': cov}) + if file: + data[-1]['href'] = self.env.href.browser(config.path, file) + hdf = HDFWrapper(loadpaths=Chrome(self.env).get_all_templates_dirs()) hdf['data'] = data return hdf.render('bitten_summary_coverage.cs')
--- a/bitten/trac_ext/templates/bitten_summary_coverage.cs +++ b/bitten/trac_ext/templates/bitten_summary_coverage.cs @@ -3,8 +3,10 @@ <thead><tr><th class="name">Unit</th><th class="loc">Lines of Code</th> <th clsas="cov">Coverage</th></tr></thead> <tbody><?cs - each:item = data ?><tr><td class="name"><a href="<?cs - var:item.href ?>"><?cs var:item.name ?></a></td><td class="loc"><?cs + each:item = data ?><tr><td class="name"><?cs + if:item.href ?><a href="<?cs var:item.href ?>"><?cs var:item.name ?></a><?cs + else ?><?cs var:item.name ?><?cs + /if ?></td><td class="loc"><?cs var:item.loc ?></td><td class="cov"><?cs var:item.cov ?>%</td></tr><?cs /each ?></tbody> </table>
--- a/bitten/trac_ext/templates/bitten_summary_tests.cs +++ b/bitten/trac_ext/templates/bitten_summary_tests.cs @@ -5,8 +5,10 @@ <th>Failures</th><th>Errors</th> </tr></thead> <tbody><?cs - each:item = data ?><tr><td><a href="<?cs - var:item.href ?>"><?cs var:item.name ?></a></td><td><?cs + each:item = data ?><tr><td><?cs + if:item.href ?><a href="<?cs var:item.href ?>"><?cs var:item.name ?></a><?cs + else ?><?cs var:item.name ?><?cs + /if ?></td><td><?cs var:#item.num_success + #item.num_failure + #item.num_error ?></td><td><?cs var:item.num_failure ?></td><td><?cs var:item.num_error ?></td></tr><?cs
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -481,7 +481,7 @@ 'duration': pretty_timedelta(step.started, step.stopped), 'failed': step.status == BuildStep.FAILURE, 'log': self._render_log(req, build, step), - 'reports': self._render_reports(req, build, step) + 'reports': self._render_reports(req, config, build, step) }) req.hdf['build.steps'] = steps req.hdf['build.can_delete'] = req.perm.has_permission('BUILD_DELETE') @@ -550,7 +550,7 @@ items.append({'level': level, 'message': message}) return items - def _render_reports(self, req, build, step): + def _render_reports(self, req, config, build, step): summarizers = {} # keyed by report type for summarizer in self.report_summarizers: categories = summarizer.get_supported_categories() @@ -560,7 +560,7 @@ for report in Report.select(self.env, build=build.id, step=step.name): summarizer = summarizers.get(report.category) if summarizer: - summary = summarizer.render_summary(req, build, step, + summary = summarizer.render_summary(req, config, build, step, report.category) else: summary = None @@ -581,7 +581,9 @@ def _walk(node): for child in node.get_entries(): path = child.path[len(config.path) + 1:] - pattern = re.compile("([\s'\"])(%s)([\s'\"])" % re.escape(path)) + pattern = re.compile("([\s'\"])(%s|%s)([\s'\"])" + % (re.escape(path), + re.escape(path.replace('/', '\\')))) nodes.append((child.path, pattern)) if child.isdir: _walk(child)
--- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -210,9 +210,53 @@ container.close() dbenv.close(0) +def normalize_file_paths(env, db): + """Normalize the file separator in file names in reports.""" + cursor = db.cursor() + cursor.execute("SELECT report,item,value FROM bitten_report_item " + "WHERE name='file'") + rows = cursor.fetchall() + for report, item, value in rows: + if '\\' in value: + cursor.execute("UPDATE bitten_report_item SET value=%s " + "WHERE report=%s AND item=%s AND name='file'", + (value.replace('\\', '/'), report, item)) + +def fixup_generators(env, db): + """Upgrade the identifiers for the recipe commands that generated log + messages and report data.""" + from bitten.model import BuildLog, Report + + mapping = { + 'pipe': 'http://bitten.cmlenz.net/tools/sh#pipe', + 'make': 'http://bitten.cmlenz.net/tools/c#make', + 'distutils': 'http://bitten.cmlenz.net/tools/python#distutils', + 'exec_': 'http://bitten.cmlenz.net/tools/python#exec' # Ambigious + } + cursor = db.cursor() + cursor.execute("SELECT id,generator FROM bitten_log " + "WHERE generator IN (%s)" + % ','.join([repr(key) for key in mapping.keys()])) + for id, generator in cursor: + cursor.execute("UPDATE bitten_log SET generator=%s " + "WHERE id=%s", (mapping[generator], id)) + + mapping = { + 'unittest': 'http://bitten.cmlenz.net/tools/python#unittest', + 'trace': 'http://bitten.cmlenz.net/tools/python#trace', + 'pylint': 'http://bitten.cmlenz.net/tools/python#pylint' + } + cursor.execute("SELECT id,generator FROM bitten_report " + "WHERE generator IN (%s)" + % ','.join([repr(key) for key in mapping.keys()])) + for id, generator in cursor: + cursor.execute("UPDATE bitten_report SET generator=%s " + "WHERE id=%s", (mapping[generator], id)) + map = { 2: [add_log_table], 3: [add_recipe_to_config], 4: [add_config_to_reports], - 5: [add_order_to_log, add_report_tables, xmldb_to_db] + 5: [add_order_to_log, add_report_tables, xmldb_to_db], + 6: [normalize_file_paths, fixup_generators] }
--- a/setup.py +++ b/setup.py @@ -13,6 +13,8 @@ from bitten import __version__ as VERSION from bitten.util.testrunner import unittest +NS = 'http://bitten.cmlenz.net/tools/' + setup( name='Bitten', version=VERSION, author='Christopher Lenz', author_email='cmlenz@gmx.de', url='http://bitten.cmlenz.net/', @@ -26,13 +28,29 @@ 'templates/*.cs'] }, entry_points = { - 'console_scripts': ['bitten-master = bitten.master:main', - 'bitten-slave = bitten.slave:main'], - 'distutils.commands': ['unittest = bitten.util.testrunner:unittest'], - 'trac.plugins': ['bitten.main = bitten.trac_ext.main', - 'bitten.web_ui = bitten.trac_ext.web_ui', - 'bitten.summarizers = bitten.trac_ext.summarizers', - 'bitten.charts = bitten.trac_ext.charts'] + 'console_scripts': [ + 'bitten-master = bitten.master:main', + 'bitten-slave = bitten.slave:main' + ], + 'distutils.commands': [ + 'unittest = bitten.util.testrunner:unittest' + ], + 'trac.plugins': [ + 'bitten.main = bitten.trac_ext.main', + 'bitten.web_ui = bitten.trac_ext.web_ui', + 'bitten.summarizers = bitten.trac_ext.summarizers', + 'bitten.charts = bitten.trac_ext.charts' + ], + 'bitten.recipe_commands': [ + NS + 'sh#exec = bitten.build.shtools:exec_', + NS + 'sh#pipe = bitten.build.shtools:pipe', + NS + 'c#make = bitten.build.ctools:make', + NS + 'python#distutils = bitten.build.pythontools:distutils', + NS + 'python#exec = bitten.build.pythontools:exec_', + NS + 'python#pylint = bitten.build.pythontools:pylint', + NS + 'python#trace = bitten.build.pythontools:trace', + NS + 'python#unittest = bitten.build.pythontools:unittest' + ] }, test_suite='bitten.tests.suite', zip_safe=True )