Mercurial > bitten > bitten-test
changeset 258:77cdef044d48
* Improve build log formatter performance: now only matches strings using the `path:line` format, and checks the existance of files in the repository when they are encountered. Should fix (or at least improve) #54.
* More code reuse in the recipe commands code. Some minor/cosmetic cleanup.
| author | cmlenz |
|---|---|
| date | Thu, 06 Oct 2005 10:09:38 +0000 |
| parents | 5acdaab27187 |
| children | f8e20eac7df4 |
| files | bitten/build/ctools.py bitten/build/javatools.py bitten/build/pythontools.py bitten/build/shtools.py bitten/queue.py bitten/trac_ext/api.py bitten/trac_ext/htdocs/bitten.css bitten/trac_ext/web_ui.py |
| diffstat | 8 files changed, 121 insertions(+), 137 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/build/ctools.py +++ b/bitten/build/ctools.py @@ -36,18 +36,9 @@ if cxxflags: args.append('CXXFLAGS=%s' % cxxflags) - log_elem = xmlio.Fragment() - cmdline = CommandLine(ctxt.resolve(file_), args, cwd=ctxt.basedir) - for out, err in cmdline.execute(): - if out is not None: - log.info(out) - log_elem.append(xmlio.Element('message', level='info')[out]) - if err is not None: - log.error(err) - log_elem.append(xmlio.Element('message', level='error')[err]) - ctxt.log(log_elem) - - if cmdline.returncode != 0: + from bitten.build import shtools + returncode = shtools.execute(ctxt, file_=file_, args=args) + if returncode != 0: ctxt.error('configure failed (%s)' % cmdline.returncode) def make(ctxt, target=None, file_=None, keep_going=False): @@ -60,16 +51,7 @@ if target: args.append(target) - log_elem = xmlio.Fragment() - cmdline = CommandLine('make', args) - for out, err in cmdline.execute(): - if out is not None: - log.info(out) - log_elem.append(xmlio.Element('message', level='info')[out]) - if err is not None: - log.error(err) - log_elem.append(xmlio.Element('message', level='error')[err]) - ctxt.log(log_elem) - - if cmdline.returncode != 0: + from bitten.build import shtools + returncode = shtools.execute(ctxt, executable='make', args=args) + if returncode != 0: ctxt.error('make failed (%s)' % cmdline.returncode)
--- a/bitten/build/javatools.py +++ b/bitten/build/javatools.py @@ -77,31 +77,31 @@ if not error_logged and cmdline.returncode != 0: ctxt.error('Ant failed (%s)' % cmdline.returncode) -def java_src(src, cls): - return posixpath.join(src, *cls.split('.')) + '.java' - -def junit(ctxt, file_=None, src=None): +def junit(ctxt, file_=None, srcdir=None): + """Extract test results from a JUnit XML report.""" assert file_, 'Missing required attribute "file"' try: total, failed = 0, 0 results = xmlio.Fragment() - for f in glob(ctxt.resolve(file_)): - fd = open(f, 'r') + for path in glob(ctxt.resolve(file_)): + fileobj = file(path, 'r') try: - for testcase in xmlio.parse(fd).children('testcase'): + for testcase in xmlio.parse(fileobj).children('testcase'): test = xmlio.Element('test') test.attr['fixture'] = testcase.attr['classname'] - test.attr['duration'] = testcase.attr['time'] - if src: - test.attr['file'] = java_src(src, - testcase.attr['classname']) + if 'time' in testcase.attr: + test.attr['duration'] = testcase.attr['time'] + if srcdir: + cls = testcase.attr['classname'].split('.') + test.attr['file'] = posixpath.join(srcdir, *cls) + \ + '.java' result = list(testcase.children()) if result: test.attr['status'] = result[0].name test.append(xmlio.Element('traceback')[ - result[0].gettext() - ]) + result[0].gettext() + ]) failed += 1 else: test.attr['status'] = 'success' @@ -109,13 +109,12 @@ results.append(test) total += 1 finally: - fd.close() + fileobj.close() if failed: ctxt.error('%d of %d test%s failed' % (failed, total, total != 1 and 's' or '')) ctxt.report('test', results) except IOError, e: - log.warning('Error opening junit results file (%s)', e) + log.warning('Error opening JUnit results file (%s)', e) except xmlio.ParseError, e: - log.warning('Error parsing junit results file (%s)', e) - + log.warning('Error parsing JUnit results file (%s)', e)
--- a/bitten/build/pythontools.py +++ b/bitten/build/pythontools.py @@ -85,8 +85,8 @@ return from bitten.build import shtools - shtools.exec_(ctxt, executable=_python_path(ctxt), file_=file_, - output=output, args=args) + shtools.execute(ctxt, executable=_python_path(ctxt), file_=file_, + output=output, args=args) def pylint(ctxt, file_=None): """Extract data from a `pylint` run written to a file."""
--- a/bitten/build/shtools.py +++ b/bitten/build/shtools.py @@ -21,57 +21,34 @@ assert executable or file_, \ 'Either "executable" or "file" attribute required' - if args: - args = shlex.split(args) - else: - args = [] - - if executable is None: - executable = file_ - elif file_: - args[:0] = [file_] - - output_file = None - if output: - output = ctxt.resolve(output) - output_file = file(output, 'w') - - try: - cmdline = CommandLine(executable, args, cwd=ctxt.basedir) - log_elem = xmlio.Fragment() - for out, err in cmdline.execute(): - if out is not None: - log.info(out) - log_elem.append(xmlio.Element('message', level='info')[out]) - if output: - output_file.write(out + os.linesep) - if err is not None: - log.error(err) - log_elem.append(xmlio.Element('message', level='error')[err]) - if output: - output_file.write(err + os.linesep) - ctxt.log(log_elem) - finally: - if output: - output_file.close() - - if cmdline.returncode != 0: - ctxt.error('Executing %s failed (%s)' % (executable, - cmdline.returncode)) + returncode = execute(ctxt, executable=executable, file_=file_, + output=output, args=args) + if returncode != 0: + ctxt.error('Executing %s failed (%s)' % (executable or file_, + returncode)) def pipe(ctxt, executable=None, file_=None, input_=None, output=None, args=None): """Pipe the contents of a file through a script.""" assert executable or file_, \ 'Either "executable" or "file" attribute required' - assert input_, 'Missing required attribute "file"' + assert input_, 'Missing required attribute "input"' + returncode = execute(ctxt, executable=executable, file_=file_, + input_=input_, output=output, args=args) + if returncode != 0: + ctxt.error('Piping through %s failed (%s)' % (executable or file_, + returncode)) + +def execute(ctxt, executable=None, file_=None, input_=None, output=None, + args=None): + """Generic external program execution.""" if args: args = shlex.split(args) else: args = [] - if os.path.isfile(ctxt.resolve(file_)): + if file_ and os.path.isfile(ctxt.resolve(file_)): file_ = ctxt.resolve(file_) if executable is None: @@ -79,7 +56,10 @@ elif file_: args[:0] = [file_] - input_file = file(ctxt.resolve(input_), 'r') + if input_: + input_file = file(ctxt.resolve(input_), 'r') + else: + input_file = None output_file = None if output: @@ -93,20 +73,25 @@ for out, err in cmdline.execute(): if out is not None: log.info(out) - log_elem.append(xmlio.Element('message', level='info')[out]) + log_elem.append(xmlio.Element('message', level='info')[ + out.replace(ctxt.basedir + os.sep, '') + .replace(ctxt.basedir, '') + ]) if output: output_file.write(out + os.linesep) if err is not None: log.error(err) - log_elem.append(xmlio.Element('message', level='error')[err]) + log_elem.append(xmlio.Element('message', level='error')[ + err.replace(ctxt.basedir + os.sep, '') + .replace(ctxt.basedir, '') + ]) if output: output_file.write(err + os.linesep) ctxt.log(log_elem) finally: - input_file.close() + if input_: + input_file.close() if output: output_file.close() - if cmdline.returncode != 0: - ctxt.error('Piping through %s failed (%s)' % (executable, - cmdline.returncode)) + return cmdline.returncode
--- a/bitten/queue.py +++ b/bitten/queue.py @@ -18,7 +18,7 @@ log = logging.getLogger('bitten.queue') -def collect_changes(repos, config): +def collect_changes(repos, config, db=None): """Collect all changes for a build configuration that either have already been built, or still need to be built. @@ -27,6 +27,8 @@ the changeset, and `build` is a `Build` object or `None`. """ env = config.env + if not db: + db = env.get_db_cnx() node = repos.get_node(config.path) for path, rev, chg in node.get_history(): @@ -53,8 +55,9 @@ # For every target platform, check whether there's a build # of this revision - for platform in TargetPlatform.select(env, config.name): - builds = list(Build.select(env, config.name, rev, platform.id)) + for platform in TargetPlatform.select(env, config.name, db=db): + builds = list(Build.select(env, config.name, rev, platform.id, + db=db)) if builds: build = builds[0] else: @@ -150,8 +153,11 @@ try: repos.sync() - for config in BuildConfig.select(self.env): - for platform, rev, build in collect_changes(repos, config): + db = self.env.get_db_cnx() + build = None + insert_build = False + for config in BuildConfig.select(self.env, db=db): + for platform, rev, build in collect_changes(repos, config, db): if build is None: log.info('Enqueuing build of configuration "%s" at ' 'revision [%s] on %s', config.name, rev, @@ -161,8 +167,10 @@ build.rev = str(rev) build.rev_time = repos.get_changeset(rev).date build.platform = platform.id - build.insert() + insert_build = True break + if insert_build: + build.insert(db=db) finally: repos.close()
--- a/bitten/trac_ext/api.py +++ b/bitten/trac_ext/api.py @@ -40,11 +40,11 @@ """Extension point interface for components that format build log messages.""" - def get_formatter(req, build, step, type): + def get_formatter(req, build): """Return a function that gets called for every log message. - The function must take two positional arguments, `level` and `message`, - and return the formatted message. + The function must take four positional arguments, `step`, `generator`, + `level` and `message`, and return the formatted message. """
--- a/bitten/trac_ext/htdocs/bitten.css +++ b/bitten/trac_ext/htdocs/bitten.css @@ -57,6 +57,8 @@ #content.build div.platforms { margin-top: 2em; } #content.build form.platforms ul { list-style-type: none; padding-left: 1em; } +#content.build p.path { color: #999; font-size: smaller; margin-top: 0; } + #content.build #charts { clear: right; float: right; width: 44%; } #content.build #builds { clear: none; margin-top: 2em; table-layout: fixed;
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -8,6 +8,7 @@ # are also available at http://bitten.cmlenz.net/wiki/License. from datetime import datetime, timedelta +import posixpath import re try: set @@ -560,6 +561,15 @@ 'href': self.env.href.build(config.name) } + formatters = [] + for formatter in self.log_formatters: + formatters.append(formatter.get_formatter(req, build)) + + summarizers = {} # keyed by report type + for summarizer in self.report_summarizers: + categories = summarizer.get_supported_categories() + summarizers.update(dict([(cat, summarizer) for cat in categories])) + req.hdf['build'] = _build_to_hdf(self.env, req, build) steps = [] for step in BuildStep.select(self.env, build=build.id, db=db): @@ -568,8 +578,9 @@ 'duration': pretty_timedelta(step.started, step.stopped), 'failed': step.status == BuildStep.FAILURE, 'errors': step.errors, - 'log': self._render_log(req, build, step), - 'reports': self._render_reports(req, config, build, step) + 'log': self._render_log(req, build, formatters, step), + 'reports': self._render_reports(req, config, build, summarizers, + step) }) req.hdf['build.steps'] = steps req.hdf['build.can_delete'] = req.perm.has_permission('BUILD_DELETE') @@ -668,25 +679,16 @@ req.redirect(self.env.href.build(build.config)) - def _render_log(self, req, build, step): + def _render_log(self, req, build, formatters, step): items = [] for log in BuildLog.select(self.env, build=build.id, step=step.name): - formatters = [] - for formatter in self.log_formatters: - formatters.append(formatter.get_formatter(req, build, step, - log.generator)) for level, message in log.messages: for format in formatters: - message = format(level, message) + message = format(step, log.generator, level, message) items.append({'level': level, 'message': message}) return items - def _render_reports(self, req, config, build, step): - summarizers = {} # keyed by report type - for summarizer in self.report_summarizers: - categories = summarizer.get_supported_categories() - summarizers.update(dict([(cat, summarizer) for cat in categories])) - + def _render_reports(self, req, config, build, summarizers, step): reports = [] for report in Report.select(self.env, build=build.id, step=step.name): summarizer = summarizers.get(report.category) @@ -700,33 +702,39 @@ class SourceFileLinkFormatter(Component): - """Finds references to files and directories in the repository in the build - log and renders them as links to the repository browser.""" + """Detects references to files in the build log and renders them as links + to the repository browser.""" implements(ILogFormatter) - def get_formatter(self, req, build, step, type): - config = BuildConfig.fetch(self.env, build.config) + _fileref_re = re.compile('(?P<path>[\w.-]+(?:/[\w.-]+)+)(?P<line>(:\d+))') + + def get_formatter(self, req, build): + """Return the log message formatter function.""" + config = BuildConfig.fetch(self.env, name=build.config) repos = self.env.get_repository(req.authname) - nodes = [] - def _walk(node): - for child in node.get_entries(): - path = child.path[len(config.path) + 1:] - pattern = re.compile("([\s'\"])(%s|%s)([\s'\"])" - % (re.escape(path), - re.escape(path.replace('/', '\\')))) - nodes.append((child.path, pattern)) - if child.isdir: - _walk(child) - _walk(repos.get_node(config.path, build.rev)) - nodes.sort(lambda x, y: -cmp(len(x[0]), len(y[0]))) - - def _formatter(level, message): - for path, pattern in nodes: - def _replace(m): - return '%s<a href="%s">%s</a>%s' % (m.group(1), - self.env.href.browser(path, rev=build.rev), - m.group(2), m.group(3)) - message = pattern.sub(_replace, message) - return message + href = self.env.href.browser + cache = {} + def _replace(m): + filepath = posixpath.normpath(m.group('path').replace('\\', '/')) + if not cache.get(filepath) is True: + parts = filepath.split('/') + path = '' + for part in parts: + path = posixpath.join(path, part) + if not path in cache: + try: + self.log.debug('Cache miss for "%s" (%s)' % (path, m.group(0))) + repos.get_node(posixpath.join(config.path, path), + build.rev) + cache[path] = True + except TracError: + cache[path] = False + if cache[path] is False: + return m.group(0) + return '<a href="%s">%s</a>' % ( + href(m.group('path')) + '#L' + m.group('line')[1:], + m.group(0)) + def _formatter(step, type, level, message): + return self._fileref_re.sub(_replace, message) return _formatter