Mercurial > bitten > bitten-test
changeset 652:de04ce69da53
0.6dev: Removing code and updated docs related to Trac < 0.11 and Python < 2.4 (base requirements for Bitten 0.6).
| author | osimons |
|---|---|
| date | Tue, 25 Aug 2009 00:16:58 +0000 |
| parents | 44a862c1e559 |
| children | 166e1d272623 |
| files | README.txt bitten/build/api.py bitten/report/coverage.py bitten/slave.py |
| diffstat | 4 files changed, 202 insertions(+), 363 deletions(-) [+] |
line wrap: on
line diff
--- a/README.txt +++ b/README.txt @@ -6,21 +6,16 @@ metrics generated by builds, to enable feedback and reporting about the progress of a software project. -The Bitten software consists of three separate parts: - * The build slave, which executes builds on behalf of a local or remote - build master - * The build master, which orchestrates builds for a project across all - connected slaves, and stores the build status and results to the - database +The Bitten software consists of two separate parts: + * The build slave, which executes builds on behalf of a build master. * The web interface, which is implemented as an add-on to Trac (http://trac.edgewall.com/) and provides a build management interface as well as presentation of build results. -Both the build master and the web interface depend on Trac 0.10, and need -to be installed on the same machine, together with the Subversion -repository. The build slave only requires Python (>= 2.3), setuptools -(>= 0.6a2), as well as any tools required by the build process itself. A -build slave may be run on any machine that can connect to the server +The build master is a plugin for Trac 0.11, while the build slave can be +installed without other dependencies than Python >= 2.4 and +setuptools >= 0.6a2 in addition to any tools required by the build process +itself. A build slave may be run on any machine that can connect to the server running the Bitten build master.
--- a/bitten/build/api.py +++ b/bitten/build/api.py @@ -15,12 +15,9 @@ import os import shlex import time +import subprocess import sys -try: - import subprocess -except ImportError: - subprocess = None log = logging.getLogger('bitten.build.api') @@ -87,236 +84,99 @@ assert os.path.isdir(self.cwd) self.returncode = None - if subprocess: - - def execute(self, timeout=None): - """Execute the command, and return a generator for iterating over - the output written to the standard output and error streams. - - :param timeout: number of seconds before the external process - should be aborted (not supported on Windows without - ``subprocess`` module / Python 2.4+) - """ - from threading import Thread - from Queue import Queue, Empty - - class ReadThread(Thread): - def __init__(self, pipe, pipe_name, queue): - self.pipe = pipe - self.pipe_name = pipe_name - self.queue = queue - Thread.__init__(self) - def run(self): - while self.pipe and not self.pipe.closed: - line = self.pipe.readline() - if line == '': - break - self.queue.put((self.pipe_name, line)) - if not self.pipe.closed: - self.pipe.close() - - class WriteThread(Thread): - def __init__(self, pipe, data): - self.pipe = pipe - self.data = data - Thread.__init__(self) - def run(self): - if self.data and self.pipe and not self.pipe.closed: - self.pipe.write(self.data) - if not self.pipe.closed: - self.pipe.close() - - args = [self.executable] + self.arguments - try: - p = subprocess.Popen(args, bufsize=1, # Line buffered - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=(self.cwd or None), - shell=(os.name == 'nt' and True or False), - universal_newlines=True, - env=None) - except Exception, e: - # NT executes through shell and will not raise BuildError - raise BuildError('Error executing %s: %s %s' % (args, - e.__class__.__name__, str(e))) - - log.debug('Executing %s, (pid = %s)', args, p.pid) - - if self.input: - if isinstance(self.input, basestring): - in_data = self.input - else: - in_data = self.input.read() - else: - in_data = None - - queue = Queue() - limit = timeout and timeout + time.time() or 0 - - pipe_in = WriteThread(p.stdin, in_data) - pipe_out = ReadThread(p.stdout, 'stdout', queue) - pipe_err = ReadThread(p.stderr, 'stderr', queue) - pipe_err.start(); pipe_out.start(); pipe_in.start() - - while True: - if limit and limit < time.time(): - if hasattr(subprocess, 'kill'): # Python 2.6+ - p.kill() - raise TimeoutError('Command %s timed out' % self.executable) - if p.poll() != None and self.returncode == None: - self.returncode = p.returncode - try: - name, line = queue.get(block=True, timeout=.01) - line = line and _decode(line.rstrip().replace('\x00', '')) - if name == 'stderr': - yield (None, line) - else: - yield (line, None) - except Empty: - if self.returncode != None: - break - - pipe_out.join(); pipe_in.join(); pipe_err.join() - - log.debug('%s exited with code %s', self.executable, - self.returncode) - - elif os.name == 'nt': # windows - - def execute(self, timeout=None): - """Execute the command, and return a generator for iterating over - the output written to the standard output and error streams. - - :param timeout: number of seconds before the external process - should be aborted (not supported on Windows without - ``subprocess`` module / Python 2.4+) - """ - args = [self.executable] + self.arguments - for idx, arg in enumerate(args): - if arg.find(' ') >= 0: - args[idx] = '"%s"' % arg - log.debug('Executing %s', args) - - if self.cwd: - old_cwd = os.getcwd() - os.chdir(self.cwd) - import tempfile - in_name = None - if self.input: - if isinstance(self.input, basestring): - in_file, in_name = tempfile.mkstemp(prefix='bitten_', - suffix='.pipe') - os.write(in_file, self.input) - os.close(in_file) - in_redirect = '< "%s" ' % in_name - else: - in_redirect = '< "%s" ' % self.input.name - else: - in_redirect = '' - - out_file, out_name = tempfile.mkstemp(prefix='bitten_', - suffix='.pipe') - os.close(out_file) - - try: - # NT without subprocess joins output from stdout & stderr - cmd = '( %s ) > "%s" %s 2>&1' % (' '.join(args), out_name, - in_redirect) - log.info("running: %s", cmd) - self.returncode = os.system(cmd) - log.debug('Exited with code %s', self.returncode) - - out_file = file(out_name, 'r') - out_lines = out_file.readlines() - err_lines = [] - out_file.close() - finally: - if in_name: - os.unlink(in_name) - if out_name: - os.unlink(out_name) - if self.cwd: - os.chdir(old_cwd) - - for out_line, err_line in _combine(out_lines, err_lines): - yield out_line and _decode( - out_line.rstrip().replace('\x00', '')), \ - err_line and _decode( - err_line.rstrip().replace('\x00', '')) - - if self.cwd: - os.chdir(old_cwd) - - else: # posix + def execute(self, timeout=None): + """Execute the command, and return a generator for iterating over + the output written to the standard output and error streams. + + :param timeout: number of seconds before the external process + should be aborted (not supported on Windows without + ``subprocess`` module / Python 2.4+) + """ + from threading import Thread + from Queue import Queue, Empty - def execute(self, timeout=None): - """Execute the command, and return a generator for iterating over - the output written to the standard output and error streams. - - :param timeout: number of seconds before the external process - should be aborted (not supported on Windows without - ``subprocess`` module / Python 2.4+) - """ - import popen2, select - if self.cwd: - old_cwd = os.getcwd() - os.chdir(self.cwd) - - log.debug('Executing %s', [self.executable] + self.arguments) - pipe = popen2.Popen3([self.executable] + self.arguments, - capturestderr=True) - if self.input: - if isinstance(self.input, basestring): - in_data = self.input - else: - in_data = self.input.read() - else: - pipe.tochild.close() - in_data = '' + class ReadThread(Thread): + def __init__(self, pipe, pipe_name, queue): + self.pipe = pipe + self.pipe_name = pipe_name + self.queue = queue + Thread.__init__(self) + def run(self): + while self.pipe and not self.pipe.closed: + line = self.pipe.readline() + if line == '': + break + self.queue.put((self.pipe_name, line)) + if not self.pipe.closed: + self.pipe.close() - out_data, err_data = [], [] - in_eof = out_eof = err_eof = False - if not in_data: - in_eof = True - while not out_eof or not err_eof: - readable = [pipe.fromchild] * (not out_eof) + \ - [pipe.childerr] * (not err_eof) - writable = [pipe.tochild] * (not in_eof) - ready = select.select(readable, writable, [], timeout) - if not (ready[0] or ready[1]): - raise TimeoutError('Command %s timed out' % self.executable) - if pipe.tochild in ready[1]: - sent = os.write(pipe.tochild.fileno(), in_data) - in_data = in_data[sent:] - if not in_data: - pipe.tochild.close() - in_eof = True - if pipe.fromchild in ready[0]: - data = os.read(pipe.fromchild.fileno(), 1024) - if data: - out_data.append(data) - else: - out_eof = True - if pipe.childerr in ready[0]: - data = os.read(pipe.childerr.fileno(), 1024) - if data: - err_data.append(data) - else: - err_eof = True - out_lines = self._extract_lines(out_data) - err_lines = self._extract_lines(err_data) - for out_line, err_line in _combine(out_lines, err_lines): - yield out_line and _decode(out_line), \ - err_line and _decode(err_line) - time.sleep(.1) - self.returncode = pipe.wait() - log.debug('%s exited with code %s', self.executable, - self.returncode) + class WriteThread(Thread): + def __init__(self, pipe, data): + self.pipe = pipe + self.data = data + Thread.__init__(self) + def run(self): + if self.data and self.pipe and not self.pipe.closed: + self.pipe.write(self.data) + if not self.pipe.closed: + self.pipe.close() - if self.cwd: - os.chdir(old_cwd) + args = [self.executable] + self.arguments + try: + p = subprocess.Popen(args, bufsize=1, # Line buffered + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=(self.cwd or None), + shell=(os.name == 'nt' and True or False), + universal_newlines=True, + env=None) + except Exception, e: + # NT executes through shell and will not raise BuildError + raise BuildError('Error executing %s: %s %s' % (args, + e.__class__.__name__, str(e))) + + log.debug('Executing %s, (pid = %s)', args, p.pid) + + if self.input: + if isinstance(self.input, basestring): + in_data = self.input + else: + in_data = self.input.read() + else: + in_data = None + + queue = Queue() + limit = timeout and timeout + time.time() or 0 + + pipe_in = WriteThread(p.stdin, in_data) + pipe_out = ReadThread(p.stdout, 'stdout', queue) + pipe_err = ReadThread(p.stderr, 'stderr', queue) + pipe_err.start(); pipe_out.start(); pipe_in.start() + + while True: + if limit and limit < time.time(): + if hasattr(subprocess, 'kill'): # Python 2.6+ + p.kill() + raise TimeoutError('Command %s timed out' % self.executable) + if p.poll() != None and self.returncode == None: + self.returncode = p.returncode + try: + name, line = queue.get(block=True, timeout=.01) + line = line and _decode(line.rstrip().replace('\x00', '')) + if name == 'stderr': + yield (None, line) + else: + yield (line, None) + except Empty: + if self.returncode != None: + break + + pipe_out.join(); pipe_in.join(); pipe_err.join() + + log.debug('%s exited with code %s', self.executable, + self.returncode) def _extract_lines(self, data): extracted = []
--- a/bitten/report/coverage.py +++ b/bitten/report/coverage.py @@ -127,113 +127,111 @@ } -# Coverage annotation requires the new interface from 0.11 -if hasattr(IHTMLPreviewAnnotator, 'get_annotation_data'): - class TestCoverageAnnotator(Component): - """ - >>> from genshi.builder import tag - >>> from trac.test import Mock, MockPerm - >>> from trac.mimeview import Context - >>> from trac.web.href import Href - >>> from bitten.model import BuildConfig, Build, Report - >>> from bitten.report.tests.coverage import env_stub_with_tables - >>> env = env_stub_with_tables() - - >>> BuildConfig(env, name='trunk', path='trunk').insert() - >>> Build(env, rev=123, config='trunk', rev_time=12345, platform=1).insert() - >>> rpt = Report(env, build=1, step='test', category='coverage') - >>> rpt.items.append({'file': 'foo.py', 'line_hits': '5 - 0'}) - >>> rpt.insert() - - >>> ann = TestCoverageAnnotator(env) - >>> req = Mock(href=Href('/'), perm=MockPerm(), chrome={}) - - Version in the branch should not match: - >>> context = Context.from_request(req, 'source', '/branches/blah/foo.py', 123) - >>> ann.get_annotation_data(context) - [] - - Version in the trunk should match: - >>> context = Context.from_request(req, 'source', '/trunk/foo.py', 123) - >>> data = ann.get_annotation_data(context) - >>> print data - [u'5', u'-', u'0'] - - >>> def annotate_row(lineno, line): - ... row = tag.tr() - ... ann.annotate_row(context, row, lineno, line, data) - ... return row.generate().render('html') - - >>> annotate_row(1, 'x = 1') - '<tr><th class="covered">5</th></tr>' - >>> annotate_row(2, '') - '<tr><th></th></tr>' - >>> annotate_row(3, 'y = x') - '<tr><th class="uncovered">0</th></tr>' - """ - implements(IRequestFilter, IHTMLPreviewAnnotator) - - # IRequestFilter methods - - def pre_process_request(self, req, handler): - return handler +class TestCoverageAnnotator(Component): + """ + >>> from genshi.builder import tag + >>> from trac.test import Mock, MockPerm + >>> from trac.mimeview import Context + >>> from trac.web.href import Href + >>> from bitten.model import BuildConfig, Build, Report + >>> from bitten.report.tests.coverage import env_stub_with_tables + >>> env = env_stub_with_tables() - def post_process_request(self, req, template, data, content_type): - """ Adds a 'Coverage' context navigation menu item. """ - resource = data and data.get('context') \ - and data.get('context').resource or None - if resource and isinstance(resource, Resource) \ - and resource.realm=='source' and data.get('file') \ - and not req.args.get('annotate'): - add_ctxtnav(req, 'Coverage', - title='Annotate file with test coverage ' - 'data (if available)', - href=req.href.browser(resource.id, - annotate='coverage', rev=data.get('rev'))) - return template, data, content_type - - # IHTMLPreviewAnnotator methods - - def get_annotation_type(self): - return 'coverage', 'Cov', 'Code coverage' - - def get_annotation_data(self, context): - add_stylesheet(context.req, 'bitten/bitten_coverage.css') + >>> BuildConfig(env, name='trunk', path='trunk').insert() + >>> Build(env, rev=123, config='trunk', rev_time=12345, platform=1).insert() + >>> rpt = Report(env, build=1, step='test', category='coverage') + >>> rpt.items.append({'file': 'foo.py', 'line_hits': '5 - 0'}) + >>> rpt.insert() - resource = context.resource - self.log.debug("Looking for coverage report for %s@%s..." % ( - resource.id, str(resource.version))) - builds = Build.select(self.env, rev=resource.version) - reports = [] - for build in builds: - config = BuildConfig.fetch(self.env, build.config) - if not resource.id.startswith('/' + config.path.lstrip('/')): - continue - reports = Report.select(self.env, build=build.id, - category='coverage') - path_in_config = resource.id[len(config.path)+1:].lstrip('/') - for report in reports: - for item in report.items: - if item.get('file') == path_in_config: - coverage = item.get('line_hits', '').split() - if coverage: - # Return first result with line data - self.log.debug( - "Coverage annotate for %s@%s: %s" % \ - (resource.id, resource.version, coverage)) - return coverage - return [] + >>> ann = TestCoverageAnnotator(env) + >>> req = Mock(href=Href('/'), perm=MockPerm(), chrome={}) - def annotate_row(self, context, row, lineno, line, data): - from genshi.builder import tag - lineno -= 1 # 0-based index for data - if lineno >= len(data): - row.append(tag.th()) - return - row_data = data[lineno] - if row_data == '-': - row.append(tag.th()) - elif row_data == '0': - row.append(tag.th(row_data, class_='uncovered')) - else: - row.append(tag.th(row_data, class_='covered')) + Version in the branch should not match: + >>> context = Context.from_request(req, 'source', '/branches/blah/foo.py', 123) + >>> ann.get_annotation_data(context) + [] + + Version in the trunk should match: + >>> context = Context.from_request(req, 'source', '/trunk/foo.py', 123) + >>> data = ann.get_annotation_data(context) + >>> print data + [u'5', u'-', u'0'] + + >>> def annotate_row(lineno, line): + ... row = tag.tr() + ... ann.annotate_row(context, row, lineno, line, data) + ... return row.generate().render('html') + + >>> annotate_row(1, 'x = 1') + '<tr><th class="covered">5</th></tr>' + >>> annotate_row(2, '') + '<tr><th></th></tr>' + >>> annotate_row(3, 'y = x') + '<tr><th class="uncovered">0</th></tr>' + """ + implements(IRequestFilter, IHTMLPreviewAnnotator) + + # IRequestFilter methods + + def pre_process_request(self, req, handler): + return handler + + def post_process_request(self, req, template, data, content_type): + """ Adds a 'Coverage' context navigation menu item. """ + resource = data and data.get('context') \ + and data.get('context').resource or None + if resource and isinstance(resource, Resource) \ + and resource.realm=='source' and data.get('file') \ + and not req.args.get('annotate'): + add_ctxtnav(req, 'Coverage', + title='Annotate file with test coverage ' + 'data (if available)', + href=req.href.browser(resource.id, + annotate='coverage', rev=data.get('rev'))) + return template, data, content_type + + # IHTMLPreviewAnnotator methods + + def get_annotation_type(self): + return 'coverage', 'Cov', 'Code coverage' + + def get_annotation_data(self, context): + add_stylesheet(context.req, 'bitten/bitten_coverage.css') + + resource = context.resource + self.log.debug("Looking for coverage report for %s@%s..." % ( + resource.id, str(resource.version))) + builds = Build.select(self.env, rev=resource.version) + reports = [] + for build in builds: + config = BuildConfig.fetch(self.env, build.config) + if not resource.id.startswith('/' + config.path.lstrip('/')): + continue + reports = Report.select(self.env, build=build.id, + category='coverage') + path_in_config = resource.id[len(config.path)+1:].lstrip('/') + for report in reports: + for item in report.items: + if item.get('file') == path_in_config: + coverage = item.get('line_hits', '').split() + if coverage: + # Return first result with line data + self.log.debug( + "Coverage annotate for %s@%s: %s" % \ + (resource.id, resource.version, coverage)) + return coverage + return [] + + def annotate_row(self, context, row, lineno, line, data): + from genshi.builder import tag + lineno -= 1 # 0-based index for data + if lineno >= len(data): + row.append(tag.th()) + return + row_data = data[lineno] + if row_data == '-': + row.append(tag.th()) + elif row_data == '0': + row.append(tag.th(row_data, class_='uncovered')) + else: + row.append(tag.th(row_data, class_='covered'))
--- a/bitten/slave.py +++ b/bitten/slave.py @@ -61,20 +61,6 @@ else: return False -# Python 2.3 doesn't include HTTPErrorProcessor in urllib2. So instead of deriving we just make our own one -class SaneHTTPErrorProcessor(urllib2.BaseHandler): - "The HTTPErrorProcessor defined in urllib needs some love." - - handler_order = 1000 - - def http_response(self, request, response): - code, msg, hdrs = response.code, response.msg, response.info() - if code >= 300: - response = self.parent.error( - 'http', request, response, code, msg, hdrs) - return response - - https_response = http_response class SaneHTTPRequest(urllib2.Request): @@ -163,7 +149,7 @@ self.auth_map = dict(map(lambda x: (x, False), urls)) def _get_opener(self): - opener = urllib2.build_opener(SaneHTTPErrorProcessor) + opener = urllib2.build_opener(urllib2.HTTPErrorProcessor()) opener.add_handler(urllib2.HTTPBasicAuthHandler(self.password_mgr)) opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr)) opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar))