# HG changeset patch
# User osimons
# Date 1251159418 0
# Node ID de04ce69da53bcac4bf987ea3c2113f23d4c0ed5
# Parent 44a862c1e559fe66d6a935af49539135df732732
0.6dev: Removing code and updated docs related to Trac < 0.11 and Python < 2.4 (base requirements for Bitten 0.6).
diff --git a/README.txt b/README.txt
--- a/README.txt
+++ b/README.txt
@@ -6,21 +6,16 @@
metrics generated by builds, to enable feedback and reporting about
the progress of a software project.
-The Bitten software consists of three separate parts:
- * The build slave, which executes builds on behalf of a local or remote
- build master
- * The build master, which orchestrates builds for a project across all
- connected slaves, and stores the build status and results to the
- database
+The Bitten software consists of two separate parts:
+ * The build slave, which executes builds on behalf of a build master.
* The web interface, which is implemented as an add-on to Trac
(http://trac.edgewall.com/) and provides a build management interface
as well as presentation of build results.
-Both the build master and the web interface depend on Trac 0.10, and need
-to be installed on the same machine, together with the Subversion
-repository. The build slave only requires Python (>= 2.3), setuptools
-(>= 0.6a2), as well as any tools required by the build process itself. A
-build slave may be run on any machine that can connect to the server
+The build master is a plugin for Trac 0.11, while the build slave can be
+installed without other dependencies than Python >= 2.4 and
+setuptools >= 0.6a2 in addition to any tools required by the build process
+itself. A build slave may be run on any machine that can connect to the server
running the Bitten build master.
diff --git a/bitten/build/api.py b/bitten/build/api.py
--- a/bitten/build/api.py
+++ b/bitten/build/api.py
@@ -15,12 +15,9 @@
import os
import shlex
import time
+import subprocess
import sys
-try:
- import subprocess
-except ImportError:
- subprocess = None
log = logging.getLogger('bitten.build.api')
@@ -87,236 +84,99 @@
assert os.path.isdir(self.cwd)
self.returncode = None
- if subprocess:
-
- def execute(self, timeout=None):
- """Execute the command, and return a generator for iterating over
- the output written to the standard output and error streams.
-
- :param timeout: number of seconds before the external process
- should be aborted (not supported on Windows without
- ``subprocess`` module / Python 2.4+)
- """
- from threading import Thread
- from Queue import Queue, Empty
-
- class ReadThread(Thread):
- def __init__(self, pipe, pipe_name, queue):
- self.pipe = pipe
- self.pipe_name = pipe_name
- self.queue = queue
- Thread.__init__(self)
- def run(self):
- while self.pipe and not self.pipe.closed:
- line = self.pipe.readline()
- if line == '':
- break
- self.queue.put((self.pipe_name, line))
- if not self.pipe.closed:
- self.pipe.close()
-
- class WriteThread(Thread):
- def __init__(self, pipe, data):
- self.pipe = pipe
- self.data = data
- Thread.__init__(self)
- def run(self):
- if self.data and self.pipe and not self.pipe.closed:
- self.pipe.write(self.data)
- if not self.pipe.closed:
- self.pipe.close()
-
- args = [self.executable] + self.arguments
- try:
- p = subprocess.Popen(args, bufsize=1, # Line buffered
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- cwd=(self.cwd or None),
- shell=(os.name == 'nt' and True or False),
- universal_newlines=True,
- env=None)
- except Exception, e:
- # NT executes through shell and will not raise BuildError
- raise BuildError('Error executing %s: %s %s' % (args,
- e.__class__.__name__, str(e)))
-
- log.debug('Executing %s, (pid = %s)', args, p.pid)
-
- if self.input:
- if isinstance(self.input, basestring):
- in_data = self.input
- else:
- in_data = self.input.read()
- else:
- in_data = None
-
- queue = Queue()
- limit = timeout and timeout + time.time() or 0
-
- pipe_in = WriteThread(p.stdin, in_data)
- pipe_out = ReadThread(p.stdout, 'stdout', queue)
- pipe_err = ReadThread(p.stderr, 'stderr', queue)
- pipe_err.start(); pipe_out.start(); pipe_in.start()
-
- while True:
- if limit and limit < time.time():
- if hasattr(subprocess, 'kill'): # Python 2.6+
- p.kill()
- raise TimeoutError('Command %s timed out' % self.executable)
- if p.poll() != None and self.returncode == None:
- self.returncode = p.returncode
- try:
- name, line = queue.get(block=True, timeout=.01)
- line = line and _decode(line.rstrip().replace('\x00', ''))
- if name == 'stderr':
- yield (None, line)
- else:
- yield (line, None)
- except Empty:
- if self.returncode != None:
- break
-
- pipe_out.join(); pipe_in.join(); pipe_err.join()
-
- log.debug('%s exited with code %s', self.executable,
- self.returncode)
-
- elif os.name == 'nt': # windows
-
- def execute(self, timeout=None):
- """Execute the command, and return a generator for iterating over
- the output written to the standard output and error streams.
-
- :param timeout: number of seconds before the external process
- should be aborted (not supported on Windows without
- ``subprocess`` module / Python 2.4+)
- """
- args = [self.executable] + self.arguments
- for idx, arg in enumerate(args):
- if arg.find(' ') >= 0:
- args[idx] = '"%s"' % arg
- log.debug('Executing %s', args)
-
- if self.cwd:
- old_cwd = os.getcwd()
- os.chdir(self.cwd)
- import tempfile
- in_name = None
- if self.input:
- if isinstance(self.input, basestring):
- in_file, in_name = tempfile.mkstemp(prefix='bitten_',
- suffix='.pipe')
- os.write(in_file, self.input)
- os.close(in_file)
- in_redirect = '< "%s" ' % in_name
- else:
- in_redirect = '< "%s" ' % self.input.name
- else:
- in_redirect = ''
-
- out_file, out_name = tempfile.mkstemp(prefix='bitten_',
- suffix='.pipe')
- os.close(out_file)
-
- try:
- # NT without subprocess joins output from stdout & stderr
- cmd = '( %s ) > "%s" %s 2>&1' % (' '.join(args), out_name,
- in_redirect)
- log.info("running: %s", cmd)
- self.returncode = os.system(cmd)
- log.debug('Exited with code %s', self.returncode)
-
- out_file = file(out_name, 'r')
- out_lines = out_file.readlines()
- err_lines = []
- out_file.close()
- finally:
- if in_name:
- os.unlink(in_name)
- if out_name:
- os.unlink(out_name)
- if self.cwd:
- os.chdir(old_cwd)
-
- for out_line, err_line in _combine(out_lines, err_lines):
- yield out_line and _decode(
- out_line.rstrip().replace('\x00', '')), \
- err_line and _decode(
- err_line.rstrip().replace('\x00', ''))
-
- if self.cwd:
- os.chdir(old_cwd)
-
- else: # posix
+ def execute(self, timeout=None):
+ """Execute the command, and return a generator for iterating over
+ the output written to the standard output and error streams.
+
+ :param timeout: number of seconds before the external process
+ should be aborted (not supported on Windows without
+ ``subprocess`` module / Python 2.4+)
+ """
+ from threading import Thread
+ from Queue import Queue, Empty
- def execute(self, timeout=None):
- """Execute the command, and return a generator for iterating over
- the output written to the standard output and error streams.
-
- :param timeout: number of seconds before the external process
- should be aborted (not supported on Windows without
- ``subprocess`` module / Python 2.4+)
- """
- import popen2, select
- if self.cwd:
- old_cwd = os.getcwd()
- os.chdir(self.cwd)
-
- log.debug('Executing %s', [self.executable] + self.arguments)
- pipe = popen2.Popen3([self.executable] + self.arguments,
- capturestderr=True)
- if self.input:
- if isinstance(self.input, basestring):
- in_data = self.input
- else:
- in_data = self.input.read()
- else:
- pipe.tochild.close()
- in_data = ''
+ class ReadThread(Thread):
+ def __init__(self, pipe, pipe_name, queue):
+ self.pipe = pipe
+ self.pipe_name = pipe_name
+ self.queue = queue
+ Thread.__init__(self)
+ def run(self):
+ while self.pipe and not self.pipe.closed:
+ line = self.pipe.readline()
+ if line == '':
+ break
+ self.queue.put((self.pipe_name, line))
+ if not self.pipe.closed:
+ self.pipe.close()
- out_data, err_data = [], []
- in_eof = out_eof = err_eof = False
- if not in_data:
- in_eof = True
- while not out_eof or not err_eof:
- readable = [pipe.fromchild] * (not out_eof) + \
- [pipe.childerr] * (not err_eof)
- writable = [pipe.tochild] * (not in_eof)
- ready = select.select(readable, writable, [], timeout)
- if not (ready[0] or ready[1]):
- raise TimeoutError('Command %s timed out' % self.executable)
- if pipe.tochild in ready[1]:
- sent = os.write(pipe.tochild.fileno(), in_data)
- in_data = in_data[sent:]
- if not in_data:
- pipe.tochild.close()
- in_eof = True
- if pipe.fromchild in ready[0]:
- data = os.read(pipe.fromchild.fileno(), 1024)
- if data:
- out_data.append(data)
- else:
- out_eof = True
- if pipe.childerr in ready[0]:
- data = os.read(pipe.childerr.fileno(), 1024)
- if data:
- err_data.append(data)
- else:
- err_eof = True
- out_lines = self._extract_lines(out_data)
- err_lines = self._extract_lines(err_data)
- for out_line, err_line in _combine(out_lines, err_lines):
- yield out_line and _decode(out_line), \
- err_line and _decode(err_line)
- time.sleep(.1)
- self.returncode = pipe.wait()
- log.debug('%s exited with code %s', self.executable,
- self.returncode)
+ class WriteThread(Thread):
+ def __init__(self, pipe, data):
+ self.pipe = pipe
+ self.data = data
+ Thread.__init__(self)
+ def run(self):
+ if self.data and self.pipe and not self.pipe.closed:
+ self.pipe.write(self.data)
+ if not self.pipe.closed:
+ self.pipe.close()
- if self.cwd:
- os.chdir(old_cwd)
+ args = [self.executable] + self.arguments
+ try:
+ p = subprocess.Popen(args, bufsize=1, # Line buffered
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ cwd=(self.cwd or None),
+ shell=(os.name == 'nt' and True or False),
+ universal_newlines=True,
+ env=None)
+ except Exception, e:
+ # NT executes through shell and will not raise BuildError
+ raise BuildError('Error executing %s: %s %s' % (args,
+ e.__class__.__name__, str(e)))
+
+ log.debug('Executing %s, (pid = %s)', args, p.pid)
+
+ if self.input:
+ if isinstance(self.input, basestring):
+ in_data = self.input
+ else:
+ in_data = self.input.read()
+ else:
+ in_data = None
+
+ queue = Queue()
+ limit = timeout and timeout + time.time() or 0
+
+ pipe_in = WriteThread(p.stdin, in_data)
+ pipe_out = ReadThread(p.stdout, 'stdout', queue)
+ pipe_err = ReadThread(p.stderr, 'stderr', queue)
+ pipe_err.start(); pipe_out.start(); pipe_in.start()
+
+ while True:
+ if limit and limit < time.time():
+ if hasattr(subprocess, 'kill'): # Python 2.6+
+ p.kill()
+ raise TimeoutError('Command %s timed out' % self.executable)
+ if p.poll() != None and self.returncode == None:
+ self.returncode = p.returncode
+ try:
+ name, line = queue.get(block=True, timeout=.01)
+ line = line and _decode(line.rstrip().replace('\x00', ''))
+ if name == 'stderr':
+ yield (None, line)
+ else:
+ yield (line, None)
+ except Empty:
+ if self.returncode != None:
+ break
+
+ pipe_out.join(); pipe_in.join(); pipe_err.join()
+
+ log.debug('%s exited with code %s', self.executable,
+ self.returncode)
def _extract_lines(self, data):
extracted = []
diff --git a/bitten/report/coverage.py b/bitten/report/coverage.py
--- a/bitten/report/coverage.py
+++ b/bitten/report/coverage.py
@@ -127,113 +127,111 @@
}
-# Coverage annotation requires the new interface from 0.11
-if hasattr(IHTMLPreviewAnnotator, 'get_annotation_data'):
- class TestCoverageAnnotator(Component):
- """
- >>> from genshi.builder import tag
- >>> from trac.test import Mock, MockPerm
- >>> from trac.mimeview import Context
- >>> from trac.web.href import Href
- >>> from bitten.model import BuildConfig, Build, Report
- >>> from bitten.report.tests.coverage import env_stub_with_tables
- >>> env = env_stub_with_tables()
-
- >>> BuildConfig(env, name='trunk', path='trunk').insert()
- >>> Build(env, rev=123, config='trunk', rev_time=12345, platform=1).insert()
- >>> rpt = Report(env, build=1, step='test', category='coverage')
- >>> rpt.items.append({'file': 'foo.py', 'line_hits': '5 - 0'})
- >>> rpt.insert()
-
- >>> ann = TestCoverageAnnotator(env)
- >>> req = Mock(href=Href('/'), perm=MockPerm(), chrome={})
-
- Version in the branch should not match:
- >>> context = Context.from_request(req, 'source', '/branches/blah/foo.py', 123)
- >>> ann.get_annotation_data(context)
- []
-
- Version in the trunk should match:
- >>> context = Context.from_request(req, 'source', '/trunk/foo.py', 123)
- >>> data = ann.get_annotation_data(context)
- >>> print data
- [u'5', u'-', u'0']
-
- >>> def annotate_row(lineno, line):
- ... row = tag.tr()
- ... ann.annotate_row(context, row, lineno, line, data)
- ... return row.generate().render('html')
-
- >>> annotate_row(1, 'x = 1')
- '| 5 |
|---|
'
- >>> annotate_row(2, '')
- ' |
|---|
'
- >>> annotate_row(3, 'y = x')
- '| 0 |
|---|
'
- """
- implements(IRequestFilter, IHTMLPreviewAnnotator)
-
- # IRequestFilter methods
-
- def pre_process_request(self, req, handler):
- return handler
+class TestCoverageAnnotator(Component):
+ """
+ >>> from genshi.builder import tag
+ >>> from trac.test import Mock, MockPerm
+ >>> from trac.mimeview import Context
+ >>> from trac.web.href import Href
+ >>> from bitten.model import BuildConfig, Build, Report
+ >>> from bitten.report.tests.coverage import env_stub_with_tables
+ >>> env = env_stub_with_tables()
- def post_process_request(self, req, template, data, content_type):
- """ Adds a 'Coverage' context navigation menu item. """
- resource = data and data.get('context') \
- and data.get('context').resource or None
- if resource and isinstance(resource, Resource) \
- and resource.realm=='source' and data.get('file') \
- and not req.args.get('annotate'):
- add_ctxtnav(req, 'Coverage',
- title='Annotate file with test coverage '
- 'data (if available)',
- href=req.href.browser(resource.id,
- annotate='coverage', rev=data.get('rev')))
- return template, data, content_type
-
- # IHTMLPreviewAnnotator methods
-
- def get_annotation_type(self):
- return 'coverage', 'Cov', 'Code coverage'
-
- def get_annotation_data(self, context):
- add_stylesheet(context.req, 'bitten/bitten_coverage.css')
+ >>> BuildConfig(env, name='trunk', path='trunk').insert()
+ >>> Build(env, rev=123, config='trunk', rev_time=12345, platform=1).insert()
+ >>> rpt = Report(env, build=1, step='test', category='coverage')
+ >>> rpt.items.append({'file': 'foo.py', 'line_hits': '5 - 0'})
+ >>> rpt.insert()
- resource = context.resource
- self.log.debug("Looking for coverage report for %s@%s..." % (
- resource.id, str(resource.version)))
- builds = Build.select(self.env, rev=resource.version)
- reports = []
- for build in builds:
- config = BuildConfig.fetch(self.env, build.config)
- if not resource.id.startswith('/' + config.path.lstrip('/')):
- continue
- reports = Report.select(self.env, build=build.id,
- category='coverage')
- path_in_config = resource.id[len(config.path)+1:].lstrip('/')
- for report in reports:
- for item in report.items:
- if item.get('file') == path_in_config:
- coverage = item.get('line_hits', '').split()
- if coverage:
- # Return first result with line data
- self.log.debug(
- "Coverage annotate for %s@%s: %s" % \
- (resource.id, resource.version, coverage))
- return coverage
- return []
+ >>> ann = TestCoverageAnnotator(env)
+ >>> req = Mock(href=Href('/'), perm=MockPerm(), chrome={})
- def annotate_row(self, context, row, lineno, line, data):
- from genshi.builder import tag
- lineno -= 1 # 0-based index for data
- if lineno >= len(data):
- row.append(tag.th())
- return
- row_data = data[lineno]
- if row_data == '-':
- row.append(tag.th())
- elif row_data == '0':
- row.append(tag.th(row_data, class_='uncovered'))
- else:
- row.append(tag.th(row_data, class_='covered'))
+ Version in the branch should not match:
+ >>> context = Context.from_request(req, 'source', '/branches/blah/foo.py', 123)
+ >>> ann.get_annotation_data(context)
+ []
+
+ Version in the trunk should match:
+ >>> context = Context.from_request(req, 'source', '/trunk/foo.py', 123)
+ >>> data = ann.get_annotation_data(context)
+ >>> print data
+ [u'5', u'-', u'0']
+
+ >>> def annotate_row(lineno, line):
+ ... row = tag.tr()
+ ... ann.annotate_row(context, row, lineno, line, data)
+ ... return row.generate().render('html')
+
+ >>> annotate_row(1, 'x = 1')
+ '| 5 |
|---|
'
+ >>> annotate_row(2, '')
+ ' |
|---|
'
+ >>> annotate_row(3, 'y = x')
+ '| 0 |
|---|
'
+ """
+ implements(IRequestFilter, IHTMLPreviewAnnotator)
+
+ # IRequestFilter methods
+
+ def pre_process_request(self, req, handler):
+ return handler
+
+ def post_process_request(self, req, template, data, content_type):
+ """ Adds a 'Coverage' context navigation menu item. """
+ resource = data and data.get('context') \
+ and data.get('context').resource or None
+ if resource and isinstance(resource, Resource) \
+ and resource.realm=='source' and data.get('file') \
+ and not req.args.get('annotate'):
+ add_ctxtnav(req, 'Coverage',
+ title='Annotate file with test coverage '
+ 'data (if available)',
+ href=req.href.browser(resource.id,
+ annotate='coverage', rev=data.get('rev')))
+ return template, data, content_type
+
+ # IHTMLPreviewAnnotator methods
+
+ def get_annotation_type(self):
+ return 'coverage', 'Cov', 'Code coverage'
+
+ def get_annotation_data(self, context):
+ add_stylesheet(context.req, 'bitten/bitten_coverage.css')
+
+ resource = context.resource
+ self.log.debug("Looking for coverage report for %s@%s..." % (
+ resource.id, str(resource.version)))
+ builds = Build.select(self.env, rev=resource.version)
+ reports = []
+ for build in builds:
+ config = BuildConfig.fetch(self.env, build.config)
+ if not resource.id.startswith('/' + config.path.lstrip('/')):
+ continue
+ reports = Report.select(self.env, build=build.id,
+ category='coverage')
+ path_in_config = resource.id[len(config.path)+1:].lstrip('/')
+ for report in reports:
+ for item in report.items:
+ if item.get('file') == path_in_config:
+ coverage = item.get('line_hits', '').split()
+ if coverage:
+ # Return first result with line data
+ self.log.debug(
+ "Coverage annotate for %s@%s: %s" % \
+ (resource.id, resource.version, coverage))
+ return coverage
+ return []
+
+ def annotate_row(self, context, row, lineno, line, data):
+ from genshi.builder import tag
+ lineno -= 1 # 0-based index for data
+ if lineno >= len(data):
+ row.append(tag.th())
+ return
+ row_data = data[lineno]
+ if row_data == '-':
+ row.append(tag.th())
+ elif row_data == '0':
+ row.append(tag.th(row_data, class_='uncovered'))
+ else:
+ row.append(tag.th(row_data, class_='covered'))
diff --git a/bitten/slave.py b/bitten/slave.py
--- a/bitten/slave.py
+++ b/bitten/slave.py
@@ -61,20 +61,6 @@
else:
return False
-# Python 2.3 doesn't include HTTPErrorProcessor in urllib2. So instead of deriving we just make our own one
-class SaneHTTPErrorProcessor(urllib2.BaseHandler):
- "The HTTPErrorProcessor defined in urllib needs some love."
-
- handler_order = 1000
-
- def http_response(self, request, response):
- code, msg, hdrs = response.code, response.msg, response.info()
- if code >= 300:
- response = self.parent.error(
- 'http', request, response, code, msg, hdrs)
- return response
-
- https_response = http_response
class SaneHTTPRequest(urllib2.Request):
@@ -163,7 +149,7 @@
self.auth_map = dict(map(lambda x: (x, False), urls))
def _get_opener(self):
- opener = urllib2.build_opener(SaneHTTPErrorProcessor)
+ opener = urllib2.build_opener(urllib2.HTTPErrorProcessor())
opener.add_handler(urllib2.HTTPBasicAuthHandler(self.password_mgr))
opener.add_handler(urllib2.HTTPDigestAuthHandler(self.password_mgr))
opener.add_handler(urllib2.HTTPCookieProcessor(self.cookiejar))