Mercurial > bitten > bitten-test
annotate bitten/master.py @ 568:84a45f7c9833
0.6dev: Adding Bitten version info to About Trac.
| author | osimons |
|---|---|
| date | Sun, 05 Jul 2009 11:23:22 +0000 |
| parents | b4d3d9cbf200 |
| children | dcee8ff20e82 |
| rev | line source |
|---|---|
| 379 | 1 # -*- coding: utf-8 -*- |
|
13
21aa17f97522
Initial code for build master and slave... these don't do a lot yet.
cmlenz
parents:
diff
changeset
|
2 # |
|
408
933105ab516b
Update file headers and other stuff pointing to the old home.
cmlenz
parents:
402
diff
changeset
|
3 # Copyright (C) 2007 Edgewall Software |
|
933105ab516b
Update file headers and other stuff pointing to the old home.
cmlenz
parents:
402
diff
changeset
|
4 # Copyright (C) 2005-2007 Christopher Lenz <cmlenz@gmx.de> |
| 163 | 5 # All rights reserved. |
|
13
21aa17f97522
Initial code for build master and slave... these don't do a lot yet.
cmlenz
parents:
diff
changeset
|
6 # |
| 163 | 7 # This software is licensed as described in the file COPYING, which |
| 8 # you should have received as part of this distribution. The terms | |
|
408
933105ab516b
Update file headers and other stuff pointing to the old home.
cmlenz
parents:
402
diff
changeset
|
9 # are also available at http://bitten.edgewall.org/wiki/License. |
|
13
21aa17f97522
Initial code for build master and slave... these don't do a lot yet.
cmlenz
parents:
diff
changeset
|
10 |
| 392 | 11 """Build master implementation.""" |
| 313 | 12 |
| 312 | 13 import calendar |
| 392 | 14 import re |
|
47
083e848088ee
* Improvements to the model classes, and a couple of unit tests.
cmlenz
parents:
45
diff
changeset
|
15 import time |
|
18
591a5a836ecc
* {{{beep.Listener}}} now has an event loop (based on code mostly from medusa)
cmlenz
parents:
15
diff
changeset
|
16 |
|
516
2f3b7c17d3c3
Switch to storing log messages in files rather than in database rows:
dfraser
parents:
494
diff
changeset
|
17 from trac.config import BoolOption, IntOption, PathOption |
| 392 | 18 from trac.core import * |
| 19 from trac.web import IRequestHandler, HTTPBadRequest, HTTPConflict, \ | |
| 20 HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, \ | |
| 21 RequestDone | |
| 22 | |
|
554
2c27f3581100
Supply the target platform as a variable `platform` for build recipes, and adjust test accordingly - fixes #376
dfraser
parents:
542
diff
changeset
|
23 from bitten.model import BuildConfig, Build, BuildStep, BuildLog, Report, \ |
|
2c27f3581100
Supply the target platform as a variable `platform` for build recipes, and adjust test accordingly - fixes #376
dfraser
parents:
542
diff
changeset
|
24 TargetPlatform |
|
2c27f3581100
Supply the target platform as a variable `platform` for build recipes, and adjust test accordingly - fixes #376
dfraser
parents:
542
diff
changeset
|
25 |
|
410
7930cdd83d13
More restructuring: got rid of the `trac_ext` subpackage, which makes no sense now that the master is also coupled to Trac.
cmlenz
parents:
408
diff
changeset
|
26 from bitten.main import BuildSystem |
|
227
014bc6c29dff
* Factor build queue logic into a class separate from the build master.
cmlenz
parents:
213
diff
changeset
|
27 from bitten.queue import BuildQueue |
| 392 | 28 from bitten.recipe import Recipe |
| 29 from bitten.util import xmlio | |
|
83
42970c14524a
Perform slave/platform matching at slave registration. Use builtin {{{set}}} type on Python >= 2.4.
cmlenz
parents:
82
diff
changeset
|
30 |
|
411
a169d2e96463
Use reStructuredText as the API documentation syntax.
cmlenz
parents:
410
diff
changeset
|
31 __all__ = ['BuildMaster'] |
|
a169d2e96463
Use reStructuredText as the API documentation syntax.
cmlenz
parents:
410
diff
changeset
|
32 __docformat__ = 'restructuredtext en' |
|
a169d2e96463
Use reStructuredText as the API documentation syntax.
cmlenz
parents:
410
diff
changeset
|
33 |
| 56 | 34 |
| 392 | 35 class BuildMaster(Component): |
| 542 | 36 """Trac request handler implementation for the build master.""" |
|
13
21aa17f97522
Initial code for build master and slave... these don't do a lot yet.
cmlenz
parents:
diff
changeset
|
37 |
| 392 | 38 implements(IRequestHandler) |
|
51
5caccd7b247e
Proper archive format negotiation; improved representation of parsed XML content in {{{bitten.util.xmlio}}}.
cmlenz
parents:
49
diff
changeset
|
39 |
| 392 | 40 # Configuration options |
|
51
5caccd7b247e
Proper archive format negotiation; improved representation of parsed XML content in {{{bitten.util.xmlio}}}.
cmlenz
parents:
49
diff
changeset
|
41 |
| 392 | 42 adjust_timestamps = BoolOption('bitten', 'adjust_timestamps', False, doc= |
| 432 | 43 """Whether the timestamps of builds should be adjusted to be close |
| 392 | 44 to the timestamps of the corresponding changesets.""") |
| 45 | |
| 46 build_all = BoolOption('bitten', 'build_all', False, doc= | |
| 47 """Whether to request builds of older revisions even if a younger | |
| 48 revision has already been built.""") | |
|
468
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
49 |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
50 stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc= |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
51 """The time in seconds to wait for the repository to stabilize before |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
52 queuing up a new build. This allows time for developers to check in |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
53 a group of related changes back to back without spawning multiple |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
54 builds.""") |
| 392 | 55 |
| 56 slave_timeout = IntOption('bitten', 'slave_timeout', 3600, doc= | |
| 57 """The time in seconds after which a build is cancelled if the slave | |
| 58 does not report progress.""") | |
| 59 | |
|
516
2f3b7c17d3c3
Switch to storing log messages in files rather than in database rows:
dfraser
parents:
494
diff
changeset
|
60 logs_dir = PathOption('bitten', 'logs_dir', "log/bitten", doc= |
|
2f3b7c17d3c3
Switch to storing log messages in files rather than in database rows:
dfraser
parents:
494
diff
changeset
|
61 """The directory on the server in which client log files will be stored.""") |
|
2f3b7c17d3c3
Switch to storing log messages in files rather than in database rows:
dfraser
parents:
494
diff
changeset
|
62 |
|
557
b4d3d9cbf200
Alter the appearance of the ''Build Status'' button, to show the current build status [eblot] - fixes #373
dfraser
parents:
554
diff
changeset
|
63 quick_status = BoolOption('bitten', 'quick_status', False, doc= |
|
b4d3d9cbf200
Alter the appearance of the ''Build Status'' button, to show the current build status [eblot] - fixes #373
dfraser
parents:
554
diff
changeset
|
64 """Whether to show the current build status withing the Trac main |
|
b4d3d9cbf200
Alter the appearance of the ''Build Status'' button, to show the current build status [eblot] - fixes #373
dfraser
parents:
554
diff
changeset
|
65 navigation bar""") |
|
b4d3d9cbf200
Alter the appearance of the ''Build Status'' button, to show the current build status [eblot] - fixes #373
dfraser
parents:
554
diff
changeset
|
66 |
| 568 | 67 def __init__(self): |
| 68 self.env.systeminfo.append(('Bitten', | |
| 69 __import__('bitten', ['__version__']).__version__)) | |
| 70 | |
| 392 | 71 # IRequestHandler methods |
| 72 | |
| 73 def match_request(self, req): | |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
74 match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$', |
| 392 | 75 req.path_info) |
| 76 if match: | |
| 77 if match.group(1): | |
| 78 req.args['id'] = match.group(1) | |
| 79 req.args['collection'] = match.group(2) | |
| 80 req.args['member'] = match.group(3) | |
| 81 return True | |
| 82 | |
| 83 def process_request(self, req): | |
| 84 req.perm.assert_permission('BUILD_EXEC') | |
| 85 | |
| 86 if 'id' not in req.args: | |
| 87 if req.method != 'POST': | |
| 88 raise HTTPMethodNotAllowed('Method not allowed') | |
| 89 return self._process_build_creation(req) | |
| 90 | |
| 91 build = Build.fetch(self.env, req.args['id']) | |
| 92 if not build: | |
| 93 raise HTTPNotFound('No such build') | |
| 94 config = BuildConfig.fetch(self.env, build.config) | |
| 95 | |
| 96 if not req.args['collection']: | |
|
420
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
97 if req.method == 'DELETE': |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
98 return self._process_build_cancellation(req, config, build) |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
99 else: |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
100 return self._process_build_initiation(req, config, build) |
| 392 | 101 |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
102 if req.method != 'POST': |
| 392 | 103 raise HTTPMethodNotAllowed('Method not allowed') |
| 104 | |
| 105 if req.args['collection'] == 'steps': | |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
106 return self._process_build_step(req, config, build) |
| 392 | 107 else: |
| 108 raise HTTPNotFound('No such collection') | |
| 109 | |
| 110 def _process_build_creation(self, req): | |
|
468
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
111 queue = BuildQueue(self.env, build_all=self.build_all, |
|
44c2b4ac6157
Add stabilization time parameter to build master. Closes #189. Many thanks to Allen Bierbaum for the patch.
cmlenz
parents:
466
diff
changeset
|
112 stabilize_wait=self.stabilize_wait, |
|
419
b72802dc0632
Fix resetting of builds when multiple slaves are building simultaneously, and implement the `slave_timeout` trac.ini option.
cmlenz
parents:
411
diff
changeset
|
113 timeout=self.slave_timeout) |
| 392 | 114 queue.populate() |
| 115 | |
| 116 try: | |
| 117 elem = xmlio.parse(req.read()) | |
| 118 except xmlio.ParseError, e: | |
|
492
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
119 self.log.error('Error parsing build initialization request: %s', e, |
|
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
120 exc_info=True) |
| 392 | 121 raise HTTPBadRequest('XML parser error') |
| 122 | |
| 426 | 123 slavename = elem.attr['name'] |
|
463
977a6c122205
Make slave names available for use in target platform rules, and added some documentation to the admin panel. Closes #190.
cmlenz
parents:
459
diff
changeset
|
124 properties = {'name': slavename, Build.IP_ADDRESS: req.remote_addr} |
| 426 | 125 self.log.info('Build slave %r connected from %s', slavename, |
| 126 req.remote_addr) | |
| 392 | 127 |
| 128 for child in elem.children(): | |
| 129 if child.name == 'platform': | |
| 130 properties[Build.MACHINE] = child.gettext() | |
| 131 properties[Build.PROCESSOR] = child.attr.get('processor') | |
| 132 elif child.name == 'os': | |
| 133 properties[Build.OS_NAME] = child.gettext() | |
| 134 properties[Build.OS_FAMILY] = child.attr.get('family') | |
| 135 properties[Build.OS_VERSION] = child.attr.get('version') | |
| 136 elif child.name == 'package': | |
| 137 for name, value in child.attr.items(): | |
| 138 if name == 'name': | |
| 139 continue | |
| 140 properties[child.attr['name'] + '.' + name] = value | |
| 141 | |
|
444
22d6a7da8777
Another minor improvement to logging in the build master.
cmlenz
parents:
436
diff
changeset
|
142 self.log.debug('Build slave configuration: %r', properties) |
|
22d6a7da8777
Another minor improvement to logging in the build master.
cmlenz
parents:
436
diff
changeset
|
143 |
| 426 | 144 build = queue.get_build_for_slave(slavename, properties) |
| 392 | 145 if not build: |
| 146 req.send_response(204) | |
|
420
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
147 req.write('') |
| 392 | 148 raise RequestDone |
| 149 | |
| 150 req.send_response(201) | |
| 151 req.send_header('Content-Type', 'text/plain') | |
| 152 req.send_header('Location', req.abs_href.builds(build.id)) | |
| 153 req.write('Build pending') | |
| 154 raise RequestDone | |
| 155 | |
|
420
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
156 def _process_build_cancellation(self, req, config, build): |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
157 self.log.info('Build slave %r cancelled build %d', build.slave, |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
158 build.id) |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
159 build.status = Build.PENDING |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
160 build.slave = None |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
161 build.slave_info = {} |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
162 build.started = 0 |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
163 db = self.env.get_db_cnx() |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
164 for step in list(BuildStep.select(self.env, build=build.id, db=db)): |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
165 step.delete(db=db) |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
166 build.update(db=db) |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
167 db.commit() |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
168 |
| 458 | 169 for listener in BuildSystem(self.env).listeners: |
| 170 listener.build_aborted(build) | |
| 171 | |
|
420
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
172 req.send_response(204) |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
173 req.write('') |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
174 raise RequestDone |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
175 |
| 392 | 176 def _process_build_initiation(self, req, config, build): |
|
420
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
177 self.log.info('Build slave %r initiated build %d', build.slave, |
|
23de253435b8
Slaves now attempt to explicitly cancel builds when they are interrupted.
cmlenz
parents:
419
diff
changeset
|
178 build.id) |
| 392 | 179 build.started = int(time.time()) |
| 180 build.update() | |
| 277 | 181 |
| 458 | 182 for listener in BuildSystem(self.env).listeners: |
| 183 listener.build_started(build) | |
| 184 | |
|
287
6abd43d0cd8a
The build slave now stores snapshot archives and the corresponding work directories in project folders of the main work folder, to keep build configurations from different projects that share the same name separate. This also requires transmitting the project name (simply the name of the environment directory) with the build initiation.
cmlenz
parents:
284
diff
changeset
|
185 xml = xmlio.parse(config.recipe) |
| 392 | 186 xml.attr['path'] = config.path |
| 187 xml.attr['revision'] = build.rev | |
|
466
79be3c00ae69
Applied patch to #188 for stable/configurable names of build directories. Thanks to Allen Bierbaum for the patch.
cmlenz
parents:
463
diff
changeset
|
188 xml.attr['config'] = config.name |
|
79be3c00ae69
Applied patch to #188 for stable/configurable names of build directories. Thanks to Allen Bierbaum for the patch.
cmlenz
parents:
463
diff
changeset
|
189 xml.attr['build'] = str(build.id) |
|
554
2c27f3581100
Supply the target platform as a variable `platform` for build recipes, and adjust test accordingly - fixes #376
dfraser
parents:
542
diff
changeset
|
190 target_platform = TargetPlatform.fetch(self.env, build.platform) |
|
2c27f3581100
Supply the target platform as a variable `platform` for build recipes, and adjust test accordingly - fixes #376
dfraser
parents:
542
diff
changeset
|
191 xml.attr['platform'] = target_platform.name |
| 392 | 192 body = str(xml) |
|
374
446092a2d2fe
Don't accept build results from a slave if the build has been invalidated or it's being built by another slave. Closes #100.
wbell
parents:
373
diff
changeset
|
193 |
|
473
b1d346c4e539
Only accept build results from the slave that's supposed to be processing this build. Additional logging.
wbell
parents:
468
diff
changeset
|
194 self.log.info('Build slave %r initiated build %d', build.slave, |
|
b1d346c4e539
Only accept build results from the slave that's supposed to be processing this build. Additional logging.
wbell
parents:
468
diff
changeset
|
195 build.id) |
|
b1d346c4e539
Only accept build results from the slave that's supposed to be processing this build. Additional logging.
wbell
parents:
468
diff
changeset
|
196 |
| 392 | 197 req.send_response(200) |
| 198 req.send_header('Content-Type', 'application/x-bitten+xml') | |
| 199 req.send_header('Content-Length', str(len(body))) | |
| 200 req.send_header('Content-Disposition', | |
| 201 'attachment; filename=recipe_%s_r%s.xml' % | |
| 202 (config.name, build.rev)) | |
| 203 req.write(body) | |
| 204 raise RequestDone | |
|
227
014bc6c29dff
* Factor build queue logic into a class separate from the build master.
cmlenz
parents:
213
diff
changeset
|
205 |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
206 def _process_build_step(self, req, config, build): |
|
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
207 try: |
|
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
208 elem = xmlio.parse(req.read()) |
|
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
209 except xmlio.ParseError, e: |
|
492
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
210 self.log.error('Error parsing build step result: %s', e, |
|
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
211 exc_info=True) |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
212 raise HTTPBadRequest('XML parser error') |
|
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
213 stepname = elem.attr['step'] |
|
473
b1d346c4e539
Only accept build results from the slave that's supposed to be processing this build. Additional logging.
wbell
parents:
468
diff
changeset
|
214 |
|
479
07148aa7667e
Don't accept build step results for invalidated builds or from the wrong slave. Remerge of [520] with fixed tests and new functional tests.
wbell
parents:
475
diff
changeset
|
215 # make sure it's the right slave. |
|
494
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
216 if build.status != Build.IN_PROGRESS or \ |
|
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
217 build.slave_info.get(Build.IP_ADDRESS) != req.remote_addr: |
|
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
218 raise HTTPForbidden('Build %s has been invalidated for host %s.' |
|
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
219 % (build.id, req.remote_addr)) |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
220 |
| 392 | 221 step = BuildStep.fetch(self.env, build=build.id, name=stepname) |
| 222 if step: | |
| 223 raise HTTPConflict('Build step already exists') | |
| 253 | 224 |
| 392 | 225 recipe = Recipe(xmlio.parse(config.recipe)) |
| 226 index = None | |
| 227 current_step = None | |
| 228 for num, recipe_step in enumerate(recipe): | |
| 229 if recipe_step.id == stepname: | |
| 230 index = num | |
| 231 current_step = recipe_step | |
| 232 if index is None: | |
| 233 raise HTTPForbidden('No such build step') | |
| 234 last_step = index == num | |
|
200
692924ffed80
Changes to the BDB XML report store to support transactions. Closes #47.
cmlenz
parents:
197
diff
changeset
|
235 |
|
494
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
236 self.log.debug('Slave %s (build %d) completed step %d (%s) with ' |
|
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
237 'status %s', build.slave, build.id, index, stepname, |
|
9c9bf7a69bf0
Apply patch by Emmanuel Blot to allow a single slave to connect to multiple masters. Closes #271.
cmlenz
parents:
492
diff
changeset
|
238 elem.attr['status']) |
| 392 | 239 |
| 240 db = self.env.get_db_cnx() | |
| 241 | |
| 242 step = BuildStep(self.env, build=build.id, name=stepname) | |
| 243 try: | |
| 244 step.started = int(_parse_iso_datetime(elem.attr['time'])) | |
| 245 step.stopped = step.started + float(elem.attr['duration']) | |
| 246 except ValueError, e: | |
|
492
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
247 self.log.error('Error parsing build step timestamp: %s', e, |
|
56f0ad35c60a
Log errors when build master encounters an XML parse error.
cmlenz
parents:
479
diff
changeset
|
248 exc_info=True) |
| 392 | 249 raise HTTPBadRequest(e.args[0]) |
| 250 if elem.attr['status'] == 'failure': | |
| 251 self.log.warning('Build %s step %s failed', build.id, stepname) | |
|
109
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
252 step.status = BuildStep.FAILURE |
| 459 | 253 if current_step.onerror == 'fail': |
| 254 last_step = True | |
|
109
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
255 else: |
|
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
256 step.status = BuildStep.SUCCESS |
| 277 | 257 step.errors += [error.gettext() for error in elem.children('error')] |
|
112
a38eabd4b6e1
* Store build logs in a structured way, for example to highlight messages on the error stream.
cmlenz
parents:
110
diff
changeset
|
258 step.insert(db=db) |
|
109
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
259 |
| 392 | 260 # Collect log messages from the request body |
|
203
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
261 for idx, log_elem in enumerate(elem.children('log')): |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
262 build_log = BuildLog(self.env, build=build.id, step=stepname, |
|
203
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
263 generator=log_elem.attr.get('generator'), |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
264 orderno=idx) |
|
115
16d69eb6e047
Add support for XML fragments to the {{{xmlio}}} module, so that build output and reports don't need to be nested in a meaningless element (such as {{{<log type="distutils"><messages><message ...>}}}).
cmlenz
parents:
112
diff
changeset
|
265 for message_elem in log_elem.children('message'): |
|
16d69eb6e047
Add support for XML fragments to the {{{xmlio}}} module, so that build output and reports don't need to be nested in a meaningless element (such as {{{<log type="distutils"><messages><message ...>}}}).
cmlenz
parents:
112
diff
changeset
|
266 build_log.messages.append((message_elem.attr['level'], |
|
16d69eb6e047
Add support for XML fragments to the {{{xmlio}}} module, so that build output and reports don't need to be nested in a meaningless element (such as {{{<log type="distutils"><messages><message ...>}}}).
cmlenz
parents:
112
diff
changeset
|
267 message_elem.gettext())) |
|
112
a38eabd4b6e1
* Store build logs in a structured way, for example to highlight messages on the error stream.
cmlenz
parents:
110
diff
changeset
|
268 build_log.insert(db=db) |
|
109
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
269 |
| 392 | 270 # Collect report data from the request body |
|
203
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
271 for report_elem in elem.children('report'): |
|
401
a10942252ebc
Use POST for submitting step results, instead of PUT.
cmlenz
parents:
392
diff
changeset
|
272 report = Report(self.env, build=build.id, step=stepname, |
|
213
25f84dd9f159
* Refactoring of build recipes, the file format has changed slightly:
cmlenz
parents:
204
diff
changeset
|
273 category=report_elem.attr.get('category'), |
|
25f84dd9f159
* Refactoring of build recipes, the file format has changed slightly:
cmlenz
parents:
204
diff
changeset
|
274 generator=report_elem.attr.get('generator')) |
|
203
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
275 for item_elem in report_elem.children(): |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
276 item = {'type': item_elem.name} |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
277 item.update(item_elem.attr) |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
278 for child_elem in item_elem.children(): |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
279 item[child_elem.name] = child_elem.gettext() |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
280 report.items.append(item) |
|
e6ddca1e5712
Huge refactoring to remove dependency on BDB XML. Report data is now stored in the Trac database (SQLite/PostgreSQL).
cmlenz
parents:
202
diff
changeset
|
281 report.insert(db=db) |
| 116 | 282 |
| 392 | 283 # If this was the last step in the recipe we mark the build as |
| 284 # completed | |
| 459 | 285 if last_step: |
| 392 | 286 self.log.info('Slave %s completed build %d ("%s" as of [%s])', |
| 287 build.slave, build.id, build.config, build.rev) | |
| 288 build.stopped = step.stopped | |
| 289 | |
| 290 # Determine overall outcome of the build by checking the outcome | |
| 291 # of the individual steps against the "onerror" specification of | |
| 292 # each step in the recipe | |
| 293 for num, recipe_step in enumerate(recipe): | |
| 294 step = BuildStep.fetch(self.env, build.id, recipe_step.id) | |
| 295 if step.status == BuildStep.FAILURE: | |
| 296 if recipe_step.onerror != 'ignore': | |
| 297 build.status = Build.FAILURE | |
| 298 break | |
| 299 else: | |
| 300 build.status = Build.SUCCESS | |
| 301 | |
| 302 build.update(db=db) | |
| 303 | |
|
200
692924ffed80
Changes to the BDB XML report store to support transactions. Closes #47.
cmlenz
parents:
197
diff
changeset
|
304 db.commit() |
|
692924ffed80
Changes to the BDB XML report store to support transactions. Closes #47.
cmlenz
parents:
197
diff
changeset
|
305 |
| 459 | 306 if last_step: |
| 392 | 307 for listener in BuildSystem(self.env).listeners: |
| 308 listener.build_completed(build) | |
| 253 | 309 |
| 392 | 310 body = 'Build step processed' |
|
402
08801667f00f
Switch to urllib2 in slave so that we can support basic and digest authentication.
cmlenz
parents:
401
diff
changeset
|
311 req.send_response(201) |
| 392 | 312 req.send_header('Content-Type', 'text/plain') |
| 313 req.send_header('Content-Length', str(len(body))) | |
|
402
08801667f00f
Switch to urllib2 in slave so that we can support basic and digest authentication.
cmlenz
parents:
401
diff
changeset
|
314 req.send_header('Location', req.abs_href.builds(build.id, 'steps', |
|
08801667f00f
Switch to urllib2 in slave so that we can support basic and digest authentication.
cmlenz
parents:
401
diff
changeset
|
315 stepname)) |
| 392 | 316 req.write(body) |
| 317 raise RequestDone | |
|
109
5bf22bb87915
Transmit build log and generated data back to the build master in XML format. Closes #23.
cmlenz
parents:
96
diff
changeset
|
318 |
|
13
21aa17f97522
Initial code for build master and slave... these don't do a lot yet.
cmlenz
parents:
diff
changeset
|
319 |
|
82
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
320 def _parse_iso_datetime(string): |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
321 """Minimal parser for ISO date-time strings. |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
322 |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
323 Return the time as floating point number. Only handles UTC timestamps |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
324 without time zone information.""" |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
325 try: |
|
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
326 string = string.split('.', 1)[0] # strip out microseconds |
| 312 | 327 return calendar.timegm(time.strptime(string, '%Y-%m-%dT%H:%M:%S')) |
|
82
01200c88ddb0
Include timing information in the build messages transmitted from slave to master for better accuracy.
cmlenz
parents:
80
diff
changeset
|
328 except ValueError, e: |
| 392 | 329 raise ValueError('Invalid ISO date/time %r' % string) |