# HG changeset patch # User osimons # Date 1250030875 0 # Node ID f3bb52da9e3cfa1aa2aa493112a5a503acbba60f # Parent 05686657e98953f40a06afb90601afe4f0e34de1 0.6dev: Adding support for attachments to configurations and build - full web implementation that mirrors what is available in Ticket and Wiki. Also added a new generic `` command that enables attaching files to be part of a recipe and uploaded by slaves as part of build. Tests and updated documentation included. Closes #132. diff --git a/bitten/main.py b/bitten/main.py --- a/bitten/main.py +++ b/bitten/main.py @@ -12,10 +12,12 @@ import os import textwrap +from trac.attachment import ILegacyAttachmentPolicyDelegate from trac.core import * from trac.db import DatabaseManager from trac.env import IEnvironmentSetupParticipant from trac.perm import IPermissionRequestor +from trac.resource import IResourceManager from trac.wiki import IWikiSyntaxProvider from bitten.api import IBuildListener from bitten.model import schema, schema_version, Build, BuildConfig @@ -27,7 +29,8 @@ class BuildSystem(Component): implements(IEnvironmentSetupParticipant, IPermissionRequestor, - IWikiSyntaxProvider) + IWikiSyntaxProvider, IResourceManager, + ILegacyAttachmentPolicyDelegate) listeners = ExtensionPoint(IBuildListener) @@ -106,3 +109,58 @@ label) return label yield 'build', _format_link + + # IResourceManager methods + + def get_resource_realms(self): + yield 'build' + + def get_resource_url(self, resource, href, **kwargs): + config_name, build_id = self._parse_resource(resource.id) + return href.build(config_name, build_id) + + def get_resource_description(self, resource, format=None, context=None, + **kwargs): + config_name, build_id = self._parse_resource(resource.id) + config = BuildConfig.fetch(self.env, config_name) + config_label = config.label or config_name + if context: + if build_id: + return tag.a('Build %d ("%s")' % (build_id, config_label), + href=href.build(config_name, build_id)) + elif config_name: + return tag.a('Build Configuration "%s"' % config_label, + href=href.build(config_name, build_id)) + else: + if build_id: + return 'Build %d ("%s")' % (build_id, config_label) + elif config_name: + return 'Build Configuration "%s"' % config_label + self.log.error("Unknown build/config resource.id: %s" % resource.id) + return 'Unknown Build or Config' + + def _parse_resource(self, resource_id): + """ Returns a (config_name, build_id) tuple. """ + r = resource_id.split('/', 1) + if len(r) == 1: + return r[0], None + elif len(r) == 2: + try: + return r[0], int(r[1]) + except: + return r[0], None + return None, None + + # ILegacyAttachmentPolicyDelegate methods + + def check_attachment_permission(self, action, username, resource, perm): + """ Respond to the various actions into the legacy attachment + permissions used by the Attachment module. """ + if resource.parent.realm == 'build': + if action == 'ATTACHMENT_VIEW': + return 'BUILD_VIEW' in perm(resource.parent) + elif action == 'ATTACHMENT_CREATE': + return 'BUILD_MODIFY' in perm(resource.parent) \ + or 'BUILD_CREATE' in perm(resource.parent) + elif action == 'ATTACHMENT_DELETE': + return 'BUILD_DELETE' in perm(resource.parent) diff --git a/bitten/master.py b/bitten/master.py --- a/bitten/master.py +++ b/bitten/master.py @@ -13,9 +13,12 @@ import calendar import re import time +from StringIO import StringIO +from trac.attachment import Attachment from trac.config import BoolOption, IntOption, Option from trac.core import * +from trac.resource import ResourceNotFound from trac.web import IRequestHandler, HTTPBadRequest, HTTPConflict, \ HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, \ RequestDone @@ -281,6 +284,24 @@ report.items.append(item) report.insert(db=db) + # Collect attachments from the request body + for attach_elem in elem.children(Recipe.ATTACH): + attach_elem = list(attach_elem.children('file'))[0] # One file only + filename = attach_elem.attr.get('filename') + resource_id = attach_elem.attr.get('resource') == 'config' \ + and build.config or build.resource.id + try: # Delete attachment if it already exists + old_attach = Attachment(self.env, 'build', + parent_id=resource_id, filename=filename) + old_attach.delete() + except ResourceNotFound: + pass + attachment = Attachment(self.env, 'build', parent_id=resource_id) + attachment.description = attach_elem.attr.get('description') + attachment.author = req.authname + fileobj = StringIO(attach_elem.gettext().decode('base64')) + attachment.insert(filename, fileobj, fileobj.len, db=db) + # If this was the last step in the recipe we mark the build as # completed if last_step: diff --git a/bitten/model.py b/bitten/model.py --- a/bitten/model.py +++ b/bitten/model.py @@ -10,7 +10,9 @@ """Model classes for objects persisted in the database.""" +from trac.attachment import Attachment from trac.db import Table, Column, Index +from trac.resource import Resource from trac.util.text import to_unicode import codecs import os @@ -52,6 +54,8 @@ exists = property(fget=lambda self: self._old_name is not None, doc='Whether this configuration exists in the database') + resource = property(fget=lambda self: Resource('build', '%s' % self.name), + doc='Build Config resource identification') def delete(self, db=None): """Remove a build configuration and all dependent objects from the @@ -69,6 +73,9 @@ for build in list(Build.select(self.env, config=self.name, db=db)): build.delete(db=db) + # Delete attachments + Attachment.delete_all(self.env, 'build', self.resource.id, db) + cursor = db.cursor() cursor.execute("DELETE FROM bitten_config WHERE name=%s", (self.name,)) @@ -384,6 +391,8 @@ doc='Whether the build has been completed') successful = property(fget=lambda self: self.status == Build.SUCCESS, doc='Whether the build was successful') + resource = property(fget=lambda self: Resource('build', '%s/%s' % (self.config, self.id)), + doc='Build resource identification') def delete(self, db=None): """Remove the build from the database.""" @@ -397,6 +406,9 @@ for step in list(BuildStep.select(self.env, build=self.id)): step.delete(db=db) + # Delete attachments + Attachment.delete_all(self.env, 'build', self.resource.id, db) + cursor = db.cursor() cursor.execute("DELETE FROM bitten_slave WHERE build=%s", (self.id,)) cursor.execute("DELETE FROM bitten_build WHERE id=%s", (self.id,)) diff --git a/bitten/recipe.py b/bitten/recipe.py --- a/bitten/recipe.py +++ b/bitten/recipe.py @@ -81,6 +81,8 @@ break elif name == 'report': function = Context.report_file + elif name == 'attach': + function = Context.attach if not function: raise InvalidRecipeError('Unknown recipe command %s' % qname) @@ -160,6 +162,32 @@ self.error('Failed to read %s report at %s: %s' % (category, filename, e)) + def attach(self, file_=None, description=None, resource=None): + """Attach a file to the build or build configuration. + + :param file\_: the path to the file to attach, relative to + base directory. + :param description: description saved with attachment + :resource: which resource to attach the file to, + either 'build' (default) or 'config' + :replace: non-empty to replace existing attachment with same name + """ + filename = self.resolve(file_) + try: + fileobj = file(filename, 'r') + try: + xml_elem = xmlio.Element('file', + filename=os.path.basename(filename), + description=description, + resource=resource or 'build') + xml_elem.append(fileobj.read().encode('base64')) + self.output.append((Recipe.ATTACH, None, None, xml_elem)) + + finally: + fileobj.close() + except IOError, e: + self.error('Failed to read file %s as attachment' % file_) + def resolve(self, *path): """Return the path of a file relative to the base directory. @@ -230,6 +258,7 @@ ERROR = 'error' LOG = 'log' REPORT = 'report' + ATTACH = 'attach' def __init__(self, xml, basedir=os.getcwd(), config=None): """Create the recipe. diff --git a/bitten/slave_tests/recipe.py b/bitten/slave_tests/recipe.py --- a/bitten/slave_tests/recipe.py +++ b/bitten/slave_tests/recipe.py @@ -42,6 +42,52 @@ except InvalidRecipeError, e: self.failUnless("Unsupported argument 'foo'" in str(e)) + def test_attach_file_non_existing(self): + # Verify that it raises error and that it gets logged + ctxt = Context(self.basedir, Configuration()) + ctxt.attach(file_='nonexisting.txt', + description='build build') + + self.assertEquals(1, len(ctxt.output)) + self.assertEquals(Recipe.ERROR, ctxt.output[0][0]) + self.assertEquals('Failed to read file nonexisting.txt as attachment', + ctxt.output[0][3]) + + def test_attach_file_config(self): + # Verify output from attaching a file to a config + ctxt = Context(self.basedir, Configuration()) + test_file = open(os.path.join(self.basedir, 'config.txt'), 'w') + test_file.write('hello config') + test_file.close() + + ctxt.attach(file_='config.txt', description='config config', + resource='config') + self.assertEquals(1, len(ctxt.output)) + self.assertEquals(Recipe.ATTACH, ctxt.output[0][0]) + attach_xml = ctxt.output[0][3] + self.assertEquals('' + 'aGVsbG8gY29uZmln\n' + '', str(attach_xml)) + + def test_attach_file_build(self): + # Verify output from attaching a file to a build + ctxt = Context(self.basedir, Configuration()) + test_file = open(os.path.join(self.basedir, 'build.txt'), 'w') + test_file.write('hello build') + test_file.close() + + ctxt.attach(file_='build.txt', description='build build') + self.assertEquals(1, len(ctxt.output)) + self.assertEquals(Recipe.ATTACH, ctxt.output[0][0]) + attach_xml = ctxt.output[0][3] + self.assertEquals('' + 'aGVsbG8gYnVpbGQ=\n' + '', str(attach_xml)) + class RecipeTestCase(unittest.TestCase): def setUp(self): diff --git a/bitten/templates/bitten_build.html b/bitten/templates/bitten_build.html --- a/bitten/templates/bitten_build.html +++ b/bitten/templates/bitten_build.html @@ -5,6 +5,7 @@ xmlns:xi="http://www.w3.org/2001/XInclude" xmlns:py="http://genshi.edgewall.org/"> + $title @@ -52,7 +53,10 @@ - + ${attach_file_form(build.attachments)} + + ${list_of_attachments(build.attachments, compact=True)} +

$step.name ($step.duration)

Errors

diff --git a/bitten/templates/bitten_config.html b/bitten/templates/bitten_config.html --- a/bitten/templates/bitten_config.html +++ b/bitten/templates/bitten_config.html @@ -5,6 +5,7 @@ xmlns:xi="http://www.w3.org/2001/XInclude" xmlns:py="http://genshi.edgewall.org/"> + $title @@ -125,6 +126,10 @@
$config.description
+
+ ${attach_file_form(config.attachments)} +
+ ${list_of_attachments(config.attachments, compact=True)}
$config.builds_pending pending builds ( diff --git a/bitten/tests/master.py b/bitten/tests/master.py --- a/bitten/tests/master.py +++ b/bitten/tests/master.py @@ -442,6 +442,83 @@ 'type': 'test', }, reports[0].items[0]) + def test_process_build_step_success_with_attach(self): + # Parse input and create attachments for config + build + recipe = """ + + + + +""" + BuildConfig(self.env, 'test', path='somepath', active=True, + recipe=recipe).insert() + build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42, + started=42, status=Build.IN_PROGRESS) + build.slave_info[Build.IP_ADDRESS] = '127.0.0.1'; + build.insert() + + inbody = StringIO(""" + + aGVsbG8gYmFy\n + + + aGVsbG8gYmF6\n + +""") + outheaders = {} + outbody = StringIO() + req = Mock(method='POST', base_path='', + path_info='/builds/%d/steps/' % build.id, + href=Href('/trac'), abs_href=Href('http://example.org/trac'), + remote_addr='127.0.0.1', args={}, + authname='hal', + perm=PermissionCache(self.env, 'hal'), + read=inbody.read, + send_response=lambda x: outheaders.setdefault('Status', x), + send_header=lambda x, y: outheaders.setdefault(x, y), + write=outbody.write) + module = BuildMaster(self.env) + assert module.match_request(req) + + self.assertRaises(RequestDone, module.process_request, req) + + self.assertEqual(201, outheaders['Status']) + self.assertEqual('20', outheaders['Content-Length']) + self.assertEqual('text/plain', outheaders['Content-Type']) + self.assertEqual('Build step processed', outbody.getvalue()) + + build = Build.fetch(self.env, build.id) + self.assertEqual(Build.SUCCESS, build.status) + assert build.stopped + assert build.stopped > build.started + + steps = list(BuildStep.select(self.env, build.id)) + self.assertEqual(1, len(steps)) + self.assertEqual('foo', steps[0].name) + self.assertEqual(BuildStep.SUCCESS, steps[0].status) + + from trac.attachment import Attachment + config_attachments = list(Attachment.select(self.env, 'build', 'test')) + build_attachments = list(Attachment.select(self.env, 'build', 'test/1')) + + self.assertEquals(1, len(build_attachments)) + self.assertEquals('hal', build_attachments[0].author) + self.assertEquals('bar bar', build_attachments[0].description) + self.assertEquals('bar.txt', build_attachments[0].filename) + self.assertEquals('hello bar', + build_attachments[0].open().read()) + + self.assertEquals(1, len(config_attachments)) + self.assertEquals('hal', config_attachments[0].author) + self.assertEquals('baz baz', config_attachments[0].description) + self.assertEquals('baz.txt', config_attachments[0].filename) + self.assertEquals('hello baz', + config_attachments[0].open().read()) + def test_process_build_step_wrong_slave(self): recipe = """ diff --git a/bitten/tests/web_ui.py b/bitten/tests/web_ui.py --- a/bitten/tests/web_ui.py +++ b/bitten/tests/web_ui.py @@ -97,6 +97,13 @@ self.assertEqual('view_config', data['page_mode']) assert not 'next' in req.chrome['links'] + from trac.resource import Resource + self.assertEquals(Resource('build', 'test'), data['context'].resource) + + self.assertEquals([], data['config']['attachments']['attachments']) + self.assertEquals('/trac/attachment/build/test/', + data['config']['attachments']['attach_href']) + def test_view_config_paging(self): config = BuildConfig(self.env, name='test', path='trunk') config.insert() @@ -144,6 +151,43 @@ class BuildControllerTestCase(AbstractWebUITestCase): + def test_view_build(self): + config = BuildConfig(self.env, name='test', path='trunk') + config.insert() + platform = TargetPlatform(self.env, config='test', name='any') + platform.insert() + build = Build(self.env, config='test', platform=1, rev=123, rev_time=42, + status=Build.SUCCESS, slave='hal') + build.insert() + + PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW') + req = Mock(method='GET', base_path='', cgi_location='', + path_info='/build/test/1', href=Href('/trac'), args={}, + chrome={}, authname='joe', + perm=PermissionCache(self.env, 'joe')) + + root = Mock(get_entries=lambda: ['foo'], + get_history=lambda: [('trunk', rev, 'edit') for rev in + range(123, 111, -1)]) + self.repos = Mock(get_node=lambda path, rev=None: root, + sync=lambda: None, + normalize_path=lambda path: path, + get_changeset=lambda rev: Mock(author='joe')) + self.repos.authz = Mock(has_permission=lambda path: True, assert_permission=lambda path: None) + + module = BuildController(self.env) + assert module.match_request(req) + _, data, _ = module.process_request(req) + + self.assertEqual('view_build', data['page_mode']) + + from trac.resource import Resource + self.assertEquals(Resource('build', 'test/1'), data['context'].resource) + + self.assertEquals([], data['build']['attachments']['attachments']) + self.assertEquals('/trac/attachment/build/test/1/', + data['build']['attachments']['attach_href']) + def test_raise_404(self): PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW') module = BuildController(self.env) @@ -163,6 +207,7 @@ return self.fail("This should have raised HTTPNotFound") + class SourceFileLinkFormatterTestCase(AbstractWebUITestCase): def test_format_simple_link_in_repos(self): diff --git a/bitten/web_ui.py b/bitten/web_ui.py --- a/bitten/web_ui.py +++ b/bitten/web_ui.py @@ -16,7 +16,10 @@ import pkg_resources from genshi.builder import tag +from trac.attachment import AttachmentModule from trac.core import * +from trac.mimeview.api import Context +from trac.resource import Resource from trac.timeline import ITimelineEventProvider from trac.util import escape, pretty_timedelta, format_datetime, shorten_line, \ Markup @@ -368,6 +371,10 @@ 'builds_inprogress' : len(inprogress_builds) } + context = Context.from_request(req, config.resource) + data['context'] = context + data['config']['attachments'] = AttachmentModule(self.env).attachment_data(context) + platforms = list(TargetPlatform.select(self.env, config=config_name, db=db)) data['config']['platforms'] = [ @@ -504,6 +511,10 @@ 'href': req.href.build(config.name) } + context = Context.from_request(req, build.resource) + data['context'] = context + data['build']['attachments'] = AttachmentModule(self.env).attachment_data(context) + formatters = [] for formatter in self.log_formatters: formatters.append(formatter.get_formatter(req, build)) @@ -548,6 +559,11 @@ if 'build' not in filters: return + # Attachments (will be rendered by attachment module) + for event in AttachmentModule(self.env).get_timeline_events( + req, Resource('build'), start, stop): + yield event + start = to_timestamp(start) stop = to_timestamp(stop) diff --git a/doc/commands.txt b/doc/commands.txt --- a/doc/commands.txt +++ b/doc/commands.txt @@ -47,6 +47,33 @@ Both parameters must be specified. +------------ +```` +------------ + +Attach a file to the build or configuration as regular attachment. An already +existing attachment on the same resource with same base filename will be +replaced. + +**Note:** Unless consistently building latest only, overwriting files on +config level may lead to unexpected results. + +Parameters +---------- + ++-----------------+----------------------------------------------------------+ +| Name | Description | ++=================+==========================================================+ +| ``file`` | Path to the file to attach, relative to the project | +| | directory. | ++-----------------+----------------------------------------------------------+ +| ``resource`` | The resource to attach the file to. Either | +| | 'build' (default) or 'config'. Optional. | ++-----------------+----------------------------------------------------------+ +| ``description`` | Attachment description. Optional. | ++-----------------+----------------------------------------------------------+ + + Shell Tools ===========