changeset 629:f3bb52da9e3c

0.6dev: Adding support for attachments to configurations and build - full web implementation that mirrors what is available in Ticket and Wiki. Also added a new generic `<attach/>` command that enables attaching files to be part of a recipe and uploaded by slaves as part of build. Tests and updated documentation included. Closes #132.
author osimons
date Tue, 11 Aug 2009 22:47:55 +0000
parents 05686657e989
children 042c8b49ce7f
files bitten/main.py bitten/master.py bitten/model.py bitten/recipe.py bitten/slave_tests/recipe.py bitten/templates/bitten_build.html bitten/templates/bitten_config.html bitten/tests/master.py bitten/tests/web_ui.py bitten/web_ui.py doc/commands.txt
diffstat 11 files changed, 342 insertions(+), 2 deletions(-) [+]
line wrap: on
line diff
--- a/bitten/main.py
+++ b/bitten/main.py
@@ -12,10 +12,12 @@
 import os
 import textwrap
 
+from trac.attachment import ILegacyAttachmentPolicyDelegate
 from trac.core import *
 from trac.db import DatabaseManager
 from trac.env import IEnvironmentSetupParticipant
 from trac.perm import IPermissionRequestor
+from trac.resource import IResourceManager
 from trac.wiki import IWikiSyntaxProvider
 from bitten.api import IBuildListener
 from bitten.model import schema, schema_version, Build, BuildConfig
@@ -27,7 +29,8 @@
 class BuildSystem(Component):
 
     implements(IEnvironmentSetupParticipant, IPermissionRequestor,
-               IWikiSyntaxProvider)
+               IWikiSyntaxProvider, IResourceManager,
+               ILegacyAttachmentPolicyDelegate)
 
     listeners = ExtensionPoint(IBuildListener)
 
@@ -106,3 +109,58 @@
                           label)
             return label
         yield 'build', _format_link
+
+    # IResourceManager methods
+    
+    def get_resource_realms(self):
+        yield 'build'
+
+    def get_resource_url(self, resource, href, **kwargs):
+        config_name, build_id = self._parse_resource(resource.id)
+        return href.build(config_name, build_id)
+
+    def get_resource_description(self, resource, format=None, context=None,
+                                 **kwargs):
+        config_name, build_id = self._parse_resource(resource.id)
+        config = BuildConfig.fetch(self.env, config_name)
+        config_label = config.label or config_name
+        if context:
+            if build_id:
+                return tag.a('Build %d ("%s")' % (build_id, config_label),
+                        href=href.build(config_name, build_id))
+            elif config_name:
+                return tag.a('Build Configuration "%s"' % config_label,
+                        href=href.build(config_name, build_id))
+        else:
+            if build_id:
+                return 'Build %d ("%s")' % (build_id, config_label)
+            elif config_name:
+                return 'Build Configuration  "%s"' % config_label
+        self.log.error("Unknown build/config resource.id: %s" % resource.id)
+        return 'Unknown Build or Config'
+
+    def _parse_resource(self, resource_id):
+        """ Returns a (config_name, build_id) tuple. """
+        r = resource_id.split('/', 1)
+        if len(r) == 1:
+            return r[0], None
+        elif len(r) == 2:
+            try:
+                return r[0], int(r[1])
+            except:
+                return r[0], None
+        return None, None
+
+    # ILegacyAttachmentPolicyDelegate methods
+
+    def check_attachment_permission(self, action, username, resource, perm):
+        """ Respond to the various actions into the legacy attachment
+        permissions used by the Attachment module. """
+        if resource.parent.realm == 'build':
+            if action == 'ATTACHMENT_VIEW':
+                return 'BUILD_VIEW' in perm(resource.parent)
+            elif action == 'ATTACHMENT_CREATE':
+                return 'BUILD_MODIFY' in perm(resource.parent) \
+                        or 'BUILD_CREATE' in perm(resource.parent)
+            elif action == 'ATTACHMENT_DELETE':
+                return 'BUILD_DELETE' in perm(resource.parent)
--- a/bitten/master.py
+++ b/bitten/master.py
@@ -13,9 +13,12 @@
 import calendar
 import re
 import time
+from StringIO import StringIO
 
+from trac.attachment import Attachment
 from trac.config import BoolOption, IntOption, Option
 from trac.core import *
+from trac.resource import ResourceNotFound
 from trac.web import IRequestHandler, HTTPBadRequest, HTTPConflict, \
                      HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, \
                      RequestDone
@@ -281,6 +284,24 @@
                 report.items.append(item)
             report.insert(db=db)
 
+        # Collect attachments from the request body
+        for attach_elem in elem.children(Recipe.ATTACH):
+            attach_elem = list(attach_elem.children('file'))[0] # One file only
+            filename = attach_elem.attr.get('filename')
+            resource_id = attach_elem.attr.get('resource') == 'config' \
+                                    and build.config or build.resource.id
+            try: # Delete attachment if it already exists
+                old_attach = Attachment(self.env, 'build',
+                                    parent_id=resource_id, filename=filename)
+                old_attach.delete()
+            except ResourceNotFound:
+                pass
+            attachment = Attachment(self.env, 'build', parent_id=resource_id)
+            attachment.description = attach_elem.attr.get('description')
+            attachment.author = req.authname
+            fileobj = StringIO(attach_elem.gettext().decode('base64'))
+            attachment.insert(filename, fileobj, fileobj.len, db=db)
+
         # If this was the last step in the recipe we mark the build as
         # completed
         if last_step:
--- a/bitten/model.py
+++ b/bitten/model.py
@@ -10,7 +10,9 @@
 
 """Model classes for objects persisted in the database."""
 
+from trac.attachment import Attachment
 from trac.db import Table, Column, Index
+from trac.resource import Resource
 from trac.util.text import to_unicode
 import codecs
 import os
@@ -52,6 +54,8 @@
 
     exists = property(fget=lambda self: self._old_name is not None,
                       doc='Whether this configuration exists in the database')
+    resource = property(fget=lambda self: Resource('build', '%s' % self.name),
+                        doc='Build Config resource identification')
 
     def delete(self, db=None):
         """Remove a build configuration and all dependent objects from the
@@ -69,6 +73,9 @@
         for build in list(Build.select(self.env, config=self.name, db=db)):
             build.delete(db=db)
 
+        # Delete attachments
+        Attachment.delete_all(self.env, 'build', self.resource.id, db)
+
         cursor = db.cursor()
         cursor.execute("DELETE FROM bitten_config WHERE name=%s", (self.name,))
 
@@ -384,6 +391,8 @@
                          doc='Whether the build has been completed')
     successful = property(fget=lambda self: self.status == Build.SUCCESS,
                           doc='Whether the build was successful')
+    resource = property(fget=lambda self: Resource('build', '%s/%s' % (self.config, self.id)),
+                        doc='Build resource identification')
 
     def delete(self, db=None):
         """Remove the build from the database."""
@@ -397,6 +406,9 @@
         for step in list(BuildStep.select(self.env, build=self.id)):
             step.delete(db=db)
 
+        # Delete attachments
+        Attachment.delete_all(self.env, 'build', self.resource.id, db)
+
         cursor = db.cursor()
         cursor.execute("DELETE FROM bitten_slave WHERE build=%s", (self.id,))
         cursor.execute("DELETE FROM bitten_build WHERE id=%s", (self.id,))
--- a/bitten/recipe.py
+++ b/bitten/recipe.py
@@ -81,6 +81,8 @@
                     break
             elif name == 'report':
                 function = Context.report_file
+            elif name == 'attach':
+                function = Context.attach
             if not function:
                 raise InvalidRecipeError('Unknown recipe command %s' % qname)
 
@@ -160,6 +162,32 @@
             self.error('Failed to read %s report at %s: %s'
                        % (category, filename, e))
 
+    def attach(self, file_=None, description=None, resource=None):
+        """Attach a file to the build or build configuration.
+        
+        :param file\_: the path to the file to attach, relative to
+                       base directory.
+        :param description: description saved with attachment
+        :resource: which resource to attach the file to,
+                   either 'build' (default) or 'config'
+        :replace: non-empty to replace existing attachment with same name
+        """
+        filename = self.resolve(file_)
+        try:
+            fileobj = file(filename, 'r')
+            try:
+                xml_elem = xmlio.Element('file',
+                                filename=os.path.basename(filename),
+                                description=description,
+                                resource=resource or 'build')
+                xml_elem.append(fileobj.read().encode('base64'))
+                self.output.append((Recipe.ATTACH, None, None, xml_elem))
+                
+            finally:
+                fileobj.close()
+        except IOError, e:
+            self.error('Failed to read file %s as attachment' % file_)
+
     def resolve(self, *path):
         """Return the path of a file relative to the base directory.
         
@@ -230,6 +258,7 @@
     ERROR = 'error'
     LOG = 'log'
     REPORT = 'report'
+    ATTACH = 'attach'
 
     def __init__(self, xml, basedir=os.getcwd(), config=None):
         """Create the recipe.
--- a/bitten/slave_tests/recipe.py
+++ b/bitten/slave_tests/recipe.py
@@ -42,6 +42,52 @@
         except InvalidRecipeError, e:
             self.failUnless("Unsupported argument 'foo'" in str(e))
 
+    def test_attach_file_non_existing(self):
+        # Verify that it raises error and that it gets logged
+        ctxt = Context(self.basedir, Configuration())
+        ctxt.attach(file_='nonexisting.txt',
+                      description='build build')
+
+        self.assertEquals(1, len(ctxt.output))
+        self.assertEquals(Recipe.ERROR, ctxt.output[0][0])
+        self.assertEquals('Failed to read file nonexisting.txt as attachment',
+                            ctxt.output[0][3])
+
+    def test_attach_file_config(self):
+        # Verify output from attaching a file to a config
+        ctxt = Context(self.basedir, Configuration())
+        test_file = open(os.path.join(self.basedir, 'config.txt'), 'w')
+        test_file.write('hello config')
+        test_file.close()
+
+        ctxt.attach(file_='config.txt', description='config config',
+                      resource='config')
+        self.assertEquals(1, len(ctxt.output))
+        self.assertEquals(Recipe.ATTACH, ctxt.output[0][0])
+        attach_xml = ctxt.output[0][3]
+        self.assertEquals('<file resource="config" '
+                          'description="config config" '
+                          'filename="config.txt">'
+                          'aGVsbG8gY29uZmln\n'
+                          '</file>', str(attach_xml))
+
+    def test_attach_file_build(self):
+        # Verify output from attaching a file to a build
+        ctxt = Context(self.basedir, Configuration())
+        test_file = open(os.path.join(self.basedir, 'build.txt'), 'w')
+        test_file.write('hello build')
+        test_file.close()
+
+        ctxt.attach(file_='build.txt', description='build build')
+        self.assertEquals(1, len(ctxt.output))
+        self.assertEquals(Recipe.ATTACH, ctxt.output[0][0])
+        attach_xml = ctxt.output[0][3]
+        self.assertEquals('<file resource="build" '
+                          'description="build build" '
+                          'filename="build.txt">'
+                          'aGVsbG8gYnVpbGQ=\n'
+                          '</file>', str(attach_xml))
+
 class RecipeTestCase(unittest.TestCase):
 
     def setUp(self):
--- a/bitten/templates/bitten_build.html
+++ b/bitten/templates/bitten_build.html
@@ -5,6 +5,7 @@
       xmlns:xi="http://www.w3.org/2001/XInclude"
       xmlns:py="http://genshi.edgewall.org/">
   <xi:include href="layout.html" />
+  <xi:include href="macros.html" />
   <head>
     <title>$title</title>
   </head>
@@ -52,7 +53,10 @@
           <input type="hidden" name="action" value="invalidate" />
           <input type="submit" value="Invalidate build" />
         </div></form>
-      </div><py:for each="step in build.steps">
+        ${attach_file_form(build.attachments)}
+      </div>
+      ${list_of_attachments(build.attachments, compact=True)}
+      <py:for each="step in build.steps">
       <h2 class="step" id="step_${step.name}">$step.name ($step.duration)</h2>
       <div py:if="step.errors" class="errors">
         <h3>Errors</h3>
--- a/bitten/templates/bitten_config.html
+++ b/bitten/templates/bitten_config.html
@@ -5,6 +5,7 @@
       xmlns:xi="http://www.w3.org/2001/XInclude"
       xmlns:py="http://genshi.edgewall.org/">
   <xi:include href="layout.html" />
+  <xi:include href="macros.html" />
   <head>
     <title>$title</title>
   </head>
@@ -125,6 +126,10 @@
       <div py:if="config.description" class="description">
         $config.description
       </div>
+      <div class="buttons">
+        ${attach_file_form(config.attachments)}
+      </div>
+      ${list_of_attachments(config.attachments, compact=True)}
       <py:if test="config.builds_pending">
 	<div>$config.builds_pending pending build<py:if test="config.builds_pending > 1">s</py:if>&nbsp;<i>(<py:for each="platform in config.platforms">
         <py:if test="platform.builds_pending">
--- a/bitten/tests/master.py
+++ b/bitten/tests/master.py
@@ -442,6 +442,83 @@
                 'type': 'test',
             }, reports[0].items[0])
 
+    def test_process_build_step_success_with_attach(self):
+        # Parse input and create attachments for config + build
+        recipe = """<build>
+  <step id="foo">
+  <attach file="bar.txt" description="bar bar"/>
+  <attach file="baz.txt" description="baz baz" resource="config"/>
+  </step>
+</build>"""
+        BuildConfig(self.env, 'test', path='somepath', active=True,
+                    recipe=recipe).insert()
+        build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
+                      started=42, status=Build.IN_PROGRESS)
+        build.slave_info[Build.IP_ADDRESS] = '127.0.0.1';
+        build.insert()
+
+        inbody = StringIO("""<result step="foo" status="success"
+                                     time="2007-04-01T15:30:00.0000"
+                                     duration="3.45">
+    <attach>
+        <file filename="bar.txt"
+              description="bar bar">aGVsbG8gYmFy\n</file>
+    </attach>
+    <attach>
+        <file filename="baz.txt" description="baz baz"
+            resource="config">aGVsbG8gYmF6\n</file>
+    </attach>
+</result>""")
+        outheaders = {}
+        outbody = StringIO()
+        req = Mock(method='POST', base_path='',
+                   path_info='/builds/%d/steps/' % build.id,
+                   href=Href('/trac'), abs_href=Href('http://example.org/trac'),
+                   remote_addr='127.0.0.1', args={},
+                   authname='hal',
+                   perm=PermissionCache(self.env, 'hal'),
+                   read=inbody.read,
+                   send_response=lambda x: outheaders.setdefault('Status', x),
+                   send_header=lambda x, y: outheaders.setdefault(x, y),
+                   write=outbody.write)
+        module = BuildMaster(self.env)
+        assert module.match_request(req)
+
+        self.assertRaises(RequestDone, module.process_request, req)
+
+        self.assertEqual(201, outheaders['Status'])
+        self.assertEqual('20', outheaders['Content-Length'])
+        self.assertEqual('text/plain', outheaders['Content-Type'])
+        self.assertEqual('Build step processed', outbody.getvalue())
+
+        build = Build.fetch(self.env, build.id)
+        self.assertEqual(Build.SUCCESS, build.status)
+        assert build.stopped
+        assert build.stopped > build.started
+
+        steps = list(BuildStep.select(self.env, build.id))
+        self.assertEqual(1, len(steps))
+        self.assertEqual('foo', steps[0].name)
+        self.assertEqual(BuildStep.SUCCESS, steps[0].status)
+
+        from trac.attachment import Attachment
+        config_attachments = list(Attachment.select(self.env, 'build', 'test'))
+        build_attachments = list(Attachment.select(self.env, 'build', 'test/1'))
+        
+        self.assertEquals(1, len(build_attachments))
+        self.assertEquals('hal', build_attachments[0].author)
+        self.assertEquals('bar bar', build_attachments[0].description)
+        self.assertEquals('bar.txt', build_attachments[0].filename)
+        self.assertEquals('hello bar',
+                        build_attachments[0].open().read())
+
+        self.assertEquals(1, len(config_attachments))
+        self.assertEquals('hal', config_attachments[0].author)
+        self.assertEquals('baz baz', config_attachments[0].description)
+        self.assertEquals('baz.txt', config_attachments[0].filename)
+        self.assertEquals('hello baz',
+                        config_attachments[0].open().read())
+
     def test_process_build_step_wrong_slave(self):
         recipe = """<build>
   <step id="foo">
--- a/bitten/tests/web_ui.py
+++ b/bitten/tests/web_ui.py
@@ -97,6 +97,13 @@
         self.assertEqual('view_config', data['page_mode'])
         assert not 'next' in req.chrome['links']
 
+        from trac.resource import Resource
+        self.assertEquals(Resource('build', 'test'), data['context'].resource)
+
+        self.assertEquals([], data['config']['attachments']['attachments'])
+        self.assertEquals('/trac/attachment/build/test/',
+                                data['config']['attachments']['attach_href'])
+
     def test_view_config_paging(self):
         config = BuildConfig(self.env, name='test', path='trunk')
         config.insert()
@@ -144,6 +151,43 @@
 
 class BuildControllerTestCase(AbstractWebUITestCase):
 
+    def test_view_build(self):
+        config = BuildConfig(self.env, name='test', path='trunk')
+        config.insert()
+        platform = TargetPlatform(self.env, config='test', name='any')
+        platform.insert()
+        build = Build(self.env, config='test', platform=1, rev=123, rev_time=42,
+                      status=Build.SUCCESS, slave='hal')
+        build.insert()
+
+        PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
+        req = Mock(method='GET', base_path='', cgi_location='',
+                   path_info='/build/test/1', href=Href('/trac'), args={},
+                   chrome={}, authname='joe',
+                   perm=PermissionCache(self.env, 'joe'))
+
+        root = Mock(get_entries=lambda: ['foo'],
+                    get_history=lambda: [('trunk', rev, 'edit') for rev in
+                                          range(123, 111, -1)])
+        self.repos = Mock(get_node=lambda path, rev=None: root,
+                          sync=lambda: None,
+                          normalize_path=lambda path: path,
+                          get_changeset=lambda rev: Mock(author='joe'))
+        self.repos.authz = Mock(has_permission=lambda path: True, assert_permission=lambda path: None)
+
+        module = BuildController(self.env)
+        assert module.match_request(req)
+        _, data, _ = module.process_request(req)
+
+        self.assertEqual('view_build', data['page_mode'])
+
+        from trac.resource import Resource
+        self.assertEquals(Resource('build', 'test/1'), data['context'].resource)
+
+        self.assertEquals([], data['build']['attachments']['attachments'])
+        self.assertEquals('/trac/attachment/build/test/1/',
+                                data['build']['attachments']['attach_href'])
+
     def test_raise_404(self):
         PermissionSystem(self.env).grant_permission('joe', 'BUILD_VIEW')
         module = BuildController(self.env)
@@ -163,6 +207,7 @@
             return
         self.fail("This should have raised HTTPNotFound")
 
+
 class SourceFileLinkFormatterTestCase(AbstractWebUITestCase):
 
     def test_format_simple_link_in_repos(self):
--- a/bitten/web_ui.py
+++ b/bitten/web_ui.py
@@ -16,7 +16,10 @@
 
 import pkg_resources
 from genshi.builder import tag
+from trac.attachment import AttachmentModule
 from trac.core import *
+from trac.mimeview.api import Context
+from trac.resource import Resource
 from trac.timeline import ITimelineEventProvider
 from trac.util import escape, pretty_timedelta, format_datetime, shorten_line, \
                       Markup
@@ -368,6 +371,10 @@
             'builds_inprogress' : len(inprogress_builds)
         }
 
+        context = Context.from_request(req, config.resource)
+        data['context'] = context
+        data['config']['attachments'] = AttachmentModule(self.env).attachment_data(context)
+
         platforms = list(TargetPlatform.select(self.env, config=config_name,
                                                db=db))
         data['config']['platforms'] = [
@@ -504,6 +511,10 @@
             'href': req.href.build(config.name)
         }
 
+        context = Context.from_request(req, build.resource)
+        data['context'] = context
+        data['build']['attachments'] = AttachmentModule(self.env).attachment_data(context)
+
         formatters = []
         for formatter in self.log_formatters:
             formatters.append(formatter.get_formatter(req, build))
@@ -548,6 +559,11 @@
         if 'build' not in filters:
             return
 
+        # Attachments (will be rendered by attachment module)
+        for event in AttachmentModule(self.env).get_timeline_events(
+            req, Resource('build'), start, stop):
+            yield event
+
         start = to_timestamp(start)
         stop = to_timestamp(stop)
 
--- a/doc/commands.txt
+++ b/doc/commands.txt
@@ -47,6 +47,33 @@
 Both parameters must be specified.
 
 
+------------
+``<attach>``
+------------
+
+Attach a file to the build or configuration as regular attachment. An already
+existing attachment on the same resource with same base filename will be
+replaced.
+
+**Note:** Unless consistently building latest only, overwriting files on
+config level may lead to unexpected results.
+
+Parameters
+----------
+
++-----------------+----------------------------------------------------------+
+| Name            | Description                                              |
++=================+==========================================================+
+| ``file``        | Path to the file to attach, relative to the project      |
+|                 | directory.                                               |
++-----------------+----------------------------------------------------------+
+| ``resource``    | The resource to attach the file to. Either               |
+|                 | 'build' (default) or 'config'. Optional.                 |
++-----------------+----------------------------------------------------------+
+| ``description`` | Attachment description. Optional.                        |
++-----------------+----------------------------------------------------------+
+
+
 Shell Tools
 ===========
 
Copyright (C) 2012-2017 Edgewall Software