# HG changeset patch # User cmlenz # Date 1132835667 0 # Node ID 90422699a594c6b8b498725e0c76d0e48f814abf # Parent 1016c3d12cbc4e2b58244550ab202f640f9a6f04 More and improved docstrings (using epydoc format). diff --git a/bitten/build/api.py b/bitten/build/api.py --- a/bitten/build/api.py +++ b/bitten/build/api.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Functions and classes used to simplify the implementation recipe commands.""" + import logging import fnmatch import os @@ -43,17 +45,16 @@ class CommandLine(object): """Simple helper for executing subprocesses.""" - # TODO: Use 'subprocess' module if available (Python >= 2.4) def __init__(self, executable, args, input=None, cwd=None): """Initialize the CommandLine object. - @param executable The name of the program to execute - @param args A list of arguments to pass to the executable - @param input String or file-like object containing any input data for - the program - @param cwd The working directory to change to before executing the - command + @param executable: the name of the program to execute + @param args: a list of arguments to pass to the executable + @param input: string or file-like object containing any input data for + the program + @param cwd: The working directory to change to before executing the + command """ self.executable = executable self.arguments = [str(arg) for arg in args] @@ -66,6 +67,12 @@ if os.name == 'nt': # windows def execute(self, timeout=None): + """Execute the command, and return a generator for iterating over + the output written to the standard output and error streams. + + @param timeout: number of seconds before the external process + should be aborted (not supported on Windows) + """ args = [self.executable] + self.arguments for idx, arg in enumerate(args): if arg.find(' ') >= 0: @@ -126,6 +133,12 @@ else: # posix def execute(self, timeout=None): + """Execute the command, and return a generator for iterating over + the output written to the standard output and error streams. + + @param timeout: number of seconds before the external process + should be aborted (not supported on Windows) + """ import popen2, select if self.cwd: old_cwd = os.getcwd() @@ -215,6 +228,14 @@ '.DS_Store', 'Thumbs.db'] def __init__(self, basedir, include=None, exclude=None): + """Create the file set. + + @param basedir: the base directory for all files in the set + @param include: a list of patterns that define which files should be + included in the set + @param exclude: a list of patterns that define which files should be + excluded from the set + """ self.files = [] self.basedir = basedir @@ -254,8 +275,13 @@ self.files.append(filepath) def __iter__(self): + """Iterate over the names of all files in the set.""" for filename in self.files: yield filename def __contains__(self, filename): + """Return whether the given file name is in the set. + + @param filename: the name of the file to check + """ return filename in self.files diff --git a/bitten/build/config.py b/bitten/build/config.py --- a/bitten/build/config.py +++ b/bitten/build/config.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Support for build slave configuration.""" + from ConfigParser import SafeConfigParser import logging import os @@ -88,12 +90,23 @@ self.packages[package][propname] = value def __contains__(self, key): + """Return whether the configuration contains a value for the specified + key. + + @param key: name of the configuration option using dotted notation + (for example, "python.path") + """ if '.' in key: package, propname = key.split('.', 1) return propname in self.packages.get(package, {}) return key in self.properties def __getitem__(self, key): + """Return the value for the specified configuration key. + + @param key: name of the configuration option using dotted notation + (for example, "python.path") + """ if '.' in key: package, propname = key.split('.', 1) return self.packages.get(package, {}).get(propname) @@ -102,9 +115,15 @@ def __str__(self): return str({'properties': self.properties, 'packages': self.packages}) - _VAR_RE = re.compile(r'\$\{(?P\w[\w.]*?\w)(?:\:(?P.+))?\}') + def get_dirpath(self, key): + """Return the value of the specified configuration key, but verify that + the value refers to the path of an existing directory. + + If the value does not exist, or is not a directory path, return C{None}. - def get_dirpath(self, key): + @param key: name of the configuration option using dotted notation + (for example, "ant.home") + """ dirpath = self[key] if dirpath: if os.path.isdir(dirpath): @@ -113,6 +132,14 @@ return None def get_filepath(self, key): + """Return the value of the specified configuration key, but verify that + the value refers to the path of an existing file. + + If the value does not exist, or is not a file path, return C{None}. + + @param key: name of the configuration option using dotted notation + (for example, "python.path") + """ filepath = self[key] if filepath: if os.path.isfile(filepath): @@ -120,15 +147,19 @@ log.warning('Invalid %s: %s is not a file', key, filepath) return None + _VAR_RE = re.compile(r'\$\{(?P\w[\w.]*?\w)(?:\:(?P.+))?\}') + def interpolate(self, text): """Interpolate configuration properties into a string. Properties can be referenced in the text using the notation - `${property.name}`. A default value can be provided by appending it to + C{${property.name}}. A default value can be provided by appending it to the property name separated by a colon, for example - `${property.name:defaultvalue}`. This value will be used when there's + C{${property.name:defaultvalue}}. This value will be used when there's no such property in the configuration. Otherwise, if no default is provided, the reference is not replaced at all. + + @param text: the string containing variable references """ def _replace(m): refname = m.group('ref') diff --git a/bitten/build/ctools.py b/bitten/build/ctools.py --- a/bitten/build/ctools.py +++ b/bitten/build/ctools.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Recipe commands for build tasks commonly used for C/C++ projects.""" + import logging import re import os @@ -22,7 +24,18 @@ def configure(ctxt, file_='configure', enable=None, disable=None, with=None, without=None, cflags=None, cxxflags=None): - """Run a configure script.""" + """Run a C{configure} script. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the configure script + @param enable: names of the features to enable, seperated by spaces + @param disable: names of the features to disable, separated by spaces + @param with: names of external packages to include + @param without: names of external packages to exclude + @param cflags: C{CFLAGS} to pass to the configure script + @param cxxflags: C{CXXFLAGS} to pass to the configure script + """ args = [] if enable: args += ['--enable-%s' % feature for feature in enable.split()] @@ -48,7 +61,14 @@ ctxt.error('configure failed (%s)' % returncode) def make(ctxt, target=None, file_=None, keep_going=False): - """Execute a Makefile target.""" + """Execute a Makefile target. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the Makefile + @param keep_going: whether make should keep going when errors are + encountered + """ executable = ctxt.config.get_filepath('make.path') or 'make' args = ['--directory', ctxt.basedir] @@ -65,7 +85,15 @@ ctxt.error('make failed (%s)' % returncode) def cppunit(ctxt, file_=None, srcdir=None): - """Collect CppUnit XML data.""" + """Collect CppUnit XML data. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: path of the file containing the CppUnit results; may contain + globbing wildcards to match multiple files + @param srcdir: name of the directory containing the source files, used to + link the test results to the corresponding files + """ assert file_, 'Missing required attribute "file"' try: @@ -126,7 +154,15 @@ log.warning('Error parsing CppUnit results file (%s)', e) def gcov(ctxt, include=None, exclude=None, prefix=None): - """Run `gcov` to extract coverage data where available.""" + """Run C{gcov} to extract coverage data where available. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param include: patterns of files and directories to include + @param exclude: patterns of files and directories that should be excluded + @param prefix: optional prefix name that is added to object files by the + build system + """ file_re = re.compile(r'^File \`(?P[^\']+)\'\s*$') lines_re = re.compile(r'^Lines executed:(?P\d+\.\d+)\% of (?P\d+)\s*$') diff --git a/bitten/build/javatools.py b/bitten/build/javatools.py --- a/bitten/build/javatools.py +++ b/bitten/build/javatools.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Recipe commands for tools commonly used in Java projects.""" + from glob import glob import logging import os @@ -20,7 +22,15 @@ log = logging.getLogger('bitten.build.javatools') def ant(ctxt, file_=None, target=None, keep_going=False, args=None): - """Run an Ant build.""" + """Run an Ant build. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the Ant build file + @param target: name of the target that should be executed (optional) + @param keep_going: whether Ant should keep going when errors are encountered + @param args: additional arguments to pass to Ant + """ executable = 'ant' ant_home = ctxt.config.get_dirpath('ant.home') if ant_home: @@ -83,7 +93,15 @@ ctxt.error('Ant failed (%s)' % cmdline.returncode) def junit(ctxt, file_=None, srcdir=None): - """Extract test results from a JUnit XML report.""" + """Extract test results from a JUnit XML report. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: path to the JUnit XML test results; may contain globbing + wildcards for matching multiple results files + @param srcdir: name of the directory containing the test sources, used to + link test results to the corresponding source files + """ assert file_, 'Missing required attribute "file"' try: total, failed = 0, 0 diff --git a/bitten/build/pythontools.py b/bitten/build/pythontools.py --- a/bitten/build/pythontools.py +++ b/bitten/build/pythontools.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Recipe commands for tools commonly used by Python projects.""" + import logging import os import re @@ -33,8 +35,14 @@ return python_path return sys.executable -def distutils(ctxt, command='build', file_='setup.py'): - """Execute a `distutils` command.""" +def distutils(ctxt, file_='setup.py', command='build'): + """Execute a C{distutils} command. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the file defining the distutils setup + @param command: the setup command to execute + """ cmdline = CommandLine(_python_path(ctxt), [ctxt.resolve(file_), command], cwd=ctxt.basedir) log_elem = xmlio.Fragment() @@ -61,7 +69,21 @@ ctxt.error('distutils failed (%s)' % cmdline.returncode) def exec_(ctxt, file_=None, module=None, function=None, output=None, args=None): - """Execute a python script.""" + """Execute a Python script. + + Either the C{file_} or the C{module} parameter must be provided. If + specified using the C{file_} parameter, the file must be inside the project + directory. If specified as a module, the module must either be resolvable + to a file, or the C{function} parameter must be provided + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the script file to execute + @param module: name of the Python module to execute + @param function: name of the Python function to run + @param output: name of the file to which output should be written + @param args: extra arguments to pass to the script + """ assert file_ or module, 'Either "file" or "module" attribute required' if function: assert module and not file_, '"module" attribute required for use of ' \ @@ -89,7 +111,12 @@ output=output, args=args) def pylint(ctxt, file_=None): - """Extract data from a `pylint` run written to a file.""" + """Extract data from a C{pylint} run written to a file. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the file containing the Pylint output + """ assert file_, 'Missing required attribute "file"' msg_re = re.compile(r'^(?P.+):(?P\d+): ' r'\[(?P[A-Z]\d*)(?:, (?P[\w\.]+))?\] ' @@ -125,7 +152,16 @@ log.warning('Error opening pylint results file (%s)', e) def trace(ctxt, summary=None, coverdir=None, include=None, exclude=None): - """Extract data from a `trace.py` run.""" + """Extract data from a C{trace.py} run. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param summary: path to the file containing the coverage summary + @param coverdir: name of the directory containing the per-module coverage + details + @param include: patterns of files or directories to include in the report + @param exclude: patterns of files or directories to exclude from the report + """ assert summary, 'Missing required attribute "summary"' assert coverdir, 'Missing required attribute "coverdir"' @@ -246,7 +282,12 @@ log.warning('Error opening coverage summary file (%s)', e) def unittest(ctxt, file_=None): - """Extract data from a unittest results file in XML format.""" + """Extract data from a unittest results file in XML format. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param file_: name of the file containing the test results + """ assert file_, 'Missing required attribute "file"' try: diff --git a/bitten/build/shtools.py b/bitten/build/shtools.py --- a/bitten/build/shtools.py +++ b/bitten/build/shtools.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Generic recipe commands for executing external processes.""" + import logging import os import shlex @@ -17,7 +19,17 @@ log = logging.getLogger('bitten.build.shtools') def exec_(ctxt, executable=None, file_=None, output=None, args=None): - """Execute a shell script.""" + """Execute a program or shell script. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param executable: name of the executable to run + @param file_: name of the script file, relative to the project directory, + that should be run + @param output: name of the file to which the output of the script should be + written + @param args: command-line arguments to pass to the script + """ assert executable or file_, \ 'Either "executable" or "file" attribute required' @@ -29,7 +41,19 @@ def pipe(ctxt, executable=None, file_=None, input_=None, output=None, args=None): - """Pipe the contents of a file through a script.""" + """Pipe the contents of a file through a program or shell script. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param executable: name of the executable to run + @param file_: name of the script file, relative to the project directory, + that should be run + @param input_: name of the file containing the data that should be passed + to the shell script on its standard input stream + @param output: name of the file to which the output of the script should be + written + @param args: command-line arguments to pass to the script + """ assert executable or file_, \ 'Either "executable" or "file" attribute required' assert input_, 'Missing required attribute "input"' @@ -42,7 +66,22 @@ def execute(ctxt, executable=None, file_=None, input_=None, output=None, args=None): - """Generic external program execution.""" + """Generic external program execution. + + This function is not itself bound to a recipe command, but rather used from + other commands. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param executable: name of the executable to run + @param file_: name of the script file, relative to the project directory, + that should be run + @param input_: name of the file containing the data that should be passed + to the shell script on its standard input stream + @param output: name of the file to which the output of the script should be + written + @param args: command-line arguments to pass to the script + """ if args: if isinstance(args, basestring): args = shlex.split(args) diff --git a/bitten/build/xmltools.py b/bitten/build/xmltools.py --- a/bitten/build/xmltools.py +++ b/bitten/build/xmltools.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Recipe commands for XML processing.""" + import logging import os @@ -32,7 +34,17 @@ log = logging.getLogger('bitten.build.xmltools') def transform(ctxt, src=None, dest=None, stylesheet=None): - """Apply an XSLT stylesheet to a source XML document.""" + """Apply an XSLT stylesheet to a source XML document. + + This command requires either libxslt (with Python bindings), or MSXML to + be installed. + + @param ctxt: the build context + @type ctxt: an instance of L{bitten.recipe.Context} + @param src: name of the XML input file + @param dest: name of the XML output file + @param stylesheet: name of the file containing the XSLT stylesheet + """ assert src, 'Missing required attribute "src"' assert dest, 'Missing required attribute "dest"' assert stylesheet, 'Missing required attribute "stylesheet"' diff --git a/bitten/master.py b/bitten/master.py --- a/bitten/master.py +++ b/bitten/master.py @@ -7,6 +7,13 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Build master implementation. + +This module is runnable as a script to launch the build master. The build +master starts a single process that handles connections to any number of build +slaves. +""" + import calendar from datetime import datetime, timedelta import logging @@ -30,6 +37,7 @@ class Master(beep.Listener): + """BEEP listener implementation for the build master.""" def __init__(self, envs, ip, port, adjust_timestamps=False, check_interval=DEFAULT_CHECK_INTERVAL): @@ -104,6 +112,8 @@ class OrchestrationProfileHandler(beep.ProfileHandler): """Handler for communication on the Bitten build orchestration profile from the perspective of the build master. + + An instance of this class is associated with exactly one remote build slave. """ URI = 'http://bitten.cmlenz.net/beep/orchestration' @@ -352,6 +362,7 @@ raise ValueError, 'Invalid ISO date/time %s (%s)' % (string, e) def main(): + """Main entry point for running the build master.""" from bitten import __version__ as VERSION from optparse import OptionParser diff --git a/bitten/queue.py b/bitten/queue.py --- a/bitten/queue.py +++ b/bitten/queue.py @@ -7,6 +7,17 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Implements the scheduling of builds for a project. + +This module provides the functionality for scheduling builds for a specific +Trac environment. It is used by both the build master and the web interface to +get the list of required builds (revisions not built yet). + +Furthermore, the C{BuildQueue} class is used by the build master to determine +the next pending build, and to match build slaves against configured target +platforms. +""" + from itertools import ifilter import logging import re @@ -24,6 +35,10 @@ This function is a generator that yields `(platform, rev, build)` tuples, where `platform` is a `TargetPlatform` object, `rev` is the identifier of the changeset, and `build` is a `Build` object or `None`. + + @param repos: The version control repository + @param config: The build configuration + @param db: a database connection (optional) """ env = config.env if not db: @@ -98,7 +113,7 @@ where `build` is the `Build` object and `slave` is the name of the build slave. - Otherwise, this function will return `(None, None)`. + Otherwise, this function will return C{(None, None)} """ log.debug('Checking for pending builds...') @@ -155,7 +170,7 @@ repos.close() def reset_orphaned_builds(self): - """Reset all in-progress builds to `PENDING` state. + """Reset all in-progress builds to PENDING state. This is used to cleanup after a crash of the build master process, which would leave in-progress builds in the database that aren't @@ -177,13 +192,13 @@ def register_slave(self, name, properties): """Register a build slave with the queue. - @param name: The name of the slave - @param properties: A `dict` containing the properties of the slave - @return: whether the registration was successful - This method tries to match the slave against the configured target platforms. Only if it matches at least one platform will the registration be successful. + + @param name: The name of the slave + @param properties: A dict containing the properties of the slave + @return: Whether the registration was successful """ any_match = False for config in BuildConfig.select(self.env): @@ -212,12 +227,12 @@ def unregister_slave(self, name): """Unregister a build slave. - @param name: The name of the slave - @return: `True` if the slave was registered for this build queue, - `False` otherwise - This method removes the slave from the registry, and also resets any in-progress builds by this slave to `PENDING` state. + + @param name: The name of the slave + @return: C{True} if the slave was registered for this build queue, + C{False} otherwise """ for slaves in self.slaves.values(): if name in slaves: diff --git a/bitten/recipe.py b/bitten/recipe.py --- a/bitten/recipe.py +++ b/bitten/recipe.py @@ -7,6 +7,12 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Execution of build recipes. + +This module provides various classes that can be used to process build recipes, +most importantly the L{Recipe} class. +""" + import keyword import logging import os @@ -20,27 +26,41 @@ from bitten.build.config import Configuration from bitten.util import xmlio -__all__ = ['Recipe'] +__all__ = ['Recipe', 'InvalidRecipeError'] log = logging.getLogger('bitten.recipe') class InvalidRecipeError(Exception): - """Exception raised when a recipe cannot be processed.""" + """Exception raised when a recipe is not valid.""" class Context(object): - """The context in which a recipe command or report is run.""" + """The context in which a build is executed.""" step = None # The current step generator = None # The current generator (namespace#name) def __init__(self, basedir, config=None): + """Initialize the context. + + @param basedir: a string containing the working directory for the build + @param config: the build slave configuration + @type config: an instance of L{bitten.build.config.Configuration} + """ self.basedir = os.path.realpath(basedir) self.config = config or Configuration() self.output = [] def run(self, step, namespace, name, attr): + """Run the specified recipe command. + + @param step: the build step that the command belongs to + @param namespace: the namespace URI of the command + @param name: the local tag name of the command + @param attr: a dictionary containing the attributes defined on the + command element + """ self.step = step try: @@ -73,15 +93,34 @@ self.step = None def error(self, message): + """Record an error message. + + @param message: A string containing the error message. + """ self.output.append((Recipe.ERROR, None, self.generator, message)) - def log(self, xml_elem): - self.output.append((Recipe.LOG, None, self.generator, xml_elem)) + def log(self, xml): + """Record log output. + + @param xml: an XML fragment containing the log messages + """ + self.output.append((Recipe.LOG, None, self.generator, xml)) - def report(self, category, xml_elem): - self.output.append((Recipe.REPORT, category, self.generator, xml_elem)) + def report(self, category, xml): + """Record report data. + + @param category: the name of category of the report + @param xml: an XML fragment containing the report data + """ + self.output.append((Recipe.REPORT, category, self.generator, xml)) def report_file(self, category=None, file_=None): + """Read report data from a file and record it. + + @param category: the name of the category of the report + @param file_: the path to the file containing the report data, relative + to the base directory + """ try: fileobj = file(self.resolve(file_), 'r') try: @@ -102,6 +141,11 @@ % (category, filename, e)) def resolve(self, *path): + """Return the path of a file relative to the base directory. + + Accepts any number of positional arguments, which are joined using the + system path separator to form the path. + """ return os.path.normpath(os.path.join(self.basedir, *path)) @@ -113,12 +157,22 @@ """ def __init__(self, elem): + """Create the step. + + @param elem: the XML element representing the step + @type elem: an instance of L{bitten.util.xmlio.ParsedElement} + """ self._elem = elem self.id = elem.attr['id'] self.description = elem.attr.get('description') self.onerror = elem.attr.get('onerror', 'fail') def execute(self, ctxt): + """Execute this step in the given context. + + @param ctxt: the build context + @type ctxt: an instance of L{Context} + """ for child in self._elem: ctxt.run(self, child.namespace, child.name, child.attr) @@ -138,8 +192,8 @@ class Recipe(object): """A build recipe. - Iterate over this object to get the individual build steps in the order they - have been defined in the recipe file. + Iterate over this object to get the individual build steps in the order + they have been defined in the recipe file. """ ERROR = 'error' @@ -147,16 +201,38 @@ REPORT = 'report' def __init__(self, xml, basedir=os.getcwd(), config=None): + """Create the recipe. + + @param xml: the XML document representing the recipe + @type xml: an instance of L{bitten.util.xmlio.ParsedElement} + @param basedir: the base directory for the build + @param config: the slave configuration (optional) + @type config: an instance of L{bitten.build.config.Configuration} + """ assert isinstance(xml, xmlio.ParsedElement) self.ctxt = Context(basedir, config) self._root = xml def __iter__(self): - """Provide an iterator over the individual steps of the recipe.""" + """Iterate over the individual steps of the recipe.""" for child in self._root.children('step'): yield Step(child) def validate(self): + """Validate the recipe. + + This method checks a number of constraints: + - the name of the root element must be "build" + - the only permitted child elements or the root element with the name + "step" + - the recipe must contain at least one step + - step elements must have a unique "id" attribute + - a step must contain at least one nested command + - commands must not have nested content + + @raise InvalidRecipeError: in case any of the above contraints is + violated + """ if self._root.name != 'build': raise InvalidRecipeError, 'Root element must be ' steps = list(self._root.children()) diff --git a/bitten/slave.py b/bitten/slave.py --- a/bitten/slave.py +++ b/bitten/slave.py @@ -7,6 +7,8 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. +"""Implementation of the build slave.""" + from datetime import datetime import logging import os @@ -28,10 +30,22 @@ class Slave(beep.Initiator): - """Build slave.""" + """BEEP initiator implementation for the build slave.""" def __init__(self, ip, port, name=None, config=None, dry_run=False, work_dir=None, keep_files=False): + """Create the build slave instance. + + @param ip: Host name or IP address of the build master to connect to + @param port: TCP port number of the build master to connect to + @param name: The name with which this slave should identify itself + @param config: The slave configuration + @param dry_run: Whether the build outcome should not be reported back + to the master + @param work_dir: The working directory to use for build execution + @param keep_files: Whether files and directories created for build + execution should be kept when done + """ beep.Initiator.__init__(self, ip, port) self.name = name self.config = config @@ -44,6 +58,11 @@ self.keep_files = keep_files def greeting_received(self, profiles): + """Start a channel for the build orchestration profile, if advertised + by the peer. + + Otherwise, terminate the session. + """ if OrchestrationProfileHandler.URI not in profiles: err = 'Peer does not support the Bitten orchestration profile' log.error(err) @@ -94,6 +113,12 @@ self.channel.send_msg(beep.Payload(xml), handle_reply) def handle_msg(self, msgno, payload): + """Handle either a build initiation or the transmission of a snapshot + archive. + + @param msgno: The identifier of the BEEP message + @param payload: The payload of the message + """ if payload.content_type == beep.BEEP_XML: elem = xmlio.parse(payload.body) if elem.name == 'build': @@ -120,7 +145,7 @@ shutil.copyfileobj(payload.body, archive_file) finally: archive_file.close() - basedir = self.unpack_snapshot(msgno, project_dir, archive_name) + basedir = self.unpack_snapshot(project_dir, archive_name) try: recipe = Recipe(self.build_xml, basedir, self.config) @@ -130,8 +155,12 @@ shutil.rmtree(basedir) os.remove(archive_path) - def unpack_snapshot(self, msgno, project_dir, archive_name): - """Unpack a snapshot archive.""" + def unpack_snapshot(self, project_dir, archive_name): + """Unpack a snapshot archive. + + @param project_dir: Base directory for builds for the project + @param archive_name: Name of the archive file + """ path = os.path.join(project_dir, archive_name) log.debug('Received snapshot archive: %s', path) try: @@ -158,6 +187,15 @@ raise beep.ProtocolError(550, 'Could not unpack archive (%s)' % e) def execute_build(self, msgno, recipe): + """Execute a build. + + Execute every step in the recipe, and report the outcome of each + step back to the server using an ANS message. + + @param msgno: The identifier of the snapshot transmission message + @param recipe: The recipe object + @type recipe: an instance of L{bitten.recipe.Recipe} + """ log.info('Building in directory %s', recipe.ctxt.basedir) try: if not self.session.dry_run: @@ -237,6 +275,7 @@ def main(): + """Main entry point for running the build slave.""" from bitten import __version__ as VERSION from optparse import OptionParser diff --git a/bitten/util/beep.py b/bitten/util/beep.py --- a/bitten/util/beep.py +++ b/bitten/util/beep.py @@ -7,14 +7,13 @@ # you should have received as part of this distribution. The terms # are also available at http://bitten.cmlenz.net/wiki/License. - """Minimal implementation of the BEEP protocol (IETF RFC 3080) based on the `asyncore` module. Current limitations: - * No support for the TSL and SASL profiles. - * No support for mapping frames (SEQ frames for TCP mapping). - * No localization support (xml:lang attribute). + - No support for the TLS and SASL profiles. + - No support for mapping frames (SEQ frames for TCP mapping). + - No localization support (xml:lang attribute). """ import asynchat @@ -66,6 +65,15 @@ } def __init__(self, code, message=None): + """Create the error. + + A numeric error code must be provided as the first parameter. A message + can be provided; if it is omitted, a standard error message will be + used in case the error code is known. + + @param code: The error code + @param message: An error message (optional) + """ if message is None: message = ProtocolError._default_messages.get(code) Exception.__init__(self, 'BEEP error %d (%s)' % (code, message)) @@ -74,6 +82,12 @@ self.local = True def from_xml(cls, xml): + """Create an error object from the given XML element. + + @param xml: The XML element representing the error + @type xml: An instance of L{bitten.util.xmlio.ParsedElement} + @return: The created C{ProtocolError} object + """ elem = xmlio.parse(xml) obj = cls(int(elem.attr['code']), elem.gettext()) obj.local = False @@ -81,6 +95,10 @@ from_xml = classmethod(from_xml) def to_xml(self): + """Create an XML representation of the error. + + @return: The created XML element + """ return xmlio.Element('error', code=self.code)[self.message] @@ -100,7 +118,7 @@ self.eventqueue = [] def run(self, timeout=15.0, granularity=5): - """Start listening to incoming connections.""" + """Start the event loop.""" granularity = timedelta(seconds=granularity) socket_map = asyncore.socket_map last_event_check = datetime.min @@ -137,6 +155,13 @@ communication with the connected peer. """ def __init__(self, ip, port): + """Create the listener. + + @param ip: The IP address or host name to bind to + @type ip: a string + @param port: The TCP port number to bind to + @type port: an int + """ EventLoop.__init__(self) asyncore.dispatcher.__init__(self) self.create_socket(socket.AF_INET, socket.SOCK_STREAM) @@ -167,6 +192,8 @@ first_channelno=2)) def quit(self): + """Shutdown the listener, attempting to gracefully quit all active BEEP + sessions by first closing their respective channels.""" if not self.sessions: self.close() return @@ -382,6 +409,8 @@ """ def quit(self): + """Stops the build slave, attempting to gracefully quit the session by + closing all active channels.""" self.terminate() asyncore.loop(timeout=10) @@ -392,9 +421,9 @@ def __init__(self, session, channelno, profile_cls): """Create the channel. - @param session The `Session` object that the channel belongs to - @param channelno The channel number - @param profile The associated `ProfileHandler` class + @param session: The L{Session} object that the channel belongs to + @param channelno: The channel number + @param profile_cls: The associated L{ProfileHandler} class """ self.session = session self.channelno = channelno @@ -421,9 +450,9 @@ """Process a TCP mapping frame (SEQ). @param ackno: the value of the next sequence number that the sender is - expecting to receive on this channel + expecting to receive on this channel @param window: window size, the number of payload octets per frame that - the sender is expecting to receive on this channel + the sender is expecting to receive on this channel """ self.windowsize = window @@ -557,24 +586,44 @@ def handle_disconnect(self): """Called when the channel this profile is associated with is closed.""" - def handle_msg(self, msgno, message): - """Handle a MSG frame.""" + def handle_msg(self, msgno, payload): + """Handle a MSG frame. + + @param msgno: The message identifier + @param payload: The C{Payload} of the message + """ raise NotImplementedError - def handle_rpy(self, msgno, message): - """Handle a RPY frame.""" + def handle_rpy(self, msgno, payload): + """Handle a RPY frame. + + @param msgno: The identifier of the referenced message + @param payload: The C{Payload} of the message + """ pass - def handle_err(self, msgno, message): - """Handle an ERR frame.""" + def handle_err(self, msgno, payload): + """Handle an ERR frame. + + @param msgno: The identifier of the referenced message + @param payload: The C{Payload} of the message + """ pass - def handle_ans(self, msgno, ansno, message): - """Handle an ANS frame.""" + def handle_ans(self, msgno, ansno, payload): + """Handle an ANS frame. + + @param msgno: The identifier of the referenced message + @param ansno: The answer number + @param payload: The C{Payload} of the message + """ pass def handle_nul(self, msgno): - """Handle a NUL frame.""" + """Handle a NUL frame. + + @param msgno: The identifier of the referenced message + """ pass @@ -590,10 +639,10 @@ ] self.channel.send_rpy(0, Payload(xml)) - def handle_msg(self, msgno, message): + def handle_msg(self, msgno, payload): """Handle an incoming message.""" - assert message and message.content_type == BEEP_XML - elem = xmlio.parse(message.body) + assert payload and payload.content_type == BEEP_XML + elem = xmlio.parse(payload.body) if elem.name == 'start': channelno = int(elem.attr['number']) @@ -626,27 +675,27 @@ if not self.session.channels: self.session.close() - def handle_rpy(self, msgno, message): + def handle_rpy(self, msgno, payload): """Handle a positive reply.""" - if message.content_type == BEEP_XML: - elem = xmlio.parse(message.body) + if payload.content_type == BEEP_XML: + elem = xmlio.parse(payload.body) if elem.name == 'greeting': if isinstance(self.session, Initiator): profiles = [p.attr['uri'] for p in elem.children('profile')] self.session.greeting_received(profiles) - def handle_err(self, msgno, message): + def handle_err(self, msgno, payload): """Handle a negative reply.""" # Probably an error on connect, because other errors should get handled # by the corresponding callbacks # TODO: Terminate the session, I guess - if message.content_type == BEEP_XML: - raise ProtocolError.from_xml(message.body) + if payload.content_type == BEEP_XML: + raise ProtocolError.from_xml(payload.body) def send_close(self, channelno=0, code=200, handle_ok=None, handle_error=None): """Send a request to close a channel to the peer.""" - def handle_reply(cmd, msgno, ansno, message): + def handle_reply(cmd, msgno, ansno, payload): if cmd == 'RPY': log.debug('Channel %d closed', channelno) self.session.channels[channelno].close() @@ -655,7 +704,7 @@ if handle_ok is not None: handle_ok() elif cmd == 'ERR': - error = ProtocolError.from_xml(message.body) + error = ProtocolError.from_xml(payload.body) log.debug('Peer refused to start channel %d: %s (%d)', channelno, error.message, error.code) if handle_error is not None: @@ -668,18 +717,17 @@ def send_start(self, profiles, handle_ok=None, handle_error=None): """Send a request to start a new channel to the peer. - @param profiles A list of profiles to request for the channel, each - element being an instance of a `ProfileHandler` - sub-class - @param handle_ok An optional callback function that will be invoked when - the channel has been successfully started - @param handle_error An optional callback function that will be invoked - when the peer refuses to start the channel + @param profiles: A list of profiles to request for the channel, each + element being an instance of a L{ProfileHandler} subclass + @param handle_ok: An optional callback function that will be invoked + when the channel has been successfully started + @param handle_error: An optional callback function that will be invoked + when the peer refuses to start the channel """ channelno = self.session.channelno.next() - def handle_reply(cmd, msgno, ansno, message): + def handle_reply(cmd, msgno, ansno, payload): if cmd == 'RPY': - elem = xmlio.parse(message.body) + elem = xmlio.parse(payload.body) for cls in [p for p in profiles if p.URI == elem.attr['uri']]: log.debug('Channel %d started with profile %s', channelno, elem.attr['uri']) @@ -689,7 +737,7 @@ if handle_ok is not None: handle_ok(channelno, elem.attr['uri']) elif cmd == 'ERR': - elem = xmlio.parse(message.body) + elem = xmlio.parse(payload.body) text = elem.gettext() code = int(elem.attr['code']) log.debug('Peer refused to start channel %d: %s (%d)', @@ -710,7 +758,17 @@ def __init__(self, data=None, content_type=BEEP_XML, content_disposition=None, content_encoding=None): - """Initialize the payload.""" + """Initialize the payload object. + + @param data: The body of the MIME message. This can be either: + - a string, + - a file-like object providing a C{read()} function, + - an XML element, or + - C{None} + @param content_type: the MIME type of the payload + @param content_disposition: the filename for disposition of the data + @param content_encoding: the encoding of the data + """ self._hdr_buf = None self.content_type = content_type self.content_disposition = content_disposition @@ -728,6 +786,10 @@ self.body = data def read(self, size=None): + """Return the specified range of the MIME message. + + @param size: the number of bytes to read + """ if self._hdr_buf is None: hdrs = [] if self.content_type: @@ -757,6 +819,10 @@ return ret_buf def parse(cls, string): + """Create a C{Payload} object from the given string data. + + @param string: The string containing the MIME message. + """ message = email.message_from_string(string) content_type = message.get('Content-Type') content_disposition = message.get('Content-Disposition') @@ -768,16 +834,17 @@ class FrameProducer(object): """Internal class that emits the frames of a BEEP message, based on the - `asynchat` `push_with_producer()` protocol. + C{asynchat} C{push_with_producer()} protocol. """ def __init__(self, channel, cmd, msgno, ansno=None, payload=None): """Initialize the frame producer. - @param channel the channel the message is to be sent on - @param cmd the BEEP command/keyword (MSG, RPY, ERR, ANS or NUL) - @param msgno the message number - @param ansno the answer number (only for ANS messages) - @param payload the message payload (an instance of `Payload`) + @param channel: The channel the message is to be sent on + @param cmd: The BEEP command/keyword (MSG, RPY, ERR, ANS or NUL) + @param msgno: The message number + @param ansno: The answer number (only for ANS messages) + @param payload: The message payload + @type payload: an instance of L{Payload} """ self.session = channel.session self.channel = channel