Mercurial > bitten > bitten-test
changeset 200:692924ffed80
Changes to the BDB XML report store to support transactions. Closes #47.
| author | cmlenz |
|---|---|
| date | Wed, 14 Sep 2005 18:55:51 +0000 |
| parents | 9cabfdbdb8e0 |
| children | 98eb3b8eea29 |
| files | bitten/master.py bitten/store.py bitten/tests/store.py bitten/trac_ext/web_ui.py bitten/upgrades.py |
| diffstat | 5 files changed, 148 insertions(+), 40 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/master.py +++ b/bitten/master.py @@ -140,6 +140,7 @@ build.update(db=db) store.delete(build=build) db.commit() + store.commit() def _cleanup_snapshots(self, when): log.debug('Checking for unused snapshot archives...') @@ -324,20 +325,17 @@ elif cmd == 'ANS': assert payload.content_type == beep.BEEP_XML - db = self.env.get_db_cnx() elem = xmlio.parse(payload.body) if elem.name == 'started': self._build_started(build, elem, timestamp_delta) elif elem.name == 'step': - self._build_step_completed(db, build, elem, timestamp_delta) + self._build_step_completed(build, elem, timestamp_delta) elif elem.name == 'completed': self._build_completed(build, elem, timestamp_delta) elif elem.name == 'aborted': - self._build_aborted(db, build) + self._build_aborted(build) elif elem.name == 'error': build.status = Build.FAILURE - build.update(db=db) - db.commit() snapshot_format = { ('application/tar', 'bzip2'): 'bzip2', @@ -361,10 +359,14 @@ build.status = Build.IN_PROGRESS log.info('Slave %s started build %d ("%s" as of [%s])', self.name, build.id, build.config, build.rev) + build.update() - def _build_step_completed(self, db, build, elem, timestamp_delta=None): + def _build_step_completed(self, build, elem, timestamp_delta=None): log.debug('Slave completed step "%s" with status %s', elem.attr['id'], elem.attr['result']) + + db = self.env.get_db_cnx() + step = BuildStep(self.env, build=build.id, name=elem.attr['id'], description=elem.attr.get('description')) step.started = int(_parse_iso_datetime(elem.attr['time'])) @@ -391,10 +393,14 @@ for report in elem.children('report'): store.store(build, step, report) + db.commit() + store.commit() + def _build_completed(self, build, elem, timestamp_delta=None): log.info('Slave %s completed build %d ("%s" as of [%s]) with status %s', self.name, build.id, build.config, build.rev, elem.attr['result']) + build.stopped = int(_parse_iso_datetime(elem.attr['time'])) if timestamp_delta: build.stopped -= timestamp_delta @@ -402,21 +408,28 @@ build.status = Build.FAILURE else: build.status = Build.SUCCESS + build.update() def _build_aborted(self, db, build): log.info('Slave "%s" aborted build %d ("%s" as of [%s])', self.name, build.id, build.config, build.rev) + db = self.env.get_db_cnx() + for step in BuildStep.select(self.env, build=build.id, db=db): step.delete(db=db) - store = get_store(self.env) - store.delete(build=build) - build.slave = None build.started = 0 build.status = Build.PENDING build.slave_info = {} + build.update(db=db) + + store = get_store(self.env) + store.delete(build=build) + + db.commit() + store.commit() def _parse_iso_datetime(string):
--- a/bitten/store.py +++ b/bitten/store.py @@ -16,6 +16,15 @@ class ReportStore(object): + def close(self): + raise NotImplementedError + + def commit(self): + raise NotImplementedError + + def rollback(self): + raise NotImplementedError + def delete(self, config=None, build=None, step=None, type=None): raise NotImplementedError @@ -32,6 +41,15 @@ class NullReportStore(ReportStore): + def close(self): + pass + + def commit(self): + pass + + def rollback(self): + pass + def delete(self, config=None, build=None, step=None, type=None): return @@ -47,8 +65,10 @@ try: + from bsddb3 import db import dbxml except ImportError: + db = None dbxml = None @@ -101,32 +121,94 @@ def __init__(self, path): self.path = path - self.mgr = dbxml.XmlManager() + self.env = None + self.mgr = None + self.container = None + self.xtn = None + + def _lazyinit(self, create=False): + if self.container is not None: + if self.xtn is None: + self.xtn = self.mgr.createTransaction() + return True + + exists = os.path.exists(self.path) + if not exists and not create: + return False + + self.env = db.DBEnv() + self.env.open(os.path.dirname(self.path), + db.DB_CREATE | db.DB_INIT_LOCK | db.DB_INIT_LOG | + db.DB_INIT_MPOOL | db.DB_INIT_TXN, 0) + self.mgr = dbxml.XmlManager(self.env, 0) + self.xtn = self.mgr.createTransaction() + + if not exists: + self.container = self.mgr.createContainer(self.path, + dbxml.DBXML_TRANSACTIONAL) + ctxt = self.mgr.createUpdateContext() + for name, index in self.indices: + self.container.addIndex(self.xtn, '', name, index, ctxt) + else: + self.container = self.mgr.openContainer(self.path, + dbxml.DBXML_TRANSACTIONAL) + + return True + + def __del__(self): + self.close() + + def close(self): + if self.xtn: + self.xtn.abort() + self.xtn = None + if self.container is not None: + self.container.close() + self.container = None + if self.env is not None: + self.env.close(0) + self.env = None + + def commit(self): + if not self.xtn: + return + self.xtn.commit() + self.xtn = None + + def rollback(self): + if not self.xtn: + return + self.xtn.abort() + self.xtn = None def delete(self, config=None, build=None, step=None, type=None): - container = self._open_container() + if not self._lazyinit(create=False): + return + + container = self ._open_container() if not container: return ctxt = self.mgr.createUpdateContext() for elem in self.query('return $reports', config=config, build=build, step=step, type=type): - container.deleteDocument(elem._value.asDocument(), ctxt) + container.deleteDocument(self.xtn, elem._value.asDocument(), ctxt) def store(self, build, step, xml): assert xml.name == 'report' and 'type' in xml.attr - container = self._open_container(create=True) + assert self._lazyinit(create=True) + ctxt = self.mgr.createUpdateContext() doc = self.mgr.createDocument() doc.setContent(str(xml)) doc.setMetaData('', 'config', dbxml.XmlValue(build.config)) doc.setMetaData('', 'build', dbxml.XmlValue(build.id)) doc.setMetaData('', 'step', dbxml.XmlValue(step.name)) - container.putDocument(doc, ctxt, dbxml.DBXML_GEN_NAME) + self.container.putDocument(self.xtn, doc, ctxt, dbxml.DBXML_GEN_NAME) def query(self, xquery, config=None, build=None, step=None, type=None): - container = self._open_container() - if not container: + if not self._lazyinit(create=False): return + ctxt = self.mgr.createQueryContext() constraints = [] @@ -144,25 +226,13 @@ query += '[%s]' % ' and '.join(constraints) query += '\n' + (xquery or 'return $reports') - results = self.mgr.query(query, ctxt) + results = self.mgr.query(self.xtn, query, ctxt, dbxml.DBXML_LAZY_DOCS) for value in results: yield BDBXMLReportStore.XmlValueAdapter(value) def retrieve(self, build, step=None, type=None): return self.query('', build=build, step=step, type=type) - def _open_container(self, create=False): - if not os.path.exists(self.path): - if not create: - return None - container = self.mgr.createContainer(self.path) - ctxt = self.mgr.createUpdateContext() - for name, index in self.indices: - container.addIndex('', name, index, ctxt) - else: - container = self.mgr.openContainer(self.path) - return container - def get_store(env): if dbxml is None:
--- a/bitten/tests/store.py +++ b/bitten/tests/store.py @@ -21,12 +21,12 @@ class BDBXMLReportStoreTestCase(unittest.TestCase): def setUp(self): - self.path = os.path.join(tempfile.gettempdir(), 'bitten_test.dbxml') - self.store = BDBXMLReportStore(self.path) + self.path = tempfile.mkdtemp(prefix='bitten-test') + self.store = BDBXMLReportStore(os.path.join(self.path, 'test.dbxml')) def tearDown(self): - self.store = None - os.unlink(self.path) + self.store.close() + shutil.rmtree(self.path) def test_store_report(self): """
--- a/bitten/trac_ext/web_ui.py +++ b/bitten/trac_ext/web_ui.py @@ -177,10 +177,19 @@ if 'cancel' in req.args: req.redirect(self.env.href.build(config_name)) - config = BuildConfig.fetch(self.env, config_name) + db = self.env.get_db_cnx() + + config = BuildConfig.fetch(self.env, config_name, db=db) assert config, 'Build configuration "%s" does not exist' % config_name - config.delete() + store = get_store(self.env) + store.delete(config=config) + + config.delete(db=db) + + db.commit() + store.commit() + req.redirect(self.env.href.build()) def _do_save_config(self, req, config_name): @@ -235,6 +244,10 @@ self.log.info('Deleting target platform %s of configuration %s', platform.name, platform.config) platform.delete(db=db) + + # FIXME: this should probably also delete all builds done for this + # platform, and all the associated reports + db.commit() def _do_save_platform(self, req, config_name, platform_id): @@ -523,6 +536,7 @@ build.update() db.commit() + store.commit() req.redirect(self.env.href.build(build.config))
--- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -50,6 +50,7 @@ def add_config_to_reports(env, db): from bitten.model import Build try: + from bsddb3 import db import dbxml except ImportError, e: return @@ -58,14 +59,20 @@ if not os.path.isfile(dbfile): return - mgr = dbxml.XmlManager() + dbenv = db.DBEnv() + dbenv.open(os.path.dirname(dbfile), + db.DB_CREATE | db.DB_INIT_LOCK | db.DB_INIT_LOG | + db.DB_INIT_MPOOL | db.DB_INIT_TXN, 0) + + mgr = dbxml.XmlManager(dbenv, 0) + xtn = mgr.createTransaction() container = mgr.openContainer(dbfile) uc = mgr.createUpdateContext() - container.addIndex('', 'config', 'node-metadata-equality-string', uc) + container.addIndex(xtn, '', 'config', 'node-metadata-equality-string', uc) qc = mgr.createQueryContext() - for value in mgr.query('collection("%s")/report' % dbfile, qc): + for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc): doc = value.asDocument() metaval = dbxml.XmlValue() if doc.getMetaData('', 'build', metaval): @@ -73,10 +80,14 @@ build = Build.fetch(env, id=build_id, db=db) if build: doc.setMetaData('', 'config', dbxml.XmlValue(build.config)) - container.updateDocument(doc, uc) + container.updateDocument(xtn, doc, uc) else: # an orphaned report, for whatever reason... just remove it - container.deleteDocument(doc, uc) + container.deleteDocument(xtn, doc, uc) + + xtn.commit() + container.close() + dbenv.close(0) map = { 2: [add_log_table],