Mercurial > bitten > bitten-test
changeset 516:2f3b7c17d3c3
Switch to storing log messages in files rather than in database rows:
* Levels are supported in a supplementary file for now
* Upgrading is performed but the old `bitten_log_message` table is not dropped
* A message is printed out on upgrade to encourage checking of permissions
Fixes http://bitten.edgewall.org/ticket/329
| author | dfraser |
|---|---|
| date | Fri, 13 Mar 2009 08:52:47 +0000 |
| parents | 05cfbe90237e |
| children | 6e38de6858c0 |
| files | bitten/admin.py bitten/master.py bitten/model.py bitten/templates/bitten_admin_master.html bitten/tests/admin.py bitten/tests/master.py bitten/tests/model.py bitten/upgrades.py |
| diffstat | 8 files changed, 168 insertions(+), 58 deletions(-) [+] |
line wrap: on
line diff
--- a/bitten/admin.py +++ b/bitten/admin.py @@ -71,6 +71,11 @@ self.config['bitten'].set('slave_timeout', str(slave_timeout)) changed = True + logs_dir = req.args.get('logs_dir', None) + if logs_dir != master.logs_dir: + self.config['bitten'].set('logs_dir', str(logs_dir)) + changed = True + if changed: self.config.save()
--- a/bitten/master.py +++ b/bitten/master.py @@ -14,7 +14,7 @@ import re import time -from trac.config import BoolOption, IntOption +from trac.config import BoolOption, IntOption, PathOption from trac.core import * from trac.web import IRequestHandler, HTTPBadRequest, HTTPConflict, \ HTTPForbidden, HTTPMethodNotAllowed, HTTPNotFound, \ @@ -55,6 +55,9 @@ """The time in seconds after which a build is cancelled if the slave does not report progress.""") + logs_dir = PathOption('bitten', 'logs_dir', "log/bitten", doc= + """The directory on the server in which client log files will be stored.""") + # IRequestHandler methods def match_request(self, req):
--- a/bitten/model.py +++ b/bitten/model.py @@ -11,6 +11,7 @@ """Model classes for objects persisted in the database.""" from trac.db import Table, Column, Index +import os __docformat__ = 'restructuredtext en' @@ -669,12 +670,9 @@ Table('bitten_log', key='id')[ Column('id', auto_increment=True), Column('build', type='int'), Column('step'), Column('generator'), Column('orderno', type='int'), + Column('filename'), Index(['build', 'step']) ], - Table('bitten_log_message', key=('log', 'line'))[ - Column('log', type='int'), Column('line', type='int'), - Column('level', size=1), Column('message') - ] ] # Message levels @@ -682,9 +680,10 @@ INFO = 'I' WARNING = 'W' ERROR = 'E' + UNKNOWN = '' def __init__(self, env, build=None, step=None, generator=None, - orderno=None): + orderno=None, filename=None): """Initialize a new build log with the specified attributes. To actually create this build log in the database, the `insert` method @@ -696,11 +695,21 @@ self.step = step self.generator = generator or '' self.orderno = orderno and int(orderno) or 0 + self.filename = filename or None self.messages = [] + self.logs_dir = env.config.get('bitten', 'logs_dir', 'log/bitten') + if not os.path.isabs(self.logs_dir): + self.logs_dir = os.path.join(env.path, self.logs_dir) exists = property(fget=lambda self: self.id is not None, doc='Whether this build log exists in the database') + def get_log_file(self, filename): + """Returns the full path to the log file""" + if filename != os.path.basename(filename): + raise ValueError("Filename may not contain path: %s" % (filename,)) + return os.path.join(self.logs_dir, filename) + def delete(self, db=None): """Remove the build log from the database.""" assert self.exists, 'Cannot delete a non-existing build log' @@ -711,9 +720,11 @@ handle_ta = False cursor = db.cursor() - cursor.execute("DELETE FROM bitten_log_message WHERE log=%s", - (self.id,)) + cursor.execute("SELECT filename FROM bitten_log WHERE id=%s", (self.id,)) + log_files = cursor.fetchall() or [] cursor.execute("DELETE FROM bitten_log WHERE id=%s", (self.id,)) + for log_file in log_files: + os.remove(self.get_log_file(log_file[0])) if handle_ta: db.commit() @@ -734,11 +745,13 @@ "VALUES (%s,%s,%s,%s)", (self.build, self.step, self.generator, self.orderno)) id = db.get_last_id(cursor, 'bitten_log') + log_file = "%s.log" % (id,) + cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (log_file, id)) if self.messages: - cursor.executemany("INSERT INTO bitten_log_message " - "(log,line,level,message) VALUES (%s,%s,%s,%s)", - [(id, idx, msg[0], msg[1]) for idx, msg in - enumerate(self.messages)]) + log_file_name = self.get_log_file(log_file) + level_file_name = log_file_name + ".levels" + open(log_file_name, "w").writelines([msg[1]+"\n" for msg in self.messages]) + open(level_file_name, "w").writelines([msg[0]+"\n" for msg in self.messages]) if handle_ta: db.commit() @@ -750,16 +763,27 @@ db = env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT build,step,generator,orderno FROM bitten_log " + cursor.execute("SELECT build,step,generator,orderno,filename FROM bitten_log " "WHERE id=%s", (id,)) row = cursor.fetchone() if not row: return None - log = BuildLog(env, int(row[0]), row[1], row[2], row[3]) + log = BuildLog(env, int(row[0]), row[1], row[2], row[3], row[4]) log.id = id - cursor.execute("SELECT level,message FROM bitten_log_message " - "WHERE log=%s ORDER BY line", (id,)) - log.messages = cursor.fetchall() or [] + if log.filename: + log_filename = log.get_log_file(log.filename) + if os.path.exists(log_filename): + log_lines = open(log_filename, "r").readlines() + else: + log_lines = [] + level_filename = log.filename + ".levels" + if os.path.exists(level_filename): + log_levels = dict(enumerate(open(log.get_log_file(level_filename), "r").readlines())) + else: + log_levels = {} + log.messages = [(log_levels.get(line_num, BuildLog.UNKNOWN), line.rstrip("\n")) for line_num, line in enumerate(log_lines)] + else: + log.messages = [] return log @@ -937,4 +961,4 @@ schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \ BuildStep._schema + BuildLog._schema + Report._schema -schema_version = 7 +schema_version = 8
--- a/bitten/templates/bitten_admin_master.html +++ b/bitten/templates/bitten_admin_master.html @@ -63,6 +63,16 @@ is considered aborted, in case there has been no activity from that slave in that time. </p> + <div class="field"> + <label> + Directory for storing log files: + <input type="text" id="logs_dir" name="logs_dir" + value="$master.logs_dir" size="100" /> + </label> + </div> + <p class="hint"> + The directory on the server in which client log files will be stored. + </p> </fieldset> <div class="buttons">
--- a/bitten/tests/admin.py +++ b/bitten/tests/admin.py @@ -90,6 +90,7 @@ self.assertEqual(0, master.stabilize_wait) assert not master.adjust_timestamps assert not master.build_all + self.assertEqual('log/bitten', master.logs_dir) def test_process_config_changes(self): redirected_to = []
--- a/bitten/tests/master.py +++ b/bitten/tests/master.py @@ -364,8 +364,8 @@ self.assertEqual('http://bitten.cmlenz.net/tools/python#unittest', logs[0].generator) self.assertEqual(2, len(logs[0].messages)) - self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0]) - self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1]) + self.assertEqual((u'', u'Doing stuff'), logs[0].messages[0]) + self.assertEqual((u'', u'Ouch that hurt'), logs[0].messages[1]) def test_process_build_step_success_with_report(self): recipe = """<build>
--- a/bitten/tests/model.py +++ b/bitten/tests/model.py @@ -14,6 +14,7 @@ from trac.test import EnvironmentStub from bitten.model import BuildConfig, TargetPlatform, Build, BuildStep, \ BuildLog, Report, schema +import os class BuildConfigTestCase(unittest.TestCase): @@ -446,7 +447,10 @@ self.assertEqual([], log.messages) def test_insert(self): - log = BuildLog(self.env, build=1, step='test', generator='distutils') + log = BuildLog(self.env, build=1, step='test', generator='distutils', filename='1.log') + full_file = log.get_log_file('1.log') + if os.path.exists(full_file): + os.remove(full_file) log.messages = [ (BuildLog.INFO, 'running tests'), (BuildLog.ERROR, 'tests failed') @@ -456,28 +460,33 @@ db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT build,step,generator FROM bitten_log " + cursor.execute("SELECT build,step,generator,filename FROM bitten_log " "WHERE id=%s", (log.id,)) - self.assertEqual((1, 'test', 'distutils'), cursor.fetchone()) - cursor.execute("SELECT level,message FROM bitten_log_message " - "WHERE log=%s ORDER BY line", (log.id,)) - self.assertEqual((BuildLog.INFO, 'running tests'), cursor.fetchone()) - self.assertEqual((BuildLog.ERROR, 'tests failed'), cursor.fetchone()) + self.assertEqual((1, 'test', 'distutils', '1.log'), cursor.fetchone()) + lines = open(full_file, "r").readlines() + self.assertEqual('running tests\n', lines[0]) + self.assertEqual('tests failed\n', lines[1]) + if os.path.exists(full_file): + os.remove(full_file) def test_insert_empty(self): - log = BuildLog(self.env, build=1, step='test', generator='distutils') + log = BuildLog(self.env, build=1, step='test', generator='distutils', filename="1.log") + full_file = log.get_log_file('1.log') + if os.path.exists(full_file): + os.remove(full_file) log.messages = [] log.insert() self.assertNotEqual(None, log.id) db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("SELECT build,step,generator FROM bitten_log " + cursor.execute("SELECT build,step,generator,filename FROM bitten_log " "WHERE id=%s", (log.id,)) - self.assertEqual((1, 'test', 'distutils'), cursor.fetchone()) - cursor.execute("SELECT COUNT(*) FROM bitten_log_message " - "WHERE log=%s", (log.id,)) - self.assertEqual(0, cursor.fetchone()[0]) + self.assertEqual((1, 'test', 'distutils', '1.log'), cursor.fetchone()) + file_exists = os.path.exists(full_file) + if file_exists: + os.remove(full_file) + assert not file_exists def test_insert_no_build_or_step(self): log = BuildLog(self.env, step='test') @@ -489,13 +498,12 @@ def test_delete(self): db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("INSERT INTO bitten_log (build,step,generator) " - "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + cursor.execute("INSERT INTO bitten_log (build,step,generator,filename) " + "VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log')) id = db.get_last_id(cursor, 'bitten_log') - cursor.executemany("INSERT INTO bitten_log_message " - "VALUES (%s,%s,%s,%s)", - [(id, 1, BuildLog.INFO, 'running tests'), - (id, 2, BuildLog.ERROR, 'tests failed')]) + logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten") + full_file = os.path.join(logs_dir, "1.log") + open(full_file, "w").writelines(["running tests\n", "tests failed\n"]) log = BuildLog.fetch(self.env, id=id, db=db) self.assertEqual(True, log.exists) @@ -504,8 +512,10 @@ cursor.execute("SELECT * FROM bitten_log WHERE id=%s", (id,)) self.assertEqual(True, not cursor.fetchall()) - cursor.execute("SELECT * FROM bitten_log_message WHERE log=%s", (id,)) - self.assertEqual(True, not cursor.fetchall()) + file_exists = os.path.exists(full_file) + if os.path.exists(full_file): + os.remove(full_file) + assert not file_exists def test_delete_new(self): log = BuildLog(self.env, build=1, step='test', generator='foo') @@ -514,13 +524,12 @@ def test_fetch(self): db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("INSERT INTO bitten_log (build,step,generator) " - "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + cursor.execute("INSERT INTO bitten_log (build,step,generator,filename) " + "VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log')) id = db.get_last_id(cursor, 'bitten_log') - cursor.executemany("INSERT INTO bitten_log_message " - "VALUES (%s,%s,%s,%s)", - [(id, 1, BuildLog.INFO, 'running tests'), - (id, 2, BuildLog.ERROR, 'tests failed')]) + logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten") + full_file = os.path.join(logs_dir, "1.log") + open(full_file, "w").writelines(["running tests\n", "tests failed\n"]) log = BuildLog.fetch(self.env, id=id, db=db) self.assertEqual(True, log.exists) @@ -528,19 +537,20 @@ self.assertEqual(1, log.build) self.assertEqual('test', log.step) self.assertEqual('distutils', log.generator) - self.assertEqual((BuildLog.INFO, 'running tests'), log.messages[0]) - self.assertEqual((BuildLog.ERROR, 'tests failed'), log.messages[1]) + self.assertEqual((BuildLog.UNKNOWN, 'running tests'), log.messages[0]) + self.assertEqual((BuildLog.UNKNOWN, 'tests failed'), log.messages[1]) + if os.path.exists(full_file): + os.remove(full_file) def test_select(self): db = self.env.get_db_cnx() cursor = db.cursor() - cursor.execute("INSERT INTO bitten_log (build,step,generator) " - "VALUES (%s,%s,%s)", (1, 'test', 'distutils')) + cursor.execute("INSERT INTO bitten_log (build,step,generator,filename) " + "VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log')) id = db.get_last_id(cursor, 'bitten_log') - cursor.executemany("INSERT INTO bitten_log_message " - "VALUES (%s,%s,%s,%s)", - [(id, 1, BuildLog.INFO, 'running tests'), - (id, 2, BuildLog.ERROR, 'tests failed')]) + logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten") + full_file = os.path.join(logs_dir, "1.log") + open(full_file, "w").writelines(["running tests\n", "tests failed\n"]) logs = BuildLog.select(self.env, build=1, step='test', db=db) log = logs.next() @@ -549,9 +559,11 @@ self.assertEqual(1, log.build) self.assertEqual('test', log.step) self.assertEqual('distutils', log.generator) - self.assertEqual((BuildLog.INFO, 'running tests'), log.messages[0]) - self.assertEqual((BuildLog.ERROR, 'tests failed'), log.messages[1]) + self.assertEqual((BuildLog.UNKNOWN, 'running tests'), log.messages[0]) + self.assertEqual((BuildLog.UNKNOWN, 'tests failed'), log.messages[1]) self.assertRaises(StopIteration, logs.next) + if os.path.exists(full_file): + os.remove(full_file) class ReportTestCase(unittest.TestCase):
--- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -281,11 +281,66 @@ for stmt in connector.to_sql(table): cursor.execute(stmt) +def add_filename_to_logs(env, db): + """Add filename column to log table to save where log files are stored.""" + from bitten.model import BuildLog + cursor = db.cursor() + + cursor.execute("CREATE TEMP TABLE old_log AS " + "SELECT * FROM bitten_log") + cursor.execute("DROP TABLE bitten_log") + + connector, _ = DatabaseManager(env)._get_connector() + for stmt in connector.to_sql(BuildLog._schema[0]): + cursor.execute(stmt) + + cursor.execute("INSERT INTO bitten_log (id,build,step,generator,orderno,filename) " + "SELECT id,build,step,generator,orderno,'' FROM old_log") + +def migrate_logs_to_files(env, db): + """Migrates logs that are stored in the bitten_log_messages table into files.""" + from trac.db import Table, Column + from bitten.model import BuildLog + log_table = BuildLog._schema[0] + logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") + if not os.path.isabs(logs_dir): + logs_dir = os.path.join(env.path, logs_dir) + if not os.path.exists(logs_dir): + os.mkdir(logs_dir) + + message_table = Table('bitten_log_message', key=('log', 'line'))[ + Column('log', type='int'), Column('line', type='int'), + Column('level', size=1), Column('message') + ] + + cursor = db.cursor() + message_cursor = db.cursor() + update_cursor = db.cursor() + cursor.execute("SELECT id FROM bitten_log") + for log_id, in cursor.fetchall(): + filename = "%s.log" % (log_id,) + message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,)) + full_filename = os.path.join(logs_dir, filename) + message_file = open(full_filename, "w") + level_file = open(full_filename+".level", "w") + for message, level in message_cursor.fetchall() or []: + message_file.write(message + "\n") + level_file.write(level + "\n") + message_file.close() + level_file.close() + update_cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (filename, log_id)) + env.log.info("Migrated log %s", log_id) + env.log_warning("Logs have been migrated from the database to files in %s. " + "Ensure permissions are set correctly on this file. " + "When you have confirmed that the migration worked correctly, " + "you can drop the bitten_log_message table in the database (it remains as a backup)", logs_dir) + map = { 2: [add_log_table], 3: [add_recipe_to_config], 4: [add_config_to_reports], 5: [add_order_to_log, add_report_tables, xmldb_to_db], 6: [normalize_file_paths, fixup_generators], - 7: [add_error_table] + 7: [add_error_table], + 8: [add_filename_to_logs,migrate_logs_to_files], }