# HG changeset patch # User hodgestar # Date 1266581933 0 # Node ID dc51831e6120bb5a04346cdcbdcc861015b2a1e6 # Parent 1f03b23e38ffbaf63731f66d2568e75cacd7343b 0.6dev: Merged [805,813] from trunk. diff --git a/bitten/model.py b/bitten/model.py --- a/bitten/model.py +++ b/bitten/model.py @@ -703,6 +703,7 @@ WARNING = 'W' ERROR = 'E' UNKNOWN = '' + LEVELS_SUFFIX = '.levels' def __init__(self, env, build=None, step=None, generator=None, orderno=None, filename=None): @@ -751,7 +752,7 @@ os.remove(log_file) except Exception, e: self.env.log.warning("Error removing log file %s: %s" % (log_file, e)) - level_file = log_file + '.levels' + level_file = log_file + self.LEVELS_SUFFIX if os.path.exists(level_file): try: self.env.log.debug("Deleting level file: %s" % level_file) @@ -786,7 +787,7 @@ cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (log_file, id)) if self.messages: log_file_name = self.get_log_file(log_file) - level_file_name = log_file_name + ".levels" + level_file_name = log_file_name + self.LEVELS_SUFFIX codecs.open(log_file_name, "wb", "UTF-8").writelines([to_unicode(msg[1]+"\n") for msg in self.messages]) codecs.open(level_file_name, "wb", "UTF-8").writelines([to_unicode(msg[0]+"\n") for msg in self.messages]) @@ -813,7 +814,7 @@ log_lines = codecs.open(log_filename, "rb", "UTF-8").readlines() else: log_lines = [] - level_filename = log.get_log_file(log.filename + ".levels") + level_filename = log.get_log_file(log.filename + cls.LEVELS_SUFFIX) if os.path.exists(level_filename): log_levels = dict(enumerate(codecs.open(level_filename, "rb", "UTF-8").readlines())) else: @@ -998,4 +999,4 @@ schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \ BuildStep._schema + BuildLog._schema + Report._schema -schema_version = 10 +schema_version = 11 diff --git a/bitten/tests/model.py b/bitten/tests/model.py --- a/bitten/tests/model.py +++ b/bitten/tests/model.py @@ -509,7 +509,7 @@ build_log = BuildLog.fetch(self.env, id=build_log.id) self.assertEquals(build_log.filename, "%s.log" % build_log.id) log_file = build_log.get_log_file(build_log.filename) - levels_file = log_file+'.levels' + levels_file = log_file + BuildLog.LEVELS_SUFFIX self.failUnless(os.path.exists(log_file), 'log_file does not exist') self.failUnless(os.path.exists(levels_file), 'levels_file does not exist') diff --git a/bitten/tests/upgrades.py b/bitten/tests/upgrades.py --- a/bitten/tests/upgrades.py +++ b/bitten/tests/upgrades.py @@ -8,6 +8,7 @@ # are also available at http://bitten.edgewall.org/wiki/License. import unittest +import logging import warnings warnings.filterwarnings('ignore', '^Unknown table') @@ -37,12 +38,13 @@ # upgrades._parse_scheme() self.env.config.set('trac', 'database', self.env.dburi) self.env.path = tempfile.mkdtemp() - logs_dir = self.env.config.get("bitten", "logs_dir") + logs_dir = self.env.config.get("bitten", "logs_dir", "log/bitten") if os.path.isabs(logs_dir): raise ValueError("Should not have absolute logs directory for temporary test") logs_dir = os.path.join(self.env.path, logs_dir) if not os.path.isdir(logs_dir): os.makedirs(logs_dir) + self.logs_dir = logs_dir db = self.env.get_db_cnx() cursor = db.cursor() @@ -60,9 +62,20 @@ def tearDown(self): shutil.rmtree(self.env.path) + del self.logs_dir del self.env +class LogWatcher(logging.Handler): + + def __init__(self, level=0): + logging.Handler.__init__(self, level=0) + self.records = [] + + def emit(self, record): + self.records.append(record) + + class UpgradeHelperTestCase(BaseUpgradeTestCase): schema = [ @@ -272,6 +285,57 @@ rows = cursor.fetchall() self.assertEqual(rows, [(str(model.schema_version),)]) + def test_fix_log_levels_misnaming(self): + + logfiles = { + "1.log": "", + "2.log": "", + "3.log": "", + "1.log.level": "info\n", + "2.log.levels": "info\ninfo\n", + "3.log.level": "warn\n", + "3.log.levels": "warn\nwarn\n", + "4.log.level": "error\n", + } + expected_deletions = [ + "4.log.level", + ] + + for filename, data in logfiles.items(): + path = os.path.join(self.logs_dir, filename) + logfile = open(path, "w") + logfile.write(data) + logfile.close() + + logwatch = LogWatcher(logging.INFO) + self.env.log.setLevel(logging.INFO) + self.env.log.addHandler(logwatch) + + upgrades.fix_log_levels_misnaming(self.env, None) + + filenames = sorted(os.listdir(self.logs_dir)) + for filename in filenames: + path = os.path.join(self.logs_dir, filename) + origfile = filename in logfiles and filename or filename.replace("levels", "level") + self.assertEqual(logfiles[origfile], open(path).read()) + + self.assertEqual(len(filenames), len(logfiles) - len(expected_deletions)) + + warns = [rec for rec in logwatch.records if rec.levelname == 'WARNING'] + self.assertEqual(len(warns), 1) + for warn in warns: + self.assertTrue(warn.getMessage().startswith("Error renaming")) + + infos = [rec for rec in logwatch.records if rec.levelname == 'INFO'] + self.assertEqual(len(infos), 4) + for info in infos[:1]: + self.assertTrue(info.getMessage().startswith("Renamed incorrectly named log level file")) + self.assertTrue(infos[1].getMessage().startswith( + "Deleted stray log level file 4.log.level")) + self.assertTrue(infos[2].getMessage().startswith( + "Renamed 1 incorrectly named log level files from previous migrate (1 errors)")) + self.assertTrue(infos[3].getMessage().startswith( + "Deleted 1 stray log level (0 errors)")) def suite(): suite = unittest.TestSuite() diff --git a/bitten/upgrades.py b/bitten/upgrades.py --- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -384,7 +384,8 @@ message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,)) full_filename = os.path.join(logs_dir, filename) message_file = codecs.open(full_filename, "wb", "UTF-8") - level_file = codecs.open(full_filename+".level", "wb", "UTF-8") + # Note: the original version of this code erroneously wrote to filename + ".level" instead of ".levels", producing unused level files + level_file = codecs.open(full_filename + '.levels', "wb", "UTF-8") for message, level in message_cursor.fetchall() or []: message_file.write(to_unicode(message) + "\n") level_file.write(to_unicode(level) + "\n") @@ -402,6 +403,54 @@ cursor.close() env.log.warning("We have dropped the bitten_log_message table - you may want to vaccuum/compress your database to save space") +def fix_log_levels_misnaming(env, db): + """Renames or removes *.log.level files created by older versions of migrate_logs_to_files.""" + logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") + if not os.path.isabs(logs_dir): + logs_dir = os.path.join(env.path, logs_dir) + if not os.path.isdir(logs_dir): + return + + rename_count = 0 + rename_error_count = 0 + delete_count = 0 + delete_error_count = 0 + + for wrong_filename in os.listdir(logs_dir): + if not wrong_filename.endswith('.log.level'): + continue + + log_filename = os.path.splitext(wrong_filename)[0] + right_filename = log_filename + '.levels' + full_log_filename = os.path.join(logs_dir, log_filename) + full_wrong_filename = os.path.join(logs_dir, wrong_filename) + full_right_filename = os.path.join(logs_dir, right_filename) + + if not os.path.exists(full_log_filename): + try: + os.remove(full_wrong_filename) + delete_count += 1 + env.log.info("Deleted stray log level file %s", wrong_filename) + except Exception, e: + delete_error_count += 1 + env.log.warning("Error removing stray log level file %s: %s", wrong_filename, e) + else: + if os.path.exists(full_right_filename): + env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: new filename already exists", + full_wrong_filename, full_right_filename) + rename_error_count += 1 + continue + try: + os.rename(full_wrong_filename, full_right_filename) + rename_count += 1 + env.log.info("Renamed incorrectly named log level file %s to %s", wrong_filename, right_filename) + except Exception, e: + env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: %s", full_wrong_filename, full_right_filename, e) + rename_error_count += 1 + + env.log.info("Renamed %d incorrectly named log level files from previous migrate (%d errors)", rename_count, rename_error_count) + env.log.info("Deleted %d stray log level (%d errors)", delete_count, delete_error_count) + def recreate_rule_with_int_id(env, db): """Recreates the bitten_rule table with an integer id column rather than a text one.""" from bitten.model import TargetPlatform @@ -482,4 +531,5 @@ 8: [add_filename_to_logs,migrate_logs_to_files], 9: [recreate_rule_with_int_id], 10: [add_config_platform_rev_index_to_build, fix_sequences], + 11: [fix_log_levels_misnaming], }