# HG changeset patch # User hodgestar # Date 1266580306 0 # Node ID 90903d6cc932308dcff3f80b79c30cb889fdfab0 # Parent 59e67e5df6200876a6394ca3aa0b4f63fa662c3b Add upgrade script to fix badly named .log.level files. Don't use BuildLog.LEVELS_SUFFIX in upgrade scripts. See #517. diff --git a/bitten/model.py b/bitten/model.py --- a/bitten/model.py +++ b/bitten/model.py @@ -999,4 +999,4 @@ schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \ BuildStep._schema + BuildLog._schema + Report._schema -schema_version = 10 +schema_version = 11 diff --git a/bitten/tests/upgrades.py b/bitten/tests/upgrades.py --- a/bitten/tests/upgrades.py +++ b/bitten/tests/upgrades.py @@ -8,6 +8,7 @@ # are also available at http://bitten.edgewall.org/wiki/License. import unittest +import logging import warnings warnings.filterwarnings('ignore', '^Unknown table') @@ -37,12 +38,13 @@ # upgrades._parse_scheme() self.env.config.set('trac', 'database', self.env.dburi) self.env.path = tempfile.mkdtemp() - logs_dir = self.env.config.get("bitten", "logs_dir") + logs_dir = self.env.config.get("bitten", "logs_dir", "log/bitten") if os.path.isabs(logs_dir): raise ValueError("Should not have absolute logs directory for temporary test") logs_dir = os.path.join(self.env.path, logs_dir) if not os.path.isdir(logs_dir): os.makedirs(logs_dir) + self.logs_dir = logs_dir db = self.env.get_db_cnx() cursor = db.cursor() @@ -60,9 +62,20 @@ def tearDown(self): shutil.rmtree(self.env.path) + del self.logs_dir del self.env +class LogWatcher(logging.Handler): + + def __init__(self, level=0): + logging.Handler.__init__(self, level=0) + self.records = [] + + def emit(self, record): + self.records.append(record) + + class UpgradeHelperTestCase(BaseUpgradeTestCase): schema = [ @@ -272,6 +285,57 @@ rows = cursor.fetchall() self.assertEqual(rows, [(str(model.schema_version),)]) + def test_fix_log_levels_misnaming(self): + + logfiles = { + "1.log": "", + "2.log": "", + "3.log": "", + "1.log.level": "info\n", + "2.log.levels": "info\ninfo\n", + "3.log.level": "warn\n", + "3.log.levels": "warn\nwarn\n", + "4.log.level": "error\n", + } + expected_deletions = [ + "4.log.level", + ] + + for filename, data in logfiles.items(): + path = os.path.join(self.logs_dir, filename) + logfile = open(path, "w") + logfile.write(data) + logfile.close() + + logwatch = LogWatcher(logging.INFO) + self.env.log.setLevel(logging.INFO) + self.env.log.addHandler(logwatch) + + upgrades.fix_log_levels_misnaming(self.env, None) + + filenames = sorted(os.listdir(self.logs_dir)) + for filename in filenames: + path = os.path.join(self.logs_dir, filename) + origfile = filename in logfiles and filename or filename.replace("levels", "level") + self.assertEqual(logfiles[origfile], open(path).read()) + + self.assertEqual(len(filenames), len(logfiles) - len(expected_deletions)) + + warns = [rec for rec in logwatch.records if rec.levelname == 'WARNING'] + self.assertEqual(len(warns), 1) + for warn in warns: + self.assertTrue(warn.getMessage().startswith("Error renaming")) + + infos = [rec for rec in logwatch.records if rec.levelname == 'INFO'] + self.assertEqual(len(infos), 4) + for info in infos[:1]: + self.assertTrue(info.getMessage().startswith("Renamed incorrectly named log level file")) + self.assertTrue(infos[1].getMessage().startswith( + "Deleted stray log level file 4.log.level")) + self.assertTrue(infos[2].getMessage().startswith( + "Renamed 1 incorrectly named log level files from previous migrate (1 errors)")) + self.assertTrue(infos[3].getMessage().startswith( + "Deleted 1 stray log level (0 errors)")) def suite(): suite = unittest.TestSuite() diff --git a/bitten/upgrades.py b/bitten/upgrades.py --- a/bitten/upgrades.py +++ b/bitten/upgrades.py @@ -385,7 +385,7 @@ full_filename = os.path.join(logs_dir, filename) message_file = codecs.open(full_filename, "wb", "UTF-8") # Note: the original version of this code erroneously wrote to filename + ".level" instead of ".levels", producing unused level files - level_file = codecs.open(full_filename + BuildLog.LEVELS_SUFFIX, "wb", "UTF-8") + level_file = codecs.open(full_filename + '.levels', "wb", "UTF-8") for message, level in message_cursor.fetchall() or []: message_file.write(to_unicode(message) + "\n") level_file.write(to_unicode(level) + "\n") @@ -403,6 +403,54 @@ cursor.close() env.log.warning("We have dropped the bitten_log_message table - you may want to vaccuum/compress your database to save space") +def fix_log_levels_misnaming(env, db): + """Renames or removes *.log.level files created by older versions of migrate_logs_to_files.""" + logs_dir = env.config.get("bitten", "logs_dir", "log/bitten") + if not os.path.isabs(logs_dir): + logs_dir = os.path.join(env.path, logs_dir) + if not os.path.isdir(logs_dir): + return + + rename_count = 0 + rename_error_count = 0 + delete_count = 0 + delete_error_count = 0 + + for wrong_filename in os.listdir(logs_dir): + if not wrong_filename.endswith('.log.level'): + continue + + log_filename = os.path.splitext(wrong_filename)[0] + right_filename = log_filename + '.levels' + full_log_filename = os.path.join(logs_dir, log_filename) + full_wrong_filename = os.path.join(logs_dir, wrong_filename) + full_right_filename = os.path.join(logs_dir, right_filename) + + if not os.path.exists(full_log_filename): + try: + os.remove(full_wrong_filename) + delete_count += 1 + env.log.info("Deleted stray log level file %s", wrong_filename) + except Exception, e: + delete_error_count += 1 + env.log.warning("Error removing stray log level file %s: %s", wrong_filename, e) + else: + if os.path.exists(full_right_filename): + env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: new filename already exists", + full_wrong_filename, full_right_filename) + rename_error_count += 1 + continue + try: + os.rename(full_wrong_filename, full_right_filename) + rename_count += 1 + env.log.info("Renamed incorrectly named log level file %s to %s", wrong_filename, right_filename) + except Exception, e: + env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: %s", full_wrong_filename, full_right_filename, e) + rename_error_count += 1 + + env.log.info("Renamed %d incorrectly named log level files from previous migrate (%d errors)", rename_count, rename_error_count) + env.log.info("Deleted %d stray log level (%d errors)", delete_count, delete_error_count) + def recreate_rule_with_int_id(env, db): """Recreates the bitten_rule table with an integer id column rather than a text one.""" from bitten.model import TargetPlatform @@ -483,4 +531,5 @@ 8: [add_filename_to_logs,migrate_logs_to_files], 9: [recreate_rule_with_int_id], 10: [add_config_platform_rev_index_to_build, fix_sequences], + 11: [fix_log_levels_misnaming], }