changeset 736:dc51831e6120 0.6.x

0.6dev: Merged [805,813] from trunk.
author hodgestar
date Fri, 19 Feb 2010 12:18:53 +0000
parents 1f03b23e38ff
children 8054277e55dd
files bitten/model.py bitten/tests/model.py bitten/tests/upgrades.py bitten/upgrades.py
diffstat 4 files changed, 122 insertions(+), 7 deletions(-) [+]
line wrap: on
line diff
--- a/bitten/model.py
+++ b/bitten/model.py
@@ -703,6 +703,7 @@
     WARNING = 'W'
     ERROR = 'E'
     UNKNOWN = ''
+    LEVELS_SUFFIX = '.levels'
 
     def __init__(self, env, build=None, step=None, generator=None,
                  orderno=None, filename=None):
@@ -751,7 +752,7 @@
                     os.remove(log_file)
                 except Exception, e:
                     self.env.log.warning("Error removing log file %s: %s" % (log_file, e))
-            level_file = log_file + '.levels'
+            level_file = log_file + self.LEVELS_SUFFIX
             if os.path.exists(level_file):
                 try:
                     self.env.log.debug("Deleting level file: %s" % level_file)
@@ -786,7 +787,7 @@
         cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (log_file, id))
         if self.messages:
             log_file_name = self.get_log_file(log_file)
-            level_file_name = log_file_name + ".levels"
+            level_file_name = log_file_name + self.LEVELS_SUFFIX
             codecs.open(log_file_name, "wb", "UTF-8").writelines([to_unicode(msg[1]+"\n") for msg in self.messages])
             codecs.open(level_file_name, "wb", "UTF-8").writelines([to_unicode(msg[0]+"\n") for msg in self.messages])
 
@@ -813,7 +814,7 @@
                 log_lines = codecs.open(log_filename, "rb", "UTF-8").readlines()
             else:
                 log_lines = []
-            level_filename = log.get_log_file(log.filename + ".levels")
+            level_filename = log.get_log_file(log.filename + cls.LEVELS_SUFFIX)
             if os.path.exists(level_filename):
                 log_levels = dict(enumerate(codecs.open(level_filename, "rb", "UTF-8").readlines()))
             else:
@@ -998,4 +999,4 @@
 
 schema = BuildConfig._schema + TargetPlatform._schema + Build._schema + \
          BuildStep._schema + BuildLog._schema + Report._schema
-schema_version = 10
+schema_version = 11
--- a/bitten/tests/model.py
+++ b/bitten/tests/model.py
@@ -509,7 +509,7 @@
         build_log = BuildLog.fetch(self.env, id=build_log.id)
         self.assertEquals(build_log.filename, "%s.log" % build_log.id)
         log_file = build_log.get_log_file(build_log.filename)
-        levels_file = log_file+'.levels'
+        levels_file = log_file + BuildLog.LEVELS_SUFFIX
         self.failUnless(os.path.exists(log_file), 'log_file does not exist')
         self.failUnless(os.path.exists(levels_file),
                                                 'levels_file does not exist')
--- a/bitten/tests/upgrades.py
+++ b/bitten/tests/upgrades.py
@@ -8,6 +8,7 @@
 # are also available at http://bitten.edgewall.org/wiki/License.
 
 import unittest
+import logging
 
 import warnings
 warnings.filterwarnings('ignore', '^Unknown table')
@@ -37,12 +38,13 @@
             # upgrades._parse_scheme()
             self.env.config.set('trac', 'database', self.env.dburi)
         self.env.path = tempfile.mkdtemp()
-        logs_dir = self.env.config.get("bitten", "logs_dir")
+        logs_dir = self.env.config.get("bitten", "logs_dir", "log/bitten")
         if os.path.isabs(logs_dir):
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         if not os.path.isdir(logs_dir):
             os.makedirs(logs_dir)
+        self.logs_dir = logs_dir
 
         db = self.env.get_db_cnx()
         cursor = db.cursor()
@@ -60,9 +62,20 @@
 
     def tearDown(self):
         shutil.rmtree(self.env.path)
+        del self.logs_dir
         del self.env
 
 
+class LogWatcher(logging.Handler):
+
+    def __init__(self, level=0):
+        logging.Handler.__init__(self, level=0)
+        self.records = []
+
+    def emit(self, record):
+        self.records.append(record)
+
+
 class UpgradeHelperTestCase(BaseUpgradeTestCase):
 
     schema = [
@@ -272,6 +285,57 @@
         rows = cursor.fetchall()
         self.assertEqual(rows, [(str(model.schema_version),)])
 
+    def test_fix_log_levels_misnaming(self):
+
+        logfiles = {
+            "1.log": "",
+            "2.log": "",
+            "3.log": "",
+            "1.log.level": "info\n",
+            "2.log.levels": "info\ninfo\n",
+            "3.log.level": "warn\n",
+            "3.log.levels": "warn\nwarn\n",
+            "4.log.level": "error\n",
+        }
+        expected_deletions = [
+            "4.log.level",
+        ]
+
+        for filename, data in logfiles.items():
+            path = os.path.join(self.logs_dir, filename)
+            logfile = open(path, "w")
+            logfile.write(data)
+            logfile.close()
+
+        logwatch = LogWatcher(logging.INFO)
+        self.env.log.setLevel(logging.INFO)
+        self.env.log.addHandler(logwatch)
+
+        upgrades.fix_log_levels_misnaming(self.env, None)
+
+        filenames = sorted(os.listdir(self.logs_dir))
+        for filename in filenames:
+            path = os.path.join(self.logs_dir, filename)
+            origfile = filename in logfiles and filename or filename.replace("levels", "level")
+            self.assertEqual(logfiles[origfile], open(path).read())
+
+        self.assertEqual(len(filenames), len(logfiles) - len(expected_deletions))
+
+        warns = [rec for rec in logwatch.records if rec.levelname == 'WARNING']
+        self.assertEqual(len(warns), 1)
+        for warn in warns:
+            self.assertTrue(warn.getMessage().startswith("Error renaming"))
+
+        infos = [rec for rec in logwatch.records if rec.levelname == 'INFO']
+        self.assertEqual(len(infos), 4)
+        for info in infos[:1]:
+            self.assertTrue(info.getMessage().startswith("Renamed incorrectly named log level file"))
+        self.assertTrue(infos[1].getMessage().startswith(
+            "Deleted stray log level file 4.log.level"))
+        self.assertTrue(infos[2].getMessage().startswith(
+            "Renamed 1 incorrectly named log level files from previous migrate (1 errors)"))
+        self.assertTrue(infos[3].getMessage().startswith(
+            "Deleted 1 stray log level (0 errors)"))
 
 def suite():
     suite = unittest.TestSuite()
--- a/bitten/upgrades.py
+++ b/bitten/upgrades.py
@@ -384,7 +384,8 @@
         message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,))
         full_filename = os.path.join(logs_dir, filename)
         message_file = codecs.open(full_filename, "wb", "UTF-8")
-        level_file = codecs.open(full_filename+".level", "wb", "UTF-8")
+        # Note: the original version of this code erroneously wrote to filename + ".level" instead of ".levels", producing unused level files
+        level_file = codecs.open(full_filename + '.levels', "wb", "UTF-8")
         for message, level in message_cursor.fetchall() or []:
             message_file.write(to_unicode(message) + "\n")
             level_file.write(to_unicode(level) + "\n")
@@ -402,6 +403,54 @@
     cursor.close()
     env.log.warning("We have dropped the bitten_log_message table - you may want to vaccuum/compress your database to save space")
 
+def fix_log_levels_misnaming(env, db):
+    """Renames or removes *.log.level files created by older versions of migrate_logs_to_files."""
+    logs_dir = env.config.get("bitten", "logs_dir", "log/bitten")
+    if not os.path.isabs(logs_dir):
+        logs_dir = os.path.join(env.path, logs_dir)
+    if not os.path.isdir(logs_dir):
+        return
+
+    rename_count = 0
+    rename_error_count = 0
+    delete_count = 0
+    delete_error_count = 0
+
+    for wrong_filename in os.listdir(logs_dir):
+        if not wrong_filename.endswith('.log.level'):
+            continue
+
+        log_filename = os.path.splitext(wrong_filename)[0]
+        right_filename = log_filename + '.levels'
+        full_log_filename = os.path.join(logs_dir, log_filename)
+        full_wrong_filename = os.path.join(logs_dir, wrong_filename)
+        full_right_filename = os.path.join(logs_dir, right_filename)
+
+        if not os.path.exists(full_log_filename):
+            try:
+                os.remove(full_wrong_filename)
+                delete_count += 1
+                env.log.info("Deleted stray log level file %s", wrong_filename)
+            except Exception, e:
+                delete_error_count += 1
+                env.log.warning("Error removing stray log level file %s: %s", wrong_filename, e)
+        else:
+            if os.path.exists(full_right_filename):
+                env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: new filename already exists",
+                    full_wrong_filename, full_right_filename)
+                rename_error_count += 1
+                continue
+            try:
+                os.rename(full_wrong_filename, full_right_filename)
+                rename_count += 1
+                env.log.info("Renamed incorrectly named log level file %s to %s", wrong_filename, right_filename)
+            except Exception, e:
+                env.log.warning("Error renaming %s to %s in fix_log_levels_misnaming: %s", full_wrong_filename, full_right_filename, e)
+                rename_error_count += 1
+
+    env.log.info("Renamed %d incorrectly named log level files from previous migrate (%d errors)", rename_count, rename_error_count)
+    env.log.info("Deleted %d stray log level (%d errors)", delete_count, delete_error_count)
+
 def recreate_rule_with_int_id(env, db):
         """Recreates the bitten_rule table with an integer id column rather than a text one."""
         from bitten.model import TargetPlatform
@@ -482,4 +531,5 @@
     8: [add_filename_to_logs,migrate_logs_to_files],
     9: [recreate_rule_with_int_id],
    10: [add_config_platform_rev_index_to_build, fix_sequences],
+   11: [fix_log_levels_misnaming],
 }
Copyright (C) 2012-2017 Edgewall Software