# HG changeset patch
# User dfraser
# Date 1237204999 0
# Node ID b661ea254972d0abe915eccc3a9032daf002b53e
# Parent  a87ff6fc96b9e0cc3fb2972850b41214effd5720
Ensure log files are stored and read in binary, not text format (otherwise Unicode gets confused):
 * Use the `codecs` module to write as well as read from files
 * Explicitly change the mode to binary to avoid confusion
 * Adjust tests to read in binary mode

diff --git a/bitten/model.py b/bitten/model.py
--- a/bitten/model.py
+++ b/bitten/model.py
@@ -763,8 +763,8 @@
         if self.messages:
             log_file_name = self.get_log_file(log_file)
             level_file_name = log_file_name + ".levels"
-            open(log_file_name, "w").writelines([to_unicode(msg[1]+"\n").encode("UTF-8") for msg in self.messages])
-            open(level_file_name, "w").writelines([to_unicode(msg[0]+"\n").encode("UTF-8") for msg in self.messages])
+            codecs.open(log_file_name, "wb", "UTF-8").writelines([to_unicode(msg[1]+"\n") for msg in self.messages])
+            codecs.open(level_file_name, "wb", "UTF-8").writelines([to_unicode(msg[0]+"\n") for msg in self.messages])
 
         if handle_ta:
             db.commit()
@@ -786,12 +786,12 @@
         if log.filename:
             log_filename = log.get_log_file(log.filename)
             if os.path.exists(log_filename):
-                log_lines = codecs.open(log_filename, "r", "UTF-8").readlines()
+                log_lines = codecs.open(log_filename, "rb", "UTF-8").readlines()
             else:
                 log_lines = []
             level_filename = log.filename + ".levels"
             if os.path.exists(level_filename):
-                log_levels = dict(enumerate(codecs.open(log.get_log_file(level_filename), "r", "UTF-8").readlines()))
+                log_levels = dict(enumerate(codecs.open(log.get_log_file(level_filename), "rb", "UTF-8").readlines()))
             else:
                 log_levels = {}
             log.messages = [(log_levels.get(line_num, BuildLog.UNKNOWN), line.rstrip("\n")) for line_num, line in enumerate(log_lines)]
diff --git a/bitten/tests/model.py b/bitten/tests/model.py
--- a/bitten/tests/model.py
+++ b/bitten/tests/model.py
@@ -489,7 +489,7 @@
         cursor.execute("SELECT build,step,generator,filename FROM bitten_log "
                        "WHERE id=%s", (log.id,))
         self.assertEqual((1, 'test', 'distutils', '1.log'), cursor.fetchone())
-        lines = open(full_file, "r").readlines()
+        lines = open(full_file, "rb").readlines()
         self.assertEqual('running tests\n', lines[0])
         self.assertEqual('tests failed\n', lines[1])
         if os.path.exists(full_file):
@@ -532,7 +532,7 @@
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         full_file = os.path.join(logs_dir, "1.log")
-        open(full_file, "w").writelines(["running tests\n", "tests failed\n"])
+        open(full_file, "wb").writelines(["running tests\n", "tests failed\n"])
 
         log = BuildLog.fetch(self.env, id=id, db=db)
         self.assertEqual(True, log.exists)
@@ -561,7 +561,7 @@
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         full_file = os.path.join(logs_dir, "1.log")
-        open(full_file, "w").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
+        open(full_file, "wb").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
 
         log = BuildLog.fetch(self.env, id=id, db=db)
         self.assertEqual(True, log.exists)
@@ -586,7 +586,7 @@
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         full_file = os.path.join(logs_dir, "1.log")
-        open(full_file, "w").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
+        open(full_file, "wb").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
 
         logs = BuildLog.select(self.env, build=1, step='test', db=db)
         log = logs.next()
diff --git a/bitten/upgrades.py b/bitten/upgrades.py
--- a/bitten/upgrades.py
+++ b/bitten/upgrades.py
@@ -16,6 +16,7 @@
 
 from trac.db import DatabaseManager
 from trac.util.text import to_unicode
+import codecs
 
 __docformat__ = 'restructuredtext en'
 
@@ -322,11 +323,11 @@
         filename = "%s.log" % (log_id,)
         message_cursor.execute("SELECT message, level FROM bitten_log_message WHERE log=%s ORDER BY line", (log_id,))
         full_filename = os.path.join(logs_dir, filename)
-        message_file = open(full_filename, "w")
-        level_file = open(full_filename+".level", "w")
+        message_file = codecs.open(full_filename, "wb", "UTF-8")
+        level_file = codecs.open(full_filename+".level", "wb", "UTF-8")
         for message, level in message_cursor.fetchall() or []:
-            message_file.write(to_unicode(message).encode('UTF-8') + "\n")
-            level_file.write(to_unicode(level).encode('UTF-8') + "\n")
+            message_file.write(to_unicode(message) + "\n")
+            level_file.write(to_unicode(level) + "\n")
         message_file.close()
         level_file.close()
         update_cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (filename, log_id))