# HG changeset patch
# User dfraser
# Date 1236955971 0
# Node ID 384e59137bf8d3c08a1b8d7766d751e95006fb97
# Parent  18485105d1c3e3c71bffa5a31c6abaddb9f8e41e
Support unicode by converting everything to UTF-8 on write and back to unicode on read - should fix #369

diff --git a/bitten/model.py b/bitten/model.py
--- a/bitten/model.py
+++ b/bitten/model.py
@@ -11,6 +11,8 @@
 """Model classes for objects persisted in the database."""
 
 from trac.db import Table, Column, Index
+from trac.util.text import to_unicode
+import codecs
 import os
 
 __docformat__ = 'restructuredtext en'
@@ -750,8 +752,8 @@
         if self.messages:
             log_file_name = self.get_log_file(log_file)
             level_file_name = log_file_name + ".levels"
-            open(log_file_name, "w").writelines([msg[1]+"\n" for msg in self.messages])
-            open(level_file_name, "w").writelines([msg[0]+"\n" for msg in self.messages])
+            open(log_file_name, "w").writelines([to_unicode(msg[1]+"\n").encode("UTF-8") for msg in self.messages])
+            open(level_file_name, "w").writelines([to_unicode(msg[0]+"\n").encode("UTF-8") for msg in self.messages])
 
         if handle_ta:
             db.commit()
@@ -773,12 +775,12 @@
         if log.filename:
             log_filename = log.get_log_file(log.filename)
             if os.path.exists(log_filename):
-                log_lines = open(log_filename, "r").readlines()
+                log_lines = codecs.open(log_filename, "r", "UTF-8").readlines()
             else:
                 log_lines = []
             level_filename = log.filename + ".levels"
             if os.path.exists(level_filename):
-                log_levels = dict(enumerate(open(log.get_log_file(level_filename), "r").readlines()))
+                log_levels = dict(enumerate(codecs.open(log.get_log_file(level_filename), "r", "UTF-8").readlines()))
             else:
                 log_levels = {}
             log.messages = [(log_levels.get(line_num, BuildLog.UNKNOWN), line.rstrip("\n")) for line_num, line in enumerate(log_lines)]
diff --git a/bitten/tests/model.py b/bitten/tests/model.py
--- a/bitten/tests/model.py
+++ b/bitten/tests/model.py
@@ -561,7 +561,7 @@
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         full_file = os.path.join(logs_dir, "1.log")
-        open(full_file, "w").writelines(["running tests\n", "tests failed\n"])
+        open(full_file, "w").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
 
         log = BuildLog.fetch(self.env, id=id, db=db)
         self.assertEqual(True, log.exists)
@@ -571,6 +571,7 @@
         self.assertEqual('distutils', log.generator)
         self.assertEqual((BuildLog.UNKNOWN, 'running tests'), log.messages[0])
         self.assertEqual((BuildLog.UNKNOWN, 'tests failed'), log.messages[1])
+        self.assertEqual((BuildLog.UNKNOWN, u'test unicode\xbb'), log.messages[2])
         if os.path.exists(full_file):
             os.remove(full_file)
 
@@ -585,7 +586,7 @@
             raise ValueError("Should not have absolute logs directory for temporary test")
         logs_dir = os.path.join(self.env.path, logs_dir)
         full_file = os.path.join(logs_dir, "1.log")
-        open(full_file, "w").writelines(["running tests\n", "tests failed\n"])
+        open(full_file, "w").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
 
         logs = BuildLog.select(self.env, build=1, step='test', db=db)
         log = logs.next()
@@ -596,6 +597,7 @@
         self.assertEqual('distutils', log.generator)
         self.assertEqual((BuildLog.UNKNOWN, 'running tests'), log.messages[0])
         self.assertEqual((BuildLog.UNKNOWN, 'tests failed'), log.messages[1])
+        self.assertEqual((BuildLog.UNKNOWN, u'test unicode\xbb'), log.messages[2])
         self.assertRaises(StopIteration, logs.next)
         if os.path.exists(full_file):
             os.remove(full_file)
diff --git a/bitten/upgrades.py b/bitten/upgrades.py
--- a/bitten/upgrades.py
+++ b/bitten/upgrades.py
@@ -15,6 +15,7 @@
 import sys
 
 from trac.db import DatabaseManager
+from trac.util.text import to_unicode
 
 __docformat__ = 'restructuredtext en'
 
@@ -324,8 +325,8 @@
         message_file = open(full_filename, "w")
         level_file = open(full_filename+".level", "w")
         for message, level in message_cursor.fetchall() or []:
-            message_file.write(message + "\n")
-            level_file.write(level + "\n")
+            message_file.write(to_unicode(message).encode('UTF-8') + "\n")
+            level_file.write(to_unicode(level).encode('UTF-8') + "\n")
         message_file.close()
         level_file.close()
         update_cursor.execute("UPDATE bitten_log SET filename=%s WHERE id=%s", (filename, log_id))