Various Unicode improvements.

* Convert command line arguments from UTF-8 to Unicode right away. All
  Command classes should expect Unicode strings as arguments.
* Add test case for special characters for the 'ls' subcommand. Other
  commands are still likely to be broken.

test/ListCommandTest.py

@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from six import u, PY2
 import unittest
 
 from topydo.lib.Config import config
@@ -187,5 +188,28 @@ class ListCommandTest(CommandTest):
         self.assertEqual(self.output, "")
         self.assertEqual(self.errors, command.usage() + "\n\n" + command.help() + "\n")
 
+def _utf8(p_string):
+    """ Converts a Unicode string to UTF-8 in case of Python 2. """
+    if PY2:
+        p_string = p_string.encode('utf-8')
+
+    return p_string
+
+class ListCommandUnicodeTest(CommandTest):
+    def setUp(self):
+        super(ListCommandUnicodeTest, self).setUp()
+        self.todolist = load_file_to_todolist("test/data/ListCommandUnicodeTest.txt")
+
+    def test_list_unicode1(self):
+        """ Unicode filters """
+        command = ListCommand([u("\u25c4")], self.todolist, self.out, self.error)
+        command.execute()
+
+        self.assertFalse(self.todolist.is_dirty())
+
+        expected = _utf8(u("|  1| (C) And some sp\u00e9cial tag:\u25c4\n"))
+
+        self.assertEqual(self.output, expected)
+
 if __name__ == '__main__':
     unittest.main()

test/data/ListCommandUnicodeTest.txt

+(C) And some spécial tag:◄

topydo/cli/CLIApplicationBase.py

@@ -18,6 +18,7 @@
 
 import getopt
 import sys
+from six import PY2
 from six.moves import input
 
 def usage():
@@ -113,12 +114,17 @@ class CLIApplicationBase(object):
         self.todofile = None
 
     def _usage(self):
-        print USAGE
+        usage()
         sys.exit(0)
 
     def _process_flags(self):
+        args = sys.argv[1:]
+
+        if PY2:
+            args = [arg.decode('utf-8') for arg in args]
+
         try:
-            opts, args = getopt.getopt(sys.argv[1:], "c:d:ht:v")
+            opts, args = getopt.getopt(args, "c:d:ht:v")
         except getopt.GetoptError as e:
             error(str(e))
             sys.exit(1)

topydo/lib/Filter.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import re
+from six import text_type
 
 from topydo.lib.Config import config
 from topydo.lib.RelativeDate import relative_date_to_date
@@ -63,7 +64,7 @@ class GrepFilter(Filter):
         super(GrepFilter, self).__init__()
 
         # convert to string in case we receive integers
-        self.expression = str(p_expression)
+        self.expression = text_type(p_expression)
 
         if p_case_sensitive != None:
             self.case_sensitive = p_case_sensitive

topydo/lib/TodoBase.py

@@ -20,10 +20,12 @@ This module contains the class that represents a single todo item.
 
 from datetime import date
 import re
+from six import python_2_unicode_compatible
 
 from topydo.lib.TodoParser import parse_line
 from topydo.lib.Utils import is_valid_priority
 
+@python_2_unicode_compatible
 class TodoBase(object):
     """
     This class represents a single todo item in a todo.txt file. It maintains