Merge pull request #267 from transifex/github-markdown-handle-non-unicode

Handle non Unicode spaces

Author: arisktfx

openformats/formats/github_markdown_v2.py

@@ -3,7 +3,7 @@
 import re
 
 import six
-
+import unicodedata
 from mistune import Markdown
 from yaml.reader import Reader
 
@@ -71,7 +71,7 @@ def parse(self, content, **kwargs):
         # need to do the same in order to be able to match the substrings
         template = content.expandtabs(4)
         pattern = re.compile(ensure_unicode(r'^ +$'), re.M)
-        content = pattern.sub('', template)
+        content = unicodedata.normalize('NFKC', pattern.sub('', template))
 
         stringset = []
 

openformats/tests/formats/github_markdown_v2/test_github_markdown.py

@@ -34,6 +34,13 @@ def test_parse(self):
         content_with_spaces = self.handler.parse(content=u"# foo    bar")
         self.assertEqual(content_with_tab[0], content_with_spaces[0])
 
+    def test_parse_non_unicode(self):
+        """Test parse converts tabs to spaces"""
+        content_with_non_unicode_space = self.handler.parse(content=u"# foo\xa0bar")
+        content_with_normal_space = self.handler.parse(content=u"# foo bar")
+        self.assertEqual(
+            content_with_non_unicode_space[0], content_with_normal_space[0])
+
 
 class GithubMarkdownV2CustomTestCase(unittest.TestCase):
     """Tests some additional functionality of GithubMarkdownHandlerV2.