Fluent parser

openformats/formats/fluent.py

@@ -0,0 +1,111 @@
+import itertools
+import re
+
+from fluent.syntax import ast, parse
+from openformats.strings import OpenString
+from openformats.transcribers import Transcriber
+
+from ..handlers import Handler
+
+
+class FluentHandler(Handler):
+    # https://projectfluent.org/
+    name = "FLUENT"
+    extension = "ftl"
+    EXTRACTS_RAW = True
+
+    def parse(self, source, is_source=False):
+        transcriber = Transcriber(source)
+        source = transcriber.source
+        parsed = parse(source)
+        stringset = []
+        order = itertools.count()
+        for item in parsed.body:
+            if not isinstance(item, (ast.Message, ast.Term)):
+                continue
+            key = item.id.name
+            value = source[item.value.span.start : item.value.span.end]
+            indent = re.search(r"^\s*", value).end()
+            rest = value[indent:]
+            string = OpenString(key, rest, order=next(order))
+            stringset.append(string)
+            transcriber.copy_until(item.value.span.start + indent)
+            transcriber.add(string.template_replacement)
+            transcriber.skip_until(item.value.span.end)
+            for attribute in item.attributes:
+                attribute_key = attribute.id.name
+                value = source[attribute.value.span.start : attribute.value.span.end]
+                indent = re.search(r"^\s*", value).end()
+                rest = value[indent:]
+                string = OpenString(
+                    ".".join((key, attribute_key)), rest, order=next(order)
+                )
+                stringset.append(string)
+                transcriber.copy_until(attribute.value.span.start + indent)
+                transcriber.add(string.template_replacement)
+                transcriber.skip_until(attribute.value.span.end)
+        transcriber.copy_to_end()
+        return transcriber.get_destination(), stringset
+
+    def compile(self, template, stringset, **kwargs):
+        transcriber = Transcriber(template)
+        template = transcriber.source
+        parsed = parse(template)
+        stringset = iter(stringset)
+        next_string = next(stringset, None)
+        delete = False
+        for item in parsed.body:
+            transcriber.copy_until(item.span.start)
+            transcriber.mark_section_end()
+            if delete:
+                transcriber.remove_section()
+                delete = False
+            if not isinstance(item, (ast.Message, ast.Term)):
+                continue
+            transcriber.mark_section_start()
+            value = template[item.value.span.start : item.value.span.end]
+            indent = re.search(r"^\s*", value).end()
+            rest = value[indent:]
+            if next_string is not None and rest == next_string.template_replacement:
+                transcriber.copy_until(item.value.span.start + indent)
+                transcriber.add(next_string.string)
+                next_string = next(stringset, None)
+                transcriber.skip_until(item.value.span.end)
+            else:
+                delete = True
+            for attribute in item.attributes:
+                value = template[attribute.value.span.start : attribute.value.span.end]
+                indent = re.search(r"^\s*", value).end()
+                rest = value[indent:]
+                if next_string is not None and rest == next_string.template_replacement:
+                    transcriber.copy_until(attribute.value.span.start + indent)
+                    transcriber.add(next_string.string)
+                    next_string = next(stringset, None)
+                    transcriber.skip_until(attribute.value.span.end)
+                else:
+                    transcriber.copy_until(attribute.span.start)
+                    transcriber.skip_until(attribute.span.end)
+                # else: delete segment
+        transcriber.copy_to_end()
+        transcriber.mark_section_end()
+        if delete:
+            transcriber.remove_section()
+        return transcriber.get_destination()
+
+    @staticmethod
+    def unescape(string: str):
+        lines = string.splitlines()
+        if len(lines) > 1:
+            indent = re.search(r"^\s*", lines[1]).end()
+            result = [lines[0].strip()]
+            for line in lines[1:]:
+                if not line[:indent].isspace():
+                    return string
+                result.append(line[indent:].strip())
+            return " ".join(result)
+        else:
+            return string
+
+    @staticmethod
+    def escape(string):
+        return string

openformats/tests/formats/fluent/files/1_en.ftl

@@ -0,0 +1,22 @@
+## Closing tabs
+
+tabs-close-button = Close
+tabs-close-tooltip = {$tabCount ->
+    [one] Close {$tabCount} tab
+   *[other] Close {$tabCount} tabs
+}
+tabs-close-warning =
+    You are about to close {$tabCount} tabs.
+    Are you sure you want to continue?
+
+## Syncing
+
+-sync-brand-name = Firefox Account
+
+sync-dialog-title = {-sync-brand-name}
+    .attribute = attribute value
+sync-headline-title =
+    {-sync-brand-name}: The best way to bring
+    your data always with you
+sync-signedout-title =
+    Connect with your {-sync-brand-name}

openformats/tests/formats/fluent/test_fluent.py

@@ -0,0 +1,87 @@
+import unittest
+
+from openformats.formats.fluent import FluentHandler
+from openformats.strings import OpenString
+
+
+class FluentTestCase(unittest.TestCase):
+    def setUp(self) -> None:
+        self.handler = FluentHandler()
+
+    def test_simple(self):
+        template, stringset = self.handler.parse("a = b")
+        string = stringset[0]
+        self.assertEqual(string.key, "a")
+        self.assertEqual(string.string, "b")
+        self.assertEqual(string.order, 0)
+        self.assertEqual(template, f"a = {string.template_replacement}")
+
+        compiled = self.handler.compile(template, stringset)
+        self.assertEqual(compiled, "a = b")
+
+        compiled = self.handler.compile(template, [OpenString("a", "bbb", order=0)])
+        self.assertEqual(compiled, "a = bbb")
+
+    def test_multiline(self):
+        source = "\n".join(["a = b", "  c", "  d"])
+        template, stringset = self.handler.parse(source)
+        string = stringset[0]
+        self.assertEqual(string.key, "a")
+        self.assertEqual(string.string, "\n".join(["b", "  c", "  d"]))
+        self.assertEqual(string.order, 0)
+        self.assertEqual(template, f"a = {string.template_replacement}")
+        self.assertEqual(FluentHandler.unescape(string.string), "b c d")
+
+        compiled = self.handler.compile(template, stringset)
+        self.assertEqual(compiled, source)
+
+    def test_attributes(self):
+        source = "\n".join(["a = b", "  .c = d"])
+        template, stringset = self.handler.parse(source)
+        self.assertEqual(stringset[0].key, "a")
+        self.assertEqual(stringset[0].string, "b")
+        self.assertEqual(stringset[0].order, 0)
+        self.assertEqual(stringset[1].key, "a.c")
+        self.assertEqual(stringset[1].string, "d")
+        self.assertEqual(stringset[1].order, 1)
+        self.assertEqual(
+            template,
+            "\n".join(
+                [
+                    "a = {}".format(stringset[0].template_replacement),
+                    "  .c = {}".format(stringset[1].template_replacement),
+                ]
+            ),
+        )
+        compiled = self.handler.compile(template, stringset)
+        self.assertEqual(compiled, source)
+
+    def test_skip_message(self):
+        template, stringset = self.handler.parse("\n".join(["a = b", "c = d"]))
+
+        compiled = self.handler.compile(template, stringset[1:])
+        self.assertEqual(compiled.strip(), "c = d")
+        compiled = self.handler.compile(template, stringset[:1])
+        self.assertEqual(compiled.strip(), "a = b")
+
+    def test_skip_message_with_attributes(self):
+        template, stringset = self.handler.parse(
+            "\n".join(["a = b", "  .c = d", "e = f"])
+        )
+
+        compiled = self.handler.compile(template, stringset[1:])
+        self.assertEqual(compiled.strip(), "e = f")
+
+    def test_skip_attribute(self):
+        template, stringset = self.handler.parse(
+            "\n".join(["a = b", "  .c = d", "  .e = f"])
+        )
+
+        compiled = self.handler.compile(template, stringset[:2])
+        self.assertEqual(compiled.strip(), "\n".join(["a = b", "  .c = d"]))
+
+        compiled = self.handler.compile(template, stringset[::2])
+        self.assertEqual(
+            [line for line in compiled.splitlines() if not line.isspace()],
+            ["a = b", "  .e = f"],
+        )

requirements.txt

@@ -1,11 +1,12 @@
 PyYAML==5.4
-django==1.11.29
+Django==2.2.28
 mistune==0.7.3
 polib==1.0.3
 pyparsing==2.2.0
 six
 lxml==4.6.2
 beautifulsoup4==4.9.3
+fluent.syntax==0.18.1
 
 # InDesign
 git+https://github.com/kbairak/ucflib@py3_compatibility

testbed/main/models.py

@@ -2,31 +2,29 @@
 from hashlib import md5
 
 from django.db import models
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 
 class Payload(models.Model):
-    payload_hash = models.CharField(max_length=32, unique=True,
-                                    blank=True, null=True)
+    payload_hash = models.CharField(max_length=32, unique=True, blank=True, null=True)
     payload = models.TextField()
     created = models.DateTimeField(auto_now_add=True)
     last_viewed = models.DateTimeField(blank=True, null=True)
 
     def _presave(self):
-        self.payload_hash = md5(self.payload.encode('utf-8')).hexdigest()
+        self.payload_hash = md5(self.payload.encode("utf-8")).hexdigest()
 
     def save(self):
         self._presave()
         super(Payload, self).save()
 
     def get_absolute_url(self):
-        return reverse("testbed_main",
-                       kwargs={'payload_hash': self.payload_hash})
+        return reverse("testbed_main", kwargs={"payload_hash": self.payload_hash})
 
     def __unicode__(self):
         payload = json.loads(self.payload)
-        handler = payload.get('handler', None)
+        handler = payload.get("handler", None)
         if handler:
-            return u"{}-{}".format(handler, self.payload_hash[-8:])
+            return "{}-{}".format(handler, self.payload_hash[-8:])
         else:
             return self.payload_hash[-8:]