Implement Syntax 0.6 in the tooling parser (#69)

Spec changelog: https://github.com/projectfluent/fluent/releases/tag/v0.6.0.
Based on projectfluent/fluent.js#253.

Author: stasm

fluent/syntax/ast.py

@@ -151,12 +151,7 @@ def __init__(self, body=None, **kwargs):
 
 
 class Entry(SyntaxNode):
-    def __init__(self, annotations=None, **kwargs):
-        super(Entry, self).__init__(**kwargs)
-        self.annotations = annotations or []
-
-    def add_annotation(self, annot):
-        self.annotations.append(annot)
+    """An abstract base class for useful elements of Resource.body."""
 
 
 class Message(Entry):
@@ -179,14 +174,20 @@ def __init__(self, id, value, attributes=None,
         self.comment = comment
 
 
+class VariantList(SyntaxNode):
+    def __init__(self, variants, **kwargs):
+        super(VariantList, self).__init__(**kwargs)
+        self.variants = variants
+
+
 class Pattern(SyntaxNode):
     def __init__(self, elements, **kwargs):
         super(Pattern, self).__init__(**kwargs)
         self.elements = elements
 
 
 class PatternElement(SyntaxNode):
-    pass
+    """An abstract base class for elements of Patterns."""
 
 
 class TextElement(PatternElement):
@@ -202,19 +203,18 @@ def __init__(self, expression, **kwargs):
 
 
 class Expression(SyntaxNode):
-    def __init__(self, **kwargs):
-        super(Expression, self).__init__(**kwargs)
+    """An abstract base class for expressions."""
 
 
-class StringExpression(Expression):
+class StringLiteral(Expression):
     def __init__(self, value, **kwargs):
-        super(StringExpression, self).__init__(**kwargs)
+        super(StringLiteral, self).__init__(**kwargs)
         self.value = value
 
 
-class NumberExpression(Expression):
+class NumberLiteral(Expression):
     def __init__(self, value, **kwargs):
-        super(NumberExpression, self).__init__(**kwargs)
+        super(NumberLiteral, self).__init__(**kwargs)
         self.value = value
 
 
@@ -224,23 +224,29 @@ def __init__(self, id, **kwargs):
         self.id = id
 
 
-class ExternalArgument(Expression):
+class TermReference(Expression):
+    def __init__(self, id, **kwargs):
+        super(TermReference, self).__init__(**kwargs)
+        self.id = id
+
+
+class VariableReference(Expression):
     def __init__(self, id, **kwargs):
-        super(ExternalArgument, self).__init__(**kwargs)
+        super(VariableReference, self).__init__(**kwargs)
         self.id = id
 
 
 class SelectExpression(Expression):
-    def __init__(self, expression, variants, **kwargs):
+    def __init__(self, selector, variants, **kwargs):
         super(SelectExpression, self).__init__(**kwargs)
-        self.expression = expression
+        self.selector = selector
         self.variants = variants
 
 
 class AttributeExpression(Expression):
-    def __init__(self, id, name, **kwargs):
+    def __init__(self, ref, name, **kwargs):
         super(AttributeExpression, self).__init__(**kwargs)
-        self.id = id
+        self.ref = ref
         self.name = name
 
 
@@ -252,10 +258,11 @@ def __init__(self, ref, key, **kwargs):
 
 
 class CallExpression(Expression):
-    def __init__(self, callee, args=None, **kwargs):
+    def __init__(self, callee, positional=None, named=None, **kwargs):
         super(CallExpression, self).__init__(**kwargs)
         self.callee = callee
-        self.args = args or []
+        self.positional = positional or []
+        self.named = named or []
 
 
 class Attribute(SyntaxNode):
@@ -278,16 +285,16 @@ def __init__(self, key, value, default=False, **kwargs):
 
     @property
     def sorting_key(self):
-        if isinstance(self.key, NumberExpression):
+        if isinstance(self.key, NumberLiteral):
             return self.key.value
         return self.key.name
 
 
 class NamedArgument(SyntaxNode):
-    def __init__(self, name, val, **kwargs):
+    def __init__(self, name, value, **kwargs):
         super(NamedArgument, self).__init__(**kwargs)
         self.name = name
-        self.val = val
+        self.value = value
 
 
 class Identifier(SyntaxNode):
@@ -327,10 +334,14 @@ def __init__(self, name, **kwargs):
         super(Function, self).__init__(name, **kwargs)
 
 
-class Junk(Entry):
-    def __init__(self, content=None, **kwargs):
+class Junk(SyntaxNode):
+    def __init__(self, content=None, annotations=None, **kwargs):
         super(Junk, self).__init__(**kwargs)
         self.content = content
+        self.annotations = annotations or []
+
+    def add_annotation(self, annot):
+        self.annotations.append(annot)
 
 
 class Span(BaseNode):

fluent/syntax/errors.py

@@ -51,4 +51,16 @@ def get_error_message(code, args):
         return 'Attributes of terms cannot be used as placeables'
     if code == 'E0020':
         return 'Unterminated string expression'
+    if code == 'E0021':
+        return 'Positional arguments must not follow named arguments'
+    if code == 'E0022':
+        return 'Named arguments must be unique'
+    if code == 'E0023':
+        return 'VariantLists are only allowed inside of other VariantLists.'
+    if code == 'E0024':
+        return 'Cannot access variants of a message.'
+    if code == 'E0025':
+        return 'Unknown escape sequence: {}'.format(args[0])
+    if code == 'E0026':
+        return 'Invalid Unicode escape sequence: {}'.format(args[0])
     return code

fluent/syntax/ftlstream.py

@@ -24,15 +24,17 @@ def peek_inline_ws(self):
             ch = self.peek()
 
     def skip_blank_lines(self):
+        line_count = 0
         while True:
             self.peek_inline_ws()
 
             if self.current_peek_is('\n'):
                 self.skip_to_peek()
                 self.next()
+                line_count += 1
             else:
                 self.reset_peek()
-                break
+                return line_count
 
     def peek_blank_lines(self):
         while True:
@@ -67,11 +69,11 @@ def expect_indent(self):
         self.expect_char(' ')
         self.skip_inline_ws()
 
-    def take_char_if(self, ch):
-        if self.ch == ch:
-            self.next()
+    def expect_line_end(self):
+        if self.ch is None:
+            # EOF is a valid line end in Fluent.
             return True
-        return False
+        return self.expect_char('\n')
 
     def take_char(self, f):
         ch = self.ch
@@ -88,10 +90,7 @@ def is_char_id_start(self, ch=None):
         return (cc >= 97 and cc <= 122) or \
                (cc >= 65 and cc <= 90)
 
-    def is_entry_id_start(self):
-        if self.current_is('-'):
-            self.peek()
-
+    def is_identifier_start(self):
         ch = self.current_peek()
         is_id = self.is_char_id_start(ch)
         self.reset_peek()
@@ -112,15 +111,15 @@ def is_char_pattern_continuation(self, ch):
 
         return ch not in SPECIAL_LINE_START_CHARS
 
-    def is_peek_pattern_start(self):
+    def is_peek_value_start(self):
         self.peek_inline_ws()
         ch = self.current_peek()
 
         # Inline Patterns may start with any char.
         if ch is not None and ch != '\n':
             return True
 
-        return self.is_peek_next_line_pattern_start()
+        return self.is_peek_next_line_value()
 
     def is_peek_next_line_zero_four_style_comment(self):
         if not self.current_peek_is('\n'):
@@ -150,7 +149,7 @@ def is_peek_next_line_comment(self, level=-1):
         while (i <= level or (level == -1 and i < 3)):
             self.peek()
             if not self.current_peek_is('#'):
-                if i != level and level != -1:
+                if i <= level and level != -1:
                     self.reset_peek()
                     return False
                 break
@@ -214,7 +213,7 @@ def is_peek_next_line_attribute_start(self):
         self.reset_peek()
         return False
 
-    def is_peek_next_line_pattern_start(self):
+    def is_peek_next_line_value(self):
         if not self.current_peek_is('\n'):
             return False
 
@@ -243,25 +242,21 @@ def skip_to_next_entry_start(self):
                 self.next()
 
                 if self.ch is None or \
-                   self.is_entry_id_start() or \
+                   self.is_identifier_start() or \
+                   self.current_is('-') or \
                    self.current_is('#') or \
                    (self.current_is('/') and self.peek_char_is('/')) or \
                    (self.current_is('[') and self.peek_char_is('[')):
                     break
             self.next()
 
-    def take_id_start(self, allow_term):
-        if allow_term and self.current_is('-'):
-            self.next()
-            return '-'
-
+    def take_id_start(self):
         if self.is_char_id_start(self.ch):
             ret = self.ch
             self.next()
             return ret
 
-        allowed_range = 'a-zA-Z-' if allow_term else 'a-zA-Z'
-        raise ParseError('E0004', allowed_range)
+        raise ParseError('E0004', 'a-zA-Z')
 
     def take_id_char(self):
         def closure(ch):
@@ -288,3 +283,12 @@ def closure(ch):
             cc = ord(ch)
             return (cc >= 48 and cc <= 57)
         return self.take_char(closure)
+
+    def take_hex_digit(self):
+        def closure(ch):
+            cc = ord(ch)
+            return (
+                (cc >= 48 and cc <= 57)   # 0-9
+                or (cc >= 65 and cc <= 70)  # A-F
+                or (cc >= 97 and cc <= 102))  # a-f
+        return self.take_char(closure)