Implement print_function future feature (google#23)

* Implement print_function future feature

This allows the import statement `from __future__ import print_function` to
work, by setting a flag on the parser to treat print statement/keyword as a
syntax error.

Author: paulsmith

compiler/block.py

@@ -62,6 +62,7 @@ class Block(object):
   _runtime = None
   _strings = None
   imports = None
+  _future_features = None
 
   def __init__(self, parent_block, name):
     self.parent_block = parent_block
@@ -104,6 +105,11 @@ def lines(self):
   def strings(self):
     return self._module_block._strings  # pylint: disable=protected-access
 
+  @property
+  def future_features(self):
+    # pylint: disable=protected-access
+    return self._module_block._future_features
+
   @abc.abstractmethod
   def bind_var(self, writer, name, value):
     """Writes Go statements for assigning value to named var in this block.
@@ -216,7 +222,8 @@ class ModuleBlock(Block):
     imports: A dict mapping fully qualified Go package names to Package objects.
   """
 
-  def __init__(self, full_package_name, runtime, libroot, filename, lines):
+  def __init__(self, full_package_name, runtime, libroot, filename, lines,
+               future_features):
     super(ModuleBlock, self).__init__(None, '<module>')
     self._full_package_name = full_package_name
     self._runtime = runtime
@@ -225,6 +232,7 @@ def __init__(self, full_package_name, runtime, libroot, filename, lines):
     self._lines = lines
     self._strings = set()
     self.imports = {}
+    self._future_features = future_features
 
   def bind_var(self, writer, name, value):
     writer.write_checked_call1(
@@ -440,5 +448,5 @@ def __init__(self, node):
         raise util.ParseError(node, msg)
       self.vars[name] = Var(name, Var.TYPE_PARAM, arg_index=i)
 
-  def visit_Yield(self, unused_node):
+  def visit_Yield(self, unused_node): # pylint: disable=unused-argument
     self.is_generator = True

compiler/block_test.py

@@ -21,9 +21,9 @@
 import unittest
 
 from grumpy.compiler import block
+from grumpy.compiler import stmt
 from grumpy.compiler import util
 
-
 class PackageTest(unittest.TestCase):
 
   def testCreate(self):
@@ -243,7 +243,8 @@ def testYieldExpr(self):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [])
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+                           stmt.FutureFeatures())
 
 
 def _ParseStmt(stmt_str):

compiler/expr_visitor_test.py

@@ -24,6 +24,7 @@
 from grumpy.compiler import block
 from grumpy.compiler import expr_visitor
 from grumpy.compiler import shard_test
+from grumpy.compiler import stmt
 from grumpy.compiler import util
 
 
@@ -220,7 +221,8 @@ def testUnaryOpNotImplemented(self):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [])
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+                           stmt.FutureFeatures())
 
 
 def _ParseExpr(expr):

compiler/stmt.py

@@ -32,13 +32,103 @@
 _nil_expr = expr.nil_expr
 
 
+# Parser flags, set on 'from __future__ import *', see parser_flags on
+# StatementVisitor below. Note these have the same values as CPython.
+FUTURE_DIVISION = 0x2000
+FUTURE_ABSOLUTE_IMPORT = 0x4000
+FUTURE_PRINT_FUNCTION = 0x10000
+FUTURE_UNICODE_LITERALS = 0x20000
+
+# Names for future features in 'from __future__ import *'. Map from name in the
+# import statement to a tuple of the flag for parser, and whether we've (grumpy)
+# implemented the feature yet.
+future_features = {
+    "division": (FUTURE_DIVISION, False),
+    "absolute_import": (FUTURE_ABSOLUTE_IMPORT, False),
+    "print_function": (FUTURE_PRINT_FUNCTION, True),
+    "unicode_literals": (FUTURE_UNICODE_LITERALS, False),
+}
+
+# These future features are already in the language proper as of 2.6, so
+# importing them via __future__ has no effect.
+redundant_future_features = ["generators", "with_statement", "nested_scopes"]
+
+late_future = 'from __future__ imports must occur at the beginning of the file'
+
+
+def import_from_future(node):
+  """Processes a future import statement, returning set of flags it defines."""
+  assert isinstance(node, ast.ImportFrom)
+  assert node.module == '__future__'
+  flags = 0
+  for alias in node.names:
+    name = alias.name
+    if name in future_features:
+      flag, implemented = future_features[name]
+      if not implemented:
+        msg = 'future feature {} not yet implemented by grumpy'.format(name)
+        raise util.ParseError(node, msg)
+      flags |= flag
+    elif name == 'braces':
+      raise util.ParseError(node, 'not a chance')
+    elif name not in redundant_future_features:
+      msg = 'future feature {} is not defined'.format(name)
+      raise util.ParseError(node, msg)
+  return flags
+
+
+class FutureFeatures(object):
+  def __init__(self):
+    self.parser_flags = 0
+    self.future_lineno = 0
+
+
+def visit_future(node):
+  """Accumulates a set of compiler flags for the compiler __future__ imports.
+
+  Returns an instance of FutureFeatures which encapsulates the flags and the
+  line number of the last valid future import parsed. A downstream parser can
+  use the latter to detect invalid future imports that appear too late in the
+  file.
+  """
+  # If this is the module node, do an initial pass through the module body's
+  # statements to detect future imports and process their directives (i.e.,
+  # set compiler flags), and detect ones that don't appear at the beginning of
+  # the file. The only things that can proceed a future statement are other
+  # future statements and/or a doc string.
+  assert isinstance(node, ast.Module)
+  ff = FutureFeatures()
+  done = False
+  found_docstring = False
+  for node in node.body:
+    if isinstance(node, ast.ImportFrom):
+      modname = node.module
+      if modname == '__future__':
+        if done:
+          raise util.ParseError(node, late_future)
+        ff.parser_flags |= import_from_future(node)
+        ff.future_lineno = node.lineno
+      else:
+        done = True
+    elif isinstance(node, ast.Expr) and not found_docstring:
+      e = node.value
+      if not isinstance(e, ast.Str): # pylint: disable=simplifiable-if-statement
+        done = True
+      else:
+        found_docstring = True
+    else:
+      done = True
+  return ff
+
+
 class StatementVisitor(ast.NodeVisitor):
   """Outputs Go statements to a Writer for the given Python nodes."""
 
   # pylint: disable=invalid-name,missing-docstring
 
   def __init__(self, block_):
     self.block = block_
+    self.future_features = self.block.future_features or FutureFeatures()
     self.writer = util.Writer()
     self.expr_visitor = expr_visitor.ExprVisitor(self.block, self.writer)
 
@@ -286,6 +376,12 @@ def visit_ImportFrom(self, node):
                 mod.expr, self.block.intern(name))
             self.block.bind_var(
                 self.writer, alias.asname or alias.name, member.expr)
+    elif node.module == '__future__':
+      # At this stage all future imports are done in an initial pass (see
+      # visit() above), so if they are encountered here after the last valid
+      # __future__ then it's a syntax error.
+      if node.lineno > self.future_features.future_lineno:
+        raise util.ParseError(node, late_future)
     else:
       # NOTE: Assume that the names being imported are all modules within a
       # package. E.g. "from a.b import c" is importing the module c from package
@@ -305,6 +401,8 @@ def visit_Pass(self, node):
     self._write_py_context(node.lineno)
 
   def visit_Print(self, node):
+    if self.future_features.parser_flags & FUTURE_PRINT_FUNCTION:
+      raise util.ParseError(node, 'syntax error (print is not a keyword)')
     self._write_py_context(node.lineno)
     with self.block.alloc_temp('[]*πg.Object') as args:
       self.writer.write('{} = make([]*πg.Object, {})'.format(

compiler/stmt_test.py

@@ -287,12 +287,105 @@ def testImportNativeType(self):
         from __go__.time import type_Duration as Duration
         print Duration""")))
 
-  def testPrint(self):
+  def testPrintStatement(self):
     self.assertEqual((0, 'abc 123\nfoo bar\n'), _GrumpRun(textwrap.dedent("""\
         print 'abc',
         print '123'
         print 'foo', 'bar'""")))
 
+  def testImportFromFuture(self):
+    testcases = [
+        ('from __future__ import print_function', stmt.FUTURE_PRINT_FUNCTION),
+        ('from __future__ import generators', 0),
+        ('from __future__ import generators, print_function',
+         stmt.FUTURE_PRINT_FUNCTION),
+    ]
+
+    for i, tc in enumerate(testcases):
+      source, want_flags = tc
+      mod = ast.parse(textwrap.dedent(source))
+      node = mod.body[0]
+      got = stmt.import_from_future(node)
+      msg = '#{}: want {}, got {}'.format(i, want_flags, got)
+      self.assertEqual(want_flags, got, msg=msg)
+
+  def testImportFromFutureParseError(self):
+    testcases = [
+        # NOTE: move this group to testImportFromFuture as they are implemented
+        # by grumpy
+        ('from __future__ import absolute_import',
+         r'future feature \w+ not yet implemented'),
+        ('from __future__ import division',
+         r'future feature \w+ not yet implemented'),
+        ('from __future__ import unicode_literals',
+         r'future feature \w+ not yet implemented'),
+
+        ('from __future__ import braces', 'not a chance'),
+        ('from __future__ import nonexistant_feature',
+         r'future feature \w+ is not defined'),
+    ]
+
+    for tc in testcases:
+      source, want_regexp = tc
+      mod = ast.parse(source)
+      node = mod.body[0]
+      self.assertRaisesRegexp(util.ParseError, want_regexp,
+                              stmt.import_from_future, node)
+
+  def testVisitFuture(self):
+    testcases = [
+        ('from __future__ import print_function',
+         stmt.FUTURE_PRINT_FUNCTION, 1),
+        ("""\
+        "module docstring"
+
+        from __future__ import print_function
+        """, stmt.FUTURE_PRINT_FUNCTION, 3),
+        ("""\
+        "module docstring"
+
+        from __future__ import print_function, with_statement
+        from __future__ import nested_scopes
+        """, stmt.FUTURE_PRINT_FUNCTION, 4),
+    ]
+
+    for tc in testcases:
+      source, flags, lineno = tc
+      mod = ast.parse(textwrap.dedent(source))
+      future_features = stmt.visit_future(mod)
+      self.assertEqual(future_features.parser_flags, flags)
+      self.assertEqual(future_features.future_lineno, lineno)
+
+  def testVisitFutureParseError(self):
+    testcases = [
+        # future after normal imports
+        """\
+        import os
+        from __future__ import print_function
+        """,
+        # future after non-docstring expression
+        """
+        asd = 123
+        from __future__ import print_function
+        """
+    ]
+
+    for source in testcases:
+      mod = ast.parse(textwrap.dedent(source))
+      self.assertRaisesRegexp(util.ParseError, stmt.late_future,
+                              stmt.visit_future, mod)
+
+  def testFutureFeaturePrintFunction(self):
+    want = "abc\n123\nabc 123\nabcx123\nabc 123 "
+    self.assertEqual((0, want), _GrumpRun(textwrap.dedent("""\
+        "module docstring is ok to proceed __future__"
+        from __future__ import print_function
+        print('abc')
+        print(123)
+        print('abc', 123)
+        print('abc', 123, sep='x')
+        print('abc', 123, end=' ')""")))
+
   def testRaiseExitStatus(self):
     self.assertEqual(1, _GrumpRun('raise Exception')[0])
 
@@ -465,14 +558,17 @@ def testWriteExceptDispatcherMultipleExcept(self):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [])
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+                           stmt.FutureFeatures())
 
 
 def _ParseAndVisit(source):
+  mod = ast.parse(source)
+  future_features = stmt.visit_future(mod)
   b = block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>',
-                        source.split('\n'))
+                        source.split('\n'), future_features)
   visitor = stmt.StatementVisitor(b)
-  visitor.visit(ast.parse(source))
+  visitor.visit(mod)
   return visitor
 
 

compiler/util_test.py

@@ -20,6 +20,7 @@
 
 from grumpy.compiler import block
 from grumpy.compiler import util
+from grumpy.compiler import stmt
 
 
 class WriterTest(unittest.TestCase):
@@ -34,8 +35,9 @@ def testIndentBlock(self):
 
   def testWriteBlock(self):
     writer = util.Writer()
-    writer.write_block(block.ModuleBlock(
-        '__main__', 'grumpy', 'grumpy/lib', '<test>', []), 'BODY')
+    mod_block = block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>',
+                                  [], stmt.FutureFeatures())
+    writer.write_block(mod_block, 'BODY')
     output = writer.out.getvalue()
     dispatch = 'switch πF.State() {\n\tcase 0:\n\tdefault: panic'
     self.assertIn(dispatch, output)