Move pythonparser into grumpy package (#291)

make install was broken since we moved to pythonparser because it was
not being copied into the install dir. Rather than just copying it into
the PYTHONPATH alongside grumpy, I thought it better to embed it within
the grumpy package so that it won't conflict with an installed
pythonparser.

This addresses: google/grumpy#288

Author: davidc05

Makefile

@@ -55,7 +55,7 @@ export PATH := $(ROOT_DIR)/build/bin:$(PATH)
 
 GOPATH_PY_ROOT := $(GOPATH)/src/__python__
 
-PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/%,$(wildcard third_party/pythonparser/*.py))
+PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/grumpy/%,$(wildcard third_party/pythonparser/*.py))
 
 COMPILER_BIN := build/bin/grumpc
 COMPILER_SRCS := $(addprefix $(PY_DIR)/grumpy/compiler/,$(notdir $(shell find compiler -name '*.py' -not -name '*_test.py'))) $(PY_DIR)/grumpy/__init__.py
@@ -217,8 +217,8 @@ $(PYLINT_BIN):
 	@cd build/third_party && curl -sL https://pypi.io/packages/source/p/pylint/pylint-1.6.4.tar.gz | tar -zx
 	@cd build/third_party/pylint-1.6.4 && $(PYTHON) setup.py install --prefix $(ROOT_DIR)/build
 
-pylint: $(PYLINT_BIN)
-	@$(PYTHON) $(PYLINT_BIN) compiler/*.py $(addprefix tools/,benchcmp coverparse diffrange genmake grumpc grumprun pydeps)
+pylint: $(PYLINT_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS)
+	@$(PYTHON) $(PYLINT_BIN) $(COMPILER_SRCS) $(COMPILER_BIN) $(RUNNER_BIN) $(TOOL_BINS)
 
 lint: golint pylint
 
@@ -284,7 +284,7 @@ $(eval $(foreach x,$(STDLIB_TESTS),$(call GRUMPY_STDLIB_TEST,$(x))))
 $(PY_DIR)/weetest.py: lib/weetest.py
 	@cp -f $< $@
 
-$(PYTHONPARSER_SRCS): $(PY_DIR)/%: third_party/%
+$(PYTHONPARSER_SRCS): $(PY_DIR)/grumpy/%: third_party/%
 	@mkdir -p $(@D)
 	@cp -f $< $@
 
@@ -328,4 +328,5 @@ install: $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB)
 	install -d "$(DESTDIR)"{/usr/lib/go,"$(PY_INSTALL_DIR)"}
 	cp -rv "$(PY_DIR)/grumpy" "$(DESTDIR)$(PY_INSTALL_DIR)"
 	# Go package and sources
+	install -d "$(DESTDIR)/usr/lib/go/"
 	cp -rv build/pkg build/src "$(DESTDIR)/usr/lib/go/"

compiler/block.py

@@ -22,12 +22,11 @@
 import collections
 import re
 
-from pythonparser import algorithm
-from pythonparser import ast
-from pythonparser import source
-
 from grumpy.compiler import expr
 from grumpy.compiler import util
+from grumpy.pythonparser import algorithm
+from grumpy.pythonparser import ast
+from grumpy.pythonparser import source
 
 
 _non_word_re = re.compile('[^A-Za-z0-9_]')

compiler/block_test.py

@@ -21,11 +21,10 @@
 import textwrap
 import unittest
 
-import pythonparser
-
 from grumpy.compiler import block
 from grumpy.compiler import imputil
 from grumpy.compiler import util
+from grumpy import pythonparser
 
 class PackageTest(unittest.TestCase):
 

compiler/expr_visitor.py

@@ -21,11 +21,10 @@
 import contextlib
 import textwrap
 
-from pythonparser import algorithm
-from pythonparser import ast
-
 from grumpy.compiler import expr
 from grumpy.compiler import util
+from grumpy.pythonparser import algorithm
+from grumpy.pythonparser import ast
 
 
 class ExprVisitor(algorithm.Visitor):

compiler/expr_visitor_test.py

@@ -22,12 +22,11 @@
 import textwrap
 import unittest
 
-import pythonparser
-
 from grumpy.compiler import block
 from grumpy.compiler import imputil
 from grumpy.compiler import shard_test
 from grumpy.compiler import stmt
+from grumpy import pythonparser
 
 
 def _MakeExprTest(expr):

compiler/imputil.py

@@ -23,10 +23,9 @@
 import functools
 import os
 
-from pythonparser import algorithm
-from pythonparser import ast
-
 from grumpy.compiler import util
+from grumpy.pythonparser import algorithm
+from grumpy.pythonparser import ast
 
 
 _NATIVE_MODULE_PREFIX = '__go__.'

compiler/imputil_test.py

@@ -24,10 +24,9 @@
 import textwrap
 import unittest
 
-import pythonparser
-
 from grumpy.compiler import imputil
 from grumpy.compiler import util
+from grumpy import pythonparser
 
 
 class ImportVisitorTest(unittest.TestCase):

compiler/stmt.py

@@ -21,14 +21,13 @@
 import string
 import textwrap
 
-from pythonparser import algorithm
-from pythonparser import ast
-
 from grumpy.compiler import block
 from grumpy.compiler import expr
 from grumpy.compiler import expr_visitor
 from grumpy.compiler import imputil
 from grumpy.compiler import util
+from grumpy.pythonparser import algorithm
+from grumpy.pythonparser import ast
 
 
 _NATIVE_TYPE_PREFIX = 'type_'

compiler/stmt_test.py

@@ -23,14 +23,13 @@
 import textwrap
 import unittest
 
-import pythonparser
-from pythonparser import ast
-
 from grumpy.compiler import block
 from grumpy.compiler import imputil
 from grumpy.compiler import shard_test
 from grumpy.compiler import stmt
 from grumpy.compiler import util
+from grumpy import pythonparser
+from grumpy.pythonparser import ast
 
 
 class StatementVisitorTest(unittest.TestCase):

third_party/pythonparser/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
-import sys, pythonparser.source, pythonparser.lexer, pythonparser.parser, pythonparser.diagnostic
+import sys
+from . import source as pythonparser_source, lexer as pythonparser_lexer, parser as pythonparser_parser, diagnostic as pythonparser_diagnostic
 
 def parse_buffer(buffer, mode="exec", flags=[], version=None, engine=None):
     """
@@ -15,13 +16,13 @@ def parse_buffer(buffer, mode="exec", flags=[], version=None, engine=None):
         version = sys.version_info[0:2]
 
     if engine is None:
-        engine = pythonparser.diagnostic.Engine()
+        engine = pythonparser_diagnostic.Engine()
 
-    lexer = pythonparser.lexer.Lexer(buffer, version, engine)
+    lexer = pythonparser_lexer.Lexer(buffer, version, engine)
     if mode in ("single", "eval"):
         lexer.interactive = True
 
-    parser = pythonparser.parser.Parser(lexer, version, engine)
+    parser = pythonparser_parser.Parser(lexer, version, engine)
     parser.add_flags(flags)
 
     if mode == "exec":
@@ -54,7 +55,7 @@ def parse(source, filename="<unknown>", mode="exec",
     :raise: :class:`diagnostic.Error`
         if the source code is not well-formed
     """
-    ast, comments = parse_buffer(pythonparser.source.Buffer(source, filename),
+    ast, comments = parse_buffer(pythonparser_source.Buffer(source, filename),
                                  mode, flags, version, engine)
     return ast
 

tools/grumpc

@@ -24,12 +24,11 @@ import os
 import sys
 import textwrap
 
-import pythonparser
-
 from grumpy.compiler import block
 from grumpy.compiler import imputil
 from grumpy.compiler import stmt
 from grumpy.compiler import util
+from grumpy import pythonparser
 
 
 parser = argparse.ArgumentParser()

tools/pydeps

@@ -20,11 +20,10 @@ import argparse
 import os
 import sys
 
-import pythonparser
-from pythonparser import algorithm
-
 from grumpy.compiler import imputil
 from grumpy.compiler import util
+from grumpy import pythonparser
+from grumpy.pythonparser import algorithm
 
 
 parser = argparse.ArgumentParser()