Merge pull request #170 from awslabs/develop

Merge to master - v0.9.0

Author: sriram-mv

.appveyor.yml

@@ -12,7 +12,7 @@ environment:
   matrix:
 
   - PYTHON: "C:\\Python27-x64"
-    PYTHON_VERSION: '2.7.17'
+    PYTHON_VERSION: '2.7.18'
     PYTHON_ARCH: '64'
     LINE_COVERAGE: '91'
     NEW_FLAKE8: 0
@@ -24,7 +24,7 @@ environment:
     NEW_FLAKE8: 0
     JAVA_HOME: "C:\\Program Files\\Java\\jdk11"
   - PYTHON: "C:\\Python37-x64"
-    PYTHON_VERSION: '3.7.4'
+    PYTHON_VERSION: '3.7.7'
     PYTHON_ARCH: '64'
     LINE_COVERAGE: '91'
     NEW_FLAKE8: 0
@@ -70,6 +70,9 @@ for:
     # setup Gradle
     - "choco install gradle"
 
+    # setup make
+    - "choco install make"
+
     # Echo final Path
     - "echo %PATH%"
 

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "0.8.0"
+__version__ = "0.9.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/workflows/__init__.py

@@ -10,3 +10,4 @@
 import aws_lambda_builders.workflows.java_gradle
 import aws_lambda_builders.workflows.java_maven
 import aws_lambda_builders.workflows.dotnet_clipackage
+import aws_lambda_builders.workflows.custom_make

aws_lambda_builders/workflows/custom_make/DESIGN.md

@@ -0,0 +1,72 @@
+## Provided Make Builder
+
+### Scope
+
+Provided runtimes need a flexible builder approach as the binaries required for a building for each language is very different. The introduced builder should have scope
+to be able build irrespective of the provided language, this is only possible if the logic of the build is not entirely contained within the builder itself, but that the builder 
+executes commands that are user provided. The builder should showcase a contract to the user of how those commmands need to be provided.
+
+
+### Mechanism
+
+The Mechanism proposed here is that of `Makefile` Builder that has a particular make target available based on the logical id of the function to be built.
+The `Makefile` is the manifest and is present in the same path as the original source directory.
+
+
+### Implementation
+
+When the makefile is called through a lambda builder workflow, the appropriate target is triggered and artifacts should be copied to the exposed environmental variable `$ARTIFACTS_DIR`. This environment variable is seeded by the lambda builder with a value set to it. The targets are defined as 
+
+```
+build-{Function_Logical_Id}
+```
+
+Injected Environment Variables by Makefile Lambda Builder Workflow per Function to be built. The injected environment variable has the path values modified
+based on if this build mechanism is being run on windows (powershell or git bash).
+
+``
+ARTIFACTS_DIR=/Users/noname/sam-app/.aws-sam/build/HelloWorldFunction
+``
+
+Sample Makefile:
+
+````
+build-HelloWorldFunction:
+    touch $(ARTIFACTS_DIR)/somefile
+
+build-HelloWorldFunction2:
+    touch $(ARTIFACTS_DIR)/somefile2
+````
+
+The workflow expects that the name of the function logical id is passed into the workflow using the `options` dictionary. This helps to trigger the correct build target.
+
+#### Step 1: Copy source to a scratch directory
+
+This involves the given source to a temporary scratch directory, so that mutations can be given on the files in the scratch directory, but leave the original source directory untouched.
+
+#### Step 2: Core Workflow Action - Invoke Makefile Build Target
+
+Artifacts directory is created if it doesnt exist and passed into the makefile target process.
+
+```python
+self.subprocess_make.run(["--makefile", self.manifest_path, f"build-{self.build_logical_id}"], env={"ARTIFACTS_DIR": self.artifacts_dir}, cwd=self.scratch_dir)
+```
+
+It is then the responsibility of the make target to make sure the artifacts are built in the correct manner and dropped into the artifacts directory.
+
+### Challenges
+
+* Is `make` truly platform independent?
+  * Make comes with linux subsystem on windows and can also be installed with `choco`. 
+
+* Does this become a way to introduce plugins, makefile can have any commands in it?
+  * We only care about certain build targets. so essentially this is a pluggable builder, but nothing beyond that at this point in time.
+
+* Which environment variables are usable in this makefile?
+  * There are a series of whitelisted environment variables that need to be defined and not be overriden within the Makefile to work. Currently that is just `$ARTIFACTS_DIR`
+
+* Can this be used even for runtimes that have builders associated with it? eg: python3.8?
+  * Possibly, some changes would be needed be made to way the corresponding builder is picked up in sam cli. If we changed it such that there is a makefile we pick a makefile builder and if not fall back to the specified language builder.
+
+
+

aws_lambda_builders/workflows/custom_make/__init__.py

@@ -0,0 +1,5 @@
+"""
+Builds provided runtime lambda functions using a Makefile based builder.
+"""
+
+from .workflow import CustomMakeWorkflow

aws_lambda_builders/workflows/custom_make/actions.py

@@ -0,0 +1,97 @@
+"""
+Action to build a specific Makefile target
+"""
+
+import logging
+from pathlib import Path
+
+from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError
+from .exceptions import MakeFileNotFoundError
+from .make import MakeExecutionError
+
+LOG = logging.getLogger(__name__)
+
+
+class CustomMakeAction(BaseAction):
+
+    """
+    A Lambda Builder Action that builds and packages a provided runtime project using Make.
+    """
+
+    NAME = "MakeBuild"
+    DESCRIPTION = "Running build target on Makefile"
+    PURPOSE = Purpose.COMPILE_SOURCE
+
+    def __init__(self, artifacts_dir, scratch_dir, manifest_path, osutils, subprocess_make, build_logical_id):
+        """
+        :type artifacts_dir: str
+        :param artifacts_dir: directory where artifacts needs to be stored.
+
+        :type scratch_dir: str
+        :param scratch_dir: an existing (writable) directory for temporary files
+
+        :type manifest_path: str
+        :param manifest_path: path to Makefile of an Make project with the source in same folder.
+
+        :type osutils: aws_lambda_builders.workflows.custom_make.utils.OSUtils
+        :param osutils: An instance of OS Utilities for file manipulation
+
+        :type subprocess_make aws_lambda_builders.workflows.custom_make.make.SubprocessMake
+        :param subprocess_make: An instance of the Make process wrapper
+        """
+        super(CustomMakeAction, self).__init__()
+        self.artifacts_dir = artifacts_dir
+        self.scratch_dir = scratch_dir
+        self.manifest_path = manifest_path
+        self.osutils = osutils
+        self.subprocess_make = subprocess_make
+        self.build_logical_id = build_logical_id
+
+    @property
+    def artifact_dir_path(self):
+        # This is required when running on windows to determine if we are running in linux
+        # subsystem or on native cmd or powershell.
+        if self.osutils.is_windows():
+            return Path(self.artifacts_dir).as_posix() if self.osutils.which("sh") else self.artifacts_dir
+        else:
+            return self.artifacts_dir
+
+    def manifest_check(self):
+        # Check for manifest file presence and if not present raise MakefileNotFoundError
+        if not self.osutils.exists(self.manifest_path):
+            raise MakeFileNotFoundError("Makefile not found at {}".format(self.manifest_path))
+
+    def execute(self):
+        """
+        Runs the action.
+
+        :raises lambda_builders.actions.ActionFailedError: when Make Build fails.
+        """
+
+        # Check for manifest file
+        try:
+            self.manifest_check()
+        except MakeFileNotFoundError as ex:
+            raise ActionFailedError(str(ex))
+
+        # Create the Artifacts Directory if it doesnt exist.
+        if not self.osutils.exists(self.artifacts_dir):
+            self.osutils.makedirs(self.artifacts_dir)
+
+        try:
+            current_env = self.osutils.environ()
+            LOG.info("Current Artifacts Directory : %s", self.artifact_dir_path)
+            current_env.update({"ARTIFACTS_DIR": self.artifact_dir_path})
+            # Export environmental variables that might be needed by other binaries used
+            # within the Makefile and also specify the makefile to be used as well.
+            self.subprocess_make.run(
+                [
+                    "--makefile",
+                    "{}".format(self.manifest_path),
+                    "build-{logical_id}".format(logical_id=self.build_logical_id),
+                ],
+                env=current_env,
+                cwd=self.scratch_dir,
+            )
+        except MakeExecutionError as ex:
+            raise ActionFailedError(str(ex))

aws_lambda_builders/workflows/custom_make/exceptions.py

@@ -0,0 +1,7 @@
+"""
+Series of Exceptions raised by Custom Makefile builder.
+"""
+
+
+class MakeFileNotFoundError(Exception):
+    pass

aws_lambda_builders/workflows/custom_make/make.py

@@ -0,0 +1,90 @@
+"""
+Wrapper around calling make through a subprocess.
+"""
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+class MakeExecutionError(Exception):
+
+    """
+    Exception raised in case Make execution fails.
+    It will pass on the standard error output from the Make console.
+    """
+
+    MESSAGE = "Make Failed: {message}"
+
+    def __init__(self, **kwargs):
+        Exception.__init__(self, self.MESSAGE.format(**kwargs))
+
+
+class SubProcessMake(object):
+
+    """
+    Wrapper around the Make command line utility, making it
+    easy to consume execution results.
+    """
+
+    def __init__(self, osutils, make_exe=None):
+        """
+        :type osutils: aws_lambda_builders.workflows.custom_make.utils.OSUtils
+        :param osutils: An instance of OS Utilities for file manipulation
+
+        :type make_exe: str
+        :param make_exe: Path to the Make binary. If not set,
+            the default executable path make will be used
+        """
+        self.osutils = osutils
+
+        if make_exe is None:
+            if osutils.is_windows():
+                make_exe = "make.exe"
+            else:
+                make_exe = "make"
+
+        self.make_exe = make_exe
+
+    def run(self, args, env=None, cwd=None):
+
+        """
+        Runs the action.
+
+        :type args: list
+        :param args: Command line arguments to pass to Make
+
+        :type env: dict
+        :param env : environment variables dictionary to be passed into subprocess
+
+        :type cwd: str
+        :param cwd: Directory where to execute the command (defaults to current dir)
+
+        :rtype: str
+        :return: text of the standard output from the command
+
+        :raises aws_lambda_builders.workflows.custom_make.make.MakeExecutionError:
+            when the command executes with a non-zero return code. The exception will
+            contain the text of the standard error output from the command.
+
+        :raises ValueError: if arguments are not provided, or not a list
+        """
+
+        if not isinstance(args, list):
+            raise ValueError("args must be a list")
+
+        if not args:
+            raise ValueError("requires at least one arg")
+
+        invoke_make = [self.make_exe] + args
+
+        LOG.debug("executing Make: %s", invoke_make)
+
+        p = self.osutils.popen(invoke_make, stdout=self.osutils.pipe, stderr=self.osutils.pipe, cwd=cwd, env=env)
+
+        out, err = p.communicate()
+
+        if p.returncode != 0:
+            raise MakeExecutionError(message=err.decode("utf8").strip())
+
+        return out.decode("utf8").strip()

aws_lambda_builders/workflows/custom_make/utils.py

@@ -0,0 +1,46 @@
+"""
+Commonly used utilities
+"""
+
+import os
+import platform
+import subprocess
+
+from aws_lambda_builders.utils import which
+
+
+class OSUtils(object):
+
+    """
+    Wrapper around file system functions, to make it easy to
+    unit test actions in memory
+    """
+
+    def exists(self, p):
+        return os.path.exists(p)
+
+    def makedirs(self, path):
+        return os.makedirs(path)
+
+    def popen(self, command, stdout=None, stderr=None, env=None, cwd=None):
+        p = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env, cwd=cwd)
+        return p
+
+    def environ(self):
+        return os.environ.copy()
+
+    def normpath(self, path):
+        return os.path.normpath(path)
+
+    def abspath(self, path):
+        return os.path.abspath(path)
+
+    @property
+    def pipe(self):
+        return subprocess.PIPE
+
+    def is_windows(self):
+        return platform.system().lower() == "windows"
+
+    def which(self, executable, executable_search_paths=None):
+        return which(executable, executable_search_paths=executable_search_paths)