Skip to content

Commit 0f07ee9

Browse files
sriram-mvCoshUS
andauthored
Merge from develop to master (#270)
* fix: improve support for pep517 builds (#265) * A pep517 build can declare build dependencies. Pip will then know to install these dependencies before trying to build a wheel file. * When creating a build environment, it's only guaranteed to last for the duration of the build process. It's not accessible once a pip command finishes running. * When we try to retrieve the version of a package we run a "modified" form of "python setup.py egg_info". * The problem with this is that we're not using the build environment that has all the build dependencies installed (it's already gone), so if setup.py imports a module (e.g. cython) because it expects it to be there because it declared it as a build dependency the egg_info command will fail. * We don't check the RC or have a fallback case if we can't generate egg info. * We fail with an indecipherable IndexError. We now have a fallback where if we can't import/run the setup.py file, we assume the PKG-INFO file should be in the top level directory of the sdist so we check if it's there, and if so we use that file. * Fixed Unit Test Requiring "test" Binary (#266) * fix(go version parts): remove alphabets from the version for validation (#259) * fix(go version parts): remove alphabets from the version for validation - Go versions like 1.12rc1 or 1.16beta1 are supported. - Test added. * fix: use regex for go versions * chore: aws lambda builders version set to 1.7.0 (#269) Co-authored-by: Cosh_ <[email protected]>
1 parent b4d9a37 commit 0f07ee9

File tree

7 files changed

+125
-57
lines changed

7 files changed

+125
-57
lines changed

aws_lambda_builders/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
22
AWS Lambda Builder Library
33
"""
4-
__version__ = "1.6.0"
4+
__version__ = "1.7.0"
55
RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/workflows/go_modules/validator.py

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"""
44

55
import logging
6+
import re
67
import os
78
import subprocess
89

@@ -12,9 +13,9 @@
1213

1314

1415
class GoRuntimeValidator(object):
15-
1616
LANGUAGE = "go"
1717
SUPPORTED_RUNTIMES = {"go1.x"}
18+
GO_VERSION_REGEX = re.compile("go(\\d)\\.(x|\\d+)")
1819

1920
def __init__(self, runtime):
2021
self.runtime = runtime
@@ -28,6 +29,15 @@ def has_runtime(self):
2829
"""
2930
return self.runtime in self.SUPPORTED_RUNTIMES
3031

32+
@staticmethod
33+
def get_go_versions(version_string):
34+
parts = GoRuntimeValidator.GO_VERSION_REGEX.findall(version_string)
35+
try:
36+
# NOTE(sriram-mv): The version parts need to be a list with a major and minor version.
37+
return int(parts[0][0]), int(parts[0][1])
38+
except IndexError:
39+
return 0, 0
40+
3141
def validate(self, runtime_path):
3242
"""
3343
Checks if the language supplied matches the required lambda runtime
@@ -42,16 +52,13 @@ def validate(self, runtime_path):
4252
min_expected_minor_version = 11 if expected_major_version == 1 else 0
4353

4454
p = subprocess.Popen([runtime_path, "version"], cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
45-
out, _ = p.communicate()
55+
version_string, _ = p.communicate()
4656

4757
if p.returncode == 0:
48-
out_parts = out.decode().split()
49-
if len(out_parts) >= 3:
50-
version_parts = [int(x.replace("rc", "")) for x in out_parts[2].replace(self.LANGUAGE, "").split(".")]
51-
if len(version_parts) >= 2:
52-
if version_parts[0] == expected_major_version and version_parts[1] >= min_expected_minor_version:
53-
self._valid_runtime_path = runtime_path
54-
return self._valid_runtime_path
58+
major_version, minor_version = GoRuntimeValidator.get_go_versions(version_string.decode())
59+
if major_version == expected_major_version and minor_version >= min_expected_minor_version:
60+
self._valid_runtime_path = runtime_path
61+
return self._valid_runtime_path
5562

5663
# otherwise, raise mismatch exception
5764
raise MisMatchRuntimeError(language=self.LANGUAGE, required_runtime=self.runtime, runtime_path=runtime_path)

aws_lambda_builders/workflows/python_pip/packager.py

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,14 @@ class PackageDownloadError(PackagerError):
6464
pass
6565

6666

67+
class UnsupportedPackageError(Exception):
68+
"""Unable to parse package metadata."""
69+
70+
def __init__(self, package_name):
71+
# type: (str) -> None
72+
super(UnsupportedPackageError, self).__init__("Unable to retrieve name/version for package: %s" % package_name)
73+
74+
6775
class UnsupportedPythonVersion(PackagerError):
6876
"""Generic networking error during a package download."""
6977

@@ -538,7 +546,7 @@ def _parse_pkg_info_file(self, filepath):
538546
parser.feed(data)
539547
return parser.close()
540548

541-
def _generate_egg_info(self, package_dir):
549+
def _get_pkg_info_filepath(self, package_dir):
542550
setup_py = self._osutils.joinpath(package_dir, "setup.py")
543551
script = self._SETUPTOOLS_SHIM % setup_py
544552

@@ -548,9 +556,20 @@ def _generate_egg_info(self, package_dir):
548556
p = subprocess.Popen(
549557
cmd, cwd=package_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self._osutils.original_environ()
550558
)
551-
p.communicate()
559+
_, stderr = p.communicate()
552560
info_contents = self._osutils.get_directory_contents(egg_info_dir)
553-
pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO")
561+
if p.returncode != 0:
562+
LOG.debug("Non zero rc (%s) from the setup.py egg_info command: %s", p.returncode, stderr)
563+
if info_contents:
564+
pkg_info_path = self._osutils.joinpath(egg_info_dir, info_contents[0], "PKG-INFO")
565+
else:
566+
# This might be a pep 517 package in which case this PKG-INFO file
567+
# should be available right in the top level directory of the sdist
568+
# in the case where the egg_info command fails.
569+
LOG.debug("Using fallback location for PKG-INFO file in package directory: %s", package_dir)
570+
pkg_info_path = self._osutils.joinpath(package_dir, "PKG-INFO")
571+
if not self._osutils.file_exists(pkg_info_path):
572+
raise UnsupportedPackageError(self._osutils.basename(package_dir))
554573
return pkg_info_path
555574

556575
def _unpack_sdist_into_dir(self, sdist_path, unpack_dir):
@@ -567,7 +586,7 @@ def _unpack_sdist_into_dir(self, sdist_path, unpack_dir):
567586
def get_package_name_and_version(self, sdist_path):
568587
with self._osutils.tempdir() as tempdir:
569588
package_dir = self._unpack_sdist_into_dir(sdist_path, tempdir)
570-
pkg_info_filepath = self._generate_egg_info(package_dir)
589+
pkg_info_filepath = self._get_pkg_info_filepath(package_dir)
571590
metadata = self._parse_pkg_info_file(pkg_info_filepath)
572591
name = metadata["Name"]
573592
version = metadata["Version"]

aws_lambda_builders/workflows/python_pip/utils.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,3 +102,7 @@ def mtime(self, path):
102102
@property
103103
def pipe(self):
104104
return subprocess.PIPE
105+
106+
def basename(self, path):
107+
# type: (str) -> str
108+
return os.path.basename(path)

tests/functional/workflows/python_pip/test_packager.py

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import pytest
99
import mock
1010

11-
from aws_lambda_builders.workflows.python_pip.packager import PipRunner
11+
from aws_lambda_builders.workflows.python_pip.packager import PipRunner, UnsupportedPackageError
1212
from aws_lambda_builders.workflows.python_pip.packager import DependencyBuilder
1313
from aws_lambda_builders.workflows.python_pip.packager import Package
1414
from aws_lambda_builders.workflows.python_pip.packager import MissingDependencyError
@@ -858,18 +858,24 @@ class TestSdistMetadataFetcher(object):
858858
_SETUP_PY = "%s\n" "setup(\n" ' name="%s",\n' ' version="%s"\n' ")\n"
859859
_VALID_TAR_FORMATS = ["tar.gz", "tar.bz2"]
860860

861-
def _write_fake_sdist(self, setup_py, directory, ext):
861+
def _write_fake_sdist(self, setup_py, directory, ext, pkg_info_contents=None):
862862
filename = "sdist.%s" % ext
863863
path = "%s/%s" % (directory, filename)
864864
if ext == "zip":
865865
with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as z:
866866
z.writestr("sdist/setup.py", setup_py)
867+
if pkg_info_contents is not None:
868+
z.writestr("sdist/PKG-INFO", pkg_info_contents)
867869
elif ext in self._VALID_TAR_FORMATS:
868870
compression_format = ext.split(".")[1]
869871
with tarfile.open(path, "w:%s" % compression_format) as tar:
870872
tarinfo = tarfile.TarInfo("sdist/setup.py")
871873
tarinfo.size = len(setup_py)
872874
tar.addfile(tarinfo, io.BytesIO(setup_py.encode()))
875+
if pkg_info_contents is not None:
876+
tarinfo = tarfile.TarInfo("sdist/PKG-INFO")
877+
tarinfo.size = len(pkg_info_contents)
878+
tar.addfile(tarinfo, io.BytesIO(pkg_info_contents.encode()))
873879
else:
874880
open(path, "a").close()
875881
filepath = os.path.join(directory, filename)
@@ -967,6 +973,29 @@ def test_bad_format(self, osutils, sdist_reader):
967973
with pytest.raises(InvalidSourceDistributionNameError):
968974
name, version = sdist_reader.get_package_name_and_version(filepath)
969975

976+
def test_cant_get_egg_info_filename(self, osutils, sdist_reader):
977+
# In this scenario the setup.py file will fail with an import
978+
# error so we should verify we try a fallback to look for
979+
# PKG-INFO.
980+
bad_setup_py = self._SETUP_PY % (
981+
"import some_build_dependency",
982+
"foo",
983+
"1.0",
984+
)
985+
pkg_info_file = "Name: foo\n" "Version: 1.0\n"
986+
with osutils.tempdir() as tempdir:
987+
filepath = self._write_fake_sdist(bad_setup_py, tempdir, "zip", pkg_info_file)
988+
name, version = sdist_reader.get_package_name_and_version(filepath)
989+
assert name == "foo"
990+
assert version == "1.0"
991+
992+
def test_pkg_info_fallback_fails_raises_error(self, osutils, sdist_reader):
993+
setup_py = self._SETUP_PY % ("import build_time_dependency", "foo", "1.0")
994+
with osutils.tempdir() as tempdir:
995+
filepath = self._write_fake_sdist(setup_py, tempdir, "tar.gz")
996+
with pytest.raises(UnsupportedPackageError):
997+
sdist_reader.get_package_name_and_version(filepath)
998+
970999

9711000
class TestPackage(object):
9721001
def test_same_pkg_sdist_and_wheel_collide(self, osutils, sdist_builder):

tests/unit/test_workflow.py

Lines changed: 29 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,21 @@ def setUp(self):
176176
options={"c": "d"},
177177
)
178178

179+
def mock_binaries(self):
180+
self.validator_mock = Mock()
181+
self.validator_mock.validate = Mock()
182+
self.validator_mock.validate.return_value = "/usr/bin/binary"
183+
self.resolver_mock = Mock()
184+
self.resolver_mock.exec_paths = ["/usr/bin/binary"]
185+
self.binaries_mock = Mock()
186+
self.binaries_mock.return_value = []
187+
188+
self.work.get_validators = lambda: self.validator_mock
189+
self.work.get_resolvers = lambda: self.resolver_mock
190+
self.work.binaries = {
191+
"binary": BinaryPath(resolver=self.resolver_mock, validator=self.validator_mock, binary="binary")
192+
}
193+
179194
def test_get_binaries(self):
180195
self.assertIsNotNone(self.work.binaries)
181196
for binary, binary_path in self.work.binaries.items():
@@ -187,63 +202,39 @@ def test_get_validator(self):
187202
self.assertTrue(isinstance(validator, RuntimeValidator))
188203

189204
def test_must_execute_actions_in_sequence(self):
205+
self.mock_binaries()
190206
action_mock = Mock()
191-
validator_mock = Mock()
192-
validator_mock.validate = Mock()
193-
validator_mock.validate.return_value = "/usr/bin/binary"
194-
resolver_mock = Mock()
195-
resolver_mock.exec_paths = ["/usr/bin/binary"]
196-
binaries_mock = Mock()
197-
binaries_mock.return_value = []
198-
199-
self.work.get_validators = lambda: validator_mock
200-
self.work.get_resolvers = lambda: resolver_mock
201207
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
202-
self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")}
203208
self.work.run()
204209

205210
self.assertEqual(
206211
action_mock.method_calls, [call.action1.execute(), call.action2.execute(), call.action3.execute()]
207212
)
208-
self.assertTrue(validator_mock.validate.call_count, 1)
213+
self.assertTrue(self.validator_mock.validate.call_count, 1)
209214

210215
def test_must_fail_workflow_binary_resolution_failure(self):
216+
self.mock_binaries()
211217
action_mock = Mock()
212-
validator_mock = Mock()
213-
validator_mock.validate = Mock()
214-
validator_mock.validate.return_value = None
215-
resolver_mock = Mock()
216-
resolver_mock.exec_paths = MagicMock(side_effect=ValueError("Binary could not be resolved"))
217-
binaries_mock = Mock()
218-
binaries_mock.return_value = []
219-
220-
self.work.get_validators = lambda: validator_mock
221-
self.work.get_resolvers = lambda: resolver_mock
218+
self.resolver_mock.exec_paths = MagicMock(side_effect=ValueError("Binary could not be resolved"))
219+
222220
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
223-
self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")}
224221
with self.assertRaises(WorkflowFailedError) as ex:
225222
self.work.run()
226223

227224
def test_must_fail_workflow_binary_validation_failure(self):
228-
action_mock = Mock()
229-
validator_mock = Mock()
230-
validator_mock.validate = Mock()
231-
validator_mock.validate = MagicMock(
225+
self.mock_binaries()
226+
self.validator_mock.validate = MagicMock(
232227
side_effect=MisMatchRuntimeError(language="test", required_runtime="test1", runtime_path="/usr/bin/binary")
233228
)
234-
resolver_mock = Mock()
235-
resolver_mock.exec_paths = ["/usr/bin/binary"]
236-
binaries_mock = Mock()
237-
binaries_mock.return_value = []
238229

239-
self.work.get_validators = lambda: validator_mock
240-
self.work.get_resolvers = lambda: resolver_mock
230+
action_mock = Mock()
241231
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
242-
self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")}
243232
with self.assertRaises(WorkflowFailedError) as ex:
244233
self.work.run()
245234

246235
def test_must_raise_with_no_actions(self):
236+
self.mock_binaries()
237+
247238
self.work.actions = []
248239

249240
with self.assertRaises(WorkflowFailedError) as ctx:
@@ -252,6 +243,7 @@ def test_must_raise_with_no_actions(self):
252243
self.assertIn("Workflow does not have any actions registered", str(ctx.exception))
253244

254245
def test_must_raise_if_action_failed(self):
246+
self.mock_binaries()
255247
action_mock = Mock()
256248
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
257249

@@ -264,6 +256,7 @@ def test_must_raise_if_action_failed(self):
264256
self.assertIn("testfailure", str(ctx.exception))
265257

266258
def test_must_raise_if_action_crashed(self):
259+
self.mock_binaries()
267260
action_mock = Mock()
268261
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
269262

@@ -290,6 +283,8 @@ def test_supply_executable_path(self):
290283
options={"c": "d"},
291284
)
292285
self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3]
286+
self.mock_binaries()
287+
293288
self.work.run()
294289

295290

tests/unit/workflows/go_modules/test_validator.py

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ def __init__(self, returncode, out=b"", err=b""):
1414
self.err = err
1515

1616
def communicate(self):
17-
return (self.out, self.err)
17+
return self.out, self.err
1818

1919

2020
class TestGoRuntimeValidator(TestCase):
@@ -30,36 +30,50 @@ def test_runtime_validate_unsupported_language_fail_open(self):
3030
validator = GoRuntimeValidator(runtime="go2.x")
3131
validator.validate(runtime_path="/usr/bin/go2")
3232

33-
@parameterized.expand([(b"go version go1.11.2 test",), (b"go version go1.11rc.2 test",)])
33+
@parameterized.expand(
34+
[
35+
("go1.11.2", (1, 11)),
36+
("go1.11rc.2", (1, 11)),
37+
("go1.16beta1", (1, 16)),
38+
("go%$", (0, 0)),
39+
("unknown", (0, 0)),
40+
]
41+
)
42+
def test_get_go_versions(self, version_string, version_parts):
43+
self.assertEqual(self.validator.get_go_versions(version_string), version_parts)
44+
45+
@parameterized.expand(
46+
[(b"go version go1.11.2 test",), (b"go version go1.11rc.2 test",), (b"go version go1.16beta1 test",)]
47+
)
3448
def test_runtime_validate_supported_version_runtime(self, go_version_output):
3549
with mock.patch("subprocess.Popen") as mock_subprocess:
3650
mock_subprocess.return_value = MockSubProcess(0, out=go_version_output)
3751
self.validator.validate(runtime_path="/usr/bin/go")
38-
self.assertTrue(mock_subprocess.call_count, 1)
52+
self.assertEqual(mock_subprocess.call_count, 1)
3953

4054
def test_runtime_validate_supported_higher_than_min_version_runtime(self):
4155
with mock.patch("subprocess.Popen") as mock_subprocess:
4256
mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.12 test")
4357
self.validator.validate(runtime_path="/usr/bin/go")
44-
self.assertTrue(mock_subprocess.call_count, 1)
58+
self.assertEqual(mock_subprocess.call_count, 1)
4559

4660
def test_runtime_validate_mismatch_nonzero_exit(self):
4761
with mock.patch("subprocess.Popen") as mock_subprocess:
4862
mock_subprocess.return_value = MockSubProcess(1)
4963
with self.assertRaises(MisMatchRuntimeError):
5064
self.validator.validate(runtime_path="/usr/bin/go")
51-
self.assertTrue(mock_subprocess.call_count, 1)
65+
self.assertEqual(mock_subprocess.call_count, 1)
5266

5367
def test_runtime_validate_mismatch_invalid_version(self):
5468
with mock.patch("subprocess.Popen") as mock_subprocess:
5569
mock_subprocess.return_value = MockSubProcess(0, out=b"go version")
5670
with self.assertRaises(MisMatchRuntimeError):
5771
self.validator.validate(runtime_path="/usr/bin/go")
58-
self.assertTrue(mock_subprocess.call_count, 1)
72+
self.assertEqual(mock_subprocess.call_count, 1)
5973

6074
def test_runtime_validate_mismatch_minor_version(self):
6175
with mock.patch("subprocess.Popen") as mock_subprocess:
6276
mock_subprocess.return_value = MockSubProcess(0, out=b"go version go1.10.2 test")
6377
with self.assertRaises(MisMatchRuntimeError):
6478
self.validator.validate(runtime_path="/usr/bin/go")
65-
self.assertTrue(mock_subprocess.call_count, 1)
79+
self.assertEqual(mock_subprocess.call_count, 1)

0 commit comments

Comments
 (0)