meta tests (#96)

* readme: document meta tests (for param fetching) file layout

* make: Add metatest target

* make: Add metatest against to doctest-coverage target

* make: rename doctest-coverage to the more accurate coverage

* Add some example_cache files

* Add example meta test

* fix flake8 errors in iam resource tests

* test iam_inline_policies

.travis.yml

@@ -6,4 +6,4 @@ install:
   - make install
 
 script:
-  - make doctest-coverage flake8
+  - make coverage flake8

Makefile

@@ -29,8 +29,13 @@ clean-cache: check_venv
 doctest: check_venv
 	pytest --doctest-modules -s --offline --aws-debug-calls
 
-doctest-coverage: check_venv
+coverage: check_venv
 	pytest --cov-config .coveragerc --cov=. --doctest-modules -s --offline --aws-debug-calls
+	pytest --cov-config .coveragerc --cov=. --cov-append \
+		--aws-profiles example-account \
+		--aws-regions us-east-1 \
+		-o python_files=meta_test*.py \
+		-o cache_dir=./example_cache/
 	coverage report -m
 	coverage html
 
@@ -40,6 +45,12 @@ flake8: check_venv
 install: venv
 	( . venv/bin/activate && pip install -r requirements.txt )
 
+metatest:
+	pytest --aws-profiles example-account \
+		--aws-regions us-east-1 \
+		-o python_files=meta_test*.py \
+		-o cache_dir=./example_cache/
+
 venv:
 	python3 -m venv venv
 

README.md

@@ -442,6 +442,9 @@ Additionally we want:
 * data fetching functions in a `resources.py`
 * data checking and test helpers in a `helpers.py`
 * prefix test files with `test_`
+* doctests for non test files (e.g. `helpers.py`, `resources.py`, `client.py`)
+  * tests that depend on external IO or the runtime environment (env vars, file system, HTTP) to use the prefix `meta_test_` (and probably `mock` or `pytest.monkeypatch`)
+    * JSON fixtures for anonymized cached http call in `example_cache/v/`
 * tests to have pytest markers for any services they depend on for data
 * HTTP clients should be read only and use read only credentials
 * running a test should not modify services
@@ -451,8 +454,16 @@ Additionally we want:
 ```console
 pytest-services
 ...
+├── example_cache
+│   └── v
+│       ├── cache
+│       │   └── lastfailed
+│       ├── pytest_aws:example-account:us-east-1:ec2:describe_instances::.json
+│       ├── pytest_aws:example-account:us-east-1:ec2:describe_security_groups::.json
+...
 ├── <third party service A>
 │   ├── client.py
+│   ├── meta_test_client.py
 │   ├── <subservice A (optional)>
 │   │   ├── __init__.py
 │   │   ├── helpers.py

aws/iam/meta_test_resources.py

@@ -0,0 +1,31 @@
+
+
+from aws.iam.resources import (
+    iam_users,
+    iam_inline_policies,
+)
+
+
+def test_iam_users():
+    assert iam_users() == [
+        {
+            'Arn': 'arn:aws:iam::123456789012:user/hobbes',
+            'CreateDate': '1985-11-18T00:01:10+00:00',
+            'PasswordLastUsed': '2018-01-09T20:43:00+00:00',
+            'Path': '/',
+            'UserId': 'H0BBIHMA0CZ0R0K0MN00C',
+            'UserName': 'tigerone',
+            '__pytest_meta': {'profile': 'example-account', 'region': 'us-east-1'}},
+        {
+            'Arn': 'arn:aws:iam::123456789012:user/calvin',
+            'CreateDate': '1985-11-18T00:01:10+00:00',
+            'PasswordLastUsed': '2008-01-09T20:43:00+00:00',
+            'Path': '/',
+            'UserId': 'CALCIHMA0CZ0R0K0MN00C',
+            'UserName': 'spacemanspiff',
+            '__pytest_meta': {'profile': 'example-account', 'region': 'us-east-1'}}]
+
+
+def test_iam_inline_policies_for_user_without_policies():
+    # extracts empty 'PolicyNames'
+    assert iam_inline_policies(username='tigerone') == []

example_cache/v/cache/lastfailed

@@ -0,0 +1 @@
+{}

example_cache/v/pytest_aws:example-account:us-east-1:iam:list_user_policies::UserName=spacemanspiff.json

@@ -0,0 +1,7 @@
+{
+    "PolicyNames": [],
+    "__pytest_meta": {
+        "profile": "example-account",
+        "region": "us-east-1"
+    }
+}

example_cache/v/pytest_aws:example-account:us-east-1:iam:list_user_policies::UserName=tigerone.json

@@ -0,0 +1,7 @@
+{
+    "PolicyNames": [],
+    "__pytest_meta": {
+        "profile": "example-account",
+        "region": "us-east-1"
+    }
+}

example_cache/v/pytest_aws:example-account:us-east-1:iam:list_users::.json

@@ -0,0 +1,24 @@
+{
+    "Users": [
+        {
+            "Arn": "arn:aws:iam::123456789012:user/hobbes",
+            "CreateDate": "1985-11-18T00:01:10+00:00",
+            "PasswordLastUsed": "2018-01-09T20:43:00+00:00",
+            "Path": "/",
+            "UserId": "H0BBIHMA0CZ0R0K0MN00C",
+            "UserName": "tigerone"
+        },
+        {
+            "Arn": "arn:aws:iam::123456789012:user/calvin",
+            "CreateDate": "1985-11-18T00:01:10+00:00",
+            "PasswordLastUsed": "2008-01-09T20:43:00+00:00",
+            "Path": "/",
+            "UserId": "CALCIHMA0CZ0R0K0MN00C",
+            "UserName": "spacemanspiff"
+        }
+    ],
+    "__pytest_meta": {
+        "profile": "example-account",
+        "region": "us-east-1"
+    }
+}