[EDITOR-527] E2E testing infrastructure (#591)

* Bootstrap E2E testing infrastructure

* add e2e tests in taskfile and in CI

* fix deprecation warning when running pytest

* Run e2e tests only with GUI version on macos (Github runners are basically mac mini with GUI support)"

* add first draft of tests for certs and v2 APIs

* add logs to .gitignore

* skip update, currently no way of testing this on gh runners

Co-authored-by: umbynos <[email protected]>

Author: Maurizio Branca

.github/workflows/release.yml

@@ -123,6 +123,19 @@ jobs:
         run: task build
         if: matrix.os == 'macos-10.15'
 
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+        if: matrix.os == 'macos-10.15'
+
+      - name: Run e2e tests
+        run: |
+          pip install poetry
+          task test-e2e
+        if: matrix.os == 'macos-10.15'
+
         # this will create `public/` dir with compressed full bin (<version>/<os>-<arch>.gz) and a json file
       - name: Create autoupdate files
         run: go-selfupdate arduino-create-agent${{ matrix.ext }} ${TAG_VERSION}

.github/workflows/test.yml

@@ -91,6 +91,19 @@ jobs:
         run: task build
         if: matrix.os == 'macos-10.15'
 
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
+          architecture: 'x64'
+        if: matrix.os == 'macos-10.15'
+
+      - name: Run e2e tests
+        run: |
+          pip install poetry
+          task test-e2e
+        if: matrix.os == 'macos-10.15'
+
         # config.ini is required by the executable when it's run
       - name: Upload artifacts
         uses: actions/upload-artifact@v2

.gitignore

@@ -7,9 +7,14 @@ rsrc.syso
 snapshot/*
 public/
 artifacts*
+logs/
 
 # IDEs config 
 .idea
+.vscode
 
 # macOS
 .DS_Store
+
+# Python
+__pycache__

Taskfile.yml

@@ -33,6 +33,12 @@ tasks:
     cmds:
       - go test -short -run '{{ default ".*" .TEST_REGEX }}' {{ default "-v" .GOFLAGS }} -coverprofile=coverage_unit.txt {{ default .DEFAULT_TARGETS .TARGETS }} {{.TEST_LDFLAGS}}
 
+  test-e2e:
+    desc: Run end 2 end tests
+    cmds:
+      - poetry install --no-root
+      - poetry run pytest test
+
   check:
     desc: Check fmt and lint
     cmds:

poetry.lock

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "arduino-create-agent"
+version = "0.1.0"
+description = "Project used to run end-to-end test for the Arduino Create Agent"
+authors = ["Umberto Baldi <u.baldi@arduino.cc>", "Maurizio Branca <m.branca@arduino.cc>"]
+license = "GPLv2"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+psutil = "^5.8.0"
+
+[tool.poetry.dev-dependencies]
+pytest = "^6.2.1"
+requests = "^2.25.1"
+invoke = "^1.5.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    "ignore::DeprecationWarning:invoke.loader" # https://github.com/pyinvoke/invoke/issues/675
+]

pyproject.toml

@@ -0,0 +1,45 @@
+import os
+import platform
+import signal
+import time
+from pathlib import Path
+
+import pytest
+from invoke import Local
+from invoke.context import Context
+
+
+@pytest.fixture(scope="function")
+def agent(pytestconfig):
+    
+    agent_cli = str(Path(pytestconfig.rootdir) / "arduino-create-agent")
+    env = {
+        # "ARDUINO_DATA_DIR": data_dir,
+        # "ARDUINO_DOWNLOADS_DIR": downloads_dir,
+        # "ARDUINO_SKETCHBOOK_DIR": data_dir,
+    }
+    run_context = Context()
+
+    runner = Local(run_context) # execute a command on the local filesystem
+    
+    cd_command = "cd"
+    with run_context.prefix(f'{cd_command} ..'):
+        runner.run(agent_cli, echo=True, hide=True, warn=True, env=env, asynchronous=True)
+        
+        # we give some time to the agent to start and listen to
+        # incoming requests
+        time.sleep(.5)
+
+        # we block here until the test function using this fixture has returned
+        yield runner
+
+    # Kill the runner's process as we finished our test (platform dependent)
+    os_signal = signal.SIGTERM
+    if platform.system() != "Windows":
+        os_signal = signal.SIGKILL
+    os.kill(runner.process.pid, os_signal)
+
+
+@pytest.fixture(scope="session")
+def base_url():
+    return "http://127.0.0.1:8991"

test/conftest.py

@@ -0,0 +1,18 @@
+import requests
+
+
+def test_get_cert(base_url, agent):
+    
+    resp = requests.get(f"{base_url}/certificate.crt")
+    assert resp.status_code == 200
+
+    cert = resp.text
+    assert "<!DOCTYPE html>" in cert
+
+
+def test_del_cert(base_url, agent):
+    
+    resp = requests.delete(f"{base_url}/certificate.crt")
+    assert resp.status_code == 200
+
+    # Should rm "ca.cert.pem", "ca.cert.cer", "ca.key.pem"

test/test_certs.py

@@ -0,0 +1,11 @@
+import re
+import requests
+
+
+def test_version(base_url, agent):
+    
+    resp = requests.get(f"{base_url}/info")
+    assert resp.status_code == 200
+
+    info = resp.json()
+    assert re.match("[0-9]+.[0-9]+.[0-9]+", info["version"]) is not None

test/test_info.py

@@ -0,0 +1,33 @@
+# import json
+import psutil
+import requests
+import pytest
+
+# test if the update process succeeds in terminating the binary
+@pytest.mark.skip(reason="no way of currently testing this")
+def test_update_shutdown(base_url, agent):
+
+    procs=[]
+    for p in psutil.process_iter():
+        if p.name() == "arduino-create-agent":
+            procs.append(p)
+
+    resp = requests.post(f"{base_url}/update")
+    # assert resp.status_code == 200
+    # assert "Please wait a moment while the agent reboots itself" in info['success'] # failing on macos see https://github.com/arduino/arduino-create-agent/issues/608
+    gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) # wait for "arduino-create-agent" to terminate
+
+def on_terminate(proc):
+    print("process {} terminated with exit code {}".format(proc, proc.returncode))
+    assert True
+
+# the version currently running is the latest available
+@pytest.mark.skip(reason="no way of currently testing this")
+def test_latest_version(base_url, agent):
+    resp = requests.get(f"{base_url}/info")
+    assert resp.status_code == 200
+    latest_version = requests.get("https://s3.amazonaws.com/arduino-create-static/agent-metadata/agent-version.json") # get the latest version available
+
+    version = latest_version.json()
+    info = resp.json()
+    assert info["version"] == version["Version"]

test/test_update.py

@@ -0,0 +1,9 @@
+import requests
+
+
+def test_get_tools(base_url, agent):
+    
+    resp = requests.get(f"{base_url}/v2/pkgs/tools/installed")
+    assert resp.status_code == 200
+
+    tools = resp.json()