Fix CI

Also formats the code

Signed-off-by: Hugues de Valon <[email protected]>

Author: hug-dev

.github/workflows/ci.yml

@@ -16,31 +16,28 @@ jobs:
       run: exit ${{ steps.lc.outputs.exit_code }}
 
   build:
+    name: CI tests
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        # pyyaml version we use only supporting python 3.6 upwards
-        python-version: [ 3.6, 3.7, 3.8]
-
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: 3.7
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install -r requirements.txt
+    - name: Check formatting with blake
+      run: |
+        black --check . --exclude generator/generators/protobuf
     - name: Lint with flake8
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude generator/protobuf
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude generator/generators/protobuf
+        # The GitHub editor is 127 chars wide
+        flake8 . --count --max-line-length=127 --statistics --exclude generator/generators/protobuf
     - name: Execute smoke test
       run: |
-        python parsec_mock/parsec_mock.py
+        python parsec_mock/parsec_mock.py &
         sleep 5
         python tests/mock_test.py
+        pkill python

generator/generator.py

@@ -13,9 +13,13 @@
 # Add your new generator function here
 # TODO: with some import syntax or clever moduling, it might be possible not
 # to have to do anything here?
-from generators.ping_noauth import gen as ping_noauth
-from generators.list_opcodes_bad1 import gen as list_opcodes_bad1
-from generators.list_opcodes_directauth import gen as list_opcodes_directauth
+# F401 ignored for flake8 as those methods are called through eval
+from generators.ping_noauth import gen as ping_noauth  # noqa: F401
+from generators.list_opcodes_bad1 import gen as list_opcodes_bad1  # noqa: F401
+from generators.list_opcodes_directauth import (  # noqa: F401
+    gen as list_opcodes_directauth,
+)
+
 
 class TestSpec(object):
     """Class to represent a test specification.  Used to convert
@@ -32,15 +36,16 @@ def _traverse(key, element):
         self.__dict__.update(objd)
         self.basedict = dictionary
 
+
 def read_specs(folder):
     """Read test specs from a folder"""
     specfiles = [f for f in listdir(folder) if isfile(join(folder, f))]
     specs = []
     for file in specfiles:
         print(f"Parsing spec file: {file}")
         # Only use the first part of the filename as spec name
-        name = file.split('.')[0]
-        with open(os.path.join(folder, file), 'r') as f:
+        name = file.split(".")[0]
+        with open(os.path.join(folder, file), "r") as f:
             spec = safe_load(f)
             testspec = TestSpec(spec["spec"])
             specs.append((name, testspec))
@@ -52,27 +57,32 @@ def generate_data(specs, output_folder):
     for (name, spec) in specs:
         generate_spec_data(output_folder, spec, name)
 
+
 def generate_spec_data(output_folder, spec, name):
     """Generates data for a single spec and outputs it into the specified output folder."""
     # The generator function has the same name as the test specification
     (operation, result) = eval(name)()
 
     request_auth = create_auth(spec.request.auth)
     request_content_len = spec.request.header.content_length
-    if request_content_len == 'auto':
+    if request_content_len == "auto":
         request_content_len = len(operation)
 
     request_auth_len = spec.request.header.auth_length
-    if request_auth_len == 'auto':
+    if request_auth_len == "auto":
         request_auth_len = len(request_auth)
-    request_header = pack_header(spec.request.header, request_auth_len, request_content_len)
+    request_header = pack_header(
+        spec.request.header, request_auth_len, request_content_len
+    )
 
     response_content_len = spec.response.header.content_length
-    if response_content_len == 'auto':
+    if response_content_len == "auto":
         response_content_len = len(result)
 
     response_auth_len = spec.response.header.auth_length
-    response_header = pack_header(spec.response.header, response_auth_len, response_content_len)
+    response_header = pack_header(
+        spec.response.header, response_auth_len, response_content_len
+    )
 
     request_buf = request_header + operation + request_auth
     response_buf = response_header + result
@@ -81,13 +91,13 @@ def generate_spec_data(output_folder, spec, name):
     out_data = {
         "spec": spec.basedict,
         "test_data": {
-            "request": base64.b64encode(request_buf).decode('ascii'),
-            "response": base64.b64encode(response_buf).decode('ascii'),
-        }
+            "request": base64.b64encode(request_buf).decode("ascii"),
+            "response": base64.b64encode(response_buf).decode("ascii"),
+        },
     }
     out_path = os.path.join(output_folder, name + ".test.yaml")
     print(f"Writing spec {name} test data to {out_path}")
-    with open(out_path, 'w') as f:
+    with open(out_path, "w") as f:
         dump(out_data, f, sort_keys=False)
 
 
@@ -98,44 +108,48 @@ def pack_header(header, auth_len, body_len):
     # packed field interpretation.  See struct.pack docs for details.
     # This should map to the Fixed Common Header defined here:
     # https://parallaxsecond.github.io/parsec-book/parsec_client/wire_protocol.html#the-fixed-common-header
-    return pack('<IHBBHBQBBBIHIHH',
-                header.magic_number,
-                header.header_size,
-                header.major_version_number,
-                header.minor_version_number,
-                header.flags,
-                header.provider,
-                header.session_handle,
-                header.content_type,
-                header.accept_type,
-                header.auth_type,
-                body_len,
-                auth_len,
-                header.opcode,
-                header.status,
-                0
-                )
+    return pack(
+        "<IHBBHBQBBBIHIHH",
+        header.magic_number,
+        header.header_size,
+        header.major_version_number,
+        header.minor_version_number,
+        header.flags,
+        header.provider,
+        header.session_handle,
+        header.content_type,
+        header.accept_type,
+        header.auth_type,
+        body_len,
+        auth_len,
+        header.opcode,
+        header.status,
+        0,
+    )
 
 
 def create_auth(auth_spec):
     """Creates auth body of message"""
-    if auth_spec.type == 'none':
-        return b''
-    if auth_spec.type == 'direct':
-        return auth_spec.app_name.encode('utf-8')
-    return b''
+    if auth_spec.type == "none":
+        return b""
+    if auth_spec.type == "direct":
+        return auth_spec.app_name.encode("utf-8")
+    return b""
 
 
 def main():
     print("Generating test data.")
 
-    specdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'test_specs'))
+    specdir = os.path.abspath(os.path.join(os.path.dirname(__file__), "test_specs"))
     print(f"Reading test specs from {specdir}")
     specs = read_specs(specdir)
 
-    datadir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../generator_output'))
+    datadir = os.path.abspath(
+        os.path.join(os.path.dirname(__file__), "../generator_output")
+    )
     print(f"Generating test data to {datadir}")
     generate_data(specs, datadir)
 
+
 if __name__ == "__main__":
     main()

generator/generators/list_opcodes_bad1.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from .protobuf import list_opcodes_pb2
 
+
 def gen():
     operation = list_opcodes_pb2.Operation()
     operation.provider_id = 1

generator/generators/list_opcodes_directauth.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from .protobuf import list_opcodes_pb2
 
+
 def gen():
     operation = list_opcodes_pb2.Operation()
     operation.provider_id = 1

generator/generators/ping_noauth.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from .protobuf import ping_pb2
 
+
 def gen():
     op = ping_pb2.Operation()
     result = ping_pb2.Result()

parsec_mock/parsec_mock.py

@@ -13,21 +13,25 @@
 
 
 @click.command()
-@click.option('--test-folder', default='./generator_output', help='Load all tests from folder')
-@click.option('--parsec-socket', default='./parsec.sock', help='Path to parsec unix socket')
+@click.option(
+    "--test-folder", default="./generator_output", help="Load all tests from folder"
+)
+@click.option(
+    "--parsec-socket", default="./parsec.sock", help="Path to parsec unix socket"
+)
 def run_test(test_folder, parsec_socket):
-    print('Mock parsec service listening on unix://{}.'.format(parsec_socket))
+    print("Mock parsec service listening on unix://{}.".format(parsec_socket))
 
     test_cases = load_tests_from_folder(test_folder)
 
-    print('Serving all {} tests in folder {}'.format(len(test_cases),test_folder))
+    print("Serving all {} tests in folder {}".format(len(test_cases), test_folder))
 
     # Make sure socket doesn't already exist
     try:
         os.unlink(parsec_socket)
     except OSError:
         if os.path.exists(parsec_socket):
-            print('Error removing old parsec socket, exiting')
+            print("Error removing old parsec socket, exiting")
             return
 
     # Create a unix socket
@@ -39,16 +43,16 @@ def run_test(test_folder, parsec_socket):
     while True:
         connection, client_addr = sock.accept()
         try:
-            print('Connection received from {}'.format(client_addr))
+            print("Connection received from {}".format(client_addr))
             received_data = connection.recv(4096)
-            b64_received_data = base64.b64encode(received_data).decode('ascii')
+            b64_received_data = base64.b64encode(received_data).decode("ascii")
             if b64_received_data in test_cases:
                 (name, test_case) = test_cases[b64_received_data]
-                print('Received expected request for test case {}'.format(name))
+                print("Received expected request for test case {}".format(name))
                 bin_response = base64.b64decode(test_case.test_data.response)
                 connection.sendall(bin_response)
             else:
-                print('Received unexpected request {}'.format(b64_received_data))
+                print("Received unexpected request {}".format(b64_received_data))
         finally:
             connection.close()
 
@@ -68,15 +72,16 @@ def _traverse(key, element):
         self.__dict__.update(objd)
         self.basedict = dictionary
 
+
 def load_tests_from_folder(test_folder):
     tests = {}
     """Read test specs from a folder"""
     specfiles = [f for f in listdir(test_folder) if isfile(join(test_folder, f))]
 
     for file in specfiles:
         print(f"Parsing spec file: {file}")
-        name = file.split('.')[0]
-        with open(os.path.join(test_folder, file), 'r') as f:
+        name = file.split(".")[0]
+        with open(os.path.join(test_folder, file), "r") as f:
             spec = safe_load(f)
             testspec = TestSpec(spec)
             tests[testspec.test_data.request] = (name, testspec)

requirements.txt

@@ -1,9 +1,13 @@
 attrs==20.3.0
+black==21.9b0
 click==7.1.2
 flake8==3.9.0
 iniconfig==1.1.1
 mccabe==0.6.1
+mypy-extensions==0.4.3
 packaging==20.9
+pathspec==0.9.0
+platformdirs==2.4.0
 pluggy==0.13.1
 protobuf==3.15.6
 py==1.10.0
@@ -13,6 +17,9 @@ pyflakes==2.3.1
 pyparsing==2.4.7
 pytest==6.2.2
 PyYAML==5.4.1
+regex==2021.9.24
 rope==0.18.0
 six==1.15.0
 toml==0.10.2
+tomli==1.2.1
+typing-extensions==3.10.0.2

tests/mock_test.py

@@ -6,31 +6,31 @@
 sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 
 # Connect the socket to the port where the server is listening
-server_address = './parsec.sock'
-print('connecting to %s' % server_address)
+server_address = "./parsec.sock"
+print("connecting to %s" % server_address)
 
 sock.connect(server_address)
 exit_code = 0
 
 try:
     # Send data
-    message = 'EKfAXh4AAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA'
+    message = "EKfAXh4AAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAA"
     bin_message = base64.b64decode(message)
 
-    expected_response = 'EKfAXh4AAQAAAAAAAAAAAAAAAAAAAAIAAAAAAAEAAAAAAAAACAE='
+    expected_response = "EKfAXh4AAQAAAAAAAAAAAAAAAAAAAAIAAAAAAAEAAAAAAAAACAE="
     bin_expected = base64.b64decode(expected_response)
 
     print('sending "%s"' % message)
     sock.sendall(bin_message)
 
     received_data = sock.recv(4096)
-    print('received {}'.format(base64.b64encode(received_data).decode('ascii')))
+    print("received {}".format(base64.b64encode(received_data).decode("ascii")))
     if bin_expected == received_data:
-        print('Received expected response')
+        print("Received expected response")
     else:
-        print('Did not get expected response')
+        print("Did not get expected response")
         exit_code = 1
 finally:
-    print('closing socket')
+    print("closing socket")
     sock.close()
     exit(exit_code)