Refactor the current codebase

This commit contains the following changes:
- clean the directory tree to regroup common features with more explicit
  names
- regroup CI testing into one workflow and add the mock service smoke
  test
- simplify the README.md file instruction with what is currently there
- change the way new tests are added to make it easier

Signed-off-by: Hugues de Valon <[email protected]>

Author: hug-dev

.github/workflows/build.yml

@@ -14,3 +14,33 @@ jobs:
         args: -v -r *.md
     - name: Fail if there were link errors
       run: exit ${{ steps.lc.outputs.exit_code }}
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # pyyaml version we use only supporting python 3.6 upwards
+        python-version: [ 3.6, 3.7, 3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude generator/protobuf
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude generator/generators/protobuf
+    - name: Execute smoke test
+      run: |
+        python parsec_mock/parsec_mock.py
+        sleep 5
+        python tests/mock_test.py

.github/workflows/ci.yml

@@ -1,3 +1,3 @@
 [submodule "parsec-operations"]
-	path = parsec-operations
+	path = generator/generators/parsec-operations
 	url = https://github.com/parallaxsecond/parsec-operations

.gitmodules

@@ -9,8 +9,13 @@
 from struct import pack
 import base64
 
-from generator_lib import generators
-
+# Generator functions
+# Add your new generator function here
+# TODO: with some import syntax or clever moduling, it might be possible not
+# to have to do anything here?
+from generators.ping_noauth import gen as ping_noauth
+from generators.list_opcodes_bad1 import gen as list_opcodes_bad1
+from generators.list_opcodes_directauth import gen as list_opcodes_directauth
 
 class TestSpec(object):
     """Class to represent a test specification.  Used to convert
@@ -27,39 +32,30 @@ def _traverse(key, element):
         self.__dict__.update(objd)
         self.basedict = dictionary
 
-    def is_valid(self):
-        return True
-
-
 def read_specs(folder):
     """Read test specs from a folder"""
     specfiles = [f for f in listdir(folder) if isfile(join(folder, f))]
     specs = []
     for file in specfiles:
         print(f"Parsing spec file: {file}")
+        # Only use the first part of the filename as spec name
+        name = file.split('.')[0]
         with open(os.path.join(folder, file), 'r') as f:
             spec = safe_load(f)
             testspec = TestSpec(spec["spec"])
-            if testspec.is_valid():
-                specs.append(testspec)
-            else:
-                print(f"Error loading test spec from {file}")
+            specs.append((name, testspec))
     return specs
 
 
 def generate_data(specs, output_folder):
     """Generate test data for a list of specs"""
-    for spec in specs:
-        if spec.generator in generators:
-            print(f"Generating test {spec.name}")
-            generate_spec_data(output_folder, spec, generators[spec.generator])
-        else:
-            print(f"No generator function found for spec {spec.name}, skipping...")
-
+    for (name, spec) in specs:
+        generate_spec_data(output_folder, spec, name)
 
-def generate_spec_data(output_folder, spec, generator_fn):
+def generate_spec_data(output_folder, spec, name):
     """Generates data for a single spec and outputs it into the specified output folder."""
-    (operation, result) = generator_fn()
+    # The generator function has the same name as the test specification
+    (operation, result) = eval(name)()
 
     request_auth = create_auth(spec.request.auth)
     request_content_len = spec.request.header.content_length
@@ -76,21 +72,21 @@ def generate_spec_data(output_folder, spec, generator_fn):
         response_content_len = len(result)
 
     response_auth_len = spec.response.header.auth_length
-
     response_header = pack_header(spec.response.header, response_auth_len, response_content_len)
 
     request_buf = request_header + operation + request_auth
     response_buf = response_header + result
 
+    # The generator appends the base64 data at the end of the spec file
     out_data = {
         "spec": spec.basedict,
         "test_data": {
             "request": base64.b64encode(request_buf).decode('ascii'),
             "response": base64.b64encode(response_buf).decode('ascii'),
         }
     }
-    out_path = os.path.join(output_folder, spec.name + ".test.yaml")
-    print(f"Writing spec {spec.name} test data to {out_path}")
+    out_path = os.path.join(output_folder, name + ".test.yaml")
+    print(f"Writing spec {name} test data to {out_path}")
     with open(out_path, 'w') as f:
         dump(out_data, f, sort_keys=False)
 
@@ -100,6 +96,8 @@ def pack_header(header, auth_len, body_len):
     # pack function converts arguments into binary string, based on format string.
     # < means integers are little endian.  Rest of format string is one character per input to indicate
     # packed field interpretation.  See struct.pack docs for details.
+    # This should map to the Fixed Common Header defined here:
+    # https://parallaxsecond.github.io/parsec-book/parsec_client/wire_protocol.html#the-fixed-common-header
     return pack('<IHBBHBQBBBIHIHH',
                 header.magic_number,
                 header.header_size,
@@ -129,15 +127,15 @@ def create_auth(auth_spec):
 
 
 def main():
-    specdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'testspecs'))
-    datadir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../testdata'))
     print("Generating test data.")
+
+    specdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'test_specs'))
     print(f"Reading test specs from {specdir}")
-    print(f"Generating test data to {datadir}")
     specs = read_specs(specdir)
 
+    datadir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../generator_output'))
+    print(f"Generating test data to {datadir}")
     generate_data(specs, datadir)
 
-
 if __name__ == "__main__":
     main()

Makefile

@@ -0,0 +1,11 @@
+# Copyright 2021 Contributors to the Parsec project.
+# SPDX-License-Identifier: Apache-2.0
+PROTOC_OUTPUT_FILES=$(shell find parsec-operations/protobuf/ -name "*.proto" -exec basename {} .proto \; | awk '{print "protobuf/"$$1"_pb2.py"}')
+.PHONY: protobuf all
+
+all: protobuf
+
+protobuf: ${PROTOC_OUTPUT_FILES}
+
+protobuf/%_pb2.py: parsec-operations/protobuf/%.proto
+	@protoc -I=parsec-operations/protobuf --python_out=protobuf $< > /dev/null

README.md

@@ -1,13 +1,11 @@
 # Copyright 2021 Contributors to the Parsec project.
 # SPDX-License-Identifier: Apache-2.0
-import protobuf.ping_pb2
-import protobuf.list_opcodes_pb2
+from .protobuf import list_opcodes_pb2
 
-
-def gen_list_opcodes_auth_direct():
-    operation = protobuf.list_opcodes_pb2.Operation()
+def gen():
+    operation = list_opcodes_pb2.Operation()
     operation.provider_id = 1
 
-    result = protobuf.list_opcodes_pb2.Result()
+    result = list_opcodes_pb2.Result()
     result.opcodes.extend([1, 3, 2])
     return (operation.SerializeToString(), result.SerializeToString())

generator/generator.py

@@ -0,0 +1,11 @@
+# Copyright 2021 Contributors to the Parsec project.
+# SPDX-License-Identifier: Apache-2.0
+from .protobuf import list_opcodes_pb2
+
+def gen():
+    operation = list_opcodes_pb2.Operation()
+    operation.provider_id = 1
+
+    result = list_opcodes_pb2.Result()
+    result.opcodes.extend([1, 3, 2, 6])
+    return (operation.SerializeToString(), result.SerializeToString())

generator/generator_lib.py

@@ -1,12 +1,10 @@
 # Copyright 2021 Contributors to the Parsec project.
 # SPDX-License-Identifier: Apache-2.0
-import protobuf.ping_pb2
-import protobuf.list_opcodes_pb2
+from .protobuf import ping_pb2
 
-
-def gen_ping_no_auth():
-    op = protobuf.ping_pb2.Operation()
-    result = protobuf.ping_pb2.Result()
+def gen():
+    op = ping_pb2.Operation()
+    result = ping_pb2.Result()
     result.wire_protocol_version_maj = 1
     result.wire_protocol_version_min = 0