Toolset updates for Round 20 (TechEmpower#5648)

* Add tag support

* add --list-tag

* filter out broken tests

* work on test modules

* fix test order

* :/

* work on test-modules

* fix class

* add url length checks to verifications

* update cached-query key and vagrant to ubuntu 18.04

Author: NateBrady23

deployment/vagrant/core.rb

@@ -12,7 +12,7 @@ def provision_bootstrap(config)
 def provider_libvirt(config)
   config.vm.provider :libvirt do |virt, override|
     override.vm.hostname = "TFB-all"
-    override.vm.box = "generic/ubuntu1604"
+    override.vm.box = "generic/ubuntu1804"
 
     unless ENV.fetch('TFB_SHOW_VM', false)
       virt.graphics_type = "none"
@@ -28,7 +28,7 @@ def provider_libvirt(config)
 def provider_virtualbox(config)
   config.vm.provider :virtualbox do |vb, override|
     override.vm.hostname = "TFB-all"
-    override.vm.box = "ubuntu/xenial64"
+    override.vm.box = "ubuntu/bionic64"
 
     # Allow increase in size for /dev/sda1
     # Would need plugin:

toolset/benchmark/test_types/__init__.py

@@ -133,7 +133,7 @@ def main(argv=None):
     parser.add_argument(
         '--type',
         choices=[
-            'all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update',
+            'all', 'json', 'db', 'query', 'cached-query', 'fortune', 'update',
             'plaintext'
         ],
         nargs='+',

toolset/run-tests.py

@@ -0,0 +1,15 @@
+import imp
+import re
+
+from glob import glob
+
+test_types = {}
+test_type_folders = glob("/FrameworkBenchmarks/toolset/test_types/*/")
+
+# Loads all the test_types from the folders in this directory
+for folder in test_type_folders:
+    # regex that grabs the characters between "toolset/test_types/"
+    # and the final "/" in the folder string to get the name
+    test_type_name = re.findall(r'.+\/(.+)\/$', folder, re.M)[0]
+    test_type = imp.load_source("TestType", "%s%s.py" % (folder, test_type_name))
+    test_types[test_type_name] = test_type.TestType

toolset/test_types/__init__.py

@@ -1,11 +1,11 @@
+import abc
 import copy
 import requests
 
 from colorama import Fore
 from toolset.utils.output_helper import log
 
-
-class FrameworkTestType:
+class AbstractTestType:
     '''
     Interface between a test type (json, query, plaintext, etc) and
     the rest of TFB. A test type defines a number of keys it expects
@@ -15,6 +15,7 @@ class FrameworkTestType:
     passes an argument list of ['spam'], then after parsing there will
     exist a member `X.spam = 'foobar'`.
     '''
+    __metaclass__ = abc.ABCMeta
 
     def __init__(self,
                  config,
@@ -38,6 +39,13 @@ def __init__(self,
         self.failed = None
         self.warned = None
 
+    @classmethod
+    @abc.abstractmethod
+    def url(self):
+        pass
+
+    @classmethod
+    @abc.abstractmethod
     def accept(self, content_type):
         return {
             'json':
@@ -51,7 +59,7 @@ def accept(self, content_type):
     def parse(self, test_keys):
         '''
         Takes the dict of key/value pairs describing a FrameworkTest
-        and collects all variables needed by this FrameworkTestType
+        and collects all variables needed by this AbstractTestType
 
         Raises AttributeError if required keys are missing
         '''

toolset/benchmark/test_types/framework_test_type.py renamed to toolset/test_types/abstract_test_type.py

@@ -1,17 +1,17 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import verify_query_cases
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import verify_query_cases
 
 
-class CachedQueryTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.cached_query_url = ""
         kwargs = {
-            'name': 'cached_query',
+            'name': 'cached-query',
             'accept_header': self.accept('json'),
             'requires_db': True,
             'args': ['cached_query_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.cached_query_url
@@ -31,6 +31,14 @@ def verify(self, base_url):
 
         problems = verify_query_cases(self, cases, url)
 
+        # cached_query_url should be at least "/cached-worlds/"
+        # some frameworks use a trailing slash while others use ?q=
+        if len(self.cached_query_url) < 15:
+            problems.append(
+                ("fail",
+                 "Route for cached queries must be at least 8 characters, found '{}' instead".format(self.cached_query_url),
+                 url))
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else:

toolset/benchmark/test_types/cached_query_type.py renamed to toolset/test_types/cached-query/cached-query.py

@@ -1,8 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
 
 
-class DBTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.db_url = ""
         kwargs = {
@@ -11,14 +11,14 @@ def __init__(self, config):
             'requires_db': True,
             'args': ['db_url', 'database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.db_url
 
     def verify(self, base_url):
         '''
-        Ensures body is valid JSON with a key 'id' and a key 
+        Ensures body is valid JSON with a key 'id' and a key
         'randomNumber', both of which must map to integers
         '''
 
@@ -32,6 +32,13 @@ def verify(self, base_url):
 
         response, problems = basic_body_verification(body, url)
 
+        # db should be at least "/db"
+        if len(self.db_url) < 3:
+            problems.append(
+                ("fail",
+                 "Route for db must be at least 3 characters, found '{}' instead".format(self.db_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/benchmark/test_types/db_type.py renamed to toolset/test_types/db/db.py

@@ -1,9 +1,9 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.fortune_html_parser import FortuneHTMLParser
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_queries_count
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.fortune.fortune_html_parser import FortuneHTMLParser
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_queries_count
 
 
-class FortuneTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.fortune_url = ""
         kwargs = {
@@ -12,7 +12,7 @@ def __init__(self, config):
             'requires_db': True,
             'args': ['fortune_url','database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.fortune_url
@@ -34,6 +34,13 @@ def verify(self, base_url):
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
+        # fortune_url should be at least "/fortunes"
+        if len(self.fortune_url) < 9:
+            problems.append(
+                ("fail",
+                 "Route for fortunes must be at least 9 characters, found '{}' instead".format(self.fortune_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/test_types/fortune/__init__.py

@@ -1,8 +1,7 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
 
-
-class JsonTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.json_url = ""
         kwargs = {
@@ -11,15 +10,15 @@ def __init__(self, config):
             'requires_db': False,
             'args': ['json_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.json_url
 
     def verify(self, base_url):
         '''
-        Validates the response is a JSON object of 
-        { 'message' : 'hello, world!' }. Case insensitive and 
+        Validates the response is a JSON object of
+        { 'message' : 'hello, world!' }. Case insensitive and
         quoting style is ignored
         '''
 
@@ -28,6 +27,13 @@ def verify(self, base_url):
 
         response, problems = basic_body_verification(body, url)
 
+        # json_url should be at least "/json"
+        if len(self.json_url) < 5:
+            problems.append(
+                ("fail",
+                 "Route for json must be at least 5 characters, found '{}' instead".format(self.json_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/benchmark/test_types/fortune_type.py renamed to toolset/test_types/fortune/fortune.py

@@ -1,9 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers
-from time import sleep
+from toolset.test_types.verifications import basic_body_verification, verify_headers
+from toolset.test_types.abstract_test_type import AbstractTestType
 
 
-class PlaintextTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.plaintext_url = ""
         kwargs = {
@@ -12,14 +11,21 @@ def __init__(self, config):
             'accept_header': self.accept('plaintext'),
             'args': ['plaintext_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def verify(self, base_url):
         url = base_url + self.plaintext_url
         headers, body = self.request_headers_and_body(url)
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
+        # plaintext_url should be at least "/plaintext"
+        if len(self.plaintext_url) < 10:
+            problems.append(
+                ("fail",
+                 "Route for plaintext must be at least 10 characters, found '{}' instead".format(self.plaintext_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 
@@ -55,20 +61,20 @@ def get_script_name(self):
     def get_script_variables(self, name, url):
         return {
             'max_concurrency':
-            max(self.config.concurrency_levels),
+                max(self.config.concurrency_levels),
             'name':
-            name,
+                name,
             'duration':
-            self.config.duration,
+                self.config.duration,
             'levels':
-            " ".join("{}".format(item)
-                     for item in self.config.pipeline_concurrency_levels),
+                " ".join("{}".format(item)
+                         for item in self.config.pipeline_concurrency_levels),
             'server_host':
-            self.config.server_host,
+                self.config.server_host,
             'url':
-            url,
+                url,
             'pipeline':
-            16,
+                16,
             'accept':
-            "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+                "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
         }

toolset/benchmark/fortune_html_parser.py renamed to toolset/test_types/fortune/fortune_html_parser.py

@@ -1,8 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import verify_query_cases
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import verify_query_cases
 
 
-class QueryTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.query_url = ""
         kwargs = {
@@ -11,17 +11,17 @@ def __init__(self, config):
             'requires_db': True,
             'args': ['query_url', 'database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.query_url
 
     def verify(self, base_url):
         '''
-        Validates the response is a JSON array of 
-        the proper length, each JSON Object in the array 
-        has keys 'id' and 'randomNumber', and these keys 
-        map to integers. Case insensitive and 
+        Validates the response is a JSON array of
+        the proper length, each JSON Object in the array
+        has keys 'id' and 'randomNumber', and these keys
+        map to integers. Case insensitive and
         quoting style is ignored
         '''
 
@@ -31,6 +31,14 @@ def verify(self, base_url):
 
         problems = verify_query_cases(self, cases, url, False)
 
+        # queries_url should be at least "/queries/"
+        # some frameworks use a trailing slash while others use ?q=
+        if len(self.query_url) < 9:
+            problems.append(
+                ("fail",
+                 "Route for queries must be at least 9 characters, found '{}' instead".format(self.query_url),
+                 url))
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else: