Merge pull request #36 from mbengit/master

Improve test coverage

Author: mbengit

.gitignore

@@ -1,4 +1,6 @@
 .cache
+.coverage
+.idea
 .project
 .pydevproject
 .pytest_cache
@@ -9,10 +11,9 @@
 *.pyc
 *.swp
 build
+coverage.xml
+coverage-html
 dist
 qpylib.egg-info
-RemoteSystemsTempFiles
 qpylib/version.py
-.idea/
-coverage.xml
-.coverage
+RemoteSystemsTempFiles

.pylintrc

@@ -117,10 +117,10 @@ include-naming-hint=no
 property-classes=abc.abstractproperty
 
 # Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,40}$
+function-rgx=[a-z_][a-z0-9_]{2,50}$
 
 # Naming hint for function names
-function-name-hint=[a-z_][a-z0-9_]{2,40}$
+function-name-hint=[a-z_][a-z0-9_]{2,50}$
 
 # Regular expression matching correct variable names
 variable-rgx=[a-z_][a-z0-9_]{2,30}$

CHANGELOG.md

@@ -1,7 +1,15 @@
 # Changelog
-All notable changes to this project will be documented in this file.
 
-## [Unreleased]
-### Added
-### Changed
-### Removed
+## 2.0.1
+- Non-SDK REST calls default to `verify=True`.  
+SDK default remains `verify=False` for now, but that will change in a future release.
+
+## 2.0
+- First official release of qpylib on Github.
+- Support for Python 3 and Red Hat UBI app base image.
+- abstract/sdk/live layers removed.
+- New encryption algorithm in encdec.py, plus backwards compatibility with previous versions.
+- New ariel.py module supports Ariel searches via REST API.
+- `get_app_id` now uses `QRADAR_APP_ID` environment variable instead of manifest value.
+- REST methods now pass through `kwargs`.
+- REST replaces `gethostname()` with `localhost`.

CONTRIBUTING.md

@@ -5,12 +5,12 @@ To contribute code or documentation, please submit a [pull request](https://gith
 ## Proposing new features
 
 If you would like to implement a new feature, please [raise an issue](https://github.com/ibm/qpylib/issues)
-before sending a pull request so the feature can be discussed.
+before sending a pull request so that the feature can be discussed.
 
 ## Fixing bugs
 
 To fix a bug, please [raise an issue](https://github.ibm.com/ibm/qpylib/issues) before sending a
-pull request so it can be tracked.
+pull request so that the bug can be tracked.
 
 ## Merge approval
 
@@ -22,7 +22,7 @@ A list of maintainers can be found on the [MAINTAINERS](MAINTAINERS.md) page.
 Each source file must include a license header for the Apache Software License 2.0.
 Using the SPDX format is the simplest approach. See existing source files for an example.
 
-## Setup
+## Development
 
 On your local machine you can lint, test and build in mostly
 the same way that Travis CI does. There are three scripts you can use:
@@ -33,17 +33,16 @@ the same way that Travis CI does. There are three scripts you can use:
 
 The `requirements.txt` file contains the Python packages needed to run these scripts.
 
-## Code style
+### Code style
 
-Pull requests will only be accepted if they pass the linting.
-Linting can be run either on your fork through Travis CI, or locally using `lint.sh`.
+Pull requests will be accepted only if `lint.sh` produces no warnings or errors.
 
-## Test
+### Test
 
-Pull requests will only be accepted if they pass all tests.
-Tests can be run either on your fork through Travis CI, or locally using `test.sh`.
+Pull requests will be accepted only if `test.sh` reports no test failures **and**
+sufficient test coverage.
 
-## Build
+### Build
 
 The output of `build.sh` is a tar.gz file containing the qpylib Python package.
 You can use `pip install` to install the package into your Python 3 environment.

README.md

@@ -14,9 +14,6 @@ The qpylib library hosted here is for use only with apps that have been written
 Python 3 to run on Red Hat Universal Base Image. It is not compatible with apps
 written to run on a CentOS base image.
 
-**NOTE**: until the first official qpylib release, planned for early Q3 2020,
-any code contained herein is unsupported and subject to change.
-
 ## Project details
 
 * [LICENSE](LICENSE)

clean.sh

@@ -3,10 +3,16 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-rm -rf .pytest_cache
 rm -f qpylib/version.py
-rm -rf dist qpylib.egg-info
+
+rm -rf dist
+rm -rf qpylib.egg-info
+
+rm -rf .pytest_cache
 rm -rf qpylib/__pycache__
+rm -rf qpylib/encryption/__pycache__
 rm -rf test/__pycache__
+
 rm -f .coverage
 rm -f coverage.xml
+rm -rf coverage-html

qpylib/ariel.py

@@ -44,10 +44,11 @@ def search(self, query, api_version='latest'):
         response_json = response.json()
         return (response_json['status'], response_json['search_id'])
 
-    def search_sync(self, query, timeout=60, api_version='latest'):
+    def search_sync(self, query, timeout=60, sleep_interval=10, api_version='latest'):
         ''' Initiates a synchronous Ariel search.
               query: AQL query to execute.
               timeout: number of seconds to wait for search to complete.
+              sleep_interval: number of seconds to sleep before retrying status check.
               api_version: QRadar API version to use, defaults to latest.
             Returns a tuple containing search ID and record count.
             Raises ArielError if any of these occur:
@@ -67,7 +68,7 @@ def search_sync(self, query, timeout=60, api_version='latest'):
                 raise ArielError('Ariel search {0} failed: {1}'.format(search_id, status),
                                  aql=query)
             if time.time() < end_time:
-                time.sleep(10)
+                time.sleep(sleep_interval)
                 continue
             raise ArielError('Ariel search {0} did not complete within {1}s'
                              .format(search_id, timeout), aql=query)

qpylib/asset_qpylib.py

@@ -41,7 +41,7 @@ def get_asset_json_ld(asset_id):
                                get_asset_url_full(asset_id),
                                'asset',
                                'Asset details',
-                               'Asset details for id ' + asset_id,
+                               'Asset details for id ' + str(asset_id),
                                asset_json)
 
 def get_asset_json_html(asset_id, generate_html=None):

qpylib/encdec.py

@@ -31,7 +31,6 @@ class Encryption():
     '''
     engines = {2: Enginev2, 3: Enginev3, 4: Enginev4}
     latest_engine_version = max(engines)
-    latest_engine_class = engines[latest_engine_version]
 
     def __init__(self, data):
         ''' data is an object containing two non-empty strings:
@@ -68,13 +67,13 @@ def encrypt(self, clear_text):
             Returns the encrypted value.
         '''
         self._reset_config_if_required()
-        engine = Encryption.latest_engine_class(self.config[self.name], self.app_uuid)
+        engine = self.latest_engine_class()(self.config[self.name], self.app_uuid)
 
         try:
             encrypted_secret = engine.encrypt(clear_text)
         except Exception as error:
             raise EncryptionError('Failed to encrypt secret for name {0}: {1}'
-                                  .format(self.name, error))
+                                  .format(self.name, type(error).__name__))
 
         self.config[self.name]['secret'] = encrypted_secret
         self._save_config()
@@ -99,13 +98,17 @@ def decrypt(self):
             secret = engine.decrypt()
         except Exception as error:
             raise EncryptionError('Failed to decrypt secret for name {0}: {1}'
-                                  .format(self.name, error))
+                                  .format(self.name, type(error).__name__))
 
         if engine.version != Encryption.latest_engine_version:
             self.encrypt(secret)
 
         return secret
 
+    @staticmethod
+    def latest_engine_class():
+        return Encryption.engines[Encryption.latest_engine_version]
+
     def _choose_engine(self):
         # If no version is present in the config we default to engine v2.
         try:
@@ -130,7 +133,7 @@ def _reset_config_if_required(self):
         except KeyError:
             reset_required = True
         if reset_required:
-            self.config[self.name] = Encryption.latest_engine_class.generate_config()
+            self.config[self.name] = self.latest_engine_class().generate_config()
 
     def _read_config(self):
         try:

qpylib/json_qpylib.py

@@ -5,82 +5,68 @@
 import json
 from . import app_qpylib
 
-# A dictionary of jsonld context types mapped to the type name
 JSONLD_TYPES = {}
-
 KEY_CONTEXT = '@context'
 KEY_TYPE = '@type'
 KEY_ID = '@id'
 
 def register_jsonld_endpoints():
-    manifest = app_qpylib.get_manifest_json()
-
-    services = None
-    if 'services' in manifest.keys():
-        services = manifest['services']
-    if services is None:
+    try:
+        services = app_qpylib.get_manifest_json()['services']
+    except KeyError:
         return
 
     for service in services:
-        endpoints = None
-        if 'endpoints' in service.keys():
+        try:
             endpoints = service['endpoints']
-        if endpoints is None:
+        except KeyError:
             continue
         for endpoint in endpoints:
-            jsonld_context = None
-            if 'request_mime_type' in endpoint.keys():
-                argument = endpoint
-                jsonld_context = _extract_jsonld_context(argument, 'request_mime_type', 'request_body_type')
-                register_jsonld_type_from_context(jsonld_context)
-            if 'response' in endpoint.keys():
-                argument = endpoint['response']
-                jsonld_context = _extract_jsonld_context(argument, 'mime_type', 'body_type')
-                register_jsonld_type_from_context(jsonld_context)
+            _extract_and_register_jsonld_context(endpoint, 'request_mime_type', 'request_body_type')
+            try:
+                _extract_and_register_jsonld_context(endpoint['response'], 'mime_type', 'body_type')
+            except KeyError:
+                pass
+
+def _extract_and_register_jsonld_context(endpoint_direction, mime_id, body_id):
+    try:
+        if endpoint_direction[mime_id] == 'application/json+ld':
+            register_jsonld_type_from_context(endpoint_direction[body_id])
+    except KeyError:
+        pass
 
-def _extract_jsonld_context(argument, mime_id, context_id):
-    if (mime_id in argument.keys() and
-            context_id in argument.keys() and
-            argument[mime_id] == 'application/json+ld'):
-        return argument[context_id]
-    return None
+def register_jsonld_type(jsonld_type, context):
+    global JSONLD_TYPES
+    JSONLD_TYPES[str(jsonld_type)] = context
 
 def register_jsonld_type_from_context(context):
-    if context is not None:
-        jsonld_type = _extract_type(context)
+    jsonld_type = _extract_type_from_context(context)
+    if jsonld_type:
         register_jsonld_type(jsonld_type, context)
 
-def register_jsonld_type(jsonld_type, context):
-    global JSONLD_TYPES
-    JSONLD_TYPES[str(jsonld_type)] = context
+def _extract_type_from_context(context):
+    if KEY_CONTEXT in context.keys():
+        at_context = context[KEY_CONTEXT]
+        if KEY_TYPE in at_context.keys():
+            if at_context[KEY_TYPE] == KEY_ID and KEY_ID in at_context.keys():
+                return at_context[KEY_ID]
+            return at_context[KEY_TYPE]
+    return None
 
 def get_jsonld_type(jsonld_type):
     global JSONLD_TYPES
-    if jsonld_type in JSONLD_TYPES.keys():
+    try:
         return JSONLD_TYPES[str(jsonld_type)]
-    raise ValueError('json ld key has not been registered')
+    except KeyError:
+        raise ValueError('JSON-LD type {0} has not been registered'.format(str(jsonld_type)))
 
-def _extract_type(argument):
-    type_id = None
-    if KEY_CONTEXT in argument.keys():
-        context = argument[KEY_CONTEXT]
-        if KEY_TYPE in context.keys():
-            type_id = context[KEY_TYPE]
-        if type_id == KEY_ID and KEY_ID in context.keys():
-            type_id = context[KEY_ID]
-    return type_id
-
-def render_json_ld_type(jld_type, data, jld_id=None):
+def render_jsonld_type(jld_type, data, jld_id=None):
     jld_context = get_jsonld_type(jld_type)
-    json_dict = {}
+    json_dict = {KEY_CONTEXT: jld_context[KEY_CONTEXT], KEY_TYPE: jld_type}
+    if jld_id:
+        json_dict[KEY_ID] = jld_id
     for json_key in data:
         json_dict[json_key] = data[json_key]
-
-    json_dict[KEY_CONTEXT] = jld_context[KEY_CONTEXT]
-    json_dict[KEY_TYPE] = jld_type
-    if jld_id is not None:
-        json_dict[KEY_TYPE] = jld_type
-
     return json.dumps(json_dict, sort_keys=True)
 
 # pylint: disable=too-many-arguments