Implement Register Issue

Author: RyanJField

.zenodo.json

@@ -10,7 +10,8 @@
       },
       {
         "affiliation": "The University of Sheffield",
-        "name": "Reddyhoff, Dennis"
+        "name": "Reddyhoff, Dennis",
+        "orcid": "0000-0002-4971-2346"
       }
     ],
     "access_right": "open",

src/org/fairdatapipeline/api/common/fdp_utils.py

@@ -5,6 +5,8 @@
 from datetime import datetime
 import hashlib
 import random
+from requests import api
+from requests.models import Response
 import yaml
 import uuid
 
@@ -14,7 +16,16 @@ def get_entry(
     query: dict,
     token: str = None,
 )-> list:
-
+    """
+    Internal function to retreive and item from the registy using a query
+    Args:
+        |   url: str of the registry url
+        |   enpoint: enpoint (table)
+        |   query: dict forming a query
+        |   token: (optional) str of the registry token
+    Returns:
+        |   dict: responce from registry
+    """
     headers = {}
 
     if token is not None:
@@ -54,7 +65,16 @@ def get_entity(
     id: int,
     token: str = None,
 )-> list:
-
+    """
+    Internal function to get an item from the registry using it's id
+    Args:
+        |   url: str of the registry url
+        |   enpoint: endpoint (table)
+        |   id: id of the item
+        |   token: (optional) str of the registry token
+    Returns:
+        |   dict: responce from registry
+    """
     headers = {}
 
     if token is not None:
@@ -72,7 +92,13 @@ def get_entity(
     return response.json()
 
 def extract_id(url: str)-> str:
-
+    """
+    Internal function to return the id from an api url
+    Args:
+        |   url: str of the api url
+    Returns:
+        |   str: id derrived from the url
+    """
     parse = urllib.parse.urlsplit(url).path
     extract = list(filter(None, parse.split('/')))[-1]
 
@@ -84,7 +110,16 @@ def post_entry(
     data: dict,
     token: str,
 )-> dict:
-
+    """
+    Internal function to post and entry on the registry
+    Args:
+        |   url: str of the registry url
+        |   enpoint: str of the endpoint (table)
+        |   data: a dictionary containing the data to be posted
+        |   token: str of the registry token
+    Returns:
+        |   dict: responce from registry
+    """
     headers = {
     'Authorization': 'token ' + token,
     'Content-type': 'application/json'
@@ -110,7 +145,15 @@ def patch_entry(
     data: dict,
     token: str
 )-> dict:
-
+    """
+    Internal function to patch and entry on the registry
+    Args:
+        |   url: str of the url of what to be patched
+        |   data: a dictionary containing the data to be patched
+        |   token: str of the registry token
+    Returns:
+        |   dict: responce from registry
+    """
     headers = {
     'Authorization': 'token ' + token,
     'Content-type': 'application/json'
@@ -125,6 +168,14 @@ def patch_entry(
     return response.json()
 
 def post_storage_root(url, data, token):
+    """
+    Internal function to post a storage root to the registry
+    the function first adds file:// if the root is local
+    Args:
+        |   url: str the url for the storage root e.g. https://github.com/
+    Returns:
+        |   dict: repsonse from the local registy
+    """
     if 'local' in data.keys():
         if data['local']:
             data['root'] = 'file://' + data['root']
@@ -133,22 +184,40 @@ def post_storage_root(url, data, token):
     return post_entry(url, 'storage_root', data, token)
 
 def remove_local_from_root(root: str):
+    """
+    Internal function to remove prepending file:// from a given root
+    Args:
+        |   root: the root
+    Returns:
+        |   str: the root without file://
+    """
     if 'file://' in root:
         root = root.replace('file://', '')
 
     return root
 
 
 def random_hash()-> str:
+    """
+    Internal function to generate a random unique hash
 
+    Returns:
+        |   str: 40 character randomly generated hash.
+    """
     seed = datetime.now().timestamp() * random.uniform(1, 1000000)
     seed = str(seed).encode('utf-8')
     hashed = hashlib.sha1(seed)
 
     return hashed.hexdigest()
 
 def get_file_hash(path: str)-> str:
-
+    """
+    Internal function to return a files sha1 hash
+    Args:
+        |   path: str file path
+    Returns:
+        |   str: sha1 hash
+    """
     with open(path, 'rb') as data:
         data = data.read()
     #data = data.encode('utf-8')
@@ -157,27 +226,68 @@ def get_file_hash(path: str)-> str:
     return hashed.hexdigest()
 
 def read_token(token_path: str):
+    """
+    Internal function read a token from a given file
+    Args:
+        |   token_path: path to token
+    Returns:
+        |   str: token
+    """
     with open(token_path) as token:
         token = token.readline().strip()
     return token
 
 def get_token(token_path: str):
+    """
+    Internal function alias for read_token()
+    Args:
+        |   token_path: path to token
+    Returns:
+        |   str: token
+    """
     return read_token(token_path)
 
 def is_file(filename: str):
+    """
+    Internal function to check whether a file exists
+    Args:
+        |   filename: file to check
+    Returns:
+        |   boolean: whether the file exists
+    """
     return os.path.isfile(filename)
 
 def is_yaml(filename: str):
+    """
+    Internal function to check whether a file can be opened as a YAML file
+    ! warning returns True if the file can be coerced into yaml format
+    Args:
+        |   filename: path to the yaml file
+    Returns:
+        |   boolean: can the file be coerced into a yaml format?
+    """
     try:
         with open(filename, 'r') as data:
             yaml.safe_load(data)
     except: return False
     return True
 
 def is_valid_yaml(filename: str):
+    """
+    Internal function validate whether a file exists and can be coerced into a yaml format
+    Args:
+        |   filename: path to the yaml file
+    Returns:
+        |   boolean: does the file exist and can it be coerced into a yaml format
+    """
     return is_file(filename) & is_yaml(filename)
 
 def generate_uuid():
+    """
+    Internal function similar to random hash
+    Returns:
+        |   str: a random unique identifier
+    """
     return datetime.now().strftime('%Y%m-%d%H-%M%S-') + str(uuid.uuid4())
 
 def get_handle_index_from_path(handle: dict, path: str):
@@ -198,4 +308,121 @@ def get_handle_index_from_path(handle: dict, path: str):
         for input in handle['input']:
             if handle['input'][input]['path'] == path:
                 tmp = input
-    return tmp
+    return tmp
+
+def register_issues(token: str, handle: dict):
+    """
+    Internal function, should only be called from finalise.
+    """
+
+    api_url = handle['yaml']['run_metadata']['local_data_registry_url']
+    print(handle)
+    issues = handle['issues']
+    groups = set(handle['issues'][i]['group'] for i in handle['issues'])
+
+    for group in groups:
+        component_list = []
+        issue = None
+        severity = None
+        for i in issues:
+            if issues[i]['group'] == group:
+                type = issues[i]['type']
+                issue = issues[i]['issue']
+                severity = issues[i]['severity']
+                index = issues[i]['index']
+                data_product = issues[i]['use_data_product']
+                component= issues[i]['use_component']
+                version = issues[i]['version']
+                namespace = issues[i]['use_namespace']
+
+                component_url = None
+                object_id = None
+                if type == 'config':
+                    object_id = handle['model_config']
+                elif type == 'submission_script':
+                    object_id = handle['submission_script']
+                elif type == 'github_repo':
+                    object_id = handle['code_repo']
+
+                if object_id:
+                    component_url = get_entry(
+                        url= api_url,
+                        endpoint= 'object_component',
+                        query= {
+                            'object' : extract_id(object_id),
+                            'whole_object': True
+                        }
+                    )[0]['url']
+
+                if index:
+                    if 'output' in handle.keys():
+                        for ii in handle['output']:
+                            if handle['output'][ii] == index:
+                                if 'component_url' in handle['output'][ii].keys():
+                                    component_url = handle['output'][ii]['component_url']
+                                else:
+                                    'No Component Found Please run finalise()'
+                    if 'input' in handle.keys():
+                        for ii in handle['input']:
+                            if handle['input'][ii] == index:
+                                if 'component_url' in handle['input'][ii].keys():
+                                    component_url = handle['input'][ii]['component_url']
+                                else:
+                                    'No Component Found Please run finalise()'
+
+                if data_product:
+                    current_namespace = get_entry(
+                        url = api_url,
+                        endpoint= 'namespace',
+                        query= {
+                            'name': namespace
+                        }
+                    )[0]['url']
+
+                    object = get_entry(
+                        url= api_url,
+                        endpoint= 'data_product',
+                        query= {
+                            'name': data_product,
+                            'version': version,
+                            'namespace': extract_id(current_namespace)
+                        }
+                    )[0]['object']
+                    object_id = extract_id(object)
+                    if component:
+                        component_url = get_entry(
+                            url = api_url,
+                            endpoint= 'object_component',
+                            query= {
+                                'name': component,
+                                'object': object_id
+                            }
+                        )
+                    else:
+                        component_obj = get_entry(
+                            url = api_url,
+                            endpoint= 'object_component',
+                            query= {
+                                'object': object_id,
+                                'whole_object': True
+                            }
+                        )
+                        component_url = component_obj[0]['url']
+
+                if component_url:
+                    component_list.append(component_url)
+
+        # Register the issue:
+        print('Registering issue:' + group)
+        current_issue = post_entry(
+            url = api_url,
+            endpoint= 'issue',
+            data = {
+                'severity': severity,
+                'description': issue,
+                'component_issues': component_list
+            },
+            token = token
+        )
+    
+

src/org/fairdatapipeline/api/common/finalise_pipeline.py

@@ -260,6 +260,8 @@ def finalise(token: str, handle: dict):
         for input in handle['input']:
             input_components.append(handle['input'][input]['component_url'])
 
+    fdp_utils.register_issues(token, handle)
+
     fdp_utils.patch_entry(
         token = token,
         url = handle['code_run'],