Merge pull request #30 from FAIRDataPipeline/feature/error-handling

Feature/error handling

Author: kzscisoft

codecov.yml

@@ -0,0 +1,2 @@
+ignore:
+  - "*/tests/*"

fairdatapipeline/__init__.py

@@ -6,6 +6,7 @@
     "raise_issue_by_data_product",
     "raise_issue_by_index",
     "raise_issue_with_config",
+    "raise_issue_by_type",
     "raise_issue_by_existing_data_product",
     "raise_issue_with_submission_script",
     "raise_issue_with_github_repo",
@@ -19,6 +20,7 @@
     raise_issue_by_data_product,
     raise_issue_by_existing_data_product,
     raise_issue_by_index,
+    raise_issue_by_type,
     raise_issue_with_config,
     raise_issue_with_github_repo,
     raise_issue_with_submission_script,

fairdatapipeline/fdp_utils.py

@@ -60,7 +60,6 @@ def get_entry(
             + " Query = "
             + url
         )
-
     return response.json()["results"]
 
 
@@ -105,7 +104,11 @@ def extract_id(url: str) -> str:
     Returns:
         |   str: id derrived from the url
     """
-    return list(filter(None, urlsplit(url).path.split("/")))[-1]
+
+    split_url_path = urlsplit(url).path.split("/")
+    if not split_url_path:
+        raise IndexError(f"Unable to extract ID from registry URL: {url}")
+    return [s for s in split_url_path if s != ""][-1]
 
 
 def post_entry(
@@ -132,8 +135,6 @@ def post_entry(
 
     response = requests.post(_url, _data, headers=headers)
 
-    # print(response.request.headers)
-
     if response.status_code == 409:
         logging.info("Entry Exists: Attempting to return Existing Entry")
         existing_entry = get_entry(url, endpoint, data)

fairdatapipeline/pipeline.py

@@ -92,14 +92,18 @@ def initialise(token: str, config: str, script: str) -> dict:
     config_filetype_url = config_filetype_response["url"]
 
     # Get user for registry admin account
-    user = fdp_utils.get_entry(
+    results = fdp_utils.get_entry(
         url=registry_url,
         endpoint="users",
         query={"username": "admin"},
         token=token,
         api_version=api_version,
-    )[0]
+    )
 
+    if not results:
+        raise IndexError(f"list {results} empty")
+    else:
+        user = results[0]
     # Check users exists
     if not user:
         raise ValueError(
@@ -109,15 +113,17 @@ def initialise(token: str, config: str, script: str) -> dict:
 
     user_url = user["url"]
     user_id = fdp_utils.extract_id(user_url)
-
     # Get author(s)
-    author = fdp_utils.get_entry(
+    results = fdp_utils.get_entry(
         url=registry_url,
         endpoint="user_author",
         query={"user": user_id},
         api_version=api_version,
-    )[0]
-
+    )
+    if not results:
+        raise IndexError(f"list {results} empty")
+    else:
+        author = results[0]
     # Check user author exists
     if not author:
         raise ValueError(
@@ -334,7 +340,6 @@ def finalise(token: str, handle: dict) -> None:
     api_version = handle["yaml"]["run_metadata"]["api_version"]
 
     datastore = fdp_utils.remove_local_from_root(datastore)
-
     datastore_root = fdp_utils.get_entry(
         url=registry_url,
         endpoint="storage_root",

fairdatapipeline/raise_issue.py

@@ -148,7 +148,7 @@ def raise_issue_by_type(
 # flake8: noqa C901
 def raise_issue(
     handle: dict,
-    type: str,
+    issue_type: str,
     issue: str,
     severity: int,
     index: bool = None,
@@ -159,7 +159,7 @@ def raise_issue(
     group: bool = True,
 ) -> None:
     current_group = issue + ":" + str(severity)
-    if type in [
+    if issue_type in [
         "config",
         "submission_script",
         "github_repo",
@@ -233,7 +233,7 @@ def raise_issue(
     # Write to handle and return path
     issues_dict = {
         "index": index,
-        "type": type,
+        "type": issue_type,
         "use_data_product": data_product,
         "use_component": component,
         "version": version,

pyproject.toml

@@ -77,7 +77,7 @@ line_length = 79
 line-length = 79
 
 [tool.pytest.ini_options]
-addopts = '-s -v'
+addopts = '-s -v --cov=fairdatapipeline  --cov-report=html --cov-report=term'
 markers = [
     "pipeline: tests for 'pipeline' module ",
     "issue: tests for raising issues ",

tests/test_fdp_utils.py

@@ -26,11 +26,13 @@ def write_csv_path(test_dir: str) -> str:
 
 
 # Test is_file()
+@pytest.mark.utilities
 def test_is_file_exists(test_dir: str) -> None:
     test_file = os.path.join(test_dir, "test.csv")
     assert fdp_utils.is_file(test_file)
 
 
+@pytest.mark.utilities
 @pytest.mark.parametrize(
     "file_path",
     [
@@ -43,12 +45,14 @@ def test_is_file_not_exists(file_path: str) -> None:
     assert not fdp_utils.is_file(file_path)
 
 
+@pytest.mark.utilities
 @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"])
 def test_is_yaml(file_path: str, request: FixtureRequest) -> None:
     file_path = request.getfixturevalue(file_path)
     assert fdp_utils.is_yaml(file_path)
 
 
+@pytest.mark.utilities
 @pytest.mark.parametrize(
     "file_path",
     [
@@ -62,12 +66,14 @@ def test_is_yaml_not(file_path: str) -> None:
     assert not fdp_utils.is_yaml(file_path)
 
 
+@pytest.mark.utilities
 @pytest.mark.parametrize("file_path", ["read_csv_path", "write_csv_path"])
 def test_is_valid_yaml(file_path: str, request: FixtureRequest) -> None:
     file_path = request.getfixturevalue(file_path)
     assert fdp_utils.is_yaml(file_path)
 
 
+@pytest.mark.utilities
 @pytest.mark.parametrize(
     "file_path",
     [
@@ -81,6 +87,7 @@ def test_is_valid_yaml_not(file_path: str) -> None:
     assert not fdp_utils.is_valid_yaml(file_path)
 
 
+@pytest.mark.utilities
 def test_read_token(test_dir: str) -> None:
     token = os.path.join(test_dir, "test_token")
     assert (
@@ -89,6 +96,7 @@ def test_read_token(test_dir: str) -> None:
     )
 
 
+@pytest.mark.utilities
 def test_get_token(test_dir: str) -> None:
     token = os.path.join(test_dir, "test_token")
     assert (
@@ -97,6 +105,7 @@ def test_get_token(test_dir: str) -> None:
     )
 
 
+@pytest.mark.utilities
 def test_read_token_get_token(test_dir: str) -> None:
     token = os.path.join(test_dir, "test_token")
     assert fdp_utils.read_token(token) == fdp_utils.get_token(token)
@@ -109,6 +118,7 @@ def token() -> str:
     )
 
 
+@pytest.mark.utilities
 def test_get_file_hash(test_dir: str) -> None:
     file_path = os.path.join(test_dir, "test.csv")
     if platform.system() == "Windows":
@@ -123,32 +133,47 @@ def test_get_file_hash(test_dir: str) -> None:
         )
 
 
+@pytest.mark.utilities
 def test_random_hash_is_string() -> None:
     assert type(fdp_utils.random_hash()) == str
 
 
+@pytest.mark.utilities
 def test_random_hash_length() -> None:
     assert len(fdp_utils.random_hash()) == 40
 
 
+@pytest.mark.utilities
 def test_extract_id() -> None:
     assert fdp_utils.extract_id("http://localhost:8000/api/object/85") == "85"
 
 
+@pytest.mark.utilities
+def test_extract_id_should_fail() -> None:
+    with pytest.raises(IndexError):
+        fdp_utils.extract_id("")
+
+
+@pytest.mark.utilities
 def test_get_headers() -> None:
     assert type(fdp_utils.get_headers()) == dict
+    headers = {"Accept": "application/json; version=" + "1.0.0"}
+    assert headers == fdp_utils.get_headers()
 
 
+@pytest.mark.utilities
 def test_get_headers_with_token(token: str) -> None:
     headers = fdp_utils.get_headers(token=token)
     assert headers["Authorization"] == "token " + token
 
 
+@pytest.mark.utilities
 def test_get_headers_post() -> None:
     headers = fdp_utils.get_headers(request_type="post")
     assert headers["Content-Type"] == "application/json"
 
 
+@pytest.mark.utilities
 def test_get_headers_api_version() -> None:
     headers = fdp_utils.get_headers(api_version="0.0.1")
     assert headers["Accept"] == "application/json; version=0.0.1"
@@ -232,6 +257,32 @@ def test_get_entry(url: str, token: str, storage_root_test: dict) -> None:
     assert entry[0] == storage_root_test
 
 
+@pytest.mark.utilities
+def test_get_entry_author(url: str, token: str) -> None:
+
+    results = fdp_utils.get_entry(
+        url=url,
+        query={"user": 2},
+        token=token,
+        endpoint="user_author",
+    )
+    with pytest.raises(IndexError):
+        _ = results[0]
+
+
+@pytest.mark.utilities
+def test_get_entry_users(url: str, token: str) -> None:
+
+    results = fdp_utils.get_entry(
+        url=url,
+        query={"username": "admin1"},
+        token=token,
+        endpoint="users",
+    )
+    with pytest.raises(IndexError):
+        _ = results[0]
+
+
 @pytest.mark.utilities
 def test_get_entity(url: str, storage_root_test: dict) -> None:
     entity = fdp_utils.get_entity(