rename core to service

setup.py

@@ -65,8 +65,8 @@
 
     entry_points={
         'console_scripts' : [
-            'filer = tesk_core.filer:main',
-            'taskmaster = tesk_core.taskmaster:main'
+            'filer = service.filer:main',
+            'taskmaster = service.taskmaster:main'
         ]
     },
     test_suite='tests',

taskmaster

@@ -1,3 +1,3 @@
 # TODO: Add to makefile after undertanding what its doing 😕
 
-PYTHONPATH="tesk" python tesk/tesk_core/taskmaster.py "$@"
+PYTHONPATH="tesk" python tesk/service/taskmaster.py "$@"

tesk/tesk_core/filer.py → tesk/service/filer.py

@@ -12,12 +12,12 @@
 import netrc
 import requests
 import gzip
-from tesk_core.exception import UnknownProtocol, FileProtocolDisabled
+from service.exception import UnknownProtocol, FileProtocolDisabled
 import shutil
 from glob import glob
-from tesk_core.path import containerPath, getPath, fileEnabled
-from tesk_core.transput import Type, Transput, urlparse
-from tesk_core.filer_s3 import S3Transput
+from service.path import containerPath, getPath, fileEnabled
+from service.transput import Type, Transput, urlparse
+from service.filer_s3 import S3Transput
 
 
 

tesk/tesk_core/filer_class.py → tesk/service/filer_class.py

@@ -1,6 +1,6 @@
 import json
-from tesk_core import path
-from tesk_core.path import fileEnabled
+from service import path
+from service.path import fileEnabled
 
 
 class Filer:

tesk/tesk_core/filer_s3.py → tesk/service/filer_s3.py

@@ -4,7 +4,7 @@
 import re
 import botocore
 import boto3
-from tesk_core.transput import Transput, Type
+from service.transput import Transput, Type
 
 class S3Transput(Transput):
     def __init__(self, path, url, ftype):

tesk/tesk_core/job.py → tesk/service/job.py

@@ -3,7 +3,7 @@
 from datetime import datetime, timezone
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
-from tesk_core.Util import pprint
+from service.Util import pprint
 
 
 logging.basicConfig(format='%(message)s', level=logging.INFO)

tesk/tesk_core/path.py → tesk/service/path.py

@@ -1,6 +1,6 @@
 import os
 from os.path import relpath
-from tesk_core.exception import InvalidHostPath
+from service.exception import InvalidHostPath
 try:
     from urllib.parse import urlparse
 except ImportError:

tesk/tesk_core/pvc.py → tesk/service/pvc.py

@@ -1,6 +1,6 @@
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
-from tesk_core.Util import pprint
+from service.Util import pprint
 import os
 import logging
 

tesk/tesk_core/taskmaster.py → tesk/service/taskmaster.py

@@ -8,9 +8,9 @@
 import logging
 import gzip
 from kubernetes import client, config
-from tesk_core.job import Job
-from tesk_core.pvc import PVC
-from tesk_core.filer_class import Filer
+from service.job import Job
+from service.pvc import PVC
+from service.filer_class import Filer
 
 created_jobs = []
 poll_interval = 5

tests/unit_test/tesk_core/FilerClassTest.py → tests/unit_test/service/FilerClassTest.py

@@ -2,9 +2,9 @@
 
 import unittest
 import os
-from tesk_core.filer_class import Filer
-from tesk_core import path
-from tesk_core.Util import pprint
+from service.filer_class import Filer
+from service import path
+from service.Util import pprint
 
 try:
     from unittest.mock import patch  # Python 3 @UnresolvedImport
@@ -14,9 +14,9 @@
 
 
 
-@patch('tesk_core.path.HOST_BASE_PATH'          , '/home/tfga/workspace/cwl-tes')
-@patch('tesk_core.path.CONTAINER_BASE_PATH'     , '/transfer')
-@patch('tesk_core.path.TRANSFER_PVC_NAME'       , 'transfer-pvc')
+@patch('service.path.HOST_BASE_PATH'          , '/home/tfga/workspace/cwl-tes')
+@patch('service.path.CONTAINER_BASE_PATH'     , '/transfer')
+@patch('service.path.TRANSFER_PVC_NAME'       , 'transfer-pvc')
 @patch.dict(os.environ,
             {
                 "AWS_SHARED_CREDENTIALS_FILE": "/aws/credentials",

tests/unit_test/tesk_core/TaskMasterTest.py → tests/unit_test/service/TaskMasterTest.py

@@ -1,5 +1,5 @@
 import unittest
-from tesk_core.taskmaster import newParser, run_task, newLogger
+from service.taskmaster import newParser, run_task, newLogger
 from argparse import Namespace
 import json
 import logging
@@ -64,11 +64,11 @@ def test_pullPolicyAlways(self):
 
 
 
-    @patch('tesk_core.taskmaster.args'                  , Namespace(debug=True, namespace='default', pull_policy_always=True))
-    @patch('tesk_core.taskmaster.logger'                , newLogger(logging.DEBUG))
-    @patch('tesk_core.taskmaster.PVC.create'            , pvcCreateMock)
-    @patch('tesk_core.taskmaster.PVC.delete'            , pvcDeleteMock)
-    @patch('tesk_core.taskmaster.Job.run_to_completion' , jobRunToCompletionMock)
+    @patch('service.taskmaster.args'                  , Namespace(debug=True, namespace='default', pull_policy_always=True))
+    @patch('service.taskmaster.logger'                , newLogger(logging.DEBUG))
+    @patch('service.taskmaster.PVC.create'            , pvcCreateMock)
+    @patch('service.taskmaster.PVC.delete'            , pvcDeleteMock)
+    @patch('service.taskmaster.Job.run_to_completion' , jobRunToCompletionMock)
     def test_run_task(self):
 
         with open('tests/resources/inputFile.json') as fh:

tests/unit_test/tesk_core/test_filer.py → tests/unit_test/service/test_filer.py

@@ -1,13 +1,13 @@
 import unittest
 import logging
 import os
-from tesk_core.filer import newTransput, FTPTransput, HTTPTransput, FileTransput,\
+from service.filer import newTransput, FTPTransput, HTTPTransput, FileTransput,\
     process_file, logConfig, getPath, copyDir, copyFile, ftp_check_directory,\
     subfolders_in
-from tesk_core.exception import UnknownProtocol, InvalidHostPath,\
+from service.exception import UnknownProtocol, InvalidHostPath,\
     FileProtocolDisabled
-from tesk_core.path import containerPath
-from tesk_core.filer_s3 import S3Transput
+from service.path import containerPath
+from service.filer_s3 import S3Transput
 from assertThrows import AssertThrowsMixin
 from fs.opener import open_fs
 from io import StringIO
@@ -32,16 +32,16 @@ def stripLines(txt):
     return '\n'.join([line.strip() for line in txt.splitlines()[1:]])
 
 
-@patch('tesk_core.path.HOST_BASE_PATH', '/home/tfga/workspace/cwl-tes')
-@patch('tesk_core.path.CONTAINER_BASE_PATH', '/transfer')
+@patch('service.path.HOST_BASE_PATH', '/home/tfga/workspace/cwl-tes')
+@patch('service.path.CONTAINER_BASE_PATH', '/transfer')
 class FilerTest(unittest.TestCase, AssertThrowsMixin):
 
     @classmethod
     def setUpClass(cls):
         logConfig(logging.DEBUG)  # Doesn't work...
 
-    @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.shutil.copy')
+    @patch('service.filer.copyDir')
+    @patch('service.filer.shutil.copy')
     def test_download_file(self, copyMock, copyDirMock):
         filedata = {
             "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5",
@@ -62,8 +62,8 @@ def test_download_file(self, copyMock, copyDirMock):
                                          '/var/lib/cwl/stgda974802-fa81-4f0b-'
                                          '8fe4-341d5655af4b/md5')
 
-    @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.shutil.copy')
+    @patch('service.filer.copyDir')
+    @patch('service.filer.shutil.copy')
     def test_download_dir(self, copyMock, copyDirMock):
         filedata = {
             "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/",
@@ -78,8 +78,8 @@ def test_download_dir(self, copyMock, copyDirMock):
 
         copyDirMock.assert_called_once_with('/transfer/tmphrtip1o8', '/TclSZU')
 
-    @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.shutil.copy')
+    @patch('service.filer.copyDir')
+    @patch('service.filer.shutil.copy')
     def test_upload_dir(self, copyMock, copyDirMock):
         filedata = {
             "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/",
@@ -94,8 +94,8 @@ def test_upload_dir(self, copyMock, copyDirMock):
 
         copyDirMock.assert_called_once_with('/TclSZU', '/transfer/tmphrtip1o8')
 
-    @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.copyFile')
+    @patch('service.filer.copyDir')
+    @patch('service.filer.copyFile')
     def test_upload_file(self, copyFileMock, copyDirMock):
 
         filedata = {
@@ -113,8 +113,8 @@ def test_upload_file(self, copyFileMock, copyDirMock):
                                         , '/transfer/tmphrtip1o8/md5')
 
 
-    @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.copyFile')
+    @patch('service.filer.copyDir')
+    @patch('service.filer.copyFile')
     def test_upload_file_glob(self, copyFileMock, copyDirMock):
 
         filedata = {

tests/unit_test/tesk_core/test_filer_ftp_pytest.py → tests/unit_test/service/test_filer_ftp_pytest.py

@@ -4,7 +4,7 @@
 import ftplib
 import os
 
-from tesk_core.filer import (
+from service.filer import (
     FTPTransput,
     Type,
     ftp_login,

tests/unit_test/tesk_core/test_filer_general_pytest.py → tests/unit_test/service/test_filer_general_pytest.py

@@ -7,7 +7,7 @@
 
 import pytest
 
-from tesk_core.filer import (
+from service.filer import (
     process_file,
     copyContent,
     FileProtocolDisabled
@@ -35,7 +35,7 @@ def test_process_file_with_scheme(mocker):
         'path': '.',
         'type': 'FILE',
     }
-    mock_new_Trans = mocker.patch('tesk_core.filer.newTransput')
+    mock_new_Trans = mocker.patch('service.filer.newTransput')
     process_file('inputs', filedata)
 
     mock_new_Trans.assert_called_once_with('http','www.foo.bar')

tests/unit_test/tesk_core/test_filer_http_pytest.py → tests/unit_test/service/test_filer_http_pytest.py

@@ -4,7 +4,7 @@
 import os
 from unittest import mock
 
-from tesk_core.filer import (
+from service.filer import (
     HTTPTransput,
     Type
 )

tests/unit_test/tesk_core/test_job.py → tests/unit_test/service/test_job.py

@@ -4,8 +4,8 @@
 import datetime
 from unittest.mock import patch
 from dateutil.tz import tzutc
-from tesk_core import taskmaster
-from tesk_core.job import Job
+from service import taskmaster
+from service.job import Job
 from argparse import Namespace
 from datetime import timezone
 from kubernetes.client.rest import ApiException
@@ -139,7 +139,7 @@ def test_job(self):
         self.assertEqual(job.namespace, 'default')
 
     @patch("kubernetes.client.BatchV1Api.create_namespaced_job")
-    @patch("tesk_core.job.Job.get_status", side_effect=[("Running", True),("Running", True),("Running", True),
+    @patch("service.job.Job.get_status", side_effect=[("Running", True),("Running", True),("Running", True),
                                                         ("Running", True),("Running", True),("Complete", True)])
     def test_run_to_completion_success(self, mock_get_status, mock_create_namespaced_job):
         """
@@ -152,10 +152,10 @@ def test_run_to_completion_success(self, mock_get_status, mock_create_namespaced
                                            taskmaster.args.pod_timeout)
             self.assertEqual(status, "Complete")
 
-    @patch("tesk_core.job.Job.delete")
-    @patch("tesk_core.taskmaster.check_cancelled", return_value=True)
+    @patch("service.job.Job.delete")
+    @patch("service.taskmaster.check_cancelled", return_value=True)
     @patch("kubernetes.client.BatchV1Api.create_namespaced_job")
-    @patch("tesk_core.job.Job.get_status", side_effect=[("Running", True)])
+    @patch("service.job.Job.get_status", side_effect=[("Running", True)])
     def test_run_to_completion_cancelled(self, mock_get_status, mock_create_namespaced_job, mock_check_cancelled,
                                mock_job_delete):
         """
@@ -168,9 +168,9 @@ def test_run_to_completion_cancelled(self, mock_get_status, mock_create_namespac
                                            taskmaster.args.pod_timeout)
             self.assertEqual(status, "Cancelled")
 
-    @patch("tesk_core.taskmaster.check_cancelled", return_value=False)
+    @patch("service.taskmaster.check_cancelled", return_value=False)
     @patch("kubernetes.client.BatchV1Api.create_namespaced_job", side_effect=ApiException(status=409,reason="conflict"))
-    @patch("tesk_core.job.Job.get_status", side_effect=[("Complete", True)])
+    @patch("service.job.Job.get_status", side_effect=[("Complete", True)])
     @patch("kubernetes.client.BatchV1Api.read_namespaced_job", side_effect=read_namespaced_job_running)
     def test_run_to_completion_check_conflict_exception(self, mock_get_status, mock_read_namespaced_job,
                                                         mock_check_cancelled,mock_create_namespaced_job):
@@ -199,8 +199,8 @@ def test_run_to_completion_check_other_K8_exception(self,mock_create_namespaced_
 
     @patch("kubernetes.client.CoreV1Api.list_namespaced_pod")
     @patch("kubernetes.client.BatchV1Api.read_namespaced_job", side_effect=read_namespaced_job_error)
-    @patch("tesk_core.job.Job.delete")
-    @patch("tesk_core.taskmaster.check_cancelled", return_value=False)
+    @patch("service.job.Job.delete")
+    @patch("service.taskmaster.check_cancelled", return_value=False)
     @patch("kubernetes.client.BatchV1Api.create_namespaced_job")
     def test_run_to_completion_error(self, mock_create_namespaced_job, mock_check_cancelled,
                                          mock_job_delete,mock_read_namespaced_job,mock_list_namespaced_pod  ):

tests/unit_test/tesk_core/test_s3_filer.py → tests/unit_test/service/test_s3_filer.py

@@ -1,8 +1,8 @@
 import os
 import pytest
 import boto3
-from tesk_core.filer_s3 import S3Transput
-#from tesk_core.extract_endpoint import extract_endpoint
+from service.filer_s3 import S3Transput
+#from service.extract_endpoint import extract_endpoint
 from moto import mock_s3
 from unittest.mock import patch, mock_open
 
@@ -44,7 +44,7 @@ def test_check_if_bucket_exists(moto_boto, path, url, ftype, expected):
         trans = S3Transput(path, url, ftype)
         assert trans.check_if_bucket_exists(client) == expected
 
-# @patch('tesk_core.filer.os.makedirs')
+# @patch('service.filer.os.makedirs')
 # @patch('builtins.open')
 # @patch('s3transfer.utils.OSUtils.rename_file')
 @pytest.mark.parametrize("path, url, ftype,expected", [
@@ -64,10 +64,10 @@ def test_s3_download_file( moto_boto, path, url, ftype, expected, fs, caplog):
 
 
 
-@patch('tesk_core.filer.os.makedirs')
+@patch('service.filer.os.makedirs')
 @patch('builtins.open')
 @patch('s3transfer.utils.OSUtils.rename_file')
-#@patch("tesk_core.filer_s3.extract_endpoint", return_value="http://s3.amazonaws.com")
+#@patch("service.filer_s3.extract_endpoint", return_value="http://s3.amazonaws.com")
 @pytest.mark.parametrize("path, url, ftype,expected", [
         ("filer_test/", "s3://tesk/folder1/","DIRECTORY",0),
         ("filer_test/", "s3://tesk/folder10/folder20","DIRECTORY",1)
@@ -151,7 +151,7 @@ def test_upload_directory_for_unknown_file_type(moto_boto, fs, monkeypatch, capl
     assert "Object is neither file or directory" in caplog.text
 
 
-@patch("tesk_core.filer.os.path.exists", return_value=1)
+@patch("service.filer.os.path.exists", return_value=1)
 def test_extract_url_from_config_file(mock_path_exists):
     """
     Testing extraction of endpoint url from default file location