Move scheduler constants into their own module so that packages with …

…native dependencies are not transitively included in the package start-up script. (y-scope#80)

components/clp-py-utils/clp_py_utils/initialize-orchestration-db.py

@@ -6,7 +6,7 @@
 
 from clp_py_utils.clp_config import Database
 from clp_py_utils.core import read_yaml_config_file
-from job_orchestration.scheduler.scheduler_data import JobStatus, TaskStatus
+from job_orchestration.scheduler.constants import JobStatus, TaskStatus
 from sql_adapter import SQL_Adapter
 
 # Setup logging

components/compression-job-handler/compression_job_handler/compression_job_handler.py

@@ -19,7 +19,7 @@
 from clp_py_utils.pretty_size import pretty_size
 from clp_py_utils.sql_adapter import SQL_Adapter
 from job_orchestration.job_config import PathsToCompress, InputConfig, OutputConfig, ClpIoConfig
-from job_orchestration.scheduler.scheduler_data import JobStatus
+from job_orchestration.scheduler.constants import JobStatus
 from .utils.common import JobCompletionStatus
 
 # Setup logging

components/job-orchestration/job_orchestration/executor/celeryconfig.py

@@ -1,6 +1,6 @@
 import os
 
-from job_orchestration.scheduler.scheduler_data import QueueName, TASK_QUEUE_HIGHEST_PRIORITY
+from job_orchestration.scheduler.constants import QueueName, TASK_QUEUE_HIGHEST_PRIORITY
 
 # Worker settings
 # Force workers to consume only one task at a time

components/job-orchestration/job_orchestration/executor/compression_task.py

@@ -9,9 +9,8 @@
 from job_orchestration.executor.celery import app
 from job_orchestration.executor.utils import append_message_to_task_results_queue
 from job_orchestration.job_config import ClpIoConfig, PathsToCompress
+from job_orchestration.scheduler.constants import TaskStatus, TaskUpdateType
 from job_orchestration.scheduler.scheduler_data import \
-    TaskStatus, \
-    TaskUpdateType, \
     TaskUpdate, \
     TaskFailureUpdate, \
     CompressionTaskSuccessUpdate

components/job-orchestration/job_orchestration/executor/search_task.py

@@ -7,8 +7,8 @@
 from job_orchestration.job_config import SearchConfig
 from job_orchestration.executor.celery import app
 from job_orchestration.executor.utils import append_message_to_task_results_queue
-from job_orchestration.scheduler.scheduler_data import TaskUpdate, TaskUpdateType, TaskStatus, \
-    TaskFailureUpdate
+from job_orchestration.scheduler.constants import TaskUpdateType, TaskStatus
+from job_orchestration.scheduler.scheduler_data import TaskUpdate, TaskFailureUpdate
 
 # Setup logging
 logger = get_task_logger(__name__)

components/job-orchestration/job_orchestration/scheduler/constants.py

@@ -0,0 +1,27 @@
+TASK_QUEUE_LOWEST_PRIORITY = 1
+TASK_QUEUE_HIGHEST_PRIORITY = 3
+
+
+class QueueName:
+    COMPRESSION = "compression"
+    SEARCH = "search"
+
+
+class JobStatus:
+    SCHEDULING = 'SCHEDULING'
+    SCHEDULED = 'SCHEDULED'
+    SUCCEEDED = 'SUCCEEDED'
+    FAILED = 'FAILED'
+
+
+class TaskUpdateType:
+    COMPRESSION = 'COMPRESSION'
+    SEARCH = 'SEARCH'
+
+
+class TaskStatus:
+    SUBMITTED = 'SUBMITTED'
+    SCHEDULED = 'SCHEDULED'
+    IN_PROGRESS = 'IN_PROGRESS'
+    SUCCEEDED = 'SUCCEEDED'
+    FAILED = 'FAILED'

components/job-orchestration/job_orchestration/scheduler/scheduler.py

@@ -17,19 +17,20 @@
 from clp_py_utils.sql_adapter import SQL_Adapter
 from job_orchestration.executor.compression_task import compress
 from job_orchestration.executor.search_task import search
+from job_orchestration.scheduler.constants import \
+    QueueName, \
+    JobStatus, \
+    TaskUpdateType, \
+    TaskStatus
 from job_orchestration.scheduler.results_consumer import ReconnectingResultsConsumer
 from job_orchestration.scheduler.scheduler_data import \
     CompressionJob, \
     SearchJob, \
-    JobStatus, \
     CompressionTask, \
     SearchTask, \
-    TaskStatus, \
-    TaskUpdateType, \
     TaskUpdate, \
     TaskFailureUpdate, \
-    CompressionTaskSuccessUpdate, \
-    QueueName
+    CompressionTaskSuccessUpdate
 
 # Setup logging
 # Create logger

components/job-orchestration/job_orchestration/scheduler/scheduler_data.py

@@ -8,26 +8,11 @@
 from celery.result import AsyncResult
 from pydantic import BaseModel, validator
 
-TASK_QUEUE_LOWEST_PRIORITY = 1
-TASK_QUEUE_HIGHEST_PRIORITY = 3
-
-
-class QueueName:
-    COMPRESSION = "compression"
-    SEARCH = "search"
-
-
-class TaskUpdateType:
-    COMPRESSION = 'COMPRESSION'
-    SEARCH = 'SEARCH'
-
-
-class TaskStatus:
-    SUBMITTED = 'SUBMITTED'
-    SCHEDULED = 'SCHEDULED'
-    IN_PROGRESS = 'IN_PROGRESS'
-    SUCCEEDED = 'SUCCEEDED'
-    FAILED = 'FAILED'
+from .constants import \
+    TASK_QUEUE_LOWEST_PRIORITY, \
+    TASK_QUEUE_HIGHEST_PRIORITY, \
+    TaskStatus, \
+    TaskUpdateType
 
 
 class TaskUpdate(BaseModel):
@@ -93,13 +78,6 @@ class Config:
         arbitrary_types_allowed = True
 
 
-class JobStatus:
-    SCHEDULING = 'SCHEDULING'
-    SCHEDULED = 'SCHEDULED'
-    SUCCEEDED = 'SUCCEEDED'
-    FAILED = 'FAILED'
-
-
 class CompressionJob(BaseModel):
     id: int
     status: str

components/package-template/src/sbin/native/search

@@ -67,7 +67,7 @@ from clp.package_utils import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, validate_an
 from clp_py_utils.clp_config import CLP_METADATA_TABLE_PREFIX, Database
 from clp_py_utils.sql_adapter import SQL_Adapter
 from job_orchestration.job_config import SearchConfig
-from job_orchestration.scheduler.scheduler_data import JobStatus
+from job_orchestration.scheduler.constants import JobStatus
 
 
 async def run_function_in_process(function, *args, initializer=None, init_args=None):

components/package-template/src/sbin/start-clp

@@ -79,7 +79,7 @@ from clp.package_utils import \
     validate_queue_config, \
     validate_worker_config
 from clp_py_utils.clp_config import CLPConfig
-from job_orchestration.scheduler.scheduler_data import QueueName
+from job_orchestration.scheduler.constants import QueueName
 
 
 def append_docker_port_settings_for_host_ips(hostname: str, host_port: int, container_port: int, cmd: [str]):