Remove celery

.dockerignore

@@ -5,5 +5,3 @@
 **/.gitignore
 **/.dockerignore
 **/.gitlab-ci.yml
-celerybeat-schedule.dir
-celerybeat-schedule.pag

.flake8

@@ -18,4 +18,3 @@ exclude =
     ./libcloud/,
     ./paramiko/,
     ./run_script/,
-    ./celerybeat-mongo/,

.gitignore

@@ -72,10 +72,6 @@ target/
 # pyenv
 .python-version
 
-# celery beat schedule file
-celerybeat-schedule
-celerybeat-schedule-py3.db
-
 # SageMath parsed files
 *.sage.py
 
@@ -108,9 +104,5 @@ ENV/
 /src/mist/api/templates/landing.pt
 /src/mist/api/templates/manage.pt
 
-# Celerybeat files
-/celerybeat-schedule.dir
-/celerybeat-schedule.pag
-
 # openapi spec
 /openapi/spec.yml

.gitlab-ci.yml

@@ -96,7 +96,7 @@ flake8:
   before_script:
     - cd /mist.api
   script:
-    - flake8 --ignore=E402,E722,F632,F841,W504,W605 --exclude=v2,paramiko,celerybeat-mongo,libcloud,run_script
+    - flake8 --ignore=E402,E722,F632,F841,W504,W605 --exclude=v2,paramiko,libcloud,run_script
 
 uniq:
   stage: test

.gitmodules

@@ -7,9 +7,6 @@
 [submodule "paramiko"]
 	path = paramiko
 	url = ../../mistio/paramiko.git
-[submodule "celerybeat-mongo"]
-	path = celerybeat-mongo
-	url = ../../mistio/celerybeat-mongo.git
 [submodule "v2"]
 	path = v2
 	url = ../mist-api-v2.git

Dockerfile

@@ -1,7 +1,7 @@
 FROM mist/python3:latest
 
 # Install libvirt which requires system dependencies.
-RUN apk add --update --no-cache g++ gcc libvirt libvirt-dev libxml2-dev libxslt-dev gnupg ca-certificates wget mongodb-tools
+RUN apk add --update --no-cache g++ gcc libvirt libvirt-dev libxml2-dev libxslt-dev gnupg ca-certificates wget mongodb-tools libmemcached-dev
 
 RUN wget https://dl.influxdata.com/influxdb/releases/influxdb-1.8.4-static_linux_amd64.tar.gz && \
     tar xvfz influxdb-1.8.4-static_linux_amd64.tar.gz && rm influxdb-1.8.4-static_linux_amd64.tar.gz
@@ -21,13 +21,11 @@ COPY requirements.txt /requirements-mist.api.txt
 WORKDIR /mist.api/
 
 COPY paramiko /mist.api/paramiko
-COPY celerybeat-mongo /mist.api/celerybeat-mongo
 COPY libcloud /mist.api/libcloud
 COPY v2 /mist.api/v2
 
 RUN pip install --no-cache-dir -r /mist.api/requirements.txt && \
     pip install -e paramiko/ && \
-    pip install -e celerybeat-mongo/ && \
     pip install -e libcloud/ && \
     pip install -e v2/ && \
     pip install --no-cache-dir -r v2/requirements.txt

bin/docker-init

@@ -14,7 +14,6 @@ if [ ! -e clean ]; then
     echo "{\"sha\":\"$VERSION_SHA\",\"name\":\"$VERSION_NAME\",\"repo\":\"$VERSION_REPO\",\"modified\":true}" > /mist-version.json
     pip install -e $DIR/../paramiko/
     pip install -e $DIR/../libcloud/
-    pip install -e $DIR/../celerybeat-mongo/
     pip install -e $DIR/../src/
     set +e
     diff /requirements-mist.api.txt $DIR/../requirements.txt

bin/dramatiq

@@ -2,17 +2,36 @@
 import os
 import importlib
 
-
-os.system("pip install -U 'dramatiq[rabbitmq]'")
+from mist.api import config
 
 BROKER = 'mist.api.dramatiq_app'
-QUEUES = ['dramatiq_create_machine',
-          'dramatiq_post_deploy_steps',
-          'dramatiq_ssh_tasks',
-          'dramatiq_mappings',
-          'dramatiq_schedules']
+QUEUES = os.getenv('QUEUES', '').split(',') or [
+    'default',
+    'provisioning',
+    'scripts',
+    'schedules',
+    'rules',
+    'polling',
+    'mappings',
+    'ping_probe',
+    'ssh_probe']
+
+POSSIBLE_MODULES = [] # 'mist.api.dramatiq_tasks', 'mist.rbac.dramatiq_tasks']
+
+for root, dirs, files in os.walk("src"):
+    for file in files:
+        if file.endswith('tasks.py'):
+            module = os.path.join(
+                root, file[:-3]).split('src/')[1].replace('/', '.')
+            POSSIBLE_MODULES.append(module)
+            print(module)
+
+for root, dirs, files in os.walk("/opt"):
+    for file in files:
+        if file.endswith('tasks.py'):
+            print('.'.join(os.path.join(root, file[:-3]).split('/')[3:]))
+
 
-POSSIBLE_MODULES = ['mist.api.dramatiq_tasks', 'mist.rbac.dramatiq_tasks']
 modules = []
 
 for module in POSSIBLE_MODULES:
@@ -23,6 +42,15 @@ for module in POSSIBLE_MODULES:
     modules.append(module)
     print(f"  * {module}")
 
+for plugin in config.PLUGINS:
+    module = 'mist.' + plugin + '.tasks'
+    try:
+        importlib.import_module(module)
+    except ModuleNotFoundError:
+        continue
+    modules.append(module)
+    print(f"  * {module}")
+
 print(f"Will execute command: dramatiq {BROKER} {' '.join(modules)} -Q {' '.join(QUEUES)}")
 
 os.system(f"dramatiq {BROKER} {' '.join(modules)} -Q {' '.join(QUEUES)}")