Feature/upgrade flask celery (#5)

updates

Author: michael-yin

.DS_Store

@@ -0,0 +1,3 @@
+[run]
+omit =
+    tests/*

.coveragerc

@@ -1,8 +1,7 @@
-FLASK_ENV=development
+FLASK_DEBUG=1
 FLASK_CONFIG=development
 DATABASE_URL=postgresql://flask_celery:flask_celery@db/flask_celery
 SECRET_KEY=my_precious
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
-SOCKETIO_MESSAGE_QUEUE=redis://redis:6379/0
-
+SOCKETIO_MESSAGE_QUEUE=redis://redis:6379/0

.env/.dev-sample

@@ -1,4 +1,4 @@
-FLASK_ENV=production
+FLASK_DEBUG=0
 FLASK_CONFIG=production
 DATABASE_URL=postgresql://flask_celery:flask_celery@db/flask_celery
 SECRET_KEY=my_precious

.env/.prod-sample

@@ -1,2 +1,2 @@
+/venv/
 upload
-

.gitignore

@@ -0,0 +1 @@
+Source code for the [The Definitive Guide to Celery and Flask](https://testdriven.io/courses/flask-celery/) course by [Michael Yin](https://github.com/michael-yin/)

README.md

@@ -3,16 +3,3 @@
 
 app = create_app()
 celery = ext_celery.celery
-
-
-@app.cli.command("celery_worker")
-def celery_worker():
-    from watchgod import run_process
-    import subprocess
-
-    def run_worker():
-        subprocess.call(
-            ['celery', '-A', 'celery_app.celery', 'worker', '--loglevel=info', '-Q', 'high_priority,default']
-        )
-
-    run_process('./project', run_worker)

REEADME.md

@@ -3,5 +3,6 @@
 set -o errexit
 set -o nounset
 
-# celery -A app.celery worker --loglevel=info
-flask celery_worker
+watchfiles \
+  --filter python \
+  'celery -A celery_app.celery worker --loglevel=info -Q high_priority,default'

celery_app.py

@@ -6,4 +6,4 @@ set -o nounset
 
 flask db upgrade
 # flask run --host=0.0.0.0
-python app.py
+python app.py

compose/auto_deploy_do.sh

@@ -3,4 +3,4 @@
 set -o errexit
 set -o nounset
 
-celery -A celery_app.celery beat -l info
+exec celery -A project.wsgi.celery beat -l info

compose/local/flask/celery/worker/start

@@ -13,7 +13,7 @@ until worker_ready; do
 done
 >&2 echo 'Celery workers is available'
 
-celery -A celery_app.celery \
+exec celery -A project.wsgi.celery \
     --broker="${CELERY_BROKER_URL}" \
     flower \
     --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" \

compose/local/flask/start

@@ -3,4 +3,4 @@
 set -o errexit
 set -o nounset
 
-celery -A project.wsgi.celery worker --loglevel=info
+exec celery -A project.wsgi.celery worker --loglevel=info

compose/production/flask/celery/beat/start

@@ -1,4 +1,4 @@
-FROM nginx:1.19-alpine
+FROM nginx:1.23-alpine
 
 RUN rm /etc/nginx/conf.d/default.conf
 COPY nginx.conf /etc/nginx/conf.d

compose/production/flask/celery/flower/start

@@ -20,7 +20,7 @@ server {
         client_max_body_size 20M;
     }
     location /upload/ {
-        alias /app/uploadfiles/;
+        alias /app/upload/;
     }
 
     location /socket.io {

compose/production/flask/celery/worker/start

@@ -20,7 +20,6 @@ services:
       dockerfile: ./compose/production/flask/Dockerfile
     command: /start
     volumes:
-
       - mediafiles:/app/upload
     env_file:
       - ./.env/.prod-sample
@@ -29,7 +28,7 @@ services:
       - db
 
   db:
-    image: postgres:13.0-alpine
+    image: postgres:14.0-alpine
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     environment:
@@ -38,7 +37,7 @@ services:
       - POSTGRES_PASSWORD=flask_celery
 
   redis:
-    image: redis:6-alpine
+    image: redis:7-alpine
 
   rabbitmq:
     image: rabbitmq:3-management
@@ -88,29 +87,28 @@ services:
       - redis
       - db
 
-  prometheus:
-    image: prom/prometheus
-    ports:
-      - 9090:9090
-    command:
-      - --config.file=/etc/prometheus/prometheus.yml
-    volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
-    depends_on:
-      - cadvisor
-
-  cadvisor:
-    image: google/cadvisor
-    container_name: cadvisor
-    volumes:
-      - /:/rootfs:ro
-      - /var/run:/var/run:rw
-      - /sys:/sys:ro
-      - /var/lib/docker/:/var/lib/docker:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-
+#  prometheus:
+#    image: prom/prometheus
+#    ports:
+#      - 9090:9090
+#    command:
+#      - --config.file=/etc/prometheus/prometheus.yml
+#    volumes:
+#      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+#    depends_on:
+#      - cadvisor
+#
+#  cadvisor:
+#    image: gcr.io/cadvisor/cadvisor
+#    container_name: cadvisor
+#    volumes:
+#      - /:/rootfs:ro
+#      - /var/run:/var/run:rw
+#      - /sys:/sys:ro
+#      - /var/lib/docker/:/var/lib/docker:ro
+#      - /var/run/docker.sock:/var/run/docker.sock:ro
 
 volumes:
   postgres_data:
   mediafiles:
-  flower_db:
+  flower_db:

compose/production/nginx/Dockerfile

@@ -23,7 +23,7 @@ services:
       - db
 
   db:
-    image: postgres:13-alpine
+    image: postgres:14-alpine
     volumes:
       - postgres_data:/var/lib/postgresql/data/
     environment:
@@ -32,7 +32,7 @@ services:
       - POSTGRES_PASSWORD=flask_celery
 
   redis:
-    image: redis:6-alpine
+    image: redis:7-alpine
 
   celery_worker:
     build:
@@ -42,11 +42,6 @@ services:
     command: /start-celeryworker
     volumes:
       - .:/app
-    # logging:
-    #   driver: syslog
-    #   options:
-    #     syslog-address: "tcp+tls://logs2.papertrailapp.com:45883"
-    #     tag: "{{.Name}}/{{.ID}}"
     env_file:
       - .env/.dev-sample
     environment:
@@ -75,7 +70,7 @@ services:
     build:
       context: .
       dockerfile: ./compose/local/flask/Dockerfile
-    image: flask_celery_example_celey_flower
+    image: flask_celery_example_celery_flower
     command: /start-flower
     volumes:
       - .:/app

compose/production/nginx/nginx.conf

@@ -24,14 +24,20 @@
     'sqlalchemy.url',
     str(current_app.extensions['migrate'].db.get_engine().url).replace(
         '%', '%%'))
-target_metadata = current_app.extensions['migrate'].db.metadata
+target_db = current_app.extensions['migrate'].db
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 
 
+def get_metadata():
+    if hasattr(target_db, 'metadatas'):
+        return target_db.metadatas[None]
+    return target_db.metadata
+
+
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
 
@@ -46,7 +52,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True
+        url=url, target_metadata=get_metadata(), literal_binds=True
     )
 
     with context.begin_transaction():
@@ -76,7 +82,7 @@ def process_revision_directives(context, revision, directives):
     with connectable.connect() as connection:
         context.configure(
             connection=connection,
-            target_metadata=target_metadata,
+            target_metadata=get_metadata(),
             process_revision_directives=process_revision_directives,
             **current_app.extensions['migrate'].configure_args
         )

docker-compose.prod.yml

@@ -1,17 +1,17 @@
 """empty message
 
-Revision ID: 7309fc8782a7
-Revises: 69fd8a9d569e
-Create Date: 2021-09-17 01:42:45.564119
+Revision ID: bcb268967ba0
+Revises: de643eb1222c
+Create Date: 2022-12-19 03:32:06.930018
 
 """
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '7309fc8782a7'
-down_revision = '69fd8a9d569e'
+revision = 'bcb268967ba0'
+down_revision = 'de643eb1222c'
 branch_labels = None
 depends_on = None
 

docker-compose.yml

@@ -1,16 +1,16 @@
 """Initial migration.
 
-Revision ID: 69fd8a9d569e
+Revision ID: de643eb1222c
 Revises: 
-Create Date: 2021-09-16 09:29:54.931254
+Create Date: 2022-12-16 22:47:33.117492
 
 """
 from alembic import op
 import sqlalchemy as sa
 
 
 # revision identifiers, used by Alembic.
-revision = '69fd8a9d569e'
+revision = 'de643eb1222c'
 down_revision = None
 branch_labels = None
 depends_on = None

migrations/env.py

@@ -1,5 +1,6 @@
 import functools
 
+from flask import has_app_context
 from celery import current_app as current_celery_app
 from celery import Task, shared_task
 from celery.utils.time import get_exponential_backoff_interval
@@ -25,8 +26,10 @@ class AppContextTask(Task):
 
     def __call__(self, *args, **kwargs):
         # self.app is the Celery app instance
+        if has_app_context():
+            return Task.__call__(self, *args, **kwargs)
         with self.app.flask_app.app_context():
-            Task.__call__(self, *args, **kwargs)
+            return Task.__call__(self, *args, **kwargs)
 
 
 class custom_celery_task:
@@ -47,8 +50,7 @@ def __call__(self, func):
         @functools.wraps(func)
         def wrapper_func(*args, **kwargs):
             try:
-                with task_func.app.flask_app.app_context():
-                    return func(*args, **kwargs)
+                return func(*args, **kwargs)
             except self.EXCEPTION_BLOCK_LIST:
                 # do not retry for those exceptions
                 raise
@@ -60,7 +62,6 @@ def wrapper_func(*args, **kwargs):
         task_func = shared_task(*self.task_args, **self.task_kwargs)(wrapper_func)
         return task_func
 
-
     def _get_retry_countdown(self, task_func):
         retry_backoff = int(
             self.task_kwargs.get('retry_backoff', True)