Remove RQ dependency (#230)

Author: cunla

.github/actions/test-coverage/action.yml

@@ -17,11 +17,11 @@ outputs:
 runs:
   using: "composite"
   steps:
-    - name: Run Tests with coverage
+    - name: Run regular tests with coverage
       shell: bash
       run: |
         cd testproject
-        poetry run coverage run manage.py test scheduler
+        poetry run coverage run manage.py test --exclude-tag multiprocess scheduler
     - name: Coverage report
       id: coverage_report
       shell: bash

.github/workflows/test.yml

@@ -27,14 +27,14 @@ jobs:
         run: |
           poetry run ruff check
 
-  test:
+  test-regular:
     needs: [ 'ruff' ]
     runs-on: ubuntu-latest
     name: "Run tests ${{ matrix.python-version }}/${{ matrix.django-version }}/${{ matrix.broker }}"
     strategy:
       max-parallel: 6
       matrix:
-        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+        python-version: [ '3.11', '3.12', '3.13' ]
         django-version: [ '5.0.7', '5.1.7' ]
         broker: [ 'redis', 'fakeredis', 'valkey' ]
         include:
@@ -110,7 +110,7 @@ jobs:
           else
             export BROKER_PORT=6379
           fi
-          poetry run python manage.py test scheduler
+          poetry run python manage.py test --exclude-tag multiprocess scheduler
 
       # Steps for coverage check
       - name: Run tests with coverage
@@ -141,7 +141,7 @@ jobs:
       # write permission is required for auto-labeler
       # otherwise, read permission is required at least
       pull-requests: write
-    needs: test
+    needs: test-regular
     runs-on: ubuntu-latest
     steps:
       - uses: release-drafter/release-drafter@v6

SECURITY.md

@@ -2,9 +2,9 @@
 
 ## Supported Versions
 
-| Version     | Supported          |
-|-------------|--------------------|
-| 2023.latest | :white_check_mark: |
+| Version  | Supported          |
+|----------|--------------------|
+| 4.latest | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

docs/changelog.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## v4.0.0b1 🌈
+
+### Breaking Changes
+
+This version is a full revamp of the package. The main changes are related to removing the RQ dependency.
+Worker/Queue/Job are all implemented in the package itself. This change allows for more flexibility and control over
+the tasks.
+
+Management commands:
+
+- `rqstats` => `scheduler_stats`
+- `rqworker` => `scheduler_worker`
+
+Settings:
+
+- `SCHEDULER_CONFIG` is now a `SchedulerConfiguration` object to help IDE guide settings.
+- `SCHEDULER_QUEUES` is now a list of `QueueConfiguration` objects to help IDE guide settings.
+- Configuring queue to use `SSL`/`SSL_CERT_REQS`/`SOCKET_TIMEOUT` is now done using `CONNECTION_KWARGS` in
+  `QueueConfiguration`
+  ```python
+  SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = {
+    'default': QueueConfiguration(
+        HOST='localhost',
+        PORT=6379,
+        USERNAME='some-user',
+        PASSWORD='some-password',
+        CONNECTION_KWARGS={  # Eventual additional Broker connection arguments
+            'ssl_cert_reqs': 'required',
+            'ssl':True,
+        },
+    ),
+   # ...
+   }
+   ```
+- For how to configure in `settings.py`, please see the [settings documentation](./configuration.md).
+
 ## v3.0.1 🌈
 
 ### 🐛 Bug Fixes

docs/commands.md

@@ -1,44 +1,43 @@
 # Management commands
 
-## rqworker
+## `scheduler_worker` - Create a worker
 
 Create a new worker with a scheduler for specific queues by order of priority.
 If no queues are specified, will run on default queue only.
 
 All queues must have the same redis settings on `SCHEDULER_QUEUES`.
 
 ```shell
-usage: manage.py rqworker [-h] [--pid PIDFILE] [--burst] [--name NAME] [--worker-ttl WORKER_TTL] [--max-jobs MAX_JOBS]
-                          [--fork-job-execution FORK_JOB_EXECUTION] [--job-class JOB_CLASS] [--sentry-dsn SENTRY_DSN] [--sentry-debug]
-                          [--sentry-ca-certs SENTRY_CA_CERTS] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH]
-                          [--traceback] [--no-color] [--force-color] [--skip-checks]
-                          [queues ...]
+usage: manage.py scheduler_worker [-h] [--pid PIDFILE] [--name NAME] [--worker-ttl WORKER_TTL] [--fork-job-execution FORK_JOB_EXECUTION] [--sentry-dsn SENTRY_DSN] [--sentry-debug] [--sentry-ca-certs SENTRY_CA_CERTS] [--burst]
+                                  [--max-jobs MAX_JOBS] [--max-idle-time MAX_IDLE_TIME] [--with-scheduler] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color]
+                                  [--skip-checks]
+                                  [queues ...]
 
 positional arguments:
   queues                The queues to work on, separated by space, all queues should be using the same redis
 
 options:
   -h, --help            show this help message and exit
   --pid PIDFILE         file to write the worker`s pid into
-  --burst               Run worker in burst mode
   --name NAME           Name of the worker
   --worker-ttl WORKER_TTL
                         Default worker timeout to be used
-  --max-jobs MAX_JOBS   Maximum number of jobs to execute before terminating worker
   --fork-job-execution FORK_JOB_EXECUTION
                         Fork job execution to another process
-  --job-class JOB_CLASS
-                        Jobs class to use
   --sentry-dsn SENTRY_DSN
                         Sentry DSN to use
   --sentry-debug        Enable Sentry debug mode
   --sentry-ca-certs SENTRY_CA_CERTS
                         Path to CA certs file
+  --burst               Run worker in burst mode
+  --max-jobs MAX_JOBS   Maximum number of jobs to execute before terminating worker
+  --max-idle-time MAX_IDLE_TIME
+                        Maximum number of seconds to wait for new job before terminating worker
+  --with-scheduler      Run worker with scheduler, default to True
   --version             Show program's version number and exit.
   -v {0,1,2,3}, --verbosity {0,1,2,3}
                         Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output
-  --settings SETTINGS   The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the
-                        DJANGO_SETTINGS_MODULE environment variable will be used.
+  --settings SETTINGS   The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable will be used.
   --pythonpath PYTHONPATH
                         A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".
   --traceback           Raise on CommandError exceptions.
@@ -49,7 +48,7 @@ options:
 
 
 
-## export
+## `export` - Export scheduled tasks
 
 Export all scheduled tasks from django db to json/yaml format.
 
@@ -62,7 +61,7 @@ Result should be (for json):
 ```json
 [
   {
-    "model": "ScheduledJob",
+    "model": "CronTaskType",
     "name": "Scheduled Task 1",
     "callable": "scheduler.tests.test_job",
     "callable_args": [
@@ -83,7 +82,7 @@ Result should be (for json):
 ]
 ```
 
-## import
+## `import` - Import scheduled tasks
 
 A json/yaml that was exported using the `export` command
 can be imported to django.
@@ -96,18 +95,62 @@ can be imported to django.
 python manage.py import -f {yaml,json} --filename {SOURCE-FILE}
 ```
 
-## run_job
+## `run_job` - Run a job immediately
 
 Run a method in a queue immediately.
 
 ```shell
 python manage.py run_job {callable} {callable args ...}
 ```
 
-## delete failed jobs
+## `delete_failed_jobs` - delete failed jobs
 
 Run this to empty failed jobs registry from a queue.
 
 ```shell
 python manage.py delete_failed_jobs 
 ```
+
+## `scheduler_stats` - Show scheduler stats
+
+Prints scheduler stats as a table, json, or yaml, example:
+
+```shell
+$ python manage.py scheduler_stats
+
+Django-Scheduler CLI Dashboard
+
+--------------------------------------------------------------------------------
+| Name             |    Queued |    Active |  Finished |  Canceled |   Workers |
+--------------------------------------------------------------------------------
+| default          |         0 |         0 |         0 |         0 |         0 |
+| low              |         0 |         0 |         0 |         0 |         0 |
+| high             |         0 |         0 |         0 |         0 |         0 |
+| medium           |         0 |         0 |         0 |         0 |         0 |
+| another          |         0 |         0 |         0 |         0 |         0 |
+--------------------------------------------------------------------------------
+```
+
+```shell
+usage: manage.py scheduler_stats [-h] [-j] [-y] [-i INTERVAL] [--version] [-v {0,1,2,3}] [--settings SETTINGS] [--pythonpath PYTHONPATH] [--traceback] [--no-color] [--force-color] [--skip-checks]
+
+Print statistics
+
+options:
+  -h, --help            show this help message and exit
+  -j, --json            Output statistics as JSON
+  -y, --yaml            Output statistics as YAML
+  -i INTERVAL, --interval INTERVAL
+                        Poll statistics every N seconds
+  --version             Show program's version number and exit.
+  -v {0,1,2,3}, --verbosity {0,1,2,3}
+                        Verbosity level; 0=minimal output, 1=normal output, 2=verbose output, 3=very verbose output
+  --settings SETTINGS   The Python path to a settings module, e.g. "myproject.settings.main". If this isn't provided, the DJANGO_SETTINGS_MODULE environment variable will be used.
+  --pythonpath PYTHONPATH
+                        A directory to add to the Python path, e.g. "/home/djangoprojects/myproject".
+  --traceback           Raise on CommandError exceptions.
+  --no-color            Don't colorize the command output.
+  --force-color         Force colorization of the command output.
+  --skip-checks         Skip system checks.
+
+```

docs/configuration.md

@@ -5,35 +5,41 @@
 All default settings for scheduler can be in one dictionary in `settings.py`:
 
 ```python
-SCHEDULER_CONFIG = {
-    'EXECUTIONS_IN_PAGE': 20,
-    'DEFAULT_RESULT_TTL': 500,
-    'DEFAULT_TIMEOUT': 300,  # 5 minutes
-    'SCHEDULER_INTERVAL': 10,  # 10 seconds
-    'BROKER': 'redis',
-}
-SCHEDULER_QUEUES = {
-    'default': {
-        'HOST': 'localhost',
-        'PORT': 6379,
-        'DB': 0,
-        'USERNAME': 'some-user',
-        'PASSWORD': 'some-password',
-        'DEFAULT_TIMEOUT': 360,
-        'CLIENT_KWARGS': {  # Eventual additional Redis connection arguments
-            'ssl_cert_reqs': None,
+from typing import Dict
+
+from scheduler.settings_types import SchedulerConfig, Broker, QueueConfiguration, UnixSignalDeathPenalty
+
+
+SCHEDULER_CONFIG = SchedulerConfig(
+    EXECUTIONS_IN_PAGE=20,
+    SCHEDULER_INTERVAL=10,
+    BROKER=Broker.REDIS,
+    CALLBACK_TIMEOUT=60,  # Callback timeout in seconds (success/failure/stopped)
+    # Default values, can be overriden per task/job
+    DEFAULT_SUCCESS_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep successful job results
+    DEFAULT_FAILURE_TTL=365 * 24 * 60 * 60,  # Time To Live (TTL) in seconds to keep job failure information
+    DEFAULT_JOB_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep job information
+    DEFAULT_JOB_TIMEOUT=5 * 60,  # timeout (seconds) for a job
+    # General configuration values
+    DEFAULT_WORKER_TTL=10 * 60,  # Time To Live (TTL) in seconds to keep worker information after last heartbeat
+    DEFAULT_MAINTENANCE_TASK_INTERVAL=10 * 60,  # The interval to run maintenance tasks in seconds. 10 minutes.
+    DEFAULT_JOB_MONITORING_INTERVAL=30,  # The interval to monitor jobs in seconds.
+    SCHEDULER_FALLBACK_PERIOD_SECS=120,  # Period (secs) to wait before requiring to reacquire locks
+    DEATH_PENALTY_CLASS=UnixSignalDeathPenalty,
+)
+SCHEDULER_QUEUES: Dict[str, QueueConfiguration] = {
+    'default': QueueConfiguration(
+        HOST='localhost',
+        PORT=6379,
+        USERNAME='some-user',
+        PASSWORD='some-password',
+        CONNECTION_KWARGS={  # Eventual additional Broker connection arguments
+            'ssl_cert_reqs': 'required',
+            'ssl': True,
         },
-        'TOKEN_VALIDATION_METHOD': None,  # Method to validate auth-header
-    },
-    'high': {
-        'URL': os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0'),  # If you're on Heroku
-        'DEFAULT_TIMEOUT': 500,
-    },
-    'low': {
-        'HOST': 'localhost',
-        'PORT': 6379,
-        'DB': 0,
-    }
+    ),
+    'high': QueueConfiguration(URL=os.getenv('REDISTOGO_URL', 'redis://localhost:6379/0')),
+    'low': QueueConfiguration(HOST='localhost', PORT=6379, DB=0, ASYNC=False),
 }
 ```
 

docs/drt-model.md

@@ -1,6 +1,6 @@
 # Worker related flows
 
-Running `python manage.py startworker --name 'X' --queues high default low`
+Running `python manage.py scheduler_worker --name 'X' --queues high default low`
 
 ## Register new worker for queues
 ```mermaid
@@ -48,8 +48,8 @@ sequenceDiagram
         note over worker,job: Find next job
        
         loop over queueKeys until job to run is found or all queues are empty
-            worker ->>+ queue: get next job id and remove it or None (zrange+zpop)
-            queue -->>- worker: job id / nothing
+            worker ->>+ queue: get next job name and remove it or None (zrange+zpop)
+            queue -->>- worker: job name / nothing
         end
         
         note over worker,job: Execute job or sleep