Skip to content

Commit ac9fa94

Browse files
humitoshumitos
authored
Celery: use django-celery-beat scheduler (#10647)
* Celery: use `django-celery-beat` scheduler This package allows us to store the periodic Celery tasks in the database. Hopefully, it will help us to solve the issue we are having that periodic tasks are triggered when building the AMI and also on deploys. Closes readthedocs/readthedocs-ops#1384 * Celery: use `wed` instead of `wednesday` `django-celery-beat` does not support the long version of the `day_of_week=`. However, it works fine using the three-char word of it. It fails when opening the Django Admin on Crontab. See celery/django-celery-beat#647
1 parent f1b7fd6 commit ac9fa94

File tree

6 files changed

+97
-12
lines changed

6 files changed

+97
-12
lines changed

‎readthedocs/settings/base.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,7 @@ def INSTALLED_APPS(self): # noqa
233233
'polymorphic',
234234
'simple_history',
235235
'djstripe',
236+
'django_celery_beat',
236237

237238
# our apps
238239
'readthedocs.projects',
@@ -477,6 +478,7 @@ def TEMPLATES(self):
477478
CELERY_CREATE_MISSING_QUEUES = True
478479

479480
CELERY_DEFAULT_QUEUE = 'celery'
481+
CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler'
480482
CELERYBEAT_SCHEDULE = {
481483
'quarter-finish-inactive-builds': {
482484
'task': 'readthedocs.projects.tasks.utils.finish_inactive_builds',
@@ -505,7 +507,7 @@ def TEMPLATES(self):
505507
},
506508
'weekly-delete-old-personal-audit-logs': {
507509
'task': 'readthedocs.audit.tasks.delete_old_personal_audit_logs',
508-
'schedule': crontab(day_of_week='wednesday', minute=0, hour=7),
510+
'schedule': crontab(day_of_week="wed", minute=0, hour=7),
509511
'options': {'queue': 'web'},
510512
},
511513
'every-day-resync-sso-organization-users': {
@@ -547,12 +549,12 @@ def TEMPLATES(self):
547549
},
548550
'weekly-config-file-notification': {
549551
'task': 'readthedocs.projects.tasks.utils.deprecated_config_file_used_notification',
550-
'schedule': crontab(day_of_week='wednesday', hour=11, minute=15),
552+
'schedule': crontab(day_of_week="wed", hour=11, minute=15),
551553
'options': {'queue': 'web'},
552554
},
553555
'weekly-build-image-notification': {
554556
'task': 'readthedocs.projects.tasks.utils.deprecated_build_image_notification',
555-
'schedule': crontab(day_of_week='wednesday', hour=9, minute=15),
557+
'schedule': crontab(day_of_week="wed", hour=9, minute=15),
556558
'options': {'queue': 'web'},
557559
},
558560
}

‎requirements/deploy.txt

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ botocore==1.31.17
3434
# boto3
3535
# s3transfer
3636
celery==5.2.7
37-
# via -r requirements/pip.txt
37+
# via
38+
# -r requirements/pip.txt
39+
# django-celery-beat
3840
certifi==2023.7.22
3941
# via
4042
# -r requirements/pip.txt
@@ -68,6 +70,10 @@ click-repl==0.3.0
6870
# via
6971
# -r requirements/pip.txt
7072
# celery
73+
cron-descriptor==1.4.0
74+
# via
75+
# -r requirements/pip.txt
76+
# django-celery-beat
7177
cryptography==41.0.3
7278
# via
7379
# -r requirements/pip.txt
@@ -97,6 +103,7 @@ django==4.2.4
97103
# django-allauth
98104
# django-annoying
99105
# django-cacheops
106+
# django-celery-beat
100107
# django-cors-headers
101108
# django-csp
102109
# django-debug-toolbar
@@ -107,6 +114,7 @@ django==4.2.4
107114
# django-storages
108115
# django-structlog
109116
# django-taggit
117+
# django-timezone-field
110118
# djangorestframework
111119
# jsonfield
112120
django-allauth==0.51.0
@@ -117,13 +125,15 @@ django-autoslug==1.9.9
117125
# via -r requirements/pip.txt
118126
django-cacheops==7.0.1
119127
# via -r requirements/pip.txt
128+
django-celery-beat==2.5.0
129+
# via -r requirements/pip.txt
120130
django-cors-headers==4.2.0
121131
# via -r requirements/pip.txt
122132
django-crispy-forms==1.14.0
123133
# via -r requirements/pip.txt
124134
django-csp==3.7
125135
# via -r requirements/pip.txt
126-
django-debug-toolbar==4.1.0
136+
django-debug-toolbar==4.2.0
127137
# via -r requirements/pip.txt
128138
django-elasticsearch-dsl==7.3
129139
# via -r requirements/pip.txt
@@ -151,6 +161,10 @@ django-structlog==2.2.0
151161
# via -r requirements/pip.txt
152162
django-taggit==4.0.0
153163
# via -r requirements/pip.txt
164+
django-timezone-field==5.1
165+
# via
166+
# -r requirements/pip.txt
167+
# django-celery-beat
154168
django-vanilla-views==3.0.0
155169
# via -r requirements/pip.txt
156170
djangorestframework==3.14.0
@@ -269,11 +283,16 @@ pyjwt[crypto]==2.8.0
269283
# django-allauth
270284
pyquery==2.0.0
271285
# via -r requirements/pip.txt
286+
python-crontab==3.0.0
287+
# via
288+
# -r requirements/pip.txt
289+
# django-celery-beat
272290
python-dateutil==2.8.2
273291
# via
274292
# -r requirements/pip.txt
275293
# botocore
276294
# elasticsearch-dsl
295+
# python-crontab
277296
python3-openid==3.2.0
278297
# via
279298
# -r requirements/pip.txt
@@ -282,6 +301,7 @@ pytz==2023.3
282301
# via
283302
# -r requirements/pip.txt
284303
# celery
304+
# django-timezone-field
285305
# djangorestframework
286306
pyyaml==6.0.1
287307
# via -r requirements/pip.txt
@@ -356,7 +376,9 @@ typing-extensions==4.7.1
356376
# -r requirements/pip.txt
357377
# asgiref
358378
tzdata==2023.3
359-
# via -r requirements/pip.txt
379+
# via
380+
# -r requirements/pip.txt
381+
# django-celery-beat
360382
ua-parser==0.18.0
361383
# via
362384
# -r requirements/pip.txt

‎requirements/docker.txt

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,9 @@ botocore==1.31.17
3636
cachetools==5.3.1
3737
# via tox
3838
celery==5.2.7
39-
# via -r requirements/pip.txt
39+
# via
40+
# -r requirements/pip.txt
41+
# django-celery-beat
4042
certifi==2023.7.22
4143
# via
4244
# -r requirements/pip.txt
@@ -73,6 +75,10 @@ click-repl==0.3.0
7375
# celery
7476
colorama==0.4.6
7577
# via tox
78+
cron-descriptor==1.4.0
79+
# via
80+
# -r requirements/pip.txt
81+
# django-celery-beat
7682
cryptography==41.0.3
7783
# via
7884
# -r requirements/pip.txt
@@ -106,6 +112,7 @@ django==4.2.4
106112
# django-allauth
107113
# django-annoying
108114
# django-cacheops
115+
# django-celery-beat
109116
# django-cors-headers
110117
# django-csp
111118
# django-debug-toolbar
@@ -116,6 +123,7 @@ django==4.2.4
116123
# django-storages
117124
# django-structlog
118125
# django-taggit
126+
# django-timezone-field
119127
# djangorestframework
120128
# jsonfield
121129
django-allauth==0.51.0
@@ -126,6 +134,8 @@ django-autoslug==1.9.9
126134
# via -r requirements/pip.txt
127135
django-cacheops==7.0.1
128136
# via -r requirements/pip.txt
137+
django-celery-beat==2.5.0
138+
# via -r requirements/pip.txt
129139
django-cors-headers==4.2.0
130140
# via -r requirements/pip.txt
131141
django-crispy-forms==1.14.0
@@ -160,6 +170,10 @@ django-structlog==2.2.0
160170
# via -r requirements/pip.txt
161171
django-taggit==4.0.0
162172
# via -r requirements/pip.txt
173+
django-timezone-field==5.1
174+
# via
175+
# -r requirements/pip.txt
176+
# django-celery-beat
163177
django-vanilla-views==3.0.0
164178
# via -r requirements/pip.txt
165179
djangorestframework==3.14.0
@@ -298,11 +312,16 @@ pyquery==2.0.0
298312
# via -r requirements/pip.txt
299313
pyrepl==0.9.0
300314
# via fancycompleter
315+
python-crontab==3.0.0
316+
# via
317+
# -r requirements/pip.txt
318+
# django-celery-beat
301319
python-dateutil==2.8.2
302320
# via
303321
# -r requirements/pip.txt
304322
# botocore
305323
# elasticsearch-dsl
324+
# python-crontab
306325
python3-openid==3.2.0
307326
# via
308327
# -r requirements/pip.txt
@@ -311,6 +330,7 @@ pytz==2023.3
311330
# via
312331
# -r requirements/pip.txt
313332
# celery
333+
# django-timezone-field
314334
# djangorestframework
315335
pyyaml==6.0.1
316336
# via -r requirements/pip.txt
@@ -387,7 +407,9 @@ typing-extensions==4.7.1
387407
# -r requirements/pip.txt
388408
# asgiref
389409
tzdata==2023.3
390-
# via -r requirements/pip.txt
410+
# via
411+
# -r requirements/pip.txt
412+
# django-celery-beat
391413
ua-parser==0.18.0
392414
# via
393415
# -r requirements/pip.txt

‎requirements/pip.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ redis
6767
# TypeError: MyException.__init__() missing 1 required positional argument: 'b'
6868
# >>>
6969
celery==5.2.7
70+
django-celery-beat
7071

7172
# Docker images need a dependency explicity marked for tzdata in order to have
7273
# necessary timezone data.

‎requirements/pip.txt

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ botocore==1.31.17
1919
# boto3
2020
# s3transfer
2121
celery==5.2.7
22-
# via -r requirements/pip.in
22+
# via
23+
# -r requirements/pip.in
24+
# django-celery-beat
2325
certifi==2023.7.22
2426
# via
2527
# elasticsearch
@@ -40,6 +42,8 @@ click-plugins==1.1.1
4042
# via celery
4143
click-repl==0.3.0
4244
# via celery
45+
cron-descriptor==1.4.0
46+
# via django-celery-beat
4347
cryptography==41.0.3
4448
# via pyjwt
4549
cssselect==1.2.0
@@ -59,6 +63,7 @@ django==4.2.4
5963
# django-allauth
6064
# django-annoying
6165
# django-cacheops
66+
# django-celery-beat
6267
# django-cors-headers
6368
# django-csp
6469
# django-debug-toolbar
@@ -69,6 +74,7 @@ django==4.2.4
6974
# django-storages
7075
# django-structlog
7176
# django-taggit
77+
# django-timezone-field
7278
# djangorestframework
7379
# jsonfield
7480
django-allauth==0.51.0
@@ -79,6 +85,8 @@ django-autoslug==1.9.9
7985
# via -r requirements/pip.in
8086
django-cacheops==7.0.1
8187
# via -r requirements/pip.in
88+
django-celery-beat==2.5.0
89+
# via -r requirements/pip.in
8290
django-cors-headers==4.2.0
8391
# via -r requirements/pip.in
8492
django-crispy-forms==1.14.0
@@ -111,6 +119,8 @@ django-structlog==2.2.0
111119
# via -r requirements/pip.in
112120
django-taggit==4.0.0
113121
# via -r requirements/pip.in
122+
django-timezone-field==5.1
123+
# via django-celery-beat
114124
django-vanilla-views==3.0.0
115125
# via -r requirements/pip.in
116126
djangorestframework==3.14.0
@@ -183,17 +193,21 @@ pyjwt[crypto]==2.8.0
183193
# via django-allauth
184194
pyquery==2.0.0
185195
# via -r requirements/pip.in
196+
python-crontab==3.0.0
197+
# via django-celery-beat
186198
python-dateutil==2.8.2
187199
# via
188200
# -r requirements/pip.in
189201
# botocore
190202
# elasticsearch-dsl
203+
# python-crontab
191204
python3-openid==3.2.0
192205
# via django-allauth
193206
pytz==2023.3
194207
# via
195208
# -r requirements/pip.in
196209
# celery
210+
# django-timezone-field
197211
# djangorestframework
198212
pyyaml==6.0.1
199213
# via -r requirements/pip.in
@@ -248,7 +262,9 @@ tomli==2.0.1
248262
typing-extensions==4.7.1
249263
# via asgiref
250264
tzdata==2023.3
251-
# via -r requirements/pip.in
265+
# via
266+
# -r requirements/pip.in
267+
# django-celery-beat
252268
ua-parser==0.18.0
253269
# via user-agents
254270
unicode-slugify @ git+https://github.com/mozilla/unicode-slugify@b696c37

0 commit comments

Comments
 (0)