diff --git a/README.md b/README.md index a1abbd7406..8ee2dc41d2 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ addon | version | maintainers | summary [queue_job](queue_job/) | 18.0.1.2.1 | [![guewen](https://github.com/guewen.png?size=30px)](https://github.com/guewen) | Job Queue [queue_job_batch](queue_job_batch/) | 18.0.1.0.0 | | Job Queue Batch [queue_job_cron](queue_job_cron/) | 18.0.1.1.0 | | Scheduled Actions as Queue Jobs +[queue_job_cron_jobrunner](queue_job_cron_jobrunner/) | 18.0.1.0.0 | [![ivantodorovich](https://github.com/ivantodorovich.png?size=30px)](https://github.com/ivantodorovich) | Run jobs without a dedicated JobRunner [queue_job_subscribe](queue_job_subscribe/) | 18.0.1.0.0 | | Control which users are subscribed to queue job notifications [test_queue_job](test_queue_job/) | 18.0.1.0.0 | | Queue Job Tests [test_queue_job_batch](test_queue_job_batch/) | 18.0.1.0.0 | | Test Job Queue Batch diff --git a/queue_job_batch/i18n/it.po b/queue_job_batch/i18n/it.po index c794b96254..58fecf0436 100644 --- a/queue_job_batch/i18n/it.po +++ b/queue_job_batch/i18n/it.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: Odoo Server 16.0\n" "Report-Msgid-Bugs-To: \n" -"PO-Revision-Date: 2024-01-11 13:35+0000\n" +"PO-Revision-Date: 2025-02-21 18:06+0000\n" "Last-Translator: mymage \n" "Language-Team: none\n" "Language: it\n" @@ -14,7 +14,7 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: \n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 4.17\n" +"X-Generator: Weblate 5.6.2\n" #. module: queue_job_batch #: model:ir.model.fields,field_description:queue_job_batch.field_queue_job_batch__message_needaction @@ -77,7 +77,7 @@ msgstr "In coda" #. odoo-javascript #: code:addons/queue_job_batch/static/src/components/QueueJobBatchMenu.esm.js:0 msgid "Failed" -msgstr "" +msgstr "Fallito" #. module: queue_job_batch #: model:ir.model.fields,field_description:queue_job_batch.field_queue_job_batch__failed_job_count @@ -234,7 +234,7 @@ msgstr "Scadenza mia attività" #. module: queue_job_batch #: model_terms:ir.ui.view,arch_db:queue_job_batch.view_queue_job_batch_search msgid "My batches" -msgstr "" +msgstr "I miei gruppi" #. module: queue_job_batch #: model:ir.model.fields,field_description:queue_job_batch.field_queue_job_batch__name @@ -286,13 +286,13 @@ msgstr "Numero di messaggi con errore di consegna" #: model:ir.model.fields.selection,name:queue_job_batch.selection__queue_job_batch__state__pending #: model_terms:ir.ui.view,arch_db:queue_job_batch.view_queue_job_batch_search msgid "Pending" -msgstr "" +msgstr "In sospeso" #. module: queue_job_batch #. odoo-javascript #: code:addons/queue_job_batch/static/src/components/QueueJobBatchMenu.esm.js:0 msgid "Planned" -msgstr "" +msgstr "Pianificato" #. module: queue_job_batch #: model:ir.model,name:queue_job_batch.model_queue_job @@ -307,7 +307,7 @@ msgstr "Utente responsabile" #. module: queue_job_batch #: model:ir.model.fields,field_description:queue_job_batch.field_queue_job_batch__message_has_sms_error msgid "SMS Delivery error" -msgstr "" +msgstr "Errore consegna SMS" #. module: queue_job_batch #: model:ir.model.fields,field_description:queue_job_batch.field_queue_job_batch__state @@ -332,7 +332,7 @@ msgstr "" #. odoo-javascript #: code:addons/queue_job_batch/static/src/components/QueueJobBatchMenu.esm.js:0 msgid "Total" -msgstr "" +msgstr "Totale" #. module: queue_job_batch #: model:ir.model.fields,help:queue_job_batch.field_queue_job_batch__activity_exception_decoration @@ -342,7 +342,7 @@ msgstr "Tipo di attività eccezione sul record." #. module: queue_job_batch #: model_terms:ir.ui.view,arch_db:queue_job_batch.view_queue_job_batch_search msgid "Unread" -msgstr "" +msgstr "Non letto" #. module: queue_job_batch #: model:ir.model,name:queue_job_batch.model_res_users @@ -355,7 +355,7 @@ msgstr "Utente" #. odoo-javascript #: code:addons/queue_job_batch/static/src/components/QueueJobBatchMenu.xml:0 msgid "View all job batches" -msgstr "" +msgstr "Vedi tutti i gruppi lavoro" #~ msgid "Draft" #~ msgstr "Bozza" diff --git a/queue_job_cron_jobrunner/README.rst b/queue_job_cron_jobrunner/README.rst new file mode 100644 index 0000000000..64555223a3 --- /dev/null +++ b/queue_job_cron_jobrunner/README.rst @@ -0,0 +1,143 @@ +======================== +Queue Job Cron Jobrunner +======================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:8805630c706cd2c715ed4e9086e130d580daff40f87890af0f3a2c928c01db16 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Alpha-red.png + :target: https://odoo-community.org/page/development-status + :alt: Alpha +.. |badge2| image:: https://img.shields.io/badge/licence-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fqueue-lightgray.png?logo=github + :target: https://github.com/OCA/queue/tree/18.0/queue_job_cron_jobrunner + :alt: OCA/queue +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/queue-18-0/queue-18-0-queue_job_cron_jobrunner + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/queue&target_branch=18.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module implements a simple ``queue.job`` runner using ``ir.cron`` +triggers. + +It's meant to be used on environments where the regular job runner can't +be run, like on Odoo.sh. + +Unlike the regular job runner, where jobs are dispatched to the +HttpWorkers, jobs are processed on the CronWorker threads by the job +runner crons. This is a design decision because: + +- Odoo.sh puts HttpWorkers to sleep when there's no network activity +- HttpWorkers are meant for traffic. Users shouldn't pay the price of + background tasks. + +For now, it only implements the most basic features of the ``queue_job`` +runner, notably no channel capacity nor priorities. Please check the +ROADMAP for further details. + +.. IMPORTANT:: + This is an alpha version, the data model and design can change at any time without warning. + Only for development or testing purpose, do not use in production. + `More details on development status `_ + +**Table of contents** + +.. contents:: + :local: + +Configuration +============= + +Warning + +Don't use this module if you're already running the regular +``queue_job`` runner. + +For the easiest case, no configuration is required besides installing +the module. + +To avoid CronWorker CPU timeout from abruptly stopping the job +processing cron, it's recommended to launch Odoo with +``--limit-time-real-cron=0``, to disable the CronWorker timeout +altogether. + +Note + +In Odoo.sh, this is done by default. + +Parallel execution of jobs can be achieved by leveraging multiple +``ir.cron`` records: + +- Make sure you have enough CronWorkers available (Odoo CLI + ``--max-cron-threads``) +- Duplicate the ``queue_job_cron`` cron record as many times as needed, + until you have as much records as cron workers. + +Known issues / Roadmap +====================== + +- Support channel capacity and priority. (See ``_acquire_one_job``) +- Gracefully handle CronWorker CPU timeouts. (See ``_job_runner``) +- Commit transaction after job state updated to started. (See + ``_process``) + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Camptocamp SA + +Contributors +------------ + +- `Camptocamp `__ + + - Iván Todorovich + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +.. |maintainer-ivantodorovich| image:: https://github.com/ivantodorovich.png?size=40px + :target: https://github.com/ivantodorovich + :alt: ivantodorovich + +Current `maintainer `__: + +|maintainer-ivantodorovich| + +This module is part of the `OCA/queue `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/queue_job_cron_jobrunner/__init__.py b/queue_job_cron_jobrunner/__init__.py new file mode 100644 index 0000000000..0650744f6b --- /dev/null +++ b/queue_job_cron_jobrunner/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/queue_job_cron_jobrunner/__manifest__.py b/queue_job_cron_jobrunner/__manifest__.py new file mode 100644 index 0000000000..ff54857efd --- /dev/null +++ b/queue_job_cron_jobrunner/__manifest__.py @@ -0,0 +1,17 @@ +{ + "name": "Queue Job Cron Jobrunner", + "summary": "Run jobs without a dedicated JobRunner", + "version": "18.0.1.0.0", + "development_status": "Alpha", + "author": "Camptocamp SA, Odoo Community Association (OCA)", + "maintainers": ["ivantodorovich"], + "website": "https://github.com/OCA/queue", + "license": "AGPL-3", + "category": "Others", + "depends": ["queue_job"], + "data": [ + "data/ir_cron.xml", + "views/ir_cron.xml", + ], + "installable": True, +} diff --git a/queue_job_cron_jobrunner/data/ir_cron.xml b/queue_job_cron_jobrunner/data/ir_cron.xml new file mode 100644 index 0000000000..7653a70ab2 --- /dev/null +++ b/queue_job_cron_jobrunner/data/ir_cron.xml @@ -0,0 +1,13 @@ + + + + Queue Job Runner + + code + model._job_runner() + + + 1 + days + + diff --git a/queue_job_cron_jobrunner/i18n/it.po b/queue_job_cron_jobrunner/i18n/it.po new file mode 100644 index 0000000000..84a21113d5 --- /dev/null +++ b/queue_job_cron_jobrunner/i18n/it.po @@ -0,0 +1,45 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * queue_job_cron_jobrunner +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 17.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2024-02-12 11:39+0000\n" +"Last-Translator: mymage \n" +"Language-Team: none\n" +"Language: it\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.17\n" + +#. module: queue_job_cron_jobrunner +#: model:ir.model.fields,help:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "If checked, the cron is considered to be a queue.job runner." +msgstr "Se selezionata, il cron è considerato un esecutore del queue.job." + +#. module: queue_job_cron_jobrunner +#. odoo-python +#: code:addons/queue_job_cron_jobrunner/models/queue_job.py:0 +#, python-format +msgid "Job interrupted and set to Done: nothing to do." +msgstr "Lavoro interrotto e impostato a completato: nulla da fare." + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_queue_job +msgid "Queue Job" +msgstr "Lavoro in coda" + +#. module: queue_job_cron_jobrunner +#: model:ir.actions.server,name:queue_job_cron_jobrunner.queue_job_cron_ir_actions_server +#: model:ir.model.fields,field_description:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "Queue Job Runner" +msgstr "Esecutore lavoro in coda" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_ir_cron +msgid "Scheduled Actions" +msgstr "Azioni pianificate" diff --git a/queue_job_cron_jobrunner/i18n/queue_job_cron_jobrunner.pot b/queue_job_cron_jobrunner/i18n/queue_job_cron_jobrunner.pot new file mode 100644 index 0000000000..f5d04d6ada --- /dev/null +++ b/queue_job_cron_jobrunner/i18n/queue_job_cron_jobrunner.pot @@ -0,0 +1,35 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * queue_job_cron_jobrunner +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 18.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: queue_job_cron_jobrunner +#: model:ir.model.fields,help:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "If checked, the cron is considered to be a queue.job runner." +msgstr "" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_queue_job +msgid "Queue Job" +msgstr "" + +#. module: queue_job_cron_jobrunner +#: model:ir.actions.server,name:queue_job_cron_jobrunner.queue_job_cron_ir_actions_server +#: model:ir.model.fields,field_description:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "Queue Job Runner" +msgstr "" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_ir_cron +msgid "Scheduled Actions" +msgstr "" diff --git a/queue_job_cron_jobrunner/i18n/ro.po b/queue_job_cron_jobrunner/i18n/ro.po new file mode 100644 index 0000000000..b095883dc0 --- /dev/null +++ b/queue_job_cron_jobrunner/i18n/ro.po @@ -0,0 +1,46 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * queue_job_cron_jobrunner +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 16.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2022-11-23 09:45+0000\n" +"Last-Translator: Dorin Hongu \n" +"Language-Team: none\n" +"Language: ro\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=3; plural=n==1 ? 0 : (n==0 || (n%100 > 0 && n%100 < " +"20)) ? 1 : 2;\n" +"X-Generator: Weblate 4.14.1\n" + +#. module: queue_job_cron_jobrunner +#: model:ir.model.fields,help:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "If checked, the cron is considered to be a queue.job runner." +msgstr "Dacă este bifat, cron-ul este considerat a fi un runner queue.job." + +#. module: queue_job_cron_jobrunner +#. odoo-python +#: code:addons/queue_job_cron_jobrunner/models/queue_job.py:0 +#, python-format +msgid "Job interrupted and set to Done: nothing to do." +msgstr "Job întrerupt și setat la Terminat: nimic de făcut." + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_queue_job +msgid "Queue Job" +msgstr "Coadă sarcini" + +#. module: queue_job_cron_jobrunner +#: model:ir.actions.server,name:queue_job_cron_jobrunner.queue_job_cron_ir_actions_server +#: model:ir.model.fields,field_description:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "Queue Job Runner" +msgstr "" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_ir_cron +msgid "Scheduled Actions" +msgstr "" diff --git a/queue_job_cron_jobrunner/i18n/zh_CN.po b/queue_job_cron_jobrunner/i18n/zh_CN.po new file mode 100644 index 0000000000..9052da779d --- /dev/null +++ b/queue_job_cron_jobrunner/i18n/zh_CN.po @@ -0,0 +1,45 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * queue_job_cron_jobrunner +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 17.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2024-07-02 09:47+0000\n" +"Last-Translator: xtanuiha \n" +"Language-Team: none\n" +"Language: zh_CN\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=1; plural=0;\n" +"X-Generator: Weblate 4.17\n" + +#. module: queue_job_cron_jobrunner +#: model:ir.model.fields,help:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "If checked, the cron is considered to be a queue.job runner." +msgstr "如果选中，cron任务将被视为队列.job的执行器。" + +#. module: queue_job_cron_jobrunner +#. odoo-python +#: code:addons/queue_job_cron_jobrunner/models/queue_job.py:0 +#, python-format +msgid "Job interrupted and set to Done: nothing to do." +msgstr "任务中断并设置为已完成：无需执行任何操作。" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_queue_job +msgid "Queue Job" +msgstr "队列任务" + +#. module: queue_job_cron_jobrunner +#: model:ir.actions.server,name:queue_job_cron_jobrunner.queue_job_cron_ir_actions_server +#: model:ir.model.fields,field_description:queue_job_cron_jobrunner.field_ir_cron__queue_job_runner +msgid "Queue Job Runner" +msgstr "队列任务执行器" + +#. module: queue_job_cron_jobrunner +#: model:ir.model,name:queue_job_cron_jobrunner.model_ir_cron +msgid "Scheduled Actions" +msgstr "计划的动作" diff --git a/queue_job_cron_jobrunner/models/__init__.py b/queue_job_cron_jobrunner/models/__init__.py new file mode 100644 index 0000000000..4ba9dd1a16 --- /dev/null +++ b/queue_job_cron_jobrunner/models/__init__.py @@ -0,0 +1,2 @@ +from . import ir_cron +from . import queue_job diff --git a/queue_job_cron_jobrunner/models/ir_cron.py b/queue_job_cron_jobrunner/models/ir_cron.py new file mode 100644 index 0000000000..61235dc45a --- /dev/null +++ b/queue_job_cron_jobrunner/models/ir_cron.py @@ -0,0 +1,13 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# @author Iván Todorovich +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from odoo import fields, models + + +class IrCron(models.Model): + _inherit = "ir.cron" + + queue_job_runner = fields.Boolean( + help="If checked, the cron is considered to be a queue.job runner.", + ) diff --git a/queue_job_cron_jobrunner/models/queue_job.py b/queue_job_cron_jobrunner/models/queue_job.py new file mode 100644 index 0000000000..28ed243e7c --- /dev/null +++ b/queue_job_cron_jobrunner/models/queue_job.py @@ -0,0 +1,156 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# @author Iván Todorovich +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +import logging +import traceback +from io import StringIO + +from psycopg2 import OperationalError + +from odoo import api, models, tools +from odoo.service.model import PG_CONCURRENCY_ERRORS_TO_RETRY + +from odoo.addons.queue_job.controllers.main import PG_RETRY +from odoo.addons.queue_job.exception import FailedJobError, RetryableJobError +from odoo.addons.queue_job.job import Job + +_logger = logging.getLogger(__name__) + + +class QueueJob(models.Model): + _inherit = "queue.job" + + @api.model + def _acquire_one_job(self): + """Acquire the next job to be run. + + :returns: queue.job record (locked for update) + """ + # TODO: This method should respect channel priority and capacity, + # rather than just fetching them by creation date. + self.env.flush_all() + self.env.cr.execute( + """ + SELECT id + FROM queue_job + WHERE state = 'pending' + AND (eta IS NULL OR eta <= (now() AT TIME ZONE 'UTC')) + ORDER BY date_created DESC + LIMIT 1 FOR NO KEY UPDATE SKIP LOCKED + """ + ) + row = self.env.cr.fetchone() + return self.browse(row and row[0]) + + def _process(self, commit=False): + """Process the job""" + self.ensure_one() + job = Job._load_from_db_record(self) + # Set it as started + job.set_started() + job.store() + _logger.debug("%s started", job.uuid) + # TODO: Commit the state change so that the state can be read from the UI + # while the job is processing. However, doing this will release the + # lock on the db, so we need to find another way. + # if commit: + # self.flush() + # self.env.cr.commit() + + # Actual processing + try: + try: + with self.env.cr.savepoint(): + job.perform() + job.set_done() + job.store() + except OperationalError as err: + # Automatically retry the typical transaction serialization errors + if err.pgcode not in PG_CONCURRENCY_ERRORS_TO_RETRY: + raise + message = tools.ustr(err.pgerror, errors="replace") + job.postpone(result=message, seconds=PG_RETRY) + job.set_pending(reset_retry=False) + job.store() + _logger.debug("%s OperationalError, postponed", job) + + except RetryableJobError as err: + # delay the job later, requeue + job.postpone(result=str(err), seconds=5) + job.set_pending(reset_retry=False) + job.store() + _logger.debug("%s postponed", job) + + except (FailedJobError, Exception): + with StringIO() as buff: + traceback.print_exc(file=buff) + _logger.error(buff.getvalue()) + job.set_failed(exc_info=buff.getvalue()) + job.store() + + if commit: # pragma: no cover + self.env.flush_all() + self.env.cr.commit() # pylint: disable=invalid-commit + + _logger.debug("%s enqueue depends started", job) + job.enqueue_waiting() + _logger.debug("%s enqueue depends done", job) + + @api.model + def _job_runner(self, commit=True): + """Short-lived job runner, triggered by async crons""" + job = self._acquire_one_job() + while job: + job._process(commit=commit) + job = self._acquire_one_job() + # TODO: If limit_time_real_cron is reached before all the jobs are done, + # the worker will be killed abruptly. + # Ideally, find a way to know if we're close to reaching this limit, + # stop processing, and trigger a new execution to continue. + # + # if job and limit_time_real_cron_reached_or_about_to_reach: + # self._cron_trigger() + # break + + @api.model + def _cron_trigger(self, at=None): + """Trigger the cron job runners + + Odoo will prevent concurrent cron jobs from running. + So, to support parallel execution, we'd need to have (at least) the + same number of ir.crons records as cron workers. + + All crons should be triggered at the same time. + """ + crons = self.env["ir.cron"].sudo().search([("queue_job_runner", "=", True)]) + for cron in crons: + cron._trigger(at=at) + + def _ensure_cron_trigger(self): + """Create cron triggers for these jobs""" + records = self.filtered(lambda r: r.state == "pending") + if not records: + return + # Trigger immediate runs + immediate = any(not rec.eta for rec in records) + if immediate: + self._cron_trigger() + # Trigger delayed eta runs + delayed_etas = {rec.eta for rec in records if rec.eta} + if delayed_etas: + self._cron_trigger(at=list(delayed_etas)) + + @api.model_create_multi + def create(self, vals_list): + # When jobs are created, also create the cron trigger + records = super().create(vals_list) + records._ensure_cron_trigger() + return records + + def write(self, vals): + # When a job state or eta changes, make sure a cron trigger is created + res = super().write(vals) + if "state" in vals or "eta" in vals: + self._ensure_cron_trigger() + return res diff --git a/queue_job_cron_jobrunner/pyproject.toml b/queue_job_cron_jobrunner/pyproject.toml new file mode 100644 index 0000000000..4231d0cccb --- /dev/null +++ b/queue_job_cron_jobrunner/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/queue_job_cron_jobrunner/readme/CONFIGURE.md b/queue_job_cron_jobrunner/readme/CONFIGURE.md new file mode 100644 index 0000000000..109a7654b9 --- /dev/null +++ b/queue_job_cron_jobrunner/readme/CONFIGURE.md @@ -0,0 +1,24 @@ +Warning + +Don't use this module if you're already running the regular `queue_job` +runner. + +For the easiest case, no configuration is required besides installing +the module. + +To avoid CronWorker CPU timeout from abruptly stopping the job +processing cron, it's recommended to launch Odoo with +`--limit-time-real-cron=0`, to disable the CronWorker timeout +altogether. + +Note + +In Odoo.sh, this is done by default. + +Parallel execution of jobs can be achieved by leveraging multiple +`ir.cron` records: + +- Make sure you have enough CronWorkers available (Odoo CLI + `--max-cron-threads`) +- Duplicate the `queue_job_cron` cron record as many times as needed, + until you have as much records as cron workers. diff --git a/queue_job_cron_jobrunner/readme/CONTRIBUTORS.md b/queue_job_cron_jobrunner/readme/CONTRIBUTORS.md new file mode 100644 index 0000000000..35c75cb4cb --- /dev/null +++ b/queue_job_cron_jobrunner/readme/CONTRIBUTORS.md @@ -0,0 +1,3 @@ +- [Camptocamp](https://www.camptocamp.com) + + > - Iván Todorovich \<\> diff --git a/queue_job_cron_jobrunner/readme/DESCRIPTION.md b/queue_job_cron_jobrunner/readme/DESCRIPTION.md new file mode 100644 index 0000000000..e6dcbeabfc --- /dev/null +++ b/queue_job_cron_jobrunner/readme/DESCRIPTION.md @@ -0,0 +1,17 @@ +This module implements a simple `queue.job` runner using `ir.cron` +triggers. + +It's meant to be used on environments where the regular job runner can't +be run, like on Odoo.sh. + +Unlike the regular job runner, where jobs are dispatched to the +HttpWorkers, jobs are processed on the CronWorker threads by the job +runner crons. This is a design decision because: + +- Odoo.sh puts HttpWorkers to sleep when there's no network activity +- HttpWorkers are meant for traffic. Users shouldn't pay the price of + background tasks. + +For now, it only implements the most basic features of the `queue_job` +runner, notably no channel capacity nor priorities. Please check the +ROADMAP for further details. diff --git a/queue_job_cron_jobrunner/readme/ROADMAP.md b/queue_job_cron_jobrunner/readme/ROADMAP.md new file mode 100644 index 0000000000..cf1932bcc1 --- /dev/null +++ b/queue_job_cron_jobrunner/readme/ROADMAP.md @@ -0,0 +1,4 @@ +- Support channel capacity and priority. (See `_acquire_one_job`) +- Gracefully handle CronWorker CPU timeouts. (See `_job_runner`) +- Commit transaction after job state updated to started. (See + `_process`) diff --git a/queue_job_cron_jobrunner/static/description/icon.png b/queue_job_cron_jobrunner/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/queue_job_cron_jobrunner/static/description/icon.png differ diff --git a/queue_job_cron_jobrunner/static/description/index.html b/queue_job_cron_jobrunner/static/description/index.html new file mode 100644 index 0000000000..eb4aba0576 --- /dev/null +++ b/queue_job_cron_jobrunner/static/description/index.html @@ -0,0 +1,484 @@ + + + + + +Queue Job Cron Jobrunner + + + +
+

Queue Job Cron Jobrunner

+ + +

Alpha License: AGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

+

This module implements a simple queue.job runner using ir.cron +triggers.

+

It’s meant to be used on environments where the regular job runner can’t +be run, like on Odoo.sh.

+

Unlike the regular job runner, where jobs are dispatched to the +HttpWorkers, jobs are processed on the CronWorker threads by the job +runner crons. This is a design decision because:

+
    +
  • Odoo.sh puts HttpWorkers to sleep when there’s no network activity
    • +
  • HttpWorkers are meant for traffic. Users shouldn’t pay the price of +background tasks.
    • +
+

For now, it only implements the most basic features of the queue_job +runner, notably no channel capacity nor priorities. Please check the +ROADMAP for further details.

+
+

Important

+

This is an alpha version, the data model and design can change at any time without warning. +Only for development or testing purpose, do not use in production. +More details on development status

+
+

Table of contents

+ +
+

Configuration

+

Warning

+

Don’t use this module if you’re already running the regular +queue_job runner.

+

For the easiest case, no configuration is required besides installing +the module.

+

To avoid CronWorker CPU timeout from abruptly stopping the job +processing cron, it’s recommended to launch Odoo with +--limit-time-real-cron=0, to disable the CronWorker timeout +altogether.

+

Note

+

In Odoo.sh, this is done by default.

+

Parallel execution of jobs can be achieved by leveraging multiple +ir.cron records:

+
    +
  • Make sure you have enough CronWorkers available (Odoo CLI +--max-cron-threads)
    • +
  • Duplicate the queue_job_cron cron record as many times as needed, +until you have as much records as cron workers.
    • +
+
+
+

Known issues / Roadmap

+
    +
  • Support channel capacity and priority. (See _acquire_one_job)
    • +
  • Gracefully handle CronWorker CPU timeouts. (See _job_runner)
    • +
  • Commit transaction after job state updated to started. (See +_process)
    • +
+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Camptocamp SA
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

Current maintainer:

+

ivantodorovich

+

This module is part of the OCA/queue project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/queue_job_cron_jobrunner/tests/__init__.py b/queue_job_cron_jobrunner/tests/__init__.py new file mode 100644 index 0000000000..42bd479ed0 --- /dev/null +++ b/queue_job_cron_jobrunner/tests/__init__.py @@ -0,0 +1 @@ +from . import test_queue_job diff --git a/queue_job_cron_jobrunner/tests/test_queue_job.py b/queue_job_cron_jobrunner/tests/test_queue_job.py new file mode 100644 index 0000000000..3f2e0ef637 --- /dev/null +++ b/queue_job_cron_jobrunner/tests/test_queue_job.py @@ -0,0 +1,71 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# @author Iván Todorovich +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from datetime import timedelta + +from freezegun import freeze_time + +from odoo import fields +from odoo.tests.common import TransactionCase +from odoo.tools import mute_logger + + +class TestQueueJob(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + cls.cron = cls.env.ref("queue_job_cron_jobrunner.queue_job_cron") + # Cleanup triggers just in case + cls.env["ir.cron.trigger"].search([]).unlink() + + def assertTriggerAt(self, at, message=None): + """Ensures a cron trigger is created at the given time""" + return self.assertTrue( + self.env["ir.cron.trigger"].search([("call_at", "=", at)]), + message, + ) + + @freeze_time("2022-02-22 22:22:22") + def test_queue_job_cron_trigger(self): + """Test that ir.cron triggers are created for every queue.job""" + job = self.env["res.partner"].with_delay().create({"name": "test"}) + job_record = job.db_record() + self.assertTriggerAt(fields.Datetime.now(), "Trigger should've been created") + job_record.eta = fields.Datetime.now() + timedelta(hours=1) + self.assertTriggerAt(job_record.eta, "A new trigger should've been created") + + @mute_logger("odoo.addons.queue_job_cron_jobrunner.models.queue_job") + def test_queue_job_process(self): + """Test that jobs are processed by the queue job cron""" + # Create some jobs + job1 = self.env["res.partner"].with_delay().create({"name": "test"}) + job1_record = job1.db_record() + job2 = self.env["res.partner"].with_delay().create(False) + job2_record = job2.db_record() + job3 = self.env["res.partner"].with_delay(eta=3600).create({"name": "Test"}) + job3_record = job3.db_record() + # Run the job processing cron + self.env["queue.job"]._job_runner(commit=False) + # Check that the jobs were processed + self.assertEqual(job1_record.state, "done", "Processed OK") + self.assertEqual(job2_record.state, "failed", "Has errors") + self.assertEqual(job3_record.state, "pending", "Still pending, because of eta") + + @freeze_time("2022-02-22 22:22:22") + def test_queue_job_cron_trigger_enqueue_dependencies(self): + """Test that ir.cron execution enqueue waiting dependencies""" + delayable = self.env["res.partner"].delayable().create({"name": "test"}) + delayable2 = self.env["res.partner"].delayable().create({"name": "test2"}) + delayable.on_done(delayable2) + delayable.delay() + job_record = delayable._generated_job.db_record() + job_record_depends = delayable2._generated_job.db_record() + + self.env["queue.job"]._job_runner(commit=False) + + self.assertEqual(job_record.state, "done", "Processed OK") + # if the state is "waiting_dependencies", it means the "enqueue_waiting()" + # step has not been doen when the parent job has been done + self.assertEqual(job_record_depends.state, "done", "Processed OK") diff --git a/queue_job_cron_jobrunner/views/ir_cron.xml b/queue_job_cron_jobrunner/views/ir_cron.xml new file mode 100644 index 0000000000..e87a0d2a1c --- /dev/null +++ b/queue_job_cron_jobrunner/views/ir_cron.xml @@ -0,0 +1,17 @@ + + + + + ir.cron + + + + + + + + diff --git a/setup/_metapackage/pyproject.toml b/setup/_metapackage/pyproject.toml index e22937b40c..58b1890a7b 100644 --- a/setup/_metapackage/pyproject.toml +++ b/setup/_metapackage/pyproject.toml @@ -1,10 +1,11 @@ [project] name = "odoo-addons-oca-queue" -version = "18.0.20250220.0" +version = "18.0.20250222.0" dependencies = [ "odoo-addon-queue_job==18.0.*", "odoo-addon-queue_job_batch==18.0.*", "odoo-addon-queue_job_cron==18.0.*", + "odoo-addon-queue_job_cron_jobrunner==18.0.*", "odoo-addon-queue_job_subscribe==18.0.*", "odoo-addon-test_queue_job==18.0.*", "odoo-addon-test_queue_job_batch==18.0.*",