From 533fdff281643c206e7dbf5b5084cb4f98995933 Mon Sep 17 00:00:00 2001 From: Gabriele Tramonte Date: Fri, 26 Jul 2024 14:35:19 +0200 Subject: [PATCH] setting number of thread per worker to 8 --- executor/app/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/executor/app/main.py b/executor/app/main.py index 5282fd9..e0dce23 100644 --- a/executor/app/main.py +++ b/executor/app/main.py @@ -249,8 +249,9 @@ def create_dask_cluster(self, dask_cluster_opts: dict = None): dask_cluster_opts["processes"] = True port = int(os.getenv("DASK_DASHBOARD_PORT", 8787)) dask_cluster_opts["dashboard_address"] = f":{port}" - dask_cluster_opts["n_workers"] = 4 + dask_cluster_opts["n_workers"] = 1 dask_cluster_opts["memory_limit"] = "auto" + dask_cluster_opts['thread_per_worker'] = 8 self._worker_id = self._db.create_worker( status="enabled", dask_scheduler_port=dask_cluster_opts["scheduler_port"], @@ -266,7 +267,7 @@ def create_dask_cluster(self, dask_cluster_opts: dict = None): #scheduler_port=dask_cluster_opts["scheduler_port"], #dashboard_address=dask_cluster_opts["dashboard_address"], #memory_limit=dask_cluster_opts["memory_limit"], - threads_per_worker=1 + threads_per_worker=dask_cluster_opts['thread_per_worker'], ) self._LOG.info( "not creating Dask Client...", extra={"track_id": self._worker_id}