fix(bench): Only validate the relevant site's config

agent/base.py

@@ -10,6 +10,7 @@
 
 import redis
 
+from agent.exceptions import AgentException
 from agent.job import connection
 from agent.utils import get_execution_result
 
@@ -233,8 +234,3 @@ def retrieve_log(self, name):
         log_file = os.path.join(self.logs_directory, name)
         with open(log_file) as lf:
             return lf.read()
-
-
-class AgentException(Exception):
-    def __init__(self, data):
-        self.data = data

agent/bench.py

@@ -20,7 +20,7 @@
 
 from agent.app import App
 from agent.base import AgentException, Base
-from agent.exceptions import SiteNotExistsException
+from agent.exceptions import InvalidSiteConfigException, SiteNotExistsException
 from agent.job import job, step
 from agent.site import Site
 from agent.utils import download_file, end_execution, get_execution_result, get_size
@@ -790,18 +790,24 @@ def _sites(self, validate_configs=False) -> dict[str, Site]:
                 sites[directory] = Site(directory, self)
             except json.decoder.JSONDecodeError as jde:
                 output = self.readable_jde_err(f"Error parsing JSON in {directory}", jde)
-                self.execute(
-                    f"echo '{output}';exit {int(validate_configs)}",
-                )  # exit 1 to make sure the job fails and shows output
+                try:
+                    self.execute(
+                        f"echo '{output}';exit {int(validate_configs)}",
+                    )  # exit 1 to make sure the job fails and shows output
+                except AgentException as e:
+                    raise InvalidSiteConfigException(e.data, directory) from e
             except Exception:
                 pass
         return sites
 
     def get_site(self, site):
         try:
             return self.valid_sites[site]
-        except KeyError as exc:
-            raise SiteNotExistsException(site, self.name) from exc
+        except KeyError as e:
+            raise SiteNotExistsException(site, self.name) from e
+        except InvalidSiteConfigException as e:
+            if e.site == site:
+                raise
 
     @property
     def step_record(self):

agent/exceptions.py

@@ -1,6 +1,11 @@
 from __future__ import annotations
 
 
+class AgentException(Exception):
+    def __init__(self, data):
+        self.data = data
+
+
 class BenchNotExistsException(Exception):
     def __init__(self, bench):
         self.bench = bench
@@ -16,3 +21,9 @@ def __init__(self, site, bench):
         self.message = f"Site {site} does not exist on bench {bench}"
 
         super().__init__(self.message)
+
+
+class InvalidSiteConfigException(AgentException):
+    def __init__(self, data: dict, site=None):
+        self.site = site
+        super().__init__(data)