fix(state): Gracefully handle stale service entries in state database

devservices/commands/down.py

@@ -34,6 +34,7 @@
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.services import Service
 from devservices.utils.state import ServiceRuntime
 from devservices.utils.state import State
@@ -90,9 +91,7 @@ def down(args: Namespace) -> None:
     exclude_local = getattr(args, "exclude_local", False)
 
     state = State()
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names(clean_stale_entries=True)
     if service.name not in active_services:
         console.warning(f"{service.name} is not running")
         return  # Since exit(0) is captured as an internal_error by sentry

devservices/commands/foreground.py

@@ -19,6 +19,7 @@
 from devservices.exceptions import SupervisorProcessError
 from devservices.utils.console import Console
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.state import State
 from devservices.utils.state import StateTables
 from devservices.utils.supervisor import SupervisorManager
@@ -60,9 +61,7 @@ def foreground(args: Namespace) -> None:
         )
         return
     state = State()
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names()
     if service.name not in active_services:
         console.warning(f"{service.name} is not running")
         return

devservices/commands/logs.py

@@ -29,6 +29,7 @@
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.services import Service
 from devservices.utils.state import State
 from devservices.utils.state import StateTables
@@ -84,9 +85,7 @@ def logs(args: Namespace) -> None:
     if not mode_dependencies and "default" in modes:
         mode_dependencies.update(modes["default"])
 
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    running_services = starting_services.union(started_services)
+    running_services = get_active_service_names()
     if service.name not in running_services:
         console.warning(f"Service {service.name} is not running")
         return

devservices/commands/reset.py

@@ -20,6 +20,7 @@
 from devservices.utils.docker import remove_docker_resources
 from devservices.utils.docker import stop_containers
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.state import State
 from devservices.utils.state import StateTables
 
@@ -69,9 +70,7 @@ def reset(args: Namespace) -> None:
         exit(1)
 
     state = State()
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    active_service_names = starting_services.union(started_services)
+    active_service_names = get_active_service_names(clean_stale_entries=True)
 
     # TODO: We should add threading here to speed up the process
     for active_service_name in active_service_names:

devservices/commands/status.py

@@ -35,6 +35,7 @@
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.services import Service
 from devservices.utils.state import ServiceRuntime
 from devservices.utils.state import State
@@ -95,10 +96,7 @@ def status(args: Namespace) -> None:
         console.failure(str(e))
         exit(1)
 
-    state = State()
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names()
     if service.name not in active_services:
         console.warning(f"Status unavailable. {service.name} is not running standalone")
         return  # Since exit(0) is captured as an internal_error by sentry

devservices/commands/toggle.py

@@ -20,6 +20,7 @@
 from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.services import Service
 from devservices.utils.state import ServiceRuntime
 from devservices.utils.state import State
@@ -97,9 +98,7 @@ def handle_transition_to_local_runtime(service_to_transition: Service) -> None:
     console = Console()
     state = State()
 
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names(clean_stale_entries=True)
 
     # If the service is already running standalone, we can just update the runtime
     if service_to_transition.name in active_services:
@@ -158,9 +157,7 @@ def handle_transition_to_containerized_runtime(service: Service) -> None:
     """Handle the transition to a containerized runtime for a service."""
     console = Console()
     state = State()
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names(clean_stale_entries=True)
     if service.name in active_services:
         console.warning(f"{service.name} is running, please stop it first")
         return

devservices/utils/dependencies.py

@@ -39,6 +39,7 @@
 from devservices.exceptions import UnableToCloneDependencyError
 from devservices.utils.file_lock import lock
 from devservices.utils.services import find_matching_service
+from devservices.utils.services import get_active_service_names
 from devservices.utils.services import Service
 from devservices.utils.state import ServiceRuntime
 from devservices.utils.state import State
@@ -281,12 +282,9 @@ def get_non_shared_remote_dependencies(
     exclude_local: bool,
 ) -> set[InstalledRemoteDependency]:
     state = State()
-    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
-    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
-    active_services = starting_services.union(started_services)
+    active_services = get_active_service_names(clean_stale_entries=True)
     # We don't care about the remote dependencies of the service we are stopping
-    if service_to_stop.name in active_services:
-        active_services.remove(service_to_stop.name)
+    active_services.discard(service_to_stop.name)
 
     active_modes: dict[str, list[str]] = dict()
     for active_service in active_services:

devservices/utils/services.py

@@ -3,13 +3,17 @@
 import os
 from dataclasses import dataclass
 
+from sentry_sdk import logger as sentry_logger
+
 from devservices.configs.service_config import ServiceConfig
 from devservices.exceptions import ConfigNotFoundError
 from devservices.exceptions import ConfigParseError
 from devservices.exceptions import ConfigValidationError
 from devservices.exceptions import ServiceNotFoundError
 from devservices.utils.console import Console
 from devservices.utils.devenv import get_coderoot
+from devservices.utils.state import State
+from devservices.utils.state import StateTables
 
 
 @dataclass
@@ -73,3 +77,33 @@ def find_matching_service(service_name: str | None = None) -> Service:
         )
         error_message += "\nSupported services:\n" + service_bullet_points
     raise ServiceNotFoundError(error_message)
+
+
+def get_active_service_names(clean_stale_entries: bool = False) -> set[str]:
+    """Get the names of all services currently starting or started.
+
+    Args:
+        clean_stale_entries: If True, verify each service still exists on disk.
+            Stale entries (services that no longer exist) are removed
+            from the state database and excluded from the result.
+    """
+    state = State()
+    starting_services = set(state.get_service_entries(StateTables.STARTING_SERVICES))
+    started_services = set(state.get_service_entries(StateTables.STARTED_SERVICES))
+    active_services = starting_services.union(started_services)
+
+    if not clean_stale_entries:
+        return active_services
+
+    valid_services: set[str] = set()
+    for service_name in active_services:
+        try:
+            find_matching_service(service_name)
+            valid_services.add(service_name)
+        except ServiceNotFoundError:
+            sentry_logger.warning(
+                "Stale service entry found in state database, removing",
+                extra={"service_name": service_name},
+            )
+            state.remove_stale_service_entry(service_name)
+    return valid_services

devservices/utils/state.py

@@ -174,6 +174,22 @@ def get_services_by_runtime(self, runtime: ServiceRuntime) -> list[str]:
         )
         return [row[0] for row in cursor.fetchall()]
 
+    def remove_stale_service_entry(self, service_name: str) -> None:
+        """Remove a service from all state tables.
+
+        Used to clean up entries for services that no longer exist on disk.
+        """
+        self.remove_service_entry(service_name, StateTables.STARTING_SERVICES)
+        self.remove_service_entry(service_name, StateTables.STARTED_SERVICES)
+        cursor = self.conn.cursor()
+        cursor.execute(
+            f"""
+            DELETE FROM {StateTables.SERVICE_RUNTIME.value} WHERE service_name = ?
+        """,
+            (service_name,),
+        )
+        self.conn.commit()
+
     def clear_state(self) -> None:
         cursor = self.conn.cursor()
         cursor.execute(

tests/commands/test_down.py

@@ -76,6 +76,14 @@ def test_down_starting(
             mock.patch(
                 "devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")
             ),
+            mock.patch(
+                "devservices.commands.down.get_active_service_names",
+                return_value={"example-service"},
+            ),
+            mock.patch(
+                "devservices.utils.dependencies.get_active_service_names",
+                return_value={"example-service"},
+            ),
         ):
             state = State()
             state.update_service_entry(
@@ -156,6 +164,14 @@ def test_down_started(
             mock.patch(
                 "devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")
             ),
+            mock.patch(
+                "devservices.commands.down.get_active_service_names",
+                return_value={"example-service"},
+            ),
+            mock.patch(
+                "devservices.utils.dependencies.get_active_service_names",
+                return_value={"example-service"},
+            ),
         ):
             state = State()
             state.update_service_entry(
@@ -250,6 +266,14 @@ def test_down_error(
 
     with (
         mock.patch("devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")),
+        mock.patch(
+            "devservices.commands.down.get_active_service_names",
+            return_value={"example-service"},
+        ),
+        mock.patch(
+            "devservices.utils.dependencies.get_active_service_names",
+            return_value={"example-service"},
+        ),
         pytest.raises(SystemExit),
     ):
         state = State()
@@ -317,6 +341,14 @@ def test_down_mode_simple(
             mock.patch(
                 "devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")
             ),
+            mock.patch(
+                "devservices.commands.down.get_active_service_names",
+                return_value={"example-service"},
+            ),
+            mock.patch(
+                "devservices.utils.dependencies.get_active_service_names",
+                return_value={"example-service"},
+            ),
         ):
             state = State()
             state.update_service_entry(
@@ -1387,6 +1419,14 @@ def test_down_supervisor_program_error(
             mock.patch(
                 "devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")
             ),
+            mock.patch(
+                "devservices.commands.down.get_active_service_names",
+                return_value={"example-service"},
+            ),
+            mock.patch(
+                "devservices.utils.dependencies.get_active_service_names",
+                return_value={"example-service"},
+            ),
             pytest.raises(SystemExit),
         ):
             state = State()
@@ -1451,6 +1491,14 @@ def test_down_supervisor_program_success(
             mock.patch(
                 "devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")
             ),
+            mock.patch(
+                "devservices.commands.down.get_active_service_names",
+                return_value={"example-service"},
+            ),
+            mock.patch(
+                "devservices.utils.dependencies.get_active_service_names",
+                return_value={"example-service"},
+            ),
         ):
             state = State()
             state.update_service_entry(

tests/commands/test_toggle.py

@@ -525,6 +525,10 @@ def test_handle_transition_to_local_runtime_currently_running_standalone(
 ) -> None:
     with (
         mock.patch("devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")),
+        mock.patch(
+            "devservices.commands.toggle.get_active_service_names",
+            return_value={"example-service"},
+        ),
     ):
         state = State()
         state.update_service_runtime("example-service", ServiceRuntime.CONTAINERIZED)
@@ -723,6 +727,10 @@ def test_handle_transition_to_containerized_runtime_with_service_running(
 ) -> None:
     with (
         mock.patch("devservices.utils.state.STATE_DB_FILE", str(tmp_path / "state")),
+        mock.patch(
+            "devservices.commands.toggle.get_active_service_names",
+            return_value={"redis"},
+        ),
     ):
         redis_repo_path = create_mock_git_repo("blank_repo", tmp_path / "redis")
         redis_config = {