Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions src/dstack/_internal/cli/services/presets/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
ClaudeEffort = Literal["low", "medium", "high", "xhigh", "max"]
_RESUME_DELAYS_SECONDS: tuple[int, ...] = (30, 60, 120)
_TERMINATE_GRACE_SECONDS = 3
_AGENT_ERROR_MAX_LENGTH = 200
_RESUME_PROMPT = (
"The previous agent process was interrupted. Continue where you left off. "
"Re-check the states of your runs before relying on them: time may have "
Expand Down Expand Up @@ -228,9 +229,10 @@ async def run_preset_agent(
)
if output.report_data is None and returncode != 0:
output.error = output.error or f"Claude exited with return code {returncode}"
error = output.error
# Retry any process death without a submitted report; a terminal
# failure report from the agent returns immediately.
if output.report_data is not None or output.error is None:
if output.report_data is not None or error is None:
return output
# Only reset the retry budget when the last attempt made progress; a
# run that keeps stalling exhausts its retries instead of retrying a
Expand All @@ -252,7 +254,8 @@ async def run_preset_agent(
else:
action = "retrying"
print_preset_progress(
f"Agent process exited without a report; {action} in {delay}s.",
f"Agent process exited without a report: {_format_agent_error(error)};"
f" {action} in {delay}s.",
session=session,
)
await asyncio.sleep(delay)
Expand Down Expand Up @@ -375,6 +378,15 @@ def _prepare_subprocess_command(command: list[str]) -> list[str]:
return [comspec, "/d", "/s", "/c", subprocess.list2cmdline(command)]


def _format_agent_error(error: str) -> str:
"""Squashes the agent's error onto one progress line. The error is redacted
where it is captured; collapsing whitespace and truncating cannot undo that."""
text = " ".join(error.split())
if len(text) > _AGENT_ERROR_MAX_LENGTH:
text = text[:_AGENT_ERROR_MAX_LENGTH].rstrip() + "..."
return text


def _write_trace(
session: PresetSession,
*,
Expand Down
32 changes: 18 additions & 14 deletions src/dstack/_internal/cli/services/presets/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
PresetRandomConstraints,
)
from dstack._internal.core.models.runs import RunSpec
from dstack._internal.utils.power import prevent_idle_sleep
from dstack.api import Client

_RUN_STOP_TIMEOUT_SECONDS = 10 * 60
Expand Down Expand Up @@ -413,21 +414,24 @@ def create_preset(
)
try:
resolved_configuration = _resolve_preset_env(configuration)
result = asyncio.run(
_create_preset(
api=api,
configuration=resolved_configuration,
source_configuration=configuration,
store=store,
keep_service=keep_service,
build_name=build_name,
session=session,
mode="resume" if resume_session is not None else "fresh",
user_prompt=user_prompt,
allowed_fleets=allowed_fleets,
previous=previous,
# A creation session runs unattended for hours; an idling machine would
# freeze the agent and the process supervising it alike.
with prevent_idle_sleep():
result = asyncio.run(
_create_preset(
api=api,
configuration=resolved_configuration,
source_configuration=configuration,
store=store,
keep_service=keep_service,
build_name=build_name,
session=session,
mode="resume" if resume_session is not None else "fresh",
user_prompt=user_prompt,
allowed_fleets=allowed_fleets,
previous=previous,
)
)
)
except KeyboardInterrupt:
_stop_or_detach_agent_session(session, api)
raise
Expand Down
2 changes: 2 additions & 0 deletions src/dstack/_internal/compat.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import os
import sys

IS_WINDOWS = os.name == "nt"
IS_MACOS = sys.platform == "darwin"
63 changes: 63 additions & 0 deletions src/dstack/_internal/utils/power.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""System power management: keeping the machine awake through long foreground work."""

import os
import subprocess
from contextlib import contextmanager
from typing import Iterator, Optional

from dstack._internal.compat import IS_MACOS
from dstack._internal.utils.logging import get_logger

logger = get_logger(__name__)

# Present on every macOS install, so an absolute path avoids a PATH lookup.
_CAFFEINATE_PATH = "/usr/bin/caffeinate"
_STOP_TIMEOUT_SECONDS = 5


@contextmanager
def prevent_idle_sleep() -> Iterator[bool]:
"""Keeps the system from idling into sleep for the duration of the block.

Yields whether an inhibitor was acquired. Implemented on macOS only; on any
other platform, and whenever the mechanism is unavailable, this is a no-op
that yields False, since the caller has to work the same either way.
"""
process = _start_macos_inhibitor() if IS_MACOS else None
try:
yield process is not None
finally:
if process is not None:
_stop_inhibitor(process)


def _start_macos_inhibitor() -> Optional["subprocess.Popen[bytes]"]:
# `-i` asserts PreventUserIdleSystemSleep. `-w` makes caffeinate exit on its
# own once this process is gone, so not even a crash or a SIGKILL can leave
# the machine awake indefinitely.
command = [_CAFFEINATE_PATH, "-i", "-w", str(os.getpid())]
try:
return subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
except OSError as e:
logger.debug("Could not prevent idle sleep: %s", e)
return None


def _stop_inhibitor(process: "subprocess.Popen[bytes]") -> None:
if process.poll() is not None:
return
try:
process.terminate()
try:
process.wait(timeout=_STOP_TIMEOUT_SECONDS)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=_STOP_TIMEOUT_SECONDS)
except (OSError, subprocess.SubprocessError) as e:
# The inhibitor still self-releases once this process exits.
logger.debug("Could not release the idle sleep inhibitor: %s", e)
Loading