Skip to content
This repository was archived by the owner on Aug 11, 2020. It is now read-only.

Commit 25b72bf

Browse files
mkulaczkowskimkulaczkowski
authored
Merge pull request #108 from Paperspace/experiment-logs-PS-9868
feat: Logging for experiments.
2 parents 179003a + a242ef7 commit 25b72bf

File tree

5 files changed

+200
-27
lines changed

5 files changed

+200
-27
lines changed

‎paperspace/cli/experiments.py

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,13 @@ def common_experiments_create_single_node_options(f):
235235
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
236236

237237

238+
show_logs_option = click.option(
239+
"--logs/--no-logs",
240+
"show_logs",
241+
default=False,
242+
)
243+
244+
238245
@create_experiment.command(name="multinode", help="Create multi node experiment")
239246
@common_experiments_create_options
240247
@common_experiment_create_multi_node_options
@@ -259,30 +266,42 @@ def create_single_node(api_key, **kwargs):
259266
@create_and_start_experiment.command(name="multinode", help="Create and start new multi node experiment")
260267
@common_experiments_create_options
261268
@common_experiment_create_multi_node_options
262-
def create_and_start_multi_node(api_key, **kwargs):
269+
@show_logs_option
270+
@click.pass_context
271+
def create_and_start_multi_node(ctx, api_key, show_logs, **kwargs):
263272
del_if_value_is_none(kwargs)
264273
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
265274
command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
266-
command.execute(kwargs)
275+
experiment = command.execute(kwargs)
276+
if experiment and show_logs:
277+
ctx.invoke(list_logs, experiment_id=experiment["handle"], line=0, limit=100, follow=True, api_key=api_key)
267278

268279

269280
@create_and_start_experiment.command(name="singlenode", help="Create and start new single node experiment")
270281
@common_experiments_create_options
271282
@common_experiments_create_single_node_options
272-
def create_and_start_single_node(api_key, **kwargs):
283+
@show_logs_option
284+
@click.pass_context
285+
def create_and_start_single_node(ctx, api_key, show_logs, **kwargs):
273286
kwargs["experimentType"] = constants.ExperimentType.SINGLE_NODE
274287
del_if_value_is_none(kwargs)
275288
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
276289
command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
277-
command.execute(kwargs)
290+
experiment = command.execute(kwargs)
291+
if experiment and show_logs:
292+
ctx.invoke(list_logs, experiment_id=experiment["handle"], line=0, limit=100, follow=True, api_key=api_key)
278293

279294

280295
@experiments.command("start", help="Start experiment")
281296
@click.argument("experiment-id")
282297
@api_key_option
283-
def start_experiment(experiment_id, api_key):
298+
@show_logs_option
299+
@click.pass_context
300+
def start_experiment(ctx, experiment_id, show_logs, api_key):
284301
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
285302
experiments_commands.start_experiment(experiment_id, api=experiments_api)
303+
if show_logs:
304+
ctx.invoke(list_logs, experiment_id=experiment_id, line=0, limit=100, follow=True, api_key=api_key)
286305

287306

288307
@experiments.command("stop", help="Stop experiment")
@@ -308,3 +327,34 @@ def list_experiments(project_ids, api_key):
308327
def get_experiment_details(experiment_id, api_key):
309328
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
310329
experiments_commands.get_experiment_details(experiment_id, api=experiments_api)
330+
331+
332+
@experiments.command("logs", help="List experiment logs")
333+
@click.option(
334+
"--experimentId",
335+
"experiment_id",
336+
required=True
337+
)
338+
@click.option(
339+
"--line",
340+
"line",
341+
required=False,
342+
default=0
343+
)
344+
@click.option(
345+
"--limit",
346+
"limit",
347+
required=False,
348+
default=10000
349+
)
350+
@click.option(
351+
"--follow",
352+
"follow",
353+
required=False,
354+
default=False
355+
)
356+
@api_key_option
357+
def list_logs(experiment_id, line, limit, follow, api_key=None):
358+
logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
359+
command = experiments_commands.ExperimentLogsCommand(api=logs_api)
360+
command.execute(experiment_id, line, limit, follow)

‎paperspace/cli/jobs.py

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,24 +87,45 @@ def list_jobs(api_key, **filters):
8787
@click.option("--registryUsername", "registryUsername", help="Docker registry username")
8888
@click.option("--registryPassword", "registryPassword", help="Docker registry password")
8989
@api_key_option
90-
def create_job(api_key, **kwargs):
90+
@click.pass_context
91+
def create_job(ctx, api_key, **kwargs):
9192
del_if_value_is_none(kwargs)
9293
jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
9394
command = jobs_commands.CreateJobCommand(api=jobs_api)
94-
command.execute(kwargs)
95+
job = command.execute(kwargs)
96+
if job is not None:
97+
ctx.invoke(list_logs, job_id=job["handle"], line=0, limit=100, follow=True, api_key=api_key)
9598

9699

97-
@jobs_group.command("log", help="List job logs")
100+
@jobs_group.command("logs", help="List job logs")
98101
@click.option(
99102
"--jobId",
100103
"job_id",
101104
required=True
102105
)
106+
@click.option(
107+
"--line",
108+
"line",
109+
required=False,
110+
default=0
111+
)
112+
@click.option(
113+
"--limit",
114+
"limit",
115+
required=False,
116+
default=10000
117+
)
118+
@click.option(
119+
"--follow",
120+
"follow",
121+
required=False,
122+
default=False
123+
)
103124
@api_key_option
104-
def list_logs(job_id, api_key=None):
125+
def list_logs(job_id, line, limit, follow, api_key=None):
105126
logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
106127
command = jobs_commands.JobLogsCommand(api=logs_api)
107-
command.execute(job_id)
128+
command.execute(job_id, line, limit, follow)
108129

109130

110131
@jobs_group.group("artifacts", help="Manage jobs' artifacts", cls=ClickGroup)

‎paperspace/commands/experiments.py

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
1+
import pydoc
2+
13
import terminaltables
4+
from click import style
25

36
from paperspace import logger, constants, client, config
47
from paperspace.logger import log_response
5-
from paperspace.workspace import S3WorkspaceHandler
68
from paperspace.commands import common
9+
from paperspace.utils import get_terminal_lines
10+
from paperspace.workspace import S3WorkspaceHandler
711

812
experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, headers=client.default_headers)
913

@@ -51,6 +55,9 @@ def execute(self, json_):
5155
self._log_create_experiment(response,
5256
"New experiment created and started with ID: {}",
5357
"Unknown error while creating/starting the experiment")
58+
if response.ok:
59+
return response.json()
60+
return None
5461

5562

5663
def start_experiment(experiment_id, api=experiments_api):
@@ -168,3 +175,77 @@ def get_experiment_details(experiment_id, api=experiments_api):
168175
logger.error("Error parsing response data")
169176

170177
log_response(response, details, "Unknown error while retrieving details of the experiment")
178+
179+
180+
class ExperimentLogsCommand(common.CommandBase):
181+
last_line_number = 0
182+
base_url = "/jobs/logs"
183+
184+
is_logs_complete = False
185+
186+
def execute(self, experiment_id, line, limit, follow):
187+
if follow:
188+
self.logger.log("Awaiting logs...")
189+
190+
self.last_line_number = line
191+
table_title = "Experiment %s logs" % experiment_id
192+
table_data = [("JOB ID", "LINE", "MESSAGE")]
193+
table = terminaltables.AsciiTable(table_data, title=table_title)
194+
195+
while not self.is_logs_complete:
196+
response = self._get_logs(experiment_id, self.last_line_number, limit)
197+
198+
try:
199+
data = response.json()
200+
if not response.ok:
201+
self.logger.log_error_response(data)
202+
return
203+
except (ValueError, KeyError) as e:
204+
if response.status_code == 204:
205+
continue
206+
self.logger.log("Error while parsing response data: {}".format(e))
207+
return
208+
else:
209+
self._log_logs_list(data, table, table_data, follow)
210+
211+
if not follow:
212+
self.is_logs_complete = True
213+
214+
def _get_logs(self, experiment_id, line, limit):
215+
params = {
216+
'experimentId': experiment_id,
217+
'line': line,
218+
'limit': limit
219+
};
220+
return self.api.get(self.base_url, params=params)
221+
222+
def _log_logs_list(self, data, table, table_data, follow):
223+
if not data:
224+
self.logger.log("No logs found")
225+
return
226+
if follow:
227+
# TODO track number of jobs seen to look for PSEOF
228+
if data[-1].get("message") == "PSEOF":
229+
self.is_logs_complete = True
230+
else:
231+
self.last_line_number = data[-1].get("line")
232+
for log in data:
233+
log_str = "{}\t{}\t{}"
234+
self.logger.log(log_str.format(style(fg="blue", text=str(log.get("jobId"))), style(fg="red", text=str(log.get("line"))), log.get("message")))
235+
else:
236+
table_str = self._make_table(data, table, table_data)
237+
if len(table_str.splitlines()) > get_terminal_lines():
238+
pydoc.pager(table_str)
239+
else:
240+
self.logger.log(table_str)
241+
242+
def _make_table(self, logs, table, table_data):
243+
if logs[-1].get("message") == "PSEOF":
244+
self.is_logs_complete = True
245+
else:
246+
self.last_line_number = logs[-1].get("line")
247+
248+
for log in logs:
249+
table_data.append((style(fg="blue", text=str(log.get("jobId"))), style(fg="red", text=str(log.get("line"))), log.get("message")))
250+
251+
return table.table

‎paperspace/commands/jobs.py

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -72,17 +72,21 @@ def _get_table_data(self, jobs):
7272

7373
class JobLogsCommand(common.CommandBase):
7474
last_line_number = 0
75-
base_url = "/jobs/logs?jobId={}&line={}"
75+
base_url = "/jobs/logs"
7676

7777
is_logs_complete = False
7878

79-
def execute(self, job_id):
79+
def execute(self, job_id, line, limit, follow):
80+
if follow:
81+
self.logger.log("Awaiting logs...")
82+
83+
self.last_line_number = line
8084
table_title = "Job %s logs" % job_id
8185
table_data = [("LINE", "MESSAGE")]
8286
table = terminaltables.AsciiTable(table_data, title=table_title)
8387

8488
while not self.is_logs_complete:
85-
response = self._get_logs(job_id)
89+
response = self._get_logs(job_id, self.last_line_number, limit)
8690

8791
try:
8892
data = response.json()
@@ -95,15 +99,31 @@ def execute(self, job_id):
9599
self.logger.log("Error while parsing response data: {}".format(e))
96100
return
97101
else:
98-
self._log_logs_list(data, table, table_data)
102+
self._log_logs_list(data, table, table_data, follow)
99103

100-
def _get_logs(self, job_id):
101-
url = self.base_url.format(job_id, self.last_line_number)
102-
return self.api.get(url)
104+
if not follow:
105+
self.is_logs_complete = True
103106

104-
def _log_logs_list(self, data, table, table_data):
107+
def _get_logs(self, job_id, line, limit):
108+
params = {
109+
'jobId': job_id,
110+
'line': line,
111+
'limit': limit
112+
};
113+
return self.api.get(self.base_url, params=params)
114+
115+
def _log_logs_list(self, data, table, table_data, follow):
105116
if not data:
106117
self.logger.log("No Logs found")
118+
return
119+
if follow:
120+
if data[-1].get("message") == "PSEOF":
121+
self.is_logs_complete = True
122+
else:
123+
self.last_line_number = data[-1].get("line")
124+
for log in data:
125+
log_str = "{}\t{}\t{}"
126+
self.logger.log(log_str.format(style(fg="blue", text=str(log.get("jobId"))), style(fg="red", text=str(log.get("line"))), log.get("message")))
107127
else:
108128
table_str = self._make_table(data, table, table_data)
109129
if len(table_str.splitlines()) > get_terminal_lines():

‎tests/functional/test_jobs.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -196,15 +196,16 @@ def test_should_print_proper_message_when_jobs_list_was_used_with_mutually_exclu
196196

197197

198198
class TestJobLogs(TestJobs):
199-
URL = "https://logs.paperspace.io/jobs/logs?jobId=some_job_id&line=0"
199+
URL = "https://logs.paperspace.io/jobs/logs"
200200

201201
RESPONSE_JSON_WITH_WRONG_API_TOKEN = {"status": 400, "message": "Invalid API token"}
202202
EXPECTED_RESPONSE_JSON = example_responses.LIST_OF_LOGS_FOR_JOB
203-
BASIC_COMMAND_WITHOUT_PARAMETERS = ["jobs", "log"]
204-
BASIC_COMMAND = ["jobs", "log", "--jobId", "some_job_id", "--apiKey", "some_key"]
203+
BASIC_COMMAND_WITHOUT_PARAMETERS = ["jobs", "logs"]
204+
BASIC_COMMAND = ["jobs", "logs", "--jobId", "some_job_id", "--apiKey", "some_key"]
205+
BASIC_COMMAND_PARAMS = {"jobId": "some_job_id", "line": 0, "limit": 10000}
205206

206-
EXPECTED_STDOUT_WITHOUT_PARAMETERS = """Usage: cli jobs log [OPTIONS]
207-
Try "cli jobs log --help" for help.
207+
EXPECTED_STDOUT_WITHOUT_PARAMETERS = """Usage: cli jobs logs [OPTIONS]
208+
Try "cli jobs logs --help" for help.
208209
209210
Error: Missing option "--jobId".
210211
"""
@@ -248,7 +249,7 @@ def test_should_send_valid_get_request_and_print_available_logs(self, get_patche
248249
get_patched.assert_called_with(self.URL,
249250
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
250251
json=None,
251-
params=None)
252+
params=self.BASIC_COMMAND_PARAMS)
252253

253254
assert result.output == self.EXPECTED_STDOUT
254255
assert result.exit_code == 0
@@ -263,7 +264,7 @@ def test_should_send_valid_get_request_when_log_list_was_used_with_wrong_api_key
263264
get_patched.assert_called_with(self.URL,
264265
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
265266
json=None,
266-
params=None)
267+
params=self.BASIC_COMMAND_PARAMS)
267268
assert result.output == self.EXPECTED_STDOUT_WITH_WRONG_API_TOKEN
268269
assert result.exit_code == 0
269270

@@ -277,7 +278,7 @@ def test_should_print_error_message_when_error_status_code_received_but_no_conte
277278
get_patched.assert_called_with(self.URL,
278279
headers=self.EXPECTED_HEADERS_WITH_CHANGED_API_KEY,
279280
json=None,
280-
params=None)
281+
params=self.BASIC_COMMAND_PARAMS)
281282
assert result.output == "Error while parsing response data: No JSON\n"
282283
assert result.exit_code == 0
283284

0 commit comments

Comments
 (0)