Merge remote-tracking branch 'origin/master' into run-restored

Author: kossak

README.md

@@ -17,8 +17,20 @@ Getting Started
     To install/update prerelease (Alpha/Beta) version version of paperspace-python, use:
 
     `pip install -U --pre paperspace`
-
-3. Download your api key by executing the following:
+3. Enable autocomplete:
+    
+    Add following to your `.bashrc` (or `.zshrc`) to enable autocomplete anytime you activate your shell.
+     
+    `eval $(_PAPERSPACE_PYTHON_COMPLETE=source paperspace-python)`
+    
+    For other shell (eg. `zsh` or `fish`), use proper version of `source` (`source_zsh` and `source-fish` respectively)
+     
+     Alternatively, you can create activation script by:
+     
+     `(_PAPERSPACE_PYTHON_COMPLETE=source paperspace-python > ~/paperspace_complete.sh`
+
+    More: https://click.palletsprojects.com/en/7.x/bashcomplete/
+4. Download your api key by executing the following:
 
     `paperspace-python login`
 
@@ -33,7 +45,7 @@ Getting Started
 
     `paperspace-python logout`
 
-4. Run the sample script hello.py using Python:
+5. Run the sample script hello.py using Python:
 
     `python hello.py`
 
@@ -49,7 +61,7 @@ Getting Started
 
    Note: the source is modified before transfer to the job cluster in order to remove imported `paperspace` references.
 
-5. Use paperspace-python to run a python script remotely:
+6. Use paperspace-python to run a python script remotely:
 
     `paperspace-python run myscript.py`
 

paperspace/cli/cli.py

@@ -1,9 +1,12 @@
 import click
+import click_completion
 
 from paperspace import config
 from paperspace.cli import common
 from paperspace.commands import login as login_commands
 
+click_completion.init()
+
 
 @click.group(cls=common.ClickGroup, **config.HELP_COLORS_DICT)
 def cli():

paperspace/cli/common.py

@@ -1,6 +1,7 @@
 import getpass
 
 import click
+from click import group
 from click_didyoumean import DYMMixin
 from click_help_colors import HelpColorsGroup
 
@@ -19,7 +20,27 @@ def del_if_value_is_none(dict_):
 
 
 class ClickGroup(DYMMixin, HelpColorsGroup):
-    pass
+    def group(self, *args, **kwargs):
+        aliases = []
+        _args = args
+        if args and isinstance(args[0], list):
+            # we have a list so create group aliases
+            aliases = args[0][1:]
+            _args = [args[0][0]] + list(args[1:])
+
+        if 'alias' in kwargs:
+            aliases.append(kwargs.pop('alias'))
+
+        def decorator(f):
+            cmd = group(*_args, **kwargs)(f)
+            self.add_command(cmd)
+            for alias in set(aliases):
+                alias_cmd = group(alias, **kwargs)(f)
+                self.add_command(alias_cmd)
+                alias_cmd.commands = cmd.commands
+            return cmd
+
+        return decorator
 
 
 def prompt_for_secret(prompt):

paperspace/cli/deployments.py

@@ -2,14 +2,14 @@
 
 import click
 
-from paperspace import constants, client, config
+from paperspace import client, config
 from paperspace.cli.cli import cli
 from paperspace.cli.cli_types import ChoiceType
 from paperspace.cli.common import api_key_option, del_if_value_is_none, ClickGroup
 from paperspace.commands import deployments as deployments_commands
 
 
-@cli.group("deployments", help="Manage deployments", cls=ClickGroup)
+@cli.group("deployments", alias='deployment', help="Manage deployments", cls=ClickGroup)
 def deployments():
     pass
 
@@ -22,7 +22,8 @@ def deployments():
 )
 
 DEPLOYMENT_MACHINE_TYPES = ("G1", "G6", "G12",
-    "K80", "P100", "V100")
+                            "K80", "P100", "GV100")
+
 
 @deployments.command("create", help="Create new deployment")
 @click.option(
@@ -135,4 +136,4 @@ def start_deployment(id_, api_key=None):
 def stop_deployment(id_, api_key=None):
     deployments_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = deployments_commands.StopDeploymentCommand(api=deployments_api)
-    command.execute(id_)
+    command.execute(id_)

paperspace/cli/experiments.py

@@ -17,7 +17,7 @@
 )
 
 
-@cli.group("experiments", help="Manage experiments", cls=ClickGroup)
+@cli.group(["experiments", "experiment"], help="Manage experiments", cls=ClickGroup)
 def experiments():
     pass
 
@@ -235,6 +235,13 @@ def common_experiments_create_single_node_options(f):
     return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
 
 
+show_logs_option = click.option(
+    "--logs/--no-logs",
+    "show_logs",
+    default=False,
+)
+
+
 @create_experiment.command(name="multinode", help="Create multi node experiment")
 @common_experiments_create_options
 @common_experiment_create_multi_node_options
@@ -259,30 +266,42 @@ def create_single_node(api_key, **kwargs):
 @create_and_start_experiment.command(name="multinode", help="Create and start new multi node experiment")
 @common_experiments_create_options
 @common_experiment_create_multi_node_options
-def create_and_start_multi_node(api_key, **kwargs):
+@show_logs_option
+@click.pass_context
+def create_and_start_multi_node(ctx, api_key, show_logs, **kwargs):
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
-    command.execute(kwargs)
+    experiment = command.execute(kwargs)
+    if experiment and show_logs:
+        ctx.invoke(list_logs, experiment_id=experiment["handle"], line=0, limit=100, follow=True, api_key=api_key)
 
 
 @create_and_start_experiment.command(name="singlenode", help="Create and start new single node experiment")
 @common_experiments_create_options
 @common_experiments_create_single_node_options
-def create_and_start_single_node(api_key, **kwargs):
+@show_logs_option
+@click.pass_context
+def create_and_start_single_node(ctx, api_key, show_logs, **kwargs):
     kwargs["experimentType"] = constants.ExperimentType.SINGLE_NODE
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)
-    command.execute(kwargs)
+    experiment = command.execute(kwargs)
+    if experiment and show_logs:
+        ctx.invoke(list_logs, experiment_id=experiment["handle"], line=0, limit=100, follow=True, api_key=api_key)
 
 
 @experiments.command("start", help="Start experiment")
 @click.argument("experiment-id")
 @api_key_option
-def start_experiment(experiment_id, api_key):
+@show_logs_option
+@click.pass_context
+def start_experiment(ctx, experiment_id, show_logs, api_key):
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     experiments_commands.start_experiment(experiment_id, api=experiments_api)
+    if show_logs:
+        ctx.invoke(list_logs, experiment_id=experiment_id, line=0, limit=100, follow=True, api_key=api_key)
 
 
 @experiments.command("stop", help="Stop experiment")
@@ -308,3 +327,34 @@ def list_experiments(project_ids, api_key):
 def get_experiment_details(experiment_id, api_key):
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     experiments_commands.get_experiment_details(experiment_id, api=experiments_api)
+
+
+@experiments.command("logs", help="List experiment logs")
+@click.option(
+    "--experimentId",
+    "experiment_id",
+    required=True
+)
+@click.option(
+    "--line",
+    "line",
+    required=False,
+    default=0
+)
+@click.option(
+    "--limit",
+    "limit",
+    required=False,
+    default=10000
+)
+@click.option(
+    "--follow",
+    "follow",
+    required=False,
+    default=False
+)
+@api_key_option
+def list_logs(experiment_id, line, limit, follow, api_key=None):
+    logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
+    command = experiments_commands.ExperimentLogsCommand(api=logs_api)
+    command.execute(experiment_id, line, limit, follow)

paperspace/cli/hyperparameters.py

@@ -9,7 +9,7 @@
 from paperspace.commands import hyperparameters as hyperparameters_commands
 
 
-@cli.group("hyperparameters", help="Manage hyperparameters", cls=ClickGroup)
+@cli.group(["hyperparameters", "hyperparameter"], help="Manage hyperparameters", cls=ClickGroup)
 def hyperparameters_group():
     pass
 

paperspace/cli/jobs.py

@@ -9,7 +9,7 @@
 from paperspace.commands import jobs as jobs_commands
 
 
-@cli.group("jobs", help="Manage gradient jobs", cls=ClickGroup)
+@cli.group(["jobs", "job"], help="Manage gradient jobs", cls=ClickGroup)
 def jobs_group():
     pass
 
@@ -96,24 +96,45 @@ def common_jobs_create_options(f):
 @jobs_group.command("create", help="Create job")
 @common_jobs_create_options
 @api_key_option
-def create_job(api_key, **kwargs):
+@click.pass_context
+def create_job(ctx, api_key, **kwargs):
     del_if_value_is_none(kwargs)
     jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
     command = jobs_commands.CreateJobCommand(api=jobs_api)
-    command.execute(kwargs)
+    job = command.execute(kwargs)
+    if job is not None:
+        ctx.invoke(list_logs, job_id=job["handle"], line=0, limit=100, follow=True, api_key=api_key)
 
 
-@jobs_group.command("log", help="List job logs")
+@jobs_group.command("logs", help="List job logs")
 @click.option(
     "--jobId",
     "job_id",
     required=True
 )
+@click.option(
+    "--line",
+    "line",
+    required=False,
+    default=0
+)
+@click.option(
+    "--limit",
+    "limit",
+    required=False,
+    default=10000
+)
+@click.option(
+    "--follow",
+    "follow",
+    required=False,
+    default=False
+)
 @api_key_option
-def list_logs(job_id, api_key=None):
+def list_logs(job_id, line, limit, follow, api_key=None):
     logs_api = client.API(config.CONFIG_LOG_HOST, api_key=api_key)
     command = jobs_commands.JobLogsCommand(api=logs_api)
-    command.execute(job_id)
+    command.execute(job_id, line, limit, follow)
 
 
 @jobs_group.group("artifacts", help="Manage jobs' artifacts", cls=ClickGroup)

paperspace/cli/machines.py

@@ -18,7 +18,7 @@
 )
 
 
-@cli.group("machines", help="Manage machines", cls=ClickGroup)
+@cli.group(["machines", "machine"], help="Manage machines", cls=ClickGroup)
 def machines_group():
     pass
 

paperspace/cli/models.py

@@ -6,7 +6,7 @@
 from paperspace.commands import models as models_commands
 
 
-@cli.group("models", help="Manage models", cls=common.ClickGroup)
+@cli.group(["models", "model"], help="Manage models", cls=common.ClickGroup)
 def models_group():
     pass
 

paperspace/cli/projects.py

@@ -4,7 +4,7 @@
 from paperspace.cli import common
 
 
-@cli.group("projects", help="Manage projects", cls=common.ClickGroup)
+@cli.group(["projects", "project"], help="Manage projects", cls=common.ClickGroup)
 def projects_group():
     pass
 

paperspace/commands/__init__.py

@@ -1,17 +0,0 @@
-from collections import OrderedDict
-
-from paperspace import logger
-
-
-class CommandBase(object):
-    def __init__(self, api=None, logger_=logger):
-        self.api = api
-        self.logger = logger_
-
-    def _print_dict_recursive(self, input_dict, indent=0, tabulator="  "):
-        for key, val in input_dict.items():
-            self.logger.log("%s%s:" % (tabulator * indent, key))
-            if type(val) is dict:
-                self._print_dict_recursive(OrderedDict(val), indent + 1)
-            else:
-                self.logger.log("%s%s" % (tabulator * (indent + 1), val))

paperspace/commands/common.py

@@ -1,4 +1,5 @@
 import pydoc
+from collections import OrderedDict
 
 import terminaltables
 
@@ -11,6 +12,13 @@ def __init__(self, api=None, logger_=logger):
         self.api = api
         self.logger = logger_
 
+    def _print_dict_recursive(self, input_dict, indent=0, tabulator="  "):
+        for key, val in input_dict.items():
+            self.logger.log("%s%s:" % (tabulator * indent, key))
+            if type(val) is dict:
+                self._print_dict_recursive(OrderedDict(val), indent + 1)
+            else:
+                self.logger.log("%s%s" % (tabulator * (indent + 1), val))
 
 class ListCommand(CommandBase):
     @property