Merge remote-tracking branch 'origin/master' into experiment-logs-PS-9868

# Conflicts:
#	paperspace/cli/experiments.py

Author: kossak

.gitignore

@@ -104,3 +104,5 @@ ENV/
 .vscode/*
 
 .idea
+paperspace-python.zip
+*.env

README.md

@@ -1,7 +1,7 @@
 Paperspace Python
 =================
 
-Release 0.2.0a1
+Release 0.2.0a2
 
 See [releasenotes.md](https://github.com/Paperspace/paperspace-python/blob/master/releasenotes.md) for details on the current release, as well as release history.
 
@@ -17,8 +17,20 @@ Getting Started
     To install/update prerelease (Alpha/Beta) version version of paperspace-python, use:
 
     `pip install -U --pre paperspace`
-
-3. Download your api key by executing the following:
+3. Enable autocomplete:
+    
+    Add following to your `.bashrc` (or `.zshrc`) to enable autocomplete anytime you activate your shell.
+     
+    `eval $(_PAPERSPACE_PYTHON_COMPLETE=source paperspace-python)`
+    
+    For other shell (eg. `zsh` or `fish`), use proper version of `source` (`source_zsh` and `source-fish` respectively)
+     
+     Alternatively, you can create activation script by:
+     
+     `(_PAPERSPACE_PYTHON_COMPLETE=source paperspace-python > ~/paperspace_complete.sh`
+
+    More: https://click.palletsprojects.com/en/7.x/bashcomplete/
+4. Download your api key by executing the following:
 
     `paperspace-python login`
 
@@ -33,7 +45,7 @@ Getting Started
 
     `paperspace-python logout`
 
-4. Run the sample script hello.py using Python:
+5. Run the sample script hello.py using Python:
 
     `python hello.py`
 
@@ -49,7 +61,7 @@ Getting Started
 
    Note: the source is modified before transfer to the job cluster in order to remove imported `paperspace` references.
 
-5. Use paperspace-python to run a python script remotely:
+6. Use paperspace-python to run a python script remotely:
 
     `paperspace-python run myscript.py`
 

paperspace/cli/__init__.py

@@ -5,6 +5,7 @@
 import paperspace.cli.auth
 import paperspace.cli.deployments
 import paperspace.cli.experiments
+import paperspace.cli.hyperparameters
 import paperspace.cli.jobs
 import paperspace.cli.machines
 import paperspace.cli.models

paperspace/cli/auth.py

@@ -1,38 +1,48 @@
 import click
 
-from paperspace import client, config
+from paperspace import logger
+from paperspace.cli import common
 from paperspace.cli.cli import cli
-from paperspace.cli.validators import validate_email
 from paperspace.commands import login as login_commands
 
+LOGIN_DEPRECATION_MESSAGE = """The login command is currently disabled for logging in using `--email` and `--password`.
 
-@cli.command("login", help="Log in with email and password")
+Instead, obtain an API Key from https://www.paperspace.com/console/account/api.
+
+Then use the `apiKey` command to save your API Key locally.
+
+Visit the docs @ https://docs.paperspace.com for more info!"""
+
+
+@cli.command("login", help=LOGIN_DEPRECATION_MESSAGE, hidden=True)
 @click.option(
     "--email",
     "email",
-    required=True,
-    callback=validate_email,
     help="Email used to create Paperspace account",
 )
 @click.option(
     "--password",
     "password",
-    prompt=True,
-    hide_input=True,
     help="Password used to create Paperspace account",
 )
 @click.option(
     "--apiTokenName",
     "api_token_name",
     help="Name of api token used to log in",
 )
-def login(email, password, api_token_name):
-    machines_api = client.API(config.CONFIG_HOST)
-    command = login_commands.LogInCommand(api=machines_api)
-    command.execute(email, password, api_token_name)
+def login(**kwargs):
+    logger.warning(LOGIN_DEPRECATION_MESSAGE)
 
 
 @cli.command("logout", help="Log out / remove apiKey from config file")
 def logout():
     command = login_commands.LogOutCommand()
     command.execute()
+
+
+@cli.command("apiKey", help="Save your api key")
+@click.argument("api_key", required=False, callback=common.prompt_for_secret("Enter your API Key: "))
+def save_api_key(api_key):
+    command = login_commands.SetApiKeyCommand()
+    api_key = api_key.strip()
+    command.execute(api_key)

paperspace/cli/cli.py

@@ -1,9 +1,12 @@
 import click
+import click_completion
 
 from paperspace import config
 from paperspace.cli import common
 from paperspace.commands import login as login_commands
 
+click_completion.init()
+
 
 @click.group(cls=common.ClickGroup, **config.HELP_COLORS_DICT)
 def cli():

paperspace/cli/common.py

@@ -1,3 +1,5 @@
+import getpass
+
 import click
 from click_didyoumean import DYMMixin
 from click_help_colors import HelpColorsGroup
@@ -18,3 +20,13 @@ def del_if_value_is_none(dict_):
 
 class ClickGroup(DYMMixin, HelpColorsGroup):
     pass
+
+
+def prompt_for_secret(prompt):
+    def callback_fun(ctx, param, value):
+        if value is None:
+            value = getpass.getpass(prompt)
+
+        return value
+
+    return callback_fun

paperspace/cli/deployments.py

@@ -22,7 +22,7 @@ def deployments():
 )
 
 DEPLOYMENT_MACHINE_TYPES = ("G1", "G6", "G12",
-    "K80", "P100", "V100")
+    "K80", "P100", "GV100")
 
 @deployments.command("create", help="Create new deployment")
 @click.option(

paperspace/cli/experiments.py

@@ -108,11 +108,11 @@ def common_experiments_create_options(f):
 def common_experiment_create_multi_node_options(f):
     options = [
         click.option(
-            "--experimentTypeId",
-            "experimentTypeId",
+            "--experimentType",
+            "experimentType",
             type=ChoiceType(MULTI_NODE_EXPERIMENT_TYPES_MAP, case_sensitive=False),
             required=True,
-            help="Experiment Type ID",
+            help="Experiment Type",
         ),
         click.option(
             "--workerContainer",
@@ -256,7 +256,7 @@ def create_multi_node(api_key, **kwargs):
 @common_experiments_create_options
 @common_experiments_create_single_node_options
 def create_single_node(api_key, **kwargs):
-    kwargs["experimentTypeId"] = constants.ExperimentType.SINGLE_NODE
+    kwargs["experimentType"] = constants.ExperimentType.SINGLE_NODE
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     command = experiments_commands.CreateExperimentCommand(api=experiments_api)
@@ -283,7 +283,7 @@ def create_and_start_multi_node(ctx, api_key, show_logs, **kwargs):
 @show_logs_option
 @click.pass_context
 def create_and_start_single_node(ctx, api_key, show_logs, **kwargs):
-    kwargs["experimentTypeId"] = constants.ExperimentType.SINGLE_NODE
+    kwargs["experimentType"] = constants.ExperimentType.SINGLE_NODE
     del_if_value_is_none(kwargs)
     experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
     command = experiments_commands.CreateAndStartExperimentCommand(api=experiments_api)

paperspace/cli/hyperparameters.py

@@ -0,0 +1,136 @@
+import functools
+
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import ClickGroup
+from paperspace.commands import hyperparameters as hyperparameters_commands
+
+
+@cli.group("hyperparameters", help="Manage hyperparameters", cls=ClickGroup)
+def hyperparameters_group():
+    pass
+
+
+def common_hyperparameter_create_options(f):
+    options = [
+        click.option(
+            "--name",
+            "name",
+            required=True,
+        ),
+        click.option(
+            "--projectId",
+            "projectHandle",
+            required=True,
+        ),
+        click.option(
+            "--tuningCommand",
+            "tuningCommand",
+            required=True,
+        ),
+        click.option(
+            "--workerContainer",
+            "workerContainer",
+            required=True,
+        ),
+        click.option(
+            "--workerMachineType",
+            "workerMachineType",
+            required=True,
+        ),
+        click.option(
+            "--workerCommand",
+            "workerCommand",
+            required=True,
+        ),
+        click.option(
+            "--workerCount",
+            "workerCount",
+            required=True,
+            type=int,
+        ),
+        click.option(
+            "--serverRegistryUsername",
+            "hyperparameterServerRegistryUsername",
+        ),
+        click.option(
+            "--serverRegistryPassword",
+            "hyperparameterServerRegistryPassword",
+        ),
+        click.option(
+            "--serverContainerUser",
+            "hyperparameterServerContainerUser",
+        ),
+    ]
+    return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
+
+
+@hyperparameters_group.command("create", help="Create hyperparameter")
+@common_hyperparameter_create_options
+@common.api_key_option
+def create_hyperparameter(api_key, **hyperparameter):
+    common.del_if_value_is_none(hyperparameter)
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.CreateHyperparameterCommand(api=hyperparameters_api)
+    command.execute(hyperparameter)
+
+
+@hyperparameters_group.command("createAndStart", help="Create hyperparameter")
+@common_hyperparameter_create_options
+@common.api_key_option
+def create_and_start_hyperparameter(api_key, **hyperparameter):
+    common.del_if_value_is_none(hyperparameter)
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.CreateAndStartHyperparameterCommand(api=hyperparameters_api)
+    command.execute(hyperparameter)
+
+
+@hyperparameters_group.command("list", help="List hyperparameters")
+@common.api_key_option
+def list_hyperparameters(api_key):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.ListHyperparametersCommand(api=hyperparameters_api)
+    command.execute()
+
+
+# TODO: 'unhidden' command and test it when api is updated to support deleting hyperparameters
+@hyperparameters_group.command("delete", help="Delete hyperparameter", hidden=True)
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def delete_hyperparameter(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.DeleteHyperparameterCommand(api=hyperparameters_api)
+    command.execute(id_)
+
+
+@hyperparameters_group.command("details", help="Show details of hyperparameter")
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def get_hyperparameter_details(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.HyperparameterDetailsCommand(api=hyperparameters_api)
+    command.execute(id_)
+
+
+@hyperparameters_group.command("start", help="Start hyperparameter tuning")
+@click.option(
+    "--id",
+    "id_",
+    required=True,
+)
+@common.api_key_option
+def start_hyperparameter_tuning(api_key, id_):
+    hyperparameters_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=api_key)
+    command = hyperparameters_commands.HyperparameterStartCommand(api=hyperparameters_api)
+    command.execute(id_)

paperspace/client.py

@@ -54,3 +54,12 @@ def get(self, url, json=None, params=None):
         logger.debug("Response status code: {}".format(response.status_code))
         logger.debug("Response content: {}".format(response.content))
         return response
+
+    def delete(self, url, json=None, params=None):
+        path = self.get_path(url)
+        response = requests.delete(path, params=params, headers=self.headers, json=json)
+        logger.debug("DELETE request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
+                     .format(response.url, self.headers, json, params))
+        logger.debug("Response status code: {}".format(response.status_code))
+        logger.debug("Response content: {}".format(response.content))
+        return response

paperspace/commands/experiments.py

@@ -113,7 +113,7 @@ def _get_objects(self, response, kwargs):
 
 
 def _make_details_table(experiment):
-    if experiment["experimentTypeId"] == constants.ExperimentType.SINGLE_NODE:
+    if experiment["experimentType"] == constants.ExperimentType.SINGLE_NODE:
         data = (
             ("Name", experiment["templateHistory"]["params"].get("name")),
             ("ID", experiment.get("handle")),
@@ -128,7 +128,7 @@ def _make_details_table(experiment):
             ("Model Type", experiment["templateHistory"]["params"].get("modelType")),
             ("Model Path", experiment["templateHistory"]["params"].get("modelPath")),
         )
-    elif experiment["experimentTypeId"] in (constants.ExperimentType.GRPC_MULTI_NODE,
+    elif experiment["experimentType"] in (constants.ExperimentType.GRPC_MULTI_NODE,
                                             constants.ExperimentType.MPI_MULTI_NODE):
         data = (
             ("Name", experiment["templateHistory"]["params"].get("name")),
@@ -138,7 +138,7 @@ def _make_details_table(experiment):
             ("Cluster ID", experiment["templateHistory"]["params"].get("clusterId")),
             ("Experiment Env", experiment["templateHistory"]["params"].get("experimentEnv")),
             ("Experiment Type",
-             constants.ExperimentType.get_type_str(experiment["templateHistory"]["params"].get("experimentTypeId"))),
+             constants.ExperimentType.get_type_str(experiment["templateHistory"]["params"].get("experimentType"))),
             ("Model Type", experiment["templateHistory"]["params"].get("modelType")),
             ("Model Path", experiment["templateHistory"]["params"].get("modelPath")),
             ("Parameter Server Command", experiment["templateHistory"]["params"].get("parameter_server_command")),
@@ -156,7 +156,7 @@ def _make_details_table(experiment):
             ("Workspace URL", experiment["templateHistory"]["params"].get("workspaceUrl")),
         )
     else:
-        raise ValueError("Wrong experiment type: {}".format(experiment["experimentTypeId"]))
+        raise ValueError("Wrong experiment type: {}".format(experiment["experimentType"]))
 
     ascii_table = terminaltables.AsciiTable(data)
     table_string = ascii_table.table