Command run restored

paperspace/cli/__init__.py

@@ -10,6 +10,7 @@
 import paperspace.cli.machines
 import paperspace.cli.models
 import paperspace.cli.projects
+import paperspace.cli.run
 
 
 def show(self, file=None):

paperspace/cli/jobs.py

@@ -1,3 +1,5 @@
+import functools
+
 import click
 
 from paperspace import client, config
@@ -64,28 +66,35 @@ def list_jobs(api_key, **filters):
  command.execute(filters=filters)
 
 
+def common_jobs_create_options(f):
+ options = [
+ click.option("--name", "name", help="Job name", required=True),
+ click.option("--machineType", "machineType", help="Virtual machine type"),
+ click.option("--container", "container", default="paperspace/tensorflow-python", help="Docker container"),
+ click.option("--command", "command", help="Job command/entrypoint"),
+ click.option("--ports", "ports", help="Mapped ports"),
+ click.option("--isPublic", "isPublic", help="Flag: is job public"),
+ click.option("--workspace", "workspace", required=False, help="Path to workspace directory"),
+ click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive"),
+ click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url"),
+ click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", ),
+ click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading"),
+ click.option("--experimentId", "experimentId", help="Experiment Id"),
+ click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables "),
+ click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile"),
+ click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible"),
+ click.option("--project", "project", help="Project name"),
+ click.option("--projectId", "projectHandle", help="Project ID"),
+ click.option("--startedByUserId", "startedByUserId", help="User ID"),
+ click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile"),
+ click.option("--registryUsername", "registryUsername", help="Docker registry username"),
+ click.option("--registryPassword", "registryPassword", help="Docker registry password"),
+ ]
+ return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
+
+
 @jobs_group.command("create", help="Create job")
-@click.option("--name", "name", help="Job name", required=True)
-@click.option("--machineType", "machineType", help="Virtual machine type")
-@click.option("--container", "container", help="Docker container")
-@click.option("--command", "command", help="Job command/entrypoint")
-@click.option("--ports", "ports", help="Mapped ports")
-@click.option("--isPublic", "isPublic", help="Flag: is job public")
-@click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
-@click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
-@click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
-@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
-@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
-@click.option("--experimentId", "experimentId", help="Experiment Id")
-@click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
-@click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")
-@click.option("--isPreemptible", "isPreemptible", help="Flag: isPreemptible")
-@click.option("--project", "project", help="Project name")
-@click.option("--projectId", "projectHandle", help="Project ID", required=True)
-@click.option("--startedByUserId", "startedByUserId", help="User ID")
-@click.option("--relDockerfilePath", "relDockerfilePath", help="Relative path to Dockerfile")
-@click.option("--registryUsername", "registryUsername", help="Docker registry username")
-@click.option("--registryPassword", "registryPassword", help="Docker registry password")
+@common_jobs_create_options
 @api_key_option
 def create_job(api_key, **kwargs):
  del_if_value_is_none(kwargs)

paperspace/cli/run.py

@@ -0,0 +1,23 @@
+import click
+
+from paperspace import client, config
+from paperspace.cli import common
+from paperspace.cli.cli import cli
+from paperspace.cli.common import del_if_value_is_none
+from paperspace.cli.jobs import common_jobs_create_options
+from paperspace.commands.run import RunCommand
+from paperspace.constants import RunMode
+
+
+@cli.command("new-run")
+@click.option("-c", "--python-command", "mode", flag_value=RunMode.RUN_MODE_PYTHON_COMMAND)
+@click.option("-m", "--module", "mode", flag_value=RunMode.RUN_MODE_PYTHON_MODULE)
+@click.option("-s", "--shell", "mode", flag_value=RunMode.RUN_MODE_SHELL_COMMAND)
+@common_jobs_create_options
+@click.argument("script", nargs=-1)
+@common.api_key_option
+def run(api_key, **kwargs):
+ del_if_value_is_none(kwargs)
+ jobs_api = client.API(config.CONFIG_HOST, api_key=api_key)
+ command = RunCommand(api=jobs_api)
+ command.execute(**kwargs)

paperspace/client.py

@@ -28,11 +28,11 @@ def get_path(self, url):
  template = "{}{}" if url.startswith("/") else "{}/{}"
  return template.format(api_url, url)
 
- def post(self, url, json=None, params=None, files=None):
+ def post(self, url, json=None, params=None, files=None, data=None):
  path = self.get_path(url)
- logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
- .format(path, self.headers, json, params))
- response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
+ logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}\n\tdata: {}"
+ .format(path, self.headers, json, params, data))
+ response = requests.post(path, json=json, params=params, headers=self.headers, files=files, data=data)
  logger.debug("Response status code: {}".format(response.status_code))
  logger.debug("Response content: {}".format(response.content))
  return response

paperspace/commands/experiments.py

@@ -30,7 +30,7 @@ def _log_create_experiment(self, response, success_msg_template, error_msg):
 class CreateExperimentCommand(ExperimentCommand):
 
  def execute(self, json_):
- workspace_url = self._workspace_handler.upload_workspace(json_)
+ workspace_url = self._workspace_handler.handle(json_)
  if workspace_url:
  json_['workspaceUrl'] = workspace_url
 
@@ -43,7 +43,7 @@ def execute(self, json_):
 
 class CreateAndStartExperimentCommand(ExperimentCommand):
  def execute(self, json_):
- workspace_url = self._workspace_handler.upload_workspace(json_)
+ workspace_url = self._workspace_handler.handle(json_)
  if workspace_url:
  json_['workspaceUrl'] = workspace_url
 

paperspace/commands/jobs.py

@@ -3,11 +3,10 @@
 import terminaltables
 from click import style
 
-from paperspace import config, client
 from paperspace.commands import common
 from paperspace.exceptions import BadResponseError
 from paperspace.utils import get_terminal_lines
-from paperspace.workspace import S3WorkspaceHandler
+from paperspace.workspace import WorkspaceHandler
 
 
 class JobsCommandBase(common.CommandBase):
@@ -126,22 +125,41 @@ def _make_table(self, logs, table, table_data):
 class CreateJobCommand(JobsCommandBase):
  def __init__(self, workspace_handler=None, **kwargs):
  super(CreateJobCommand, self).__init__(**kwargs)
- experiments_api = client.API(config.CONFIG_EXPERIMENTS_HOST, api_key=kwargs.get('api_key'))
- self._workspace_handler = workspace_handler or S3WorkspaceHandler(experiments_api=experiments_api,
- logger=self.logger)
+ self._workspace_handler = workspace_handler or WorkspaceHandler(logger=self.logger)
 
  def execute(self, json_):
  url = "/jobs/createJob/"
-
- workspace_url = self._workspace_handler.upload_workspace(json_)
+ files = None
+ workspace_url = self._workspace_handler.handle(json_)
  if workspace_url:
- json_['workspaceFileName'] = workspace_url
- json_['projectId'] = json_.get('projectId', json_.get('projectHandle'))
- response = self.api.post(url, json_)
+ if self._workspace_handler.archive_path:
+ archive_basename = self._workspace_handler.archive_basename
+ json_["workspaceFileName"] = archive_basename
+ self.api.headers["Content-Type"] = "multipart/form-data"
+ files = self._get_files_dict(workspace_url)
+ else:
+ json_["workspaceFileName"] = workspace_url
+
+ self.set_project(json_)
+
+ response = self.api.post(url, params=json_, files=files)
  self._log_message(response,
  "Job created",
  "Unknown error while creating job")
 
+ @staticmethod
+ def _get_files_dict(workspace_url):
+ files = {"file": open(workspace_url, "rb")}
+ return files
+
+ @staticmethod
+ def set_project(json_):
+ project_id = json_.get("projectId", json_.get("projectHandle"))
+ if not project_id:
+ json_["project"] = "paperspace-python"
+ else:
+ json_["projectId"] = project_id
+
 
 class ArtifactsDestroyCommand(JobsCommandBase):
  def execute(self, job_id, files=None):

paperspace/commands/run.py

@@ -0,0 +1,57 @@
+import os
+import sys
+
+from paperspace import client, config, logger
+from paperspace.commands.jobs import CreateJobCommand
+from paperspace.constants import RunMode
+from paperspace.workspace import WorkspaceHandler
+
+
+class RunCommand(object):
+
+ def __init__(self, api=None, logger_=logger):
+ self.api = api
+ self.logger = logger_
+
+ @staticmethod
+ def _get_executor(mode, python_version=None):
+ python_version = python_version or str(sys.version_info[0]) # defaults locally running version
+ python_bin = 'python{v}'.format(v=python_version)
+ executors = {
+ RunMode.RUN_MODE_DEFAULT: python_bin,
+ RunMode.RUN_MODE_PYTHON_COMMAND: '{python} -c'.format(python=python_bin),
+ RunMode.RUN_MODE_SHELL_COMMAND: '',
+ RunMode.RUN_MODE_PYTHON_MODULE: '{python} -m'.format(python=python_bin),
+ }
+ return executors[mode]
+
+ @staticmethod
+ def _clear_script_name(script_name, mode):
+ if mode == RunMode.RUN_MODE_DEFAULT:
+ return os.path.basename(script_name)
+ return script_name
+
+ def _create_command(self, mode, script, python_version=None):
+ command_parts = []
+ executor = self._get_executor(mode, python_version)
+ if executor:
+ command_parts.append(executor)
+
+ script_name = self._clear_script_name(script[0], mode)
+ if script_name:
+ command_parts.append(script_name)
+
+ script_params = ' '.join(script[1:])
+ if script_params:
+ command_parts.append(script_params)
+
+ command = ' '.join(command_parts)
+ return command
+
+ def execute(self, mode=None, script=None, **json_):
+ mode = mode or RunMode.RUN_MODE_DEFAULT
+ command = self._create_command(mode, script)
+ json_['command'] = command
+
+ command = CreateJobCommand(api=self.api, workspace_handler=WorkspaceHandler())
+ command.execute(json_)

paperspace/constants.py

@@ -59,3 +59,10 @@ class Region(object):
  "C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10")
 
 BILLING_TYPES = ["hourly", "monthly"]
+
+
+class RunMode:
+ RUN_MODE_DEFAULT = 1
+ RUN_MODE_PYTHON_COMMAND = 2
+ RUN_MODE_SHELL_COMMAND = 3
+ RUN_MODE_PYTHON_MODULE = 4

paperspace/workspace.py

@@ -13,15 +13,34 @@
  PresignedUrlMalformedResponseError, PresignedUrlError
 
 
-class S3WorkspaceHandler:
- def __init__(self, experiments_api, logger=None):
+class MultipartEncoder(object):
+ def __init__(self, fields):
+ s3_encoder = encoder.MultipartEncoder(fields=fields)
+ self.monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
+
+ def get_monitor(self):
+ return self.monitor
+
+ @staticmethod
+ def _create_callback(encoder_obj):
+ bar = progressbar.ProgressBar(max_value=encoder_obj.len)
+
+ def callback(monitor):
+ bar.update(monitor.bytes_read)
+
+ return callback
+
+
+class WorkspaceHandler(object):
+ def __init__(self, logger=None):
  """
 
  :param experiments_api: paperspace.client.API
  :param logger: paperspace.logger
  """
- self.experiments_api = experiments_api
  self.logger = logger or default_logger
+ self.archive_path = None
+ self.archive_basename = None
 
  @staticmethod
  def _retrieve_file_paths(dir_name, ignored_files=None):
@@ -77,41 +96,57 @@ def _zip_workspace(self, workspace_path, ignore_files):
  self.logger.log('\nFinished creating archive: %s' % zip_file_name)
  return zip_file_path
 
- @staticmethod
- def _create_callback(encoder_obj):
- bar = progressbar.ProgressBar(max_value=encoder_obj.len)
+ def handle(self, input_data):
+  workspace_archive, workspace_path, workspace_url = self._validate_input(input_data)
+ ignore_files = input_data.get('ignore_files')
 
- def callback(monitor):
- bar.update(monitor.bytes_read)
+ if workspace_url:
+ return workspace_url # nothing to do
 
- return callback
+ # Should be removed as soon it won't be necessary by PS_API
+ if workspace_path == 'none':
+ return workspace_path
 
- def upload_workspace(self, input_data):
+ if workspace_archive:
+ archive_path = os.path.abspath(workspace_archive)
+ else:
+ self.logger.log('Archiving your working directory for upload as your experiment workspace...'
+ '(See https://docs.paperspace.com/gradient/experiments/run-experiments for more information.)')
+ archive_path = self._zip_workspace(workspace_path, ignore_files)
+ self.archive_path = archive_path
+ self.archive_basename = os.path.basename(archive_path)
+ return archive_path
+
+ @staticmethod
+ def _validate_input(input_data):
  workspace_url = input_data.get('workspaceUrl')
  workspace_path = input_data.get('workspace')
  workspace_archive = input_data.get('workspaceArchive')
- ignore_files = input_data.get('ignore_files')
 
  if (workspace_archive and workspace_path) or (workspace_archive and workspace_url) or (
  workspace_path and workspace_url):
  raise click.UsageError("Use either:\n\t--workspaceUrl to point repository URL"
  "\n\t--workspace to point on project directory"
  "\n\t--workspaceArchive to point on project .zip archive"
  "\n or neither to use current directory")
+ return workspace_archive, workspace_path, workspace_url
 
- if workspace_url:
- return # nothing to do
 
- # Should be removed as soon it won't be necessary by PS_API
- if workspace_path == 'none':
- return 'none'
- if workspace_archive:
- archive_path = os.path.abspath(workspace_archive)
- else:
- self.logger.log('Archiving your working directory for upload as your experiment workspace...'
- '(See https://docs.paperspace.com/gradient/experiments/run-experiments for more information.)')
- archive_path = self._zip_workspace(workspace_path, ignore_files)
+class S3WorkspaceHandler(WorkspaceHandler):
+ def __init__(self, experiments_api, logger=None):
+ """
 
+ :param experiments_api: paperspace.client.API
+ :param logger: paperspace.logger
+ """
+ super(S3WorkspaceHandler, self).__init__(logger=logger)
+ self.experiments_api = experiments_api
+
+ def handle(self, input_data):
+ workspace = super(S3WorkspaceHandler, self).handle(input_data)
+ if not self.archive_path:
+ return workspace
+ archive_path = workspace
  file_name = os.path.basename(archive_path)
  project_handle = input_data['projectHandle']
 
@@ -133,8 +168,7 @@ def _upload(self, archive_path, s3_upload_data):
  fields = OrderedDict(s3_upload_data['fields'])
  fields.update(files)
 
- s3_encoder = encoder.MultipartEncoder(fields=fields)
- monitor = encoder.MultipartEncoderMonitor(s3_encoder, callback=self._create_callback(s3_encoder))
+ monitor = MultipartEncoder(fields).get_monitor()
  s3_response = requests.post(s3_upload_data['url'], data=monitor, headers={'Content-Type': monitor.content_type})
  self.logger.debug("S3 upload response: {}".format(s3_response.headers))
  if not s3_response.ok: