Workspace ignore files

paperspace/cli/experiments.py

@@ -58,6 +58,11 @@ def common_experiments_create_options(f):
  "workspaceUrl",
  help="Project git repository url",
  ),
+ click.option(
+ "--ignoreFiles",
+ "ignore_files",
+ help="Ignore certain files from uploading"
+ ),
  click.option(
  "--workingDirectory",
  "workingDirectory",

paperspace/cli/jobs.py

@@ -75,7 +75,8 @@ def list_jobs(api_key, **filters):
 @click.option("--workspace", "workspace", required=False, help="Path to workspace directory")
 @click.option("--workspaceArchive", "workspaceArchive", required=False, help="Path to workspace archive")
 @click.option("--workspaceUrl", "workspaceUrl", required=False, help="Project git repository url")
-@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment", )
+@click.option("--workingDirectory", "workingDirectory", help="Working directory for the experiment")
+@click.option("--ignoreFiles", "ignore_files", help="Ignore certain files from uploading")
 @click.option("--experimentId", "experimentId", help="Experiment Id")
 @click.option("--jobEnv", "envVars", type=json_string, help="Environmental variables ")
 @click.option("--useDockerfile", "useDockerfile", help="Flag: using Dockerfile")

paperspace/client.py

@@ -30,27 +30,27 @@ def get_path(self, url):
 
  def post(self, url, json=None, params=None, files=None):
  path = self.get_path(url)
- response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
  logger.debug("POST request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
- .format(response.url, self.headers, json, params))
+ .format(path, self.headers, json, params))
+ response = requests.post(path, json=json, params=params, headers=self.headers, files=files)
  logger.debug("Response status code: {}".format(response.status_code))
  logger.debug("Response content: {}".format(response.content))
  return response
 
  def put(self, url, json=None, params=None):
  path = self.get_path(url)
- response = requests.put(path, json=json, params=params, headers=self.headers)
  logger.debug("PUT request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
- .format(response.url, self.headers, json, params))
+ .format(path, self.headers, json, params))
+ response = requests.put(path, json=json, params=params, headers=self.headers)
  logger.debug("Response status code: {}".format(response.status_code))
  logger.debug("Response content: {}".format(response.content))
  return response
 
  def get(self, url, json=None, params=None):
  path = self.get_path(url)
- response = requests.get(path, params=params, headers=self.headers, json=json)
  logger.debug("GET request sent to: {} \n\theaders: {}\n\tjson: {}\n\tparams: {}"
- .format(response.url, self.headers, json, params))
+ .format(path, self.headers, json, params))
+ response = requests.get(path, params=params, headers=self.headers, json=json)
  logger.debug("Response status code: {}".format(response.status_code))
  logger.debug("Response content: {}".format(response.content))
  return response

paperspace/workspace.py

@@ -6,8 +6,8 @@
 import progressbar
 import requests
 from requests_toolbelt.multipart import encoder
-from paperspace import logger as default_logger
 
+from paperspace import logger as default_logger
 from paperspace.exceptions import S3UploadFailedError, PresignedUrlUnreachableError, \
  PresignedUrlAccessDeniedError, PresignedUrlConnectionError, ProjectAccessDeniedError, \
  PresignedUrlMalformedResponseError, PresignedUrlError
@@ -24,10 +24,11 @@ def __init__(self, experiments_api, logger=None):
  self.logger = logger or default_logger
 
  @staticmethod
- def _retrieve_file_paths(dir_name):
+ def _retrieve_file_paths(dir_name, ignored_files=''):
  # setup file paths variable
  file_paths = {}
- exclude = ['.git', '.idea', '.pytest_cache']
+ ignored_files = ignored_files.split(',')
+ exclude = ['.git', '.idea', '.pytest_cache'] + ignored_files
  # Read all directory, subdirectories and file lists
  for root, dirs, files in os.walk(dir_name, topdown=True):
  dirs[:] = [d for d in dirs if d not in exclude]
@@ -38,11 +39,12 @@ def _retrieve_file_paths(dir_name):
  file_path = filename
  else:
  file_path = os.path.join(os.path.relpath(root, dir_name), filename)
- file_paths[file_path] = os.path.join(root, filename)
+ if file_path not in ignored_files:
+ file_paths[file_path] = os.path.join(root, filename)
 
  return file_paths
 
- def _zip_workspace(self, workspace_path):
+ def _zip_workspace(self, workspace_path, ignore_files):
  if not workspace_path:
  workspace_path = '.'
  zip_file_name = os.path.basename(os.getcwd()) + '.zip'
@@ -54,8 +56,7 @@ def _zip_workspace(self, workspace_path):
  if os.path.exists(zip_file_path):
  self.logger.log('Removing existing archive')
  os.remove(zip_file_path)
-
- file_paths = self._retrieve_file_paths(workspace_path)
+ file_paths = self._retrieve_file_paths(workspace_path, ignore_files)
 
  self.logger.log('Creating zip archive: %s' % zip_file_name)
  zip_file = zipfile.ZipFile(zip_file_path, 'w')
@@ -86,6 +87,8 @@ def upload_workspace(self, input_data):
  workspace_url = input_data.get('workspaceUrl')
  workspace_path = input_data.get('workspace')
  workspace_archive = input_data.get('workspaceArchive')
+ ignore_files = input_data.get('ignore_files')
+
  if (workspace_archive and workspace_path) or (workspace_archive and workspace_url) or (
  workspace_path and workspace_url):
  raise click.UsageError("Use either:\n\t--workspaceUrl to point repository URL"
@@ -96,10 +99,13 @@ def upload_workspace(self, input_data):
  if workspace_url:
  return # nothing to do
 
+ # Should be removed as soon it won't be necessary by PS_API
+ if workspace_path == 'none':
+ return 'none'
  if workspace_archive:
  archive_path = os.path.abspath(workspace_archive)
  else:
- archive_path = self._zip_workspace(workspace_path)
+ archive_path = self._zip_workspace(workspace_path, ignore_files)
 
  file_name = os.path.basename(archive_path)
  project_handle = input_data['projectHandle']

tests/unit/test_workspace.py

@@ -10,12 +10,11 @@
 MOCK_BUCKET_NAME = 'bucket_name'
 MOCK_OBJECT_KEY = 'object_key'
 mock_upload_data = {
- "bucket_name": MOCK_BUCKET_NAME,
- "fields": {
- "key": MOCK_OBJECT_KEY
- }
+ "bucket_name": MOCK_BUCKET_NAME,
+ "fields": {
+ "key": MOCK_OBJECT_KEY
  }
-
+}
 
 mock_upload_response = {
  "message": "success",