Create working directory (#767)

* Create working directory if it does not exist before * fix flux spawner * always create working directory

Author: jan-janssen

executorlib/standalone/interactive/spawner.py

@@ -1,3 +1,4 @@
+import os
 import subprocess
 from abc import ABC, abstractmethod
 from typing import Optional
@@ -93,6 +94,8 @@ def bootup(
  Args:
  command_lst (list[str]): The command list to execute.
  """
+ if self._cwd is not None:
+ os.makedirs(self._cwd, exist_ok=True)
  self._process = subprocess.Popen(
  args=self.generate_command(command_lst=command_lst),
  cwd=self._cwd,

executorlib/task_scheduler/file/queue_spawner.py

@@ -61,7 +61,7 @@ def execute_with_pysqa(
  else:
  folder = command[-1].split("_i.h5")[0]
  cwd = os.path.join(cache_directory, folder)
-  os.makedirs(cwd, exist_ok=True)
+ os.makedirs(cwd, exist_ok=True)
  submit_kwargs = {
  "command": " ".join(command),
  "dependency_list": [str(qid) for qid in task_dependent_lst],

executorlib/task_scheduler/file/subprocess_spawner.py

@@ -56,6 +56,8 @@ def execute_in_subprocess(
  if resource_dict is None:
  resource_dict = {}
  cwd = resource_dict.get("cwd", cache_directory)
+ if cwd is not None:
+ os.makedirs(cwd, exist_ok=True)
  return subprocess.Popen(command, universal_newlines=True, cwd=cwd)
 
 

executorlib/task_scheduler/interactive/fluxspawner.py

@@ -113,6 +113,7 @@ def bootup(
  jobspec.setattr_shell_option("pmi", self._pmi_mode)
  if self._cwd is not None:
  jobspec.cwd = self._cwd
+ os.makedirs(self._cwd, exist_ok=True)
  if self._flux_log_files and self._cwd is not None:
  jobspec.stderr = os.path.join(self._cwd, "flux.err")
  jobspec.stdout = os.path.join(self._cwd, "flux.out")