Defaults: Allow out-of-tree worker modules

Attempt to load worker modules distributed with ClusterShell as well as custom worker modules distributed outside of ClusterShell. Signed-off-by: Matt Ezell <ezellma@ornl.gov>

Author: mattaezell

doc/man/man1/clush.1

@@ -281,7 +281,7 @@ limit time to connect to a node
 limit time for command to run on the node
 .TP
 .BI \-R \ WORKER\fP,\fB \ \-\-worker\fB= WORKER
-worker name to use for connection (\fBexec\fP, \fBssh\fP, \fBrsh\fP, \fBpdsh\fP), default is \fBssh\fP
+worker name to use for connection (\fBexec\fP, \fBssh\fP, \fBrsh\fP, \fBpdsh\fP, or the name of a Python worker module), default is \fBssh\fP
 .TP
 .BI \-\-remote\fB= REMOTE
 whether to enable remote execution: in tree mode, \(aqyes\(aq forces connections to the leaf nodes for execution, \(aqno\(aq establishes connections up to the leaf parent nodes for execution (default is \(aqyes\(aq)

doc/sphinx/tools/clush.rst

@@ -633,6 +633,8 @@ By default, ClusterShell supports the following worker identifiers:
  installed; doesn't provide write support (eg. you cannot ``cat file | clush
  --worker pdsh``); it is primarily an 1-to-n worker example.
 
+Worker modules distributed outside of ClusterShell are also supported by
+specifying the case-sensitive full Python module name of a worker module.
 
 .. [#] LLNL parallel remote shell utility
  (https://computing.llnl.gov/linux/pdsh.html)

lib/ClusterShell/Defaults.py

@@ -55,18 +55,31 @@ def _load_workerclass(workername):
  """
  Return the class pointer matching `workername`.
 
+ This can be the 'short' name (such as `ssh`) or a fully-qualified
+ module path (such as ClusterShell.Worker.Ssh).
+
  The module is loaded if not done yet.
  """
- modname = "ClusterShell.Worker.%s" % workername.capitalize()
 
+ # First try the worker name as a module under ClusterShell.Worker,
+ # but if that fails, try the worker name directly
+ try:
+ modname = "ClusterShell.Worker.%s" % workername.capitalize()
+ _import_module(modname)
+ except ImportError:
+ modname = workername
+ _import_module(modname)
+
+ # Get the class pointer
+ return sys.modules[modname].WORKER_CLASS
+
+def _import_module(modname):
+ """Import a python module if not done yet."""
  # Iterate over a copy of sys.modules' keys to avoid RuntimeError
  if modname.lower() not in [mod.lower() for mod in list(sys.modules)]:
  # Import module if not yet loaded
  __import__(modname)
 
- # Get the class pointer
- return sys.modules[modname].WORKER_CLASS
-
 def _local_workerclass(defaults):
  """Return default local worker class."""
  return _load_workerclass(defaults.local_workername)

tests/DefaultsTest.py

@@ -3,10 +3,14 @@
 
 """Unit test for ClusterShell.Defaults"""
 
+import os
+import sys
+import shutil
+
 from textwrap import dedent
 import unittest
 
-from TLib import make_temp_file
+from TLib import make_temp_file, make_temp_dir
 
 from ClusterShell.Defaults import Defaults, _task_print_debug
 
@@ -98,6 +102,21 @@ def test_004_workerclass(self):
  self.assertTrue(task.default("distant_worker") is WorkerSsh)
  task_terminate()
 
+ dname = make_temp_dir()
+ modfile = open(os.path.join(dname, 'OutOfTree.py'), 'w')
+ modfile.write(dedent("""
+ class OutOfTreeWorker(object):
+ pass
+ WORKER_CLASS = OutOfTreeWorker"""))
+ modfile.flush()
+ modfile.close()
+ sys.path.append(dname)
+ self.defaults.distant_workername = 'OutOfTree'
+ task = task_self(self.defaults)
+ self.assertTrue(task.default("distant_worker").__name__ is 'OutOfTreeWorker')
+ task_terminate()
+ shutil.rmtree(dname, ignore_errors=True)
+
  def test_005_misc_value_errors(self):
  """test Defaults misc value errors"""
  task_terminate()