Implement dependency mapping for environment modules, galaxy packages, and Conda.

Author: jmchilton

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -5,6 +5,9 @@
  abstractproperty,
 )
 
+import yaml
+
+from galaxy.util import listify
 from galaxy.util.dictifiable import Dictifiable
 
 from ..requirements import ToolRequirement
@@ -65,6 +68,72 @@ def _to_requirement(self, name, version=None):
  return ToolRequirement(name=name, type="package", version=version)
 
 
+class MappableDependencyResolver:
+ """Mix this into a ``DependencyResolver`` to allow mapping files.
+
+ Mapping files allow adapting generic requirements to specific local implementations.
+ """
+
+ def _setup_mapping(self, dependency_manager, **kwds):
+ mapping_files = dependency_manager.get_resolver_option(self, "mapping_file", explicit_resolver_options=kwds)
+ mappings = []
+ if mapping_files:
+ mapping_files = listify(mapping_files)
+ for mapping_file in mapping_files:
+ mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file))
+ self._mappings = mappings
+
+ @staticmethod
+ def _mapping_file_to_list(mapping_file):
+ with open(mapping_file, "r") as f:
+ raw_mapping = yaml.load(f)
+ return map(RequirementMapping.from_dict, raw_mapping)
+
+ def _expand_mappings(self, requirement):
+ for mapping in self._mappings:
+ if requirement.name == mapping.from_name:
+ if mapping.from_version is not None and mapping.from_version != requirement.version:
+ continue
+
+ requirement = requirement.copy()
+ requirement.name = mapping.to_name
+ if mapping.to_version is not None:
+ requirement.version = mapping.to_version
+
+ break
+
+ return requirement
+
+
+class RequirementMapping(object):
+
+ def __init__(self, from_name, from_version, to_name, to_version):
+ self.from_name = from_name
+ self.from_version = from_version
+ self.to_name = to_name
+ self.to_version = to_version
+
+ @staticmethod
+ def from_dict(raw_mapping):
+ from_raw = raw_mapping.get("from")
+ if isinstance(from_raw, dict):
+ from_name = from_raw.get("name")
+ from_version = str(from_raw.get("version"))
+ else:
+ from_name = from_raw
+ from_version = None
+
+ to_raw = raw_mapping.get("to")
+ if isinstance(to_raw, dict):
+ to_name = to_raw.get("name", from_name)
+ to_version = str(to_raw.get("version"))
+ else:
+ to_name = to_raw
+ to_version = None
+
+ return RequirementMapping(from_name, from_version, to_name, to_version)
+
+
 class SpecificationAwareDependencyResolver:
  """Mix this into a :class:`DependencyResolver` to implement URI specification matching.
 

lib/galaxy/tools/deps/resolvers/conda.py

@@ -29,6 +29,7 @@
  DependencyResolver,
  InstallableDependencyResolver,
  ListableDependencyResolver,
+ MappableDependencyResolver,
  MultipleDependencyResolver,
  NullDependency,
  SpecificationPatternDependencyResolver,
@@ -42,7 +43,7 @@
 log = logging.getLogger(__name__)
 
 
-class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver):
+class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
  dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
  resolver_type = "conda"
  config_options = {
@@ -57,6 +58,7 @@ class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, Li
  _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
 
  def __init__(self, dependency_manager, **kwds):
+ self._setup_mapping(dependency_manager, **kwds)
  self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
  self.dependency_manager = dependency_manager
 
@@ -146,7 +148,7 @@ def resolve_all(self, requirements, **kwds):
 
  conda_targets = []
  for requirement in requirements:
- requirement = self._expand_specs(requirement)
+ requirement = self._expand_requirement(requirement)
 
  version = requirement.version
  if self.versionless:
@@ -186,7 +188,7 @@ def merged_environment_name(self, conda_targets):
  return conda_targets[0].install_environment
 
  def resolve(self, requirement, **kwds):
- requirement = self._expand_specs(requirement)
+ requirement = self._expand_requirement(requirement)
  name, version, type = requirement.name, requirement.version, requirement.type
 
  # Check for conda just not being there, this way we can enable
@@ -236,6 +238,9 @@ def resolve(self, requirement, **kwds):
  preserve_python_environment=preserve_python_environment,
  )
 
+ def _expand_requirement(self, requirement):
+ return self._expand_specs(self._expand_mappings(requirement))
+
  def list_dependencies(self):
  for install_target in installed_conda_targets(self.conda_context):
  name = install_target.package

lib/galaxy/tools/deps/resolvers/galaxy_packages.py

@@ -16,6 +16,7 @@
  Dependency,
  DependencyResolver,
  ListableDependencyResolver,
+ MappableDependencyResolver,
  NullDependency,
 )
 
@@ -103,9 +104,17 @@ def _galaxy_package_dep( self, path, version, name, exact ):
  return NullDependency(version=version, name=name)
 
 
-class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
+class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver):
  resolver_type = "galaxy_packages"
 
+ def __init__(self, dependency_manager, **kwds):
+ super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
+ self._setup_mapping(dependency_manager, **kwds)
+
+ def resolve(self, requirement, **kwds):
+ requirement = self._expand_mappings(requirement)
+ return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds)
+
  def list_dependencies(self):
  base_path = self.base_path
  for package_name in listdir(base_path):

lib/galaxy/tools/deps/resolvers/modules.py

@@ -13,7 +13,12 @@
 
 from six import StringIO
 
-from ..resolvers import Dependency, DependencyResolver, NullDependency
+from ..resolvers import (
+ Dependency,
+ DependencyResolver,
+ MappableDependencyResolver,
+ NullDependency,
+)
 
 log = logging.getLogger( __name__ )
 
@@ -24,11 +29,12 @@
 UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s"
 
 
-class ModuleDependencyResolver(DependencyResolver):
+class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver):
  dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
  resolver_type = "modules"
 
  def __init__(self, dependency_manager, **kwds):
+ self._setup_mapping(dependency_manager, **kwds)
  self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
  find_by = kwds.get('find_by', 'avail')
  prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH))
@@ -52,6 +58,7 @@ def __default_modulespath(self):
  return module_path
 
  def resolve(self, requirement, **kwds):
+ requirement = self._expand_mappings(requirement)
  name, version, type = requirement.name, requirement.version, requirement.type
 
  if type != "package":

test/unit/tools/test_tool_deps.py

@@ -115,13 +115,7 @@ def __build_ts_test_package(base_path, script_contents=''):
 
 def test_module_dependency_resolver():
  with __test_base_path() as temp_directory:
- module_script = os.path.join(temp_directory, "modulecmd")
- __write_script(module_script, '''#!/bin/sh
-cat %s/example_output 1>&2;
-''' % temp_directory)
- with open(os.path.join(temp_directory, "example_output"), "w") as f:
- # Subset of module avail from MSI cluster.
- f.write('''
+ module_script = _setup_module_command(temp_directory, '''
 -------------------------- /soft/modules/modulefiles ---------------------------
 JAGS/3.2.0-gcc45
 JAGS/3.3.0-gcc4.7.2
@@ -141,7 +135,7 @@ def test_module_dependency_resolver():
 advisor/2013/update2 intel/11.1.080 mkl/10.2.5.035
 advisor/2013/update3 intel/12.0 mkl/10.2.7.041
 ''')
- resolver = ModuleDependencyResolver(None, modulecmd=module_script)
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script)
  module = resolver.resolve( ToolRequirement( name="R", version=None, type="package" ) )
  assert module.module_name == "R"
  assert module.module_version is None
@@ -154,6 +148,74 @@ def test_module_dependency_resolver():
  assert isinstance(module, NullDependency)
 
 
+def test_module_resolver_with_mapping():
+ with __test_base_path() as temp_directory:
+ module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.24
+''')
+ mapping_file = os.path.join(temp_directory, "mapping")
+ with open(mapping_file, "w") as f:
+ f.write('''
+- from: blast+
+ to: blast
+''')
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.24", module.module_version
+
+
+def test_module_resolver_with_mapping_versions():
+ with __test_base_path() as temp_directory:
+ module_script = _setup_module_command(temp_directory, '''
+-------------------------- /soft/modules/modulefiles ---------------------------
+blast/2.22.0-mpi
+blast/2.23
+blast/2.24.0-mpi
+''')
+ mapping_file = os.path.join(temp_directory, "mapping")
+ with open(mapping_file, "w") as f:
+ f.write('''
+- from:
+ name: blast+
+ version: 2.24
+ to:
+ name: blast
+ version: 2.24.0-mpi
+- from:
+ name: blast
+ version: 2.22
+ to:
+ version: 2.22.0-mpi
+''')
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.24", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.24.0-mpi", module.module_version
+
+ resolver = ModuleDependencyResolver(_SimpleDependencyManager(), modulecmd=module_script, mapping_file=mapping_file)
+ module = resolver.resolve( ToolRequirement( name="blast+", version="2.23", type="package" ) )
+ assert isinstance(module, NullDependency)
+
+ module = resolver.resolve( ToolRequirement( name="blast", version="2.22", type="package" ) )
+ assert module.module_name == "blast"
+ assert module.module_version == "2.22.0-mpi", module.module_version
+
+
+def _setup_module_command(temp_directory, contents):
+ module_script = os.path.join(temp_directory, "modulecmd")
+ __write_script(module_script, '''#!/bin/sh
+cat %s/example_output 1>&2;
+''' % temp_directory)
+ with open(os.path.join(temp_directory, "example_output"), "w") as f:
+ # Subset of module avail from MSI cluster.
+ f.write(contents)
+ return module_script
+
+
 def test_module_dependency():
  with __test_base_path() as temp_directory:
  # Create mock modulecmd script that just exports a variable
@@ -386,3 +448,9 @@ def __dependency_manager(xml_content):
  f.flush()
  dm = DependencyManager( default_base_path=base_path, conf_file=f.name )
  yield dm
+
+
+class _SimpleDependencyManager(object):
+
+ def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
+ return explicit_resolver_options.get(key)