[skip ci] Merge remote-tracking branch 'upstream/master' into fix/resource-monitor-revisions

Author: oesteban

CHANGES

@@ -8,7 +8,9 @@ Upcoming release
 ###### [Full changelog](https://github.com/nipy/nipype/milestone/13)
 
 * FIX+MAINT: Revision of the resource monitor (https://github.com/nipy/nipype/pull/2285)
-* ENH: Memoize version checks (https://github.com/nipy/nipype/pull/2274, https://github.com/nipy/nipype/pull/2295)
+* FIX: MultiProc mishandling crashes (https://github.com/nipy/nipype/pull/2301)
+* MAINT: Revise use of `subprocess.Popen` (https://github.com/nipy/nipype/pull/2289)
+* ENH: Memorize version checks (https://github.com/nipy/nipype/pull/2274, https://github.com/nipy/nipype/pull/2295)
 
 
 0.14.0rc1 (November 21, 2017)

nipype/algorithms/confounds.py

@@ -22,7 +22,7 @@
 from numpy.polynomial import Legendre
 from scipy import linalg
 
-from .. import logging
+from .. import config, logging
 from ..external.due import BibTeX
 from ..interfaces.base import (traits, TraitedSpec, BaseInterface,
  BaseInterfaceInputSpec, File, isdefined,
@@ -816,7 +816,7 @@ def plot_confound(tseries, figsize, name, units=None,
 
  """
  import matplotlib
- matplotlib.use('Agg')
+ matplotlib.use(config.get('execution', 'matplotlib_backend'))
  import matplotlib.pyplot as plt
  from matplotlib.gridspec import GridSpec
  from matplotlib.backends.backend_pdf import FigureCanvasPdf as FigureCanvas

nipype/algorithms/metrics.py

@@ -24,7 +24,7 @@
 from scipy.spatial.distance import cdist, euclidean, dice, jaccard
 from scipy.ndimage.measurements import center_of_mass, label
 
-from .. import logging
+from .. import config, logging
 from ..utils.misc import package_check
 
 from ..interfaces.base import (BaseInterface, traits, TraitedSpec, File,
@@ -138,6 +138,8 @@ def _eucl_mean(self, nii1, nii2, weighted=False):
 
  dist_matrix = cdist(set1_coordinates.T, set2_coordinates.T)
  min_dist_matrix = np.amin(dist_matrix, axis=0)
+ import matplotlib
+ matplotlib.use(config.get('execution', 'matplotlib_backend'))
  import matplotlib.pyplot as plt
  plt.figure()
  plt.hist(min_dist_matrix, 50, normed=1, facecolor='green')

nipype/interfaces/ants/registration.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
-"""The ants module provides basic functions for interfacing with ants functions.
+"""The ants module provides basic functions for interfacing with ants
+ functions.
 
  Change directory to provide relative paths for doctests
  >>> import os
  >>> filepath = os.path.dirname( os.path.realpath( __file__ ) )
  >>> datadir = os.path.realpath(os.path.join(filepath, '../../testing/data'))
  >>> os.chdir(datadir)
 """
-from __future__ import print_function, division, unicode_literals, absolute_import
+from __future__ import (print_function, division, unicode_literals,
+ absolute_import)
 from builtins import range, str
 import os
 
@@ -20,17 +22,19 @@ class ANTSInputSpec(ANTSCommandInputSpec):
  dimension = traits.Enum(3, 2, argstr='%d', usedefault=False,
  position=1, desc='image dimension (2 or 3)')
  fixed_image = InputMultiPath(File(exists=True), mandatory=True,
- desc=('image to which the moving image is warped'))
+ desc=('image to which the moving image is '
+ 'warped'))
  moving_image = InputMultiPath(File(exists=True), argstr='%s',
  mandatory=True,
- desc=('image to apply transformation to (generally a coregistered '
+ desc=('image to apply transformation to '
+ '(generally a coregistered'
  'functional)'))
 
 # Not all metrics are appropriate for all modalities. Also, not all metrics
-# are efficeint or appropriate at all resolution levels, Some metrics perform
-# well for gross global registraiton, but do poorly for small changes (i.e.
-# Mattes), and some metrics do well for small changes but don't work well for
-# gross level changes (i.e. 'CC').
+# are efficeint or appropriate at all resolution levels, Some metrics
+# perform well for gross global registraiton, but do poorly for small
+# changes (i.e. Mattes), and some metrics do well for small changes but
+# don't work well for gross level changes (i.e. 'CC').
 #
 # This is a two stage registration. in the first stage
 # [ 'Mattes', .................]
@@ -49,10 +53,18 @@ class ANTSInputSpec(ANTSCommandInputSpec):
  metric = traits.List(traits.Enum('CC', 'MI', 'SMI', 'PR', 'SSD',
  'MSQ', 'PSE'), mandatory=True, desc='')
 
- metric_weight = traits.List(traits.Float(), requires=['metric'], desc='')
- radius = traits.List(traits.Int(), requires=['metric'], desc='')
+ metric_weight = traits.List(traits.Float(), value=[1.0], usedefault=True,
+ requires=['metric'], mandatory=True,
+ desc='the metric weight(s) for each stage. '
+ 'The weights must sum to 1 per stage.')
 
- output_transform_prefix = Str('out', usedefault=True, argstr='--output-naming %s',
+ radius = traits.List(traits.Int(), requires=['metric'], mandatory=True,
+ desc='radius of the region (i.e. number of layers'
+ ' around a voxel point)'
+ ' that is used for computing cross correlation')
+
+ output_transform_prefix = Str('out', usedefault=True,
+ argstr='--output-naming %s',
  mandatory=True, desc='')
  transformation_model = traits.Enum('Diff', 'Elast', 'Exp', 'Greedy Exp',
  'SyN', argstr='%s', mandatory=True,

nipype/interfaces/ants/tests/test_registration.py

@@ -0,0 +1,22 @@
+# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
+# vi: set ft=python sts=4 ts=4 sw=4 et:
+from __future__ import unicode_literals
+from nipype.interfaces.ants import registration
+import os
+import pytest
+
+
+def test_ants_mand(tmpdir):
+ tmpdir.chdir()
+ filepath = os.path.dirname(os.path.realpath(__file__))
+ datadir = os.path.realpath(os.path.join(filepath, '../../../testing/data'))
+
+ ants = registration.ANTS()
+ ants.inputs.transformation_model = "SyN"
+ ants.inputs.moving_image = [os.path.join(datadir, 'resting.nii')]
+ ants.inputs.fixed_image = [os.path.join(datadir, 'T1.nii')]
+ ants.inputs.metric = ['MI']
+
+ with pytest.raises(ValueError) as er:
+ ants.run()
+ assert "ANTS requires a value for input 'radius'" in str(er.value)

nipype/interfaces/base.py

@@ -9,11 +9,10 @@
 Requires Packages to be installed
 """
 from __future__ import print_function, division, unicode_literals, absolute_import
-from future import standard_library
-standard_library.install_aliases()
+import gc
+
 from builtins import range, object, open, str, bytes
 
-from configparser import NoOptionError
 from copy import deepcopy
 import datetime
 from datetime import datetime as dt
@@ -26,7 +25,6 @@
 import select
 import subprocess as sp
 import sys
-import time
 from textwrap import wrap
 from warnings import warn
 import simplejson as json
@@ -43,6 +41,8 @@
  traits, Undefined, TraitDictObject, TraitListObject, TraitError, isdefined,
  File, Directory, DictStrStr, has_metadata, ImageFile)
 from ..external.due import due
+from future import standard_library
+standard_library.install_aliases()
 
 nipype_version = Version(__version__)
 iflogger = logging.getLogger('interface')
@@ -58,6 +58,7 @@
 class Str(traits.Unicode):
  """Replacement for the default traits.Str based in bytes"""
 
+
 traits.Str = Str
 
 
@@ -1260,6 +1261,7 @@ class SimpleInterface(BaseInterface):
  >>> os.chdir(old.strpath)
 
  """
+
  def __init__(self, from_file=None, resource_monitor=None, **inputs):
  super(SimpleInterface, self).__init__(
  from_file=from_file, resource_monitor=resource_monitor, **inputs)
@@ -1387,8 +1389,7 @@ def run_command(runtime, output=None, timeout=0.01):
  shell=True,
  cwd=runtime.cwd,
  env=env,
- close_fds=True,
- )
+ close_fds=True)
  result = {
  'stdout': [],
  'stderr': [],
@@ -1427,37 +1428,50 @@ def _process(drain=0):
  temp.sort()
  result['merged'] = [r[1] for r in temp]
 
- if output == 'allatonce':
- stdout, stderr = proc.communicate()
- result['stdout'] = read_stream(stdout, logger=iflogger)
- result['stderr'] = read_stream(stderr, logger=iflogger)
-
- elif output.startswith('file'):
+ if output.startswith('file'):
  proc.wait()
  if outfile is not None:
  stdout.flush()
  stdout.close()
  with open(outfile, 'rb') as ofh:
  stdoutstr = ofh.read()
  result['stdout'] = read_stream(stdoutstr, logger=iflogger)
+ del stdoutstr
 
  if errfile is not None:
  stderr.flush()
  stderr.close()
  with open(errfile, 'rb') as efh:
  stderrstr = efh.read()
  result['stderr'] = read_stream(stderrstr, logger=iflogger)
+ del stderrstr
 
  if output == 'file':
  result['merged'] = result['stdout']
  result['stdout'] = []
  else:
- proc.communicate() # Discard stdout and stderr
+ stdout, stderr = proc.communicate()
+ if output == 'allatonce': # Discard stdout and stderr otherwise
+ result['stdout'] = read_stream(stdout, logger=iflogger)
+ result['stderr'] = read_stream(stderr, logger=iflogger)
+
+ runtime.returncode = proc.returncode
+ try:
+ proc.terminate() # Ensure we are done
+ except OSError as error:
+ # Python 2 raises when the process is already gone
+ if error.errno != errno.ESRCH:
+ raise
+
+ # Dereference & force GC for a cleanup
+ del proc
+ del stdout
+ del stderr
+ gc.collect()
 
  runtime.stderr = '\n'.join(result['stderr'])
  runtime.stdout = '\n'.join(result['stdout'])
  runtime.merged = '\n'.join(result['merged'])
- runtime.returncode = proc.returncode
  return runtime
 
 
@@ -1467,21 +1481,26 @@ def get_dependencies(name, environ):
  Uses otool on darwin, ldd on linux. Currently doesn't support windows.
 
  """
+ cmd = None
  if sys.platform == 'darwin':
- proc = sp.Popen('otool -L `which %s`' % name,
- stdout=sp.PIPE,
- stderr=sp.PIPE,
- shell=True,
- env=environ)
+ cmd = 'otool -L `which {}`'.format
  elif 'linux' in sys.platform:
- proc = sp.Popen('ldd `which %s`' % name,
- stdout=sp.PIPE,
- stderr=sp.PIPE,
- shell=True,
- env=environ)
- else:
+ cmd = 'ldd -L `which {}`'.format
+
+ if cmd is None:
  return 'Platform %s not supported' % sys.platform
- o, e = proc.communicate()
+
+ try:
+ proc = sp.Popen(
+ cmd(name), stdout=sp.PIPE, stderr=sp.PIPE, shell=True,
+ env=environ, close_fds=True)
+ o, _ = proc.communicate()
+ proc.terminate()
+ gc.collect()
+ except:
+ iflogger.warning(
+ 'Could not get linked libraries for "%s".', name)
+ return 'Failed collecting dependencies'
  return o.rstrip()
 
 
@@ -1572,6 +1591,9 @@ def __init__(self, command=None, terminal_output=None, **inputs):
  # Set command. Input argument takes precedence
  self._cmd = command or getattr(self, '_cmd', None)
 
+ # Store dependencies in runtime object
+ self._ldd = str2bool(config.get('execution', 'get_linked_libs', 'true'))
+
  if self._cmd is None:
  raise Exception("Missing command")
 
@@ -1620,6 +1642,8 @@ def _get_environ(self):
  return getattr(self.inputs, 'environ', {})
 
  def version_from_command(self, flag='-v'):
+ iflogger.warning('version_from_command member of CommandLine was '
+ 'Deprecated in nipype-1.0.0 and deleted in 1.1.0')
  cmdname = self.cmd.split()[0]
  env = dict(os.environ)
  if _exists_in_path(cmdname, env):
@@ -1664,7 +1688,8 @@ def _run_interface(self, runtime, correct_return_codes=(0,)):
  (self.cmd.split()[0], runtime.hostname))
 
  runtime.command_path = cmd_path
- runtime.dependencies = get_dependencies(executable_name, runtime.environ)
+ runtime.dependencies = (get_dependencies(executable_name, runtime.environ)
+ if self._ldd else '<skipped>')
  runtime = run_command(runtime, output=self.terminal_output)
  if runtime.returncode is None or \
  runtime.returncode not in correct_return_codes:

nipype/pipeline/engine/workflows.py

@@ -62,6 +62,7 @@
 
 logger = logging.getLogger('workflow')
 
+
 class Workflow(EngineBase):
  """Controls the setup and execution of a pipeline of processes."""
 
@@ -196,7 +197,7 @@ def connect(self, *args, **kwargs):
  # determine their inputs/outputs depending on
  # connection settings. Skip these modules in the check
  if dest in connected_ports[destnode]:
- raise Exception("""
+ raise Exception("""\
 Trying to connect %s:%s to %s:%s but input '%s' of node '%s' is already
 connected.
 """ % (srcnode, source, destnode, dest, dest, destnode))
@@ -297,7 +298,7 @@ def disconnect(self, *args):
  remove = []
  for edge in conn:
  if edge in ed_conns:
- idx = ed_conns.index(edge)
+ # idx = ed_conns.index(edge)
  remove.append((edge[0], edge[1]))
 
  logger.debug('disconnect(): remove list %s', to_str(remove))
@@ -426,7 +427,7 @@ def write_graph(self, dotfilename='graph.dot', graph2use='hierarchical',
  base_dir = os.getcwd()
  base_dir = make_output_dir(base_dir)
  if graph2use in ['hierarchical', 'colored']:
- if self.name[:1].isdigit(): # these graphs break if int
+ if self.name[:1].isdigit():  # these graphs break if int
  raise ValueError('{} graph failed, workflow name cannot begin '
  'with a number'.format(graph2use))
  dotfilename = op.join(base_dir, dotfilename)
@@ -650,7 +651,7 @@ def _write_report_info(self, workingdir, name, graph):
  # Avoid RuntimeWarning: divide by zero encountered in log10
  num_nodes = len(nodes)
  if num_nodes > 0:
- index_name = np.ceil(np.log10(num_nodes)).astype(int)
+  index_name = np.ceil(np.log10(num_nodes)).astype(int)
  else:
  index_name = 0
  template = '%%0%dd_' % index_name
@@ -798,10 +799,10 @@ def _get_outputs(self):
  setattr(outputdict, node.name, outputs)
  return outputdict
 
- def _set_input(self, object, name, newvalue):
+ def _set_input(self, objekt, name, newvalue):
  """Trait callback function to update a node input
  """
- object.traits()[name].node.set_input(name, newvalue)
+ objekt.traits()[name].node.set_input(name, newvalue)
 
  def _set_node_input(self, node, param, source, sourceinfo):
  """Set inputs of a node given the edge connection"""