- Notifications
You must be signed in to change notification settings - Fork 536
Open
Labels
Description
Summary
Sometimes I get errors such as these. After restarting the workflow usually everything's OK again.
Traceback (most recent call last): File "<ipython-input-7-83238b59a603>", line 1, in <module> runfile('/home/visual/andche/MRI_DATA/decoding_merged_TRs.py', wdir='/home/visual/andche/MRI_DATA') File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/spyder/utils/site/sitecustomize.py", line 705, in runfile execfile(filename, namespace) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/spyder/utils/site/sitecustomize.py", line 102, in execfile exec(compile(f.read(), filename, 'exec'), namespace) File "/home/visual/andche/MRI_DATA/decoding_merged_TRs.py", line 159, in <module> decoding.run('PBS', plugin_args={'max_jobs':600, 'qsub_args' : '-l walltime=00:10:00,mem=4g', 'template':'', 'max_tries':3,'retry_timeout': 5}) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/engine/workflows.py", line 595, in run runner.run(execgraph, updatehash=updatehash, config=self.config) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/plugins/base.py", line 168, in run self._clean_queue(jobid, graph, result=result)) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/plugins/base.py", line 227, in _clean_queue raise RuntimeError("".join(result['traceback'])) RuntimeError: Traceback (most recent call last): File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/engine/nodes.py", line 480, in run result = self._run_interface(execute=True) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/engine/nodes.py", line 564, in _run_interface return self._run_command(execute) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/engine/nodes.py", line 644, in _run_command result = self._interface.run(cwd=outdir) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/core.py", line 520, in run runtime = self._run_interface(runtime) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/core.py", line 1017, in _run_interface runtime = run_command(runtime, output=self.terminal_output) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/core.py", line 784, in run_command _process() File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/core.py", line 780, in _process stream.read(drain) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/support.py", line 261, in read while self._read(drain) is not None: File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/interfaces/base/support.py", line 289, in _read iflogger.info(self._rows[idx][1]) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1285, in info self._log(INFO, msg, args, **kwargs) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1421, in _log self.handle(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1431, in handle self.callHandlers(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1493, in callHandlers hdlr.handle(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 859, in handle self.acquire() File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/external/cloghandler.py", line 194, in acquire self._openFile(self.mode) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/external/cloghandler.py", line 185, in _openFile self.stream = open(self.baseFilename, mode) OSError: [Errno 116] Stale file handle: '/home/visual/andche/MU/MainSessions/TMP/decoding_merged_TRs/pypeline.log' During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/visual/andche/MU/MainSessions/TMP/decoding_merged_TRs/_TR_4_5_6/_label_id_V2_subject_id_P01/_topn_100_tvals_type_feat_useVOI_useVOI/_feature_space_360/decoding_wrapper/mapflow/batch/pyscript_20180602_172617__decoding_wrapper10.py", line 33, in <module> result = info['node'].run(updatehash=info['updatehash']) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/pipeline/engine/nodes.py", line 482, in run logger.warning('[Node] Error on "%s" (%s)', self.fullname, outdir) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1297, in warning self._log(WARNING, msg, args, **kwargs) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1421, in _log self.handle(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1431, in handle self.callHandlers(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 1493, in callHandlers hdlr.handle(record) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/logging/__init__.py", line 859, in handle self.acquire() File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/external/cloghandler.py", line 194, in acquire self._openFile(self.mode) File "/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype/external/cloghandler.py", line 185, in _openFile self.stream = open(self.baseFilename, mode) OSError: [Errno 116] Stale file handle: '/home/visual/andche/MU/MainSessions/TMP/decoding_merged_TRs/pypeline.log'
Platform details:
{'commit_hash': '%h', 'nipype_version': '1.0.3', 'nibabel_version': '2.2.1', 'commit_source': 'archive substitution', 'pkg_path': '/project/3019005.02/conda_env/nipype_v1x/lib/python3.5/site-packages/nipype', 'numpy_version': '1.14.3', 'sys_platform': 'linux', 'traits_version': '4.6.0', 'sys_version': '3.5.5 | packaged by conda-forge | (default, Apr 6 2018, 13:41:05) \n[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]', 'networkx_version': '2.1', 'sys_executable': '/project/3019005.02/conda_env/nipype_v1x/bin/python', 'scipy_version': '1.1.0'} 1.0.3