fix hdfs download_dir (#36590)

Author: danleifeng

python/paddle/distributed/fleet/utils/fs.py

@@ -842,8 +842,8 @@ def __subprocess_download(local_path, datas):
  if self.is_file(fs_path):
  return self._try_download(fs_path, local_path)
  # download dir
- _, all_files = self.ls_dir(fs_path)
-
+ _, all_filenames = self.ls_dir(fs_path)
+ all_files = [fs_path + i for i in all_filenames]
  procs = []
  for i in range(multi_processes):
  process_datas = self._split_files(all_files, i, multi_processes)

python/paddle/fluid/tests/unittests/hdfs_test_utils.py

@@ -195,7 +195,7 @@ def _test_download_dir(self, fs):
 
  fs.download(src_file, dst_file)
  local = LocalFS()
- self.assertTrue(local.is_exist(dst_file))
+ self.assertTrue(local.is_exist(file1))
  local.delete(dst_file)
  fs.delete(src_file)
 

python/paddle/fluid/tests/unittests/test_hdfs3.py

@@ -40,6 +40,7 @@ def test_hdfs(self):
  self._test_upload(fs)
  self._test_upload_dir(fs)
  self._test_download(fs)
+ self._test_download_dir(fs)
 
  def test_local(self):
  fs = LocalFS()