Copy encoding fix (#313)

* Added unittest for remote dir copy supplying correct encoding when recursion is on - fix copy_remote_file issue with encoding when recurse is on
* Updated changelog

Author: pkittenis

Changelog.rst

@@ -10,6 +10,11 @@ Changes
 * ``user`` keyword argument no longer required on Windows - exception is raised if user cannot be identified.
 * Removed deprecated since ``2.0.0`` functions and parameters.
 
+Fixes
+-----
+
+* ``copy_remote_file`` with recurse enabled would not use a provided encoding for sub-directories - #284
+
 2.5.4
 +++++
 

pssh/clients/base/parallel.py

@@ -335,8 +335,7 @@ def join(self, output=None, consume_output=False, timeout=None):
                 timeout, finished_output, unfinished_output,
             )
 
-    def _join(self, host_out, consume_output=False, timeout=None,
-              encoding="utf-8"):
+    def _join(self, host_out, consume_output=False, timeout=None):
         if host_out is None:
             return
         client = host_out.client

pssh/clients/base/single.py

@@ -630,7 +630,7 @@ def _copy_remote_dir(self, file_list, remote_dir, local_dir, sftp,
             remote_path = os.path.join(remote_dir, file_name)
             local_path = os.path.join(local_dir, file_name)
             self.copy_remote_file(remote_path, local_path, sftp=sftp,
-                                  recurse=True)
+                                  recurse=True, encoding=encoding)
 
     def _make_local_dir(self, dirpath):
         if os.path.exists(dirpath):

tests/native/test_parallel_client.py

@@ -487,50 +487,6 @@ def test_pssh_copy_file_per_host_args(self):
                 os.unlink(remote_file_abspath)
                 os.unlink(local_file_path)
 
-    def test_pssh_client_directory_relative_path(self):
-        """Tests copying multiple directories with SSH client. Copy all the files from
-        local directory to server, then make sure they are all present."""
-        client = ParallelSSHClient([self.host], port=self.port,
-                                   pkey=self.user_key)
-        test_file_data = 'test'
-        local_test_path = 'directory_test'
-        remote_test_path = 'directory_test_copied'
-        dir_name = os.path.dirname(__file__)
-        remote_test_path_rel = os.sep.join(
-            (dir_name.replace(os.path.expanduser('~') + os.sep, ''),
-             remote_test_path))
-        remote_test_path_abs = os.sep.join((dir_name, remote_test_path))
-        for path in [local_test_path, remote_test_path_abs]:
-            try:
-                shutil.rmtree(path)
-            except OSError:
-                pass
-        os.mkdir(local_test_path)
-        remote_file_paths = []
-        for i in range(0, 10):
-            local_file_path_dir = os.path.join(
-                local_test_path, 'sub_dir1', 'sub_dir2', 'dir_foo' + str(i))
-            os.makedirs(local_file_path_dir)
-            local_file_path = os.path.join(local_file_path_dir, 'foo' + str(i))
-            remote_file_path = os.path.join(
-                remote_test_path, 'sub_dir1', 'sub_dir2', 'dir_foo' + str(i), 'foo' + str(i))
-            remote_file_paths.append(
-                os.sep.join((os.path.dirname(__file__), remote_file_path)))
-            test_file = open(local_file_path, 'w')
-            test_file.write(test_file_data)
-            test_file.close()
-        cmds = client.copy_file(local_test_path, remote_test_path_rel, recurse=True)
-        try:
-            joinall(cmds, raise_error=True)
-            for path in remote_file_paths:
-                self.assertTrue(os.path.isfile(path))
-        finally:
-            for _path in (local_test_path, remote_test_path_abs):
-                try:
-                    shutil.rmtree(_path)
-                except Exception:
-                    pass
-
     def test_pssh_client_directory_abs_path(self):
         client = ParallelSSHClient([self.host], port=self.port,
                                    pkey=self.user_key)

tests/native/test_single_client.py

@@ -19,6 +19,7 @@
 import subprocess
 import shutil
 import tempfile
+from unittest.mock import MagicMock, call
 from hashlib import sha256
 from datetime import datetime
 
@@ -935,6 +936,29 @@ def _disc():
         client._connect_init_session_retry(1)
         client.disconnect()
 
+    def test_copy_remote_dir_encoding(self):
+        client = SSHClient(self.host, port=self.port,
+                           pkey=self.user_key,
+                           num_retries=1)
+        remote_file_mock = MagicMock()
+        suffix = b"\xbc"
+        encoding = 'latin-1'
+        encoded_fn = suffix.decode(encoding)
+        file_list = [suffix + b"1", suffix + b"2"]
+        client.copy_remote_file = remote_file_mock
+        local_dir = (b"l_dir" + suffix).decode(encoding)
+        remote_dir = (b"r_dir" + suffix).decode(encoding)
+        client._copy_remote_dir(
+            file_list, local_dir, remote_dir, None, encoding=encoding)
+        call_args = [call(local_dir + "/" + file_list[0].decode(encoding),
+                          remote_dir + "/" + file_list[0].decode(encoding),
+                          recurse=True, sftp=None, encoding=encoding),
+                     call(local_dir + "/" + file_list[1].decode(encoding),
+                          remote_dir + "/" + file_list[1].decode(encoding),
+                          recurse=True, sftp=None, encoding=encoding)
+                     ]
+        self.assertListEqual(remote_file_mock.call_args_list, call_args)
+
 
     # TODO
     # * read output callback