Fixed handling of byte paths

- path entries are always saved as strings
- byte paths are converted to string paths for most functions
- fixes #517

Author: mrbean-bremen

CHANGES.md

@@ -20,6 +20,8 @@ the proposed changes so you can be ready.
     functions
 
 #### Fixes
+  * Fixed handling of byte string paths 
+    (see [#517](../../issues/517))
   * Fixed `os.walk` if path ends with path separator
     (see [#512](../../issues/512))
   * Fixed handling of empty path in `os.makedirs`

pyfakefs/fake_filesystem.py

@@ -112,7 +112,7 @@
 from pyfakefs.helpers import (
     FakeStatResult, FileBufferIO, NullFileBufferIO,
     is_int_type, is_byte_string, is_unicode_string,
-    make_string_path, IS_WIN)
+    make_string_path, IS_WIN, to_string)
 
 __pychecker__ = 'no-reimportself'
 
@@ -640,10 +640,11 @@ def add_entry(self, path_object):
                 not self.filesystem.is_windows_fs):
             raise OSError(errno.EACCES, 'Permission Denied', self.path)
 
-        if path_object.name in self.contents:
+        path_object_name = to_string(path_object.name)
+        if path_object_name in self.contents:
             self.filesystem.raise_os_error(errno.EEXIST, self.path)
 
-        self.contents[path_object.name] = path_object
+        self.contents[path_object_name] = path_object
         path_object.parent_dir = self
         self.st_nlink += 1
         path_object.st_nlink += 1
@@ -665,7 +666,7 @@ def get_entry(self, pathname_name):
             KeyError: if no child exists by the specified name.
         """
         pathname_name = self._normalized_entryname(pathname_name)
-        return self.contents[pathname_name]
+        return self.contents[to_string(pathname_name)]
 
     def _normalized_entryname(self, pathname_name):
         if not self.filesystem.is_case_sensitive:
@@ -712,7 +713,7 @@ def remove_entry(self, pathname_name, recursive=True):
         entry.st_nlink -= 1
         assert entry.st_nlink >= 0
 
-        del self.contents[pathname_name]
+        del self.contents[to_string(pathname_name)]
 
     @property
     def size(self):
@@ -1736,12 +1737,6 @@ def exists(self, file_path, check_link=False):
                 return False
         return True
 
-    @staticmethod
-    def _to_string(path):
-        if isinstance(path, bytes):
-            path = path.decode(locale.getpreferredencoding(False))
-        return path
-
     def resolve_path(self, file_path, allow_fd=False, raw_io=True):
         """Follow a path, resolving symlinks.
 
@@ -1787,7 +1782,6 @@ def resolve_path(self, file_path, allow_fd=False, raw_io=True):
         if file_path is None:
             # file.open(None) raises TypeError, so mimic that.
             raise TypeError('Expected file system path string, received None')
-        file_path = self._to_string(file_path)
         if not file_path or not self._valid_relative_path(file_path):
             # file.open('') raises OSError, so mimic that, and validate that
             # all parts of a relative path exist.
@@ -2793,6 +2787,7 @@ def makedirs(self, dir_name, mode=PERM_DEF, exist_ok=False):
         """
         if not dir_name:
             self.raise_os_error(errno.ENOENT, '')
+        dir_name = to_string(dir_name)
         ends_with_sep = self.ends_with_path_separator(dir_name)
         dir_name = self.absnormpath(dir_name)
         if (ends_with_sep and self.is_macos and
@@ -4128,6 +4123,7 @@ def makedirs(self, name, mode=PERM_DEF, exist_ok=None):
 
     def _path_with_dir_fd(self, path, fct, dir_fd):
         """Return the path considering dir_fd. Raise on invalid parameters."""
+        path = to_string(path)
         if dir_fd is not None:
             # check if fd is supported for the built-in real function
             real_fct = getattr(os, fct.__name__)

pyfakefs/fake_filesystem_unittest.py

@@ -436,20 +436,24 @@ def _is_fs_function(self, fct):
             return False
 
     def _def_values(self, item):
+        """Find default arguments that are file-system functions to be
+        patched in top-level functions and members of top-level classes."""
         # check for module-level functions
-        if inspect.isfunction(item) and item.__defaults__:
-            for i, d in enumerate(item.__defaults__):
-                if self._is_fs_function(d):
-                    yield item, i, d
-        if inspect.isclass(item):
+        if inspect.isfunction(item):
+            if item.__defaults__:
+                for i, d in enumerate(item.__defaults__):
+                    if self._is_fs_function(d):
+                        yield item, i, d
+        elif inspect.isclass(item):
             # check for methods in class (nested classes are ignored for now)
             try:
                 for m in inspect.getmembers(item,
                                             predicate=inspect.isfunction):
-                    if m[1].__defaults__:
-                        for i, d in enumerate(m[1].__defaults__):
+                    m = m[1]
+                    if m.__defaults__:
+                        for i, d in enumerate(m.__defaults__):
                             if self._is_fs_function(d):
-                                yield m[1], i, d
+                                yield m, i, d
             except ImportError:
                 # Ignore ImportError: No module named '_gdbm'
                 pass

pyfakefs/fake_scandir.py

@@ -20,6 +20,7 @@
 import sys
 
 from pyfakefs.extra_packages import use_scandir_package
+from pyfakefs.helpers import to_string
 
 if sys.version_info >= (3, 6):
     BaseClass = os.PathLike
@@ -131,7 +132,7 @@ def __init__(self, filesystem, path):
             self.path = ''
         else:
             self.abspath = self.filesystem.absnormpath(path)
-            self.path = path
+            self.path = to_string(path)
         contents = self.filesystem.confirmdir(self.abspath).contents
         self.contents_iter = iter(contents)
 
@@ -249,7 +250,7 @@ def do_walk(top_dir, top_most=False):
             if not topdown:
                 yield top_contents
 
-    return do_walk(top, top_most=True)
+    return do_walk(to_string(top), top_most=True)
 
 
 class FakeScanDirModule:

pyfakefs/helpers.py

@@ -49,6 +49,14 @@ def make_string_path(dir_name):
     return dir_name
 
 
+def to_string(path):
+    """Return the string representation of a byte string using the preferred
+     encoding, or the string itself if path is a str."""
+    if isinstance(path, bytes):
+        return path.decode(locale.getpreferredencoding(False))
+    return path
+
+
 class FakeStatResult:
     """Mimics os.stat_result for use as return type of `stat()` and similar.
     This is needed as `os.stat_result` has no possibility to set
@@ -85,17 +93,17 @@ def use_float(self, val):
 
     def __eq__(self, other):
         return (
-            isinstance(other, FakeStatResult) and
-            self._st_atime_ns == other._st_atime_ns and
-            self._st_ctime_ns == other._st_ctime_ns and
-            self._st_mtime_ns == other._st_mtime_ns and
-            self.st_size == other.st_size and
-            self.st_gid == other.st_gid and
-            self.st_uid == other.st_uid and
-            self.st_nlink == other.st_nlink and
-            self.st_dev == other.st_dev and
-            self.st_ino == other.st_ino and
-            self.st_mode == other.st_mode
+                isinstance(other, FakeStatResult) and
+                self._st_atime_ns == other._st_atime_ns and
+                self._st_ctime_ns == other._st_ctime_ns and
+                self._st_mtime_ns == other._st_mtime_ns and
+                self.st_size == other.st_size and
+                self.st_gid == other.st_gid and
+                self.st_uid == other.st_uid and
+                self.st_nlink == other.st_nlink and
+                self.st_dev == other.st_dev and
+                self.st_ino == other.st_ino and
+                self.st_mode == other.st_mode
         )
 
     def __ne__(self, other):
@@ -272,6 +280,7 @@ class FileBufferIO:
     Uses an io.BytesIO stream for the raw data and adds handling of encoding
     and newlines.
     """
+
     def __init__(self, contents=None, linesep='\n', binary=False,
                  newline=None, encoding=None, errors='strict'):
         self._newline = newline
@@ -418,5 +427,6 @@ def __getattr__(self, name):
 
 class NullFileBufferIO(FileBufferIO):
     """Special stream for null device. Does nothing on writing."""
+
     def putvalue(self, s):
         pass

pyfakefs/tests/fake_os_test.py

@@ -335,6 +335,18 @@ def test_lstat_trailing_sep(self):
         self.assertEqual(stat, self.os.lstat(
             self.base_path + self.path_separator() + self.path_separator()))
 
+    def test_stat_with_byte_string(self):
+        stat_str = self.os.stat(self.base_path)
+        base_path_bytes = self.base_path.encode('utf8')
+        stat_bytes = self.os.stat(base_path_bytes)
+        self.assertEqual(stat_bytes, stat_str)
+
+    def test_lstat_with_byte_string(self):
+        stat_str = self.os.lstat(self.base_path)
+        base_path_bytes = self.base_path.encode('utf8')
+        stat_bytes = self.os.lstat(base_path_bytes)
+        self.assertEqual(stat_bytes, stat_str)
+
     def test_exists_with_trailing_sep(self):
         # regression test for #364
         file_path = self.make_path('alpha')

pyfakefs/tests/test_utils.py

@@ -25,7 +25,7 @@
 import unittest
 
 from pyfakefs import fake_filesystem
-from pyfakefs.helpers import is_byte_string
+from pyfakefs.helpers import is_byte_string, to_string
 
 
 class DummyTime:
@@ -265,17 +265,13 @@ def make_path(self, *args):
         Always use to compose absolute paths for tests also running in the
         real FS.
         """
-        if isinstance(args[0], bytes):
-            base_path = self.base_path.encode()
-        else:
-            base_path = self.base_path
-
         if isinstance(args[0], (list, tuple)):
-            path = base_path
+            path = self.base_path
             for arg in args[0]:
-                path = self.os.path.join(path, arg)
+                path = self.os.path.join(path, to_string(arg))
             return path
-        return self.os.path.join(base_path, *args)
+        args = [to_string(arg) for arg in args]
+        return self.os.path.join(self.base_path, *args)
 
     def create_dir(self, dir_path):
         """Create the directory at `dir_path`, including subdirectories.