Added some support for io.open_code in Python 3.8

- added new argument 'patch_open_code' to allow patching the function
- see pytest-dev#554

Author: mrbean-bremen

CHANGES.md

@@ -6,7 +6,10 @@ The released versions correspond to PyPi releases.
 #### New Features
   * add support for the `buffering` parameter in `open` 
     (see [#549](../../issues/549))
-
+  * add possibility to patch `io.open_code` using the new argument 
+    `patch_open_code` (since Python 3.8)
+    (see [#554](../../issues/554))
+    
 #### Fixes
   * do not call fake `open` if called from skipped module  
     (see [#552](../../issues/552))

docs/usage.rst

@@ -386,6 +386,22 @@ the default value of ``use_known_patches`` should be used, but it is present
 to allow users to disable this patching in case it causes any problems. It
 may be removed or replaced by more fine-grained arguments in future releases.
 
+patch_open_code
+~~~~~~~~~~~~~~~
+Since Python 3.8, the ``io`` module has the function ``open_code``, which
+opens a file read-only and is used to open Python code files. By default, this
+function is not patched, because the files it opens usually belong to the
+executed library code and are not present in the fake file system.
+Under some circumstances, this may not be the case, and the opened file
+lives in the fake filesystem. For these cases, you can set ``patch_open_code``
+to ``True``.
+
+.. note:: There is no possibility to change this setting based on affected
+  files. Depending on the upcoming use cases, this may be changed in future
+  versions of ``pyfakefs``, and this argument may be changed or removed in a
+  later version.
+
+
 Using convenience methods
 -------------------------
 While ``pyfakefs`` can be used just with the standard Python file system

pyfakefs/fake_filesystem.py

@@ -888,6 +888,8 @@ def __init__(self, path_separator=os.path.sep, total_size=None,
         self.add_mount_point(self.root.name, total_size)
         self._add_standard_streams()
         self.dev_null = FakeNullFile(self)
+        # set from outside if needed
+        self.patch_open_code = False
 
     @property
     def is_linux(self):
@@ -3438,21 +3440,21 @@ def dir():
         """Return the list of patched function names. Used for patching
         functions imported from the module.
         """
-        dir = [
+        _dir = [
             'access', 'chdir', 'chmod', 'chown', 'close', 'fstat', 'fsync',
             'getcwd', 'lchmod', 'link', 'listdir', 'lstat', 'makedirs',
             'mkdir', 'mknod', 'open', 'read', 'readlink', 'remove',
             'removedirs', 'rename', 'rmdir', 'stat', 'symlink', 'umask',
             'unlink', 'utime', 'walk', 'write', 'getcwdb', 'replace'
         ]
         if sys.platform.startswith('linux'):
-            dir += [
+            _dir += [
                 'fdatasync', 'getxattr', 'listxattr',
                 'removexattr', 'setxattr'
             ]
         if use_scandir:
-            dir += ['scandir']
-        return dir
+            _dir += ['scandir']
+        return _dir
 
     def __init__(self, filesystem):
         """Also exposes self.path (to fake os.path).
@@ -4468,7 +4470,10 @@ def dir():
         """Return the list of patched function names. Used for patching
         functions imported from the module.
         """
-        return 'open',
+        _dir = ['open']
+        if sys.version_info >= (3, 8):
+            _dir.append('open_code')
+        return _dir
 
     def __init__(self, filesystem):
         """
@@ -4498,6 +4503,21 @@ def open(self, file, mode='r', buffering=-1, encoding=None,
         return fake_open(file, mode, buffering, encoding, errors,
                          newline, closefd, opener)
 
+    if sys.version_info >= (3, 8):
+        def open_code(self, path):
+            """Redirect the call to open. Note that the behavior of the real
+            function may be overridden by an earlier call to the
+            PyFile_SetOpenCodeHook(). This behavior is not reproduced here.
+            """
+            if not isinstance(path, str):
+                raise TypeError(
+                    "open_code() argument 'path' must be str, not int")
+            if not self.filesystem.patch_open_code:
+                # mostly this is used for compiled code -
+                # don't patch these, as the files are probably in the real fs
+                return self._io_module.open_code(path)
+            return self.open(path, mode='rb')
+
     def __getattr__(self, name):
         """Forwards any unfaked calls to the standard io module."""
         return getattr(self._io_module, name)

pyfakefs/fake_filesystem_unittest.py

@@ -78,7 +78,8 @@ def patchfs(_func=None, *,
             modules_to_reload=None,
             modules_to_patch=None,
             allow_root_user=True,
-            use_known_patches=True):
+            use_known_patches=True,
+            patch_open_code=False):
     """Convenience decorator to use patcher with additional parameters in a
     test function.
 
@@ -101,7 +102,8 @@ def wrapped(*args, **kwargs):
                     modules_to_reload=modules_to_reload,
                     modules_to_patch=modules_to_patch,
                     allow_root_user=allow_root_user,
-                    use_known_patches=use_known_patches) as p:
+                    use_known_patches=use_known_patches,
+                    patch_open_code=patch_open_code) as p:
                 kwargs['fs'] = p.fs
                 return f(*args, **kwargs)
 
@@ -123,7 +125,8 @@ def load_doctests(loader, tests, ignore, module,
                   modules_to_reload=None,
                   modules_to_patch=None,
                   allow_root_user=True,
-                  use_known_patches=True):  # pylint: disable=unused-argument
+                  use_known_patches=True,
+                  patch_open_code=False):  # pylint: disable=unused-argument
     """Load the doctest tests for the specified module into unittest.
         Args:
             loader, tests, ignore : arguments passed in from `load_tests()`
@@ -136,7 +139,8 @@ def load_doctests(loader, tests, ignore, module,
                        modules_to_reload=modules_to_reload,
                        modules_to_patch=modules_to_patch,
                        allow_root_user=allow_root_user,
-                       use_known_patches=use_known_patches)
+                       use_known_patches=use_known_patches,
+                       patch_open_code=patch_open_code)
     globs = _patcher.replace_globs(vars(module))
     tests.addTests(doctest.DocTestSuite(module,
                                         globs=globs,
@@ -162,8 +166,6 @@ class TestCaseMixin:
         modules_to_patch: A dictionary of fake modules mapped to the
             fully qualified patched module names. Can be used to add patching
             of modules not provided by `pyfakefs`.
-        use_known_patches: If True (the default), some patches for commonly
-            used packges are applied which make them usable with pyfakes.
 
     If you specify some of these attributes here and you have DocTests,
     consider also specifying the same arguments to :py:func:`load_doctests`.
@@ -200,7 +202,8 @@ def setUpPyfakefs(self,
                       modules_to_reload=None,
                       modules_to_patch=None,
                       allow_root_user=True,
-                      use_known_patches=True):
+                      use_known_patches=True,
+                      patch_open_code=False):
         """Bind the file-related modules to the :py:class:`pyfakefs` fake file
         system instead of the real file system.  Also bind the fake `open()`
         function.
@@ -223,7 +226,8 @@ def setUpPyfakefs(self,
             modules_to_reload=modules_to_reload,
             modules_to_patch=modules_to_patch,
             allow_root_user=allow_root_user,
-            use_known_patches=use_known_patches
+            use_known_patches=use_known_patches,
+            patch_open_code=patch_open_code
         )
 
         self._stubber.setUp()
@@ -257,9 +261,7 @@ class TestCase(unittest.TestCase, TestCaseMixin):
     def __init__(self, methodName='runTest',
                  additional_skip_names=None,
                  modules_to_reload=None,
-                 modules_to_patch=None,
-                 allow_root_user=True,
-                 use_known_patches=True):
+                 modules_to_patch=None):
         """Creates the test class instance and the patcher used to stub out
         file system related modules.
 
@@ -272,8 +274,6 @@ def __init__(self, methodName='runTest',
         self.additional_skip_names = additional_skip_names
         self.modules_to_reload = modules_to_reload
         self.modules_to_patch = modules_to_patch
-        self.allow_root_user = allow_root_user
-        self.use_known_patches = use_known_patches
 
     @Deprecator('add_real_file')
     def copyRealFile(self, real_file_path, fake_file_path=None,
@@ -358,8 +358,32 @@ class Patcher:
 
     def __init__(self, additional_skip_names=None,
                  modules_to_reload=None, modules_to_patch=None,
-                 allow_root_user=True, use_known_patches=True):
-        """For a description of the arguments, see TestCase.__init__"""
+                 allow_root_user=True, use_known_patches=True,
+                 patch_open_code=False):
+        """
+        Args:
+            additional_skip_names: names of modules inside of which no module
+                replacement shall be performed, in addition to the names in
+                :py:attr:`fake_filesystem_unittest.Patcher.SKIPNAMES`.
+                Instead of the module names, the modules themselves
+                may be used.
+            modules_to_reload: A list of modules that need to be reloaded
+                to be patched dynamically; may be needed if the module
+                imports file system modules under an alias
+
+                .. caution:: Reloading modules may have unwanted side effects.
+            modules_to_patch: A dictionary of fake modules mapped to the
+                fully qualified patched module names. Can be used to add
+                patching of modules not provided by `pyfakefs`.
+            allow_root_user: If True (default), if the test is run as root
+                user, the user in the fake file system is also considered a
+                root user, otherwise it is always considered a regular user.
+            use_known_patches: If True (the default), some patches for commonly
+                used packages are applied which make them usable with pyfakefs.
+            patch_open_code: If True, `io.open_code` is patched. The default
+                is not to patch it, as it mostly is used to load compiled
+                modules that are not in the fake file system.
+        """
 
         if not allow_root_user:
             # set non-root IDs even if the real user is root
@@ -370,6 +394,7 @@ def __init__(self, additional_skip_names=None,
         # save the original open function for use in pytest plugin
         self.original_open = open
         self.fake_open = None
+        self.patch_open_code = patch_open_code
 
         if additional_skip_names is not None:
             skip_names = [m.__name__ if inspect.ismodule(m) else m
@@ -589,6 +614,7 @@ def _refresh(self):
         self._stubs = mox3_stubout.StubOutForTesting()
 
         self.fs = fake_filesystem.FakeFilesystem(patcher=self)
+        self.fs.patch_open_code = self.patch_open_code
         for name in self._fake_module_classes:
             self.fake_modules[name] = self._fake_module_classes[name](self.fs)
             if hasattr(self.fake_modules[name], 'skip_names'):

pyfakefs/tests/fake_open_test.py

@@ -21,15 +21,24 @@
 import locale
 import os
 import stat
+import sys
 import time
 import unittest
 
 from pyfakefs import fake_filesystem
-from pyfakefs.fake_filesystem import is_root, PERM_READ
+from pyfakefs.fake_filesystem import is_root, PERM_READ, FakeIoModule
 from pyfakefs.tests.test_utils import RealFsTestCase
 
 
 class FakeFileOpenTestBase(RealFsTestCase):
+    def setUp(self):
+        super(FakeFileOpenTestBase, self).setUp()
+        if self.use_real_fs():
+            self.open = io.open
+        else:
+            self.fake_io_module = FakeIoModule(self.filesystem)
+            self.open = self.fake_io_module.open
+
     def path_separator(self):
         return '!'
 
@@ -89,7 +98,7 @@ def test_unicode_contents(self):
             contents = f.read()
         self.assertEqual(contents, text_fractions)
 
-    def test_byte_contents_py3(self):
+    def test_byte_contents(self):
         file_path = self.make_path('foo')
         byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96'
         with self.open(file_path, 'wb') as f:
@@ -110,15 +119,6 @@ def test_write_str_read_bytes(self):
         self.assertEqual(str_contents, contents.decode(
             locale.getpreferredencoding(False)))
 
-    def test_byte_contents(self):
-        file_path = self.make_path('foo')
-        byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96'
-        with self.open(file_path, 'wb') as f:
-            f.write(byte_fractions)
-        with self.open(file_path, 'rb') as f:
-            contents = f.read()
-        self.assertEqual(contents, byte_fractions)
-
     def test_open_valid_file(self):
         contents = [
             'I am he as\n',
@@ -926,6 +926,55 @@ def use_real_fs(self):
         return True
 
 
+@unittest.skipIf(sys.version_info < (3, 8),
+                 'open_code only present since Python 3.8')
+class FakeFileOpenCodeTest(FakeFileOpenTestBase):
+
+    def setUp(self):
+        super(FakeFileOpenCodeTest, self).setUp()
+        if self.use_real_fs():
+            self.open_code = io.open_code
+        else:
+            self.open_code = self.fake_io_module.open_code
+
+    def tearDown(self):
+        self.filesystem.patch_open_code = False
+        super(FakeFileOpenCodeTest, self).tearDown()
+
+    def test_invalid_path(self):
+        self.filesystem.patch_open_code = True
+        with self.assertRaises(TypeError):
+            self.open_code(4)
+
+    def test_byte_contents_open_code(self):
+        self.filesystem.patch_open_code = True
+        byte_fractions = b'\xe2\x85\x93 \xe2\x85\x94 \xe2\x85\x95 \xe2\x85\x96'
+        file_path = self.make_path('foo')
+        self.create_file(file_path, contents=byte_fractions)
+        with self.open_code(file_path) as f:
+            contents = f.read()
+        self.assertEqual(contents, byte_fractions)
+
+    def test_not_patched_open_code_in_fake_fs(self):
+        file_path = self.make_path('foo')
+        self.create_file(file_path)
+
+        with self.assertRaises(OSError):
+            self.open_code(file_path)
+
+    def test_not_patched_open_code_in_real_fs(self):
+        file_path = __file__
+
+        with self.open_code(file_path) as f:
+            contents = f.read()
+        self.assertTrue(len(contents) > 100)
+
+
+class RealFileOpenCodeTest(FakeFileOpenCodeTest):
+    def use_real_fs(self):
+        return True
+
+
 class BufferingModeTest(FakeFileOpenTestBase):
     def test_no_buffering(self):
         file_path = self.make_path("buffertest.bin")
@@ -1168,10 +1217,6 @@ class OpenFileWithEncodingTest(FakeFileOpenTestBase):
 
     def setUp(self):
         super(OpenFileWithEncodingTest, self).setUp()
-        if self.use_real_fs():
-            self.open = io.open
-        else:
-            self.open = fake_filesystem.FakeFileOpen(self.filesystem)
         self.file_path = self.make_path('foo')
 
     def test_write_str_read_bytes(self):
@@ -1451,10 +1496,6 @@ def use_real_fs(self):
 class FakeFileOpenLineEndingWithEncodingTest(FakeFileOpenTestBase):
     def setUp(self):
         super(FakeFileOpenLineEndingWithEncodingTest, self).setUp()
-        if self.use_real_fs():
-            self.open = io.open
-        else:
-            self.open = fake_filesystem.FakeFileOpen(self.filesystem)
 
     def test_read_standard_newline_mode(self):
         file_path = self.make_path('some_file')