Added automatic patching of builtin and io open as other name

- in Python 2, builtin open is now handled by FakeBuiltinModule
- in Python 3, builtin open is handled via io.open
- adapted pytest plugin to ensure that open is not patched
  for linecache and dependent tokenize modules
- importing builtin open as other name does not work with PyPy2

Author: mrbean-bremen

CHANGES.md

@@ -5,7 +5,7 @@ The release versions are PyPi releases.
 
 #### New Features
   * automatically patch file system methods imported as another name like 
-    `from os.path import exists as my_exists`
+    `from os.path import exists as my_exists`, including builtin `open` 
 
 #### Fixes
 

pyfakefs/fake_filesystem.py

@@ -4528,6 +4528,13 @@ class FakeIoModule(object):
     my_io_module = fake_filesystem.FakeIoModule(filesystem)
     """
 
+    @staticmethod
+    def dir():
+        """Return the list of patched function names. Used for patching
+        functions imported from the module.
+        """
+        return 'open',
+
     def __init__(self, filesystem):
         """
         Args:
@@ -4553,6 +4560,24 @@ def __getattr__(self, name):
         return getattr(self._io_module, name)
 
 
+class FakeBuiltinModule(object):
+    """Uses FakeFilesystem to provide a fake built-in open replacement.
+    """
+    def __init__(self, filesystem):
+        """
+        Args:
+            filesystem: FakeFilesystem used to provide
+                file system information.
+        """
+        self.filesystem = filesystem
+
+    def open(self, *args, **kwargs):
+        """Redirect the call to FakeFileOpen.
+        See FakeFileOpen.call() for description.
+        """
+        return FakeFileOpen(self.filesystem)(*args, **kwargs)
+
+
 class FakeFileWrapper(object):
     """Wrapper for a stream object for use by a FakeFile object.
 

pyfakefs/fake_filesystem_unittest.py

@@ -43,6 +43,8 @@
 import unittest
 import zipfile  # noqa: F401 make sure it gets correctly stubbed, see #427
 
+from pyfakefs.helpers import IS_PY2, IS_PYPY
+
 from pyfakefs.deprecator import Deprecator
 
 try:
@@ -79,6 +81,7 @@
 
 OS_MODULE = 'nt' if sys.platform == 'win32' else 'posix'
 PATH_MODULE = 'ntpath' if sys.platform == 'win32' else 'posixpath'
+BUILTIN_MODULE = '__builtin__' if IS_PY2 else 'buildins'
 
 
 def load_doctests(loader, tests, ignore, module,
@@ -319,6 +322,8 @@ def __init__(self, additional_skip_names=None,
         """For a description of the arguments, see TestCase.__init__"""
 
         self._skipNames = self.SKIPNAMES.copy()
+        # save the original open function for use in pytest plugin
+        self.original_open = open
 
         if additional_skip_names is not None:
             self._skipNames.update(additional_skip_names)
@@ -339,6 +344,15 @@ def __init__(self, additional_skip_names=None,
             'shutil': fake_filesystem_shutil.FakeShutilModule,
             'io': fake_filesystem.FakeIoModule,
         }
+        if IS_PY2 or IS_PYPY:
+            # in Python 2 io.open, the module is referenced as _io
+            self._fake_module_classes['_io'] = fake_filesystem.FakeIoModule
+
+        # No need to patch builtins in Python 3 - builtin open
+        # is an alias for io.open
+        if IS_PY2 or IS_PYPY:
+            self._fake_module_classes[
+                BUILTIN_MODULE] = fake_filesystem.FakeBuiltinModule
 
         # class modules maps class names against a list of modules they can
         # be contained in - this allows for alternative modules like
@@ -364,32 +378,37 @@ def __init__(self, additional_skip_names=None,
         # each patched function name has to be looked up separately
         self._fake_module_functions = {}
         for mod_name, fake_module in self._fake_module_classes.items():
-            modnames = (
-                (mod_name, OS_MODULE) if mod_name == 'os' else (mod_name,)
-            )
             if (hasattr(fake_module, 'dir') and
                     inspect.isfunction(fake_module.dir)):
                 for fct_name in fake_module.dir():
-                    self._fake_module_functions[fct_name] = (
-                        modnames,
-                        getattr(fake_module, fct_name),
-                        mod_name
-                    )
+                    module_attr = (getattr(fake_module, fct_name), mod_name)
+                    self._fake_module_functions.setdefault(
+                        fct_name, {})[mod_name] = module_attr
+                    if mod_name == 'os':
+                        self._fake_module_functions.setdefault(
+                            fct_name, {})[OS_MODULE] = module_attr
+
         # special handling for functions in os.path
         fake_module = fake_filesystem.FakePathModule
         for fct_name in fake_module.dir():
-            self._fake_module_functions[fct_name] = (
-                ('genericpath', PATH_MODULE),
-                getattr(fake_module, fct_name),
-                PATH_MODULE
-            )
+            module_attr = (getattr(fake_module, fct_name), PATH_MODULE)
+            self._fake_module_functions.setdefault(
+                fct_name, {})['genericpath'] = module_attr
+            self._fake_module_functions.setdefault(
+                fct_name, {})[PATH_MODULE] = module_attr
+
+        # special handling for built-in open
+        self._fake_module_functions.setdefault(
+            'open', {})[BUILTIN_MODULE] = (
+            fake_filesystem.FakeBuiltinModule.open,
+            BUILTIN_MODULE
+        )
 
         # Attributes set by _refresh()
         self._modules = {}
         self._fct_modules = {}
         self._stubs = None
         self.fs = None
-        self.fake_open = None
         self.fake_modules = {}
         self._dyn_patcher = None
 
@@ -441,10 +460,10 @@ def _find_modules(self):
                          inspect.isbuiltin(fct)) and
                          fct.__name__ in self._fake_module_functions and
                          fct.__module__ in self._fake_module_functions[
-                             fct.__name__][0]}
+                             fct.__name__]}
             for name, fct in functions.items():
                 self._fct_modules.setdefault(
-                    (name, fct.__name__), set()).add(module)
+                    (name, fct.__name__, fct.__module__), set()).add(module)
 
     def _refresh(self):
         """Renew the fake file system and set the _isStale flag to `False`."""
@@ -456,7 +475,6 @@ def _refresh(self):
         for name in self._fake_module_classes:
             self.fake_modules[name] = self._fake_module_classes[name](self.fs)
         self.fake_modules[PATH_MODULE] = self.fake_modules['os'].path
-        self.fake_open = fake_filesystem.FakeFileOpen(self.fs)
 
         self._isStale = False
 
@@ -480,16 +498,21 @@ def setUp(self, doctester=None):
     def start_patching(self):
         if not self._patching:
             self._patching = True
-            if sys.version_info < (3,):
-                # file() was eliminated in Python3
-                self._stubs.smart_set(builtins, 'file', self.fake_open)
-            self._stubs.smart_set(builtins, 'open', self.fake_open)
+            if IS_PYPY:
+                # patching open via _fct_modules does not work in PyPy
+                self._stubs.smart_set(builtins, 'open',
+                                      self.fake_modules[BUILTIN_MODULE].open)
+            if IS_PY2:
+                # file is not a function and has to be handled separately 
+                self._stubs.smart_set(builtins, 'file',
+                                      self.fake_modules[BUILTIN_MODULE].open)
+
             for name, modules in self._modules.items():
                 for module, attr in modules:
                     self._stubs.smart_set(
                         module, name, self.fake_modules[attr])
-            for (name, fct_name), modules in self._fct_modules.items():
-                _, method, mod_name = self._fake_module_functions[fct_name]
+            for (name, ft_name, ft_mod), modules in self._fct_modules.items():
+                method, mod_name = self._fake_module_functions[ft_name][ft_mod]
                 fake_module = self.fake_modules[mod_name]
                 attr = method.__get__(fake_module, fake_module.__class__)
                 for module in modules:

pyfakefs/helpers.py

@@ -13,14 +13,15 @@
 """Helper classes use for fake file system implementation."""
 import io
 import locale
+import platform
 import sys
 from copy import copy
 from stat import S_IFLNK
 
 import os
 
 IS_PY2 = sys.version_info[0] < 3
-
+IS_PYPY = platform.python_implementation() == 'PyPy'
 
 try:
     text_type = unicode  # Python 2

pyfakefs/pytest_plugin.py

@@ -10,33 +10,30 @@ def my_fakefs_test(fs):
 """
 
 import linecache
-import sys
+import tokenize
 
 import py
 import pytest
 
 from pyfakefs.fake_filesystem_unittest import Patcher
 
-if sys.version_info >= (3,):
-    import builtins
-else:
-    import __builtin__ as builtins
-
 Patcher.SKIPMODULES.add(py)  # Ignore pytest components when faking filesystem
 
 # The "linecache" module is used to read the test file in case of test failure
 # to get traceback information before test tear down.
 # In order to make sure that reading the test file is not faked,
 # we both skip faking the module, and add the build-in open() function
-# as a local function in the module
+# as a local function in the module (used in Python 2).
+# In Python 3, we also have to set back the cached open function in tokenize.
 Patcher.SKIPMODULES.add(linecache)
-linecache.open = builtins.open
 
 
 @pytest.fixture
 def fs(request):
     """ Fake filesystem. """
     patcher = Patcher()
     patcher.setUp()
+    linecache.open = patcher.original_open
+    tokenize._builtin_open = patcher.original_open
     request.addfinalizer(patcher.tearDown)
     return patcher.fs

pyfakefs/tests/example.py

@@ -29,8 +29,8 @@
 
 The `open()` built-in is bound to the fake `open()`:
 
->>> open     #doctest: +ELLIPSIS
-<pyfakefs.fake_filesystem.FakeFileOpen object...>
+>>> open      #doctest: +ELLIPSIS
+<bound method Fake...Module.open of <pyfakefs.fake_filesystem.Fake...>
 
 In Python 2 the `file()` built-in is also bound to the fake `open()`.  `file()`
 was eliminated in Python 3.

pyfakefs/tests/fake_filesystem_unittest_test.py

@@ -33,6 +33,7 @@
 from pyfakefs.extra_packages import pathlib
 from pyfakefs.fake_filesystem_unittest import Patcher
 import pyfakefs.tests.import_as_example
+from pyfakefs.helpers import IS_PYPY, IS_PY2
 from pyfakefs.tests.fixtures import module_with_attributes
 
 
@@ -54,7 +55,7 @@ def setUp(self):
 class TestPyfakefsUnittest(TestPyfakefsUnittestBase):  # pylint: disable=R0904
     """Test the `pyfakefs.fake_filesystem_unittest.TestCase` base class."""
 
-    @unittest.skipIf(sys.version_info > (2,),
+    @unittest.skipIf(sys.version_info[0] > 2,
                      "file() was removed in Python 3")
     def test_file(self):
         """Fake `file()` function is bound"""
@@ -185,6 +186,19 @@ def test_import_function_from_os_as_other_name(self):
         stat_result = pyfakefs.tests.import_as_example.file_stat2(file_path)
         self.assertEqual(3, stat_result.st_size)
 
+    @unittest.skipIf(IS_PYPY and IS_PY2, 'Not working for PyPy2')
+    def test_import_open_as_other_name(self):
+        file_path = '/foo/bar'
+        self.fs.create_file(file_path, contents=b'abc')
+        contents = pyfakefs.tests.import_as_example.file_contents1(file_path)
+        self.assertEqual('abc', contents)
+
+    def test_import_io_open_as_other_name(self):
+        file_path = '/foo/bar'
+        self.fs.create_file(file_path, contents=b'abc')
+        contents = pyfakefs.tests.import_as_example.file_contents2(file_path)
+        self.assertEqual('abc', contents)
+
 
 class TestAttributesWithFakeModuleNames(TestPyfakefsUnittestBase):
     """Test that module attributes with names like `path` or `io` are not

pyfakefs/tests/import_as_example.py

@@ -20,6 +20,12 @@
 from os import stat as my_stat
 from os.path import exists
 from os.path import exists as my_exists
+from io import open as io_open
+
+try:
+    from builtins import open as bltn_open
+except ImportError:
+    from __builtin__ import open as bltn_open
 
 import sys
 
@@ -79,3 +85,13 @@ def system_stat(filepath):
     else:
         from posix import stat as system_stat
     return system_stat(filepath)
+
+
+def file_contents1(filepath):
+    with bltn_open(filepath) as f:
+        return f.read()
+
+
+def file_contents2(filepath):
+    with io_open(filepath) as f:
+        return f.read()