Merge pull request #297 from seleniumbase/csp-updates

mdmintz · web-flow · commit 4a5f53c73869 · 2019-03-16T02:58:47.000-04:00
Disable the Content Security Policy of websites by default
diff --git a/examples/github_test.py b/examples/github_test.py
@@ -9,7 +9,7 @@ class GitHubTests(BaseCase):
     # "Please wait a few minutes before you try again."
     # To avoid this, slow down Selenium actions.
     def slow_click(self, css_selector):
-        time.sleep(1)
+        time.sleep(1.05)
         self.click(css_selector)
 
     def test_github(self):
diff --git a/help_docs/ReadMe.md b/help_docs/ReadMe.md
@@ -1,4 +1,4 @@
-### ![http://seleniumbase.com](https://cdn2.hubspot.net/hubfs/100006/images/super_logo_tiny.png "SeleniumBase") SeleniumBase Help Documents:
+### <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> SeleniumBase Help Documents
 
 > **Table of Contents / Navigation:**
 > - [**SeleniumBase Features List**](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/features_list.md)
diff --git a/requirements.txt b/requirements.txt
@@ -8,7 +8,7 @@ unittest2
 selenium==3.141.0
 requests==2.21.0
 urllib3==1.24.1
-pytest>=4.3.0
+pytest>=4.3.1
 pytest-cov>=2.6.1
 pytest-html>=1.20.0
 pytest-rerunfailures>=6.0
diff --git a/seleniumbase/config/settings.py b/seleniumbase/config/settings.py
@@ -19,14 +19,15 @@
 
 # If True, existing logs from past test runs will be saved and take up space.
 # If False, only the logs from the most recent test run will be saved locally.
+# You can also archive existing logs on the command line with: "--archive_logs"
 ARCHIVE_EXISTING_LOGS = False
 
 # If True, existing downloads from past runs will be saved and take up space.
 # If False, only the downloads from the most recent run will be saved locally.
 ARCHIVE_EXISTING_DOWNLOADS = False
 
 # Default names for files saved during test failures.
-# (These files will get saved to the "latest_logs/" folder)
+# (These files will get saved to the "latest_logs/" folder.)
 SCREENSHOT_NAME = "screenshot.png"
 BASIC_INFO_NAME = "basic_test_info.txt"
 PAGE_SOURCE_NAME = "page_source.html"
@@ -66,6 +67,11 @@
 # Messenger notifications appear when reaching assert statements in Demo Mode.
 DEFAULT_MESSAGE_DURATION = 2.55
 
+# If True, the Content Security Policy will be disabled on Chrome and Firefox.
+# If False, each website's default Content Security Policy will be used.
+# (A website's CSP may prevent SeleniumBase from loading custom JavaScript.)
+DISABLE_CONTENT_SECURITY_POLICY = True
+
 # If True, an Exception is raised immediately for invalid proxy string syntax.
 # If False, a Warning will appear after the test, with no proxy server used.
 # (This applies when using --proxy=[PROXY_STRING] for using a proxy server.)
diff --git a/seleniumbase/console_scripts/ReadMe.md b/seleniumbase/console_scripts/ReadMe.md
@@ -1,4 +1,4 @@
-## Console Scripts
+## <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> Console Scripts
 
 SeleniumBase console scripts help you get things done more easily, such as installing web drivers, creating a test directory with necessary configuration files, converting old Webdriver unittest scripts into SeleniumBase code, and using the Selenium Grid.
 
diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py
@@ -16,7 +16,10 @@
 from seleniumbase.fixtures import constants
 from seleniumbase.fixtures import page_utils
 from seleniumbase import drivers  # webdriver storage folder for SeleniumBase
+from seleniumbase import extensions  # browser extensions storage folder
 DRIVER_DIR = os.path.dirname(os.path.realpath(drivers.__file__))
+EXTENSIONS_DIR = os.path.dirname(os.path.realpath(extensions.__file__))
+DISABLE_CSP_ZIP_PATH = "%s/%s" % (EXTENSIONS_DIR, "disable_csp.zip")
 PROXY_ZIP_PATH = proxy_helper.PROXY_ZIP_PATH
 PROXY_ZIP_PATH_2 = proxy_helper.PROXY_ZIP_PATH_2
 PLATFORM = sys.platform
@@ -82,8 +85,16 @@ def _add_chrome_proxy_extension(
     return chrome_options
 
 
+def _add_chrome_disable_csp_extension(chrome_options):
+    """ Disable Chrome's Content-Security-Policy with a browser extension.
+        See https://github.com/PhilGrayson/chrome-csp-disable for details. """
+    disable_csp_zip = DISABLE_CSP_ZIP_PATH
+    chrome_options.add_extension(disable_csp_zip)
+    return chrome_options
+
+
 def _set_chrome_options(
-        downloads_path, proxy_string, proxy_auth,
+        downloads_path, headless, proxy_string, proxy_auth,
         proxy_user, proxy_pass, user_agent):
     chrome_options = webdriver.ChromeOptions()
     prefs = {
@@ -108,6 +119,10 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-single-click-autofill")
     chrome_options.add_argument("--disable-translate")
     chrome_options.add_argument("--disable-web-security")
+    if settings.DISABLE_CONTENT_SECURITY_POLICY and not headless:
+        # Headless Chrome doesn't support extensions, which are required
+        # for disabling the Content Security Policy on Chrome
+        chrome_options = _add_chrome_disable_csp_extension(chrome_options)
     if proxy_string:
         if proxy_auth:
             chrome_options = _add_chrome_proxy_extension(
@@ -135,7 +150,8 @@ def _create_firefox_profile(downloads_path, proxy_string, user_agent):
         profile.set_preference("general.useragent.override", user_agent)
     profile.set_preference(
         "security.mixed_content.block_active_content", False)
-    profile.set_preference("security.csp.enable", False)
+    if settings.DISABLE_CONTENT_SECURITY_POLICY:
+        profile.set_preference("security.csp.enable", False)
     profile.set_preference(
         "browser.download.manager.showAlertOnComplete", False)
     profile.set_preference("browser.privatebrowsing.autostart", True)
@@ -247,7 +263,7 @@ def get_remote_driver(
         desired_caps = capabilities_parser.get_desired_capabilities(cap_file)
     if browser_name == constants.Browser.GOOGLE_CHROME:
         chrome_options = _set_chrome_options(
-            downloads_path, proxy_string, proxy_auth,
+            downloads_path, headless, proxy_string, proxy_auth,
             proxy_user, proxy_pass, user_agent)
         if headless:
             if not proxy_auth:
@@ -458,7 +474,7 @@ def get_local_driver(
     elif browser_name == constants.Browser.GOOGLE_CHROME:
         try:
             chrome_options = _set_chrome_options(
-                downloads_path, proxy_string, proxy_auth,
+                downloads_path, headless, proxy_string, proxy_auth,
                 proxy_user, proxy_pass, user_agent)
             if headless:
                 # Headless Chrome doesn't support extensions, which are
diff --git a/seleniumbase/core/download_helper.py b/seleniumbase/core/download_helper.py
@@ -35,7 +35,10 @@ def reset_downloads_folder():
 
 def reset_downloads_folder_assistant(archived_downloads_folder):
     if not os.path.exists(archived_downloads_folder):
-        os.makedirs(archived_downloads_folder)
+        try:
+            os.makedirs(archived_downloads_folder)
+        except Exception:
+            pass  # Should only be reachable during multi-threaded test runs
     new_archived_downloads_sub_folder = "%s/downloads_%s" % (
         archived_downloads_folder, int(time.time()))
     if os.path.exists(downloads_path):
diff --git a/seleniumbase/core/report_helper.py b/seleniumbase/core/report_helper.py
@@ -66,7 +66,10 @@ def clear_out_old_report_logs(archive_past_runs=True, get_log_folder=False):
     abs_path = os.path.abspath('.')
     file_path = abs_path + "/%s" % LATEST_REPORT_DIR
     if not os.path.exists(file_path):
-        os.makedirs(file_path)
+        try:
+            os.makedirs(file_path)
+        except Exception:
+            pass  # Should only be reachable during multi-threaded runs
 
     if archive_past_runs:
         archive_timestamp = int(time.time())
diff --git a/seleniumbase/drivers/ReadMe.md b/seleniumbase/drivers/ReadMe.md
@@ -1,4 +1,4 @@
-### SeleniumBase web driver storage
+### <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> SeleniumBase web driver storage
 
 #### Usage:
 
diff --git a/seleniumbase/extensions/ReadMe.md b/seleniumbase/extensions/ReadMe.md
@@ -0,0 +1,4 @@
+### <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> SeleniumBase browser extension storage
+
+**The List:**
+* disable_csp.zip => Disable Chrome's Content-Security-Policy
diff --git a/seleniumbase/extensions/__init__.py b/seleniumbase/extensions/__init__.py
diff --git a/seleniumbase/extensions/disable_csp.zip b/seleniumbase/extensions/disable_csp.zip
diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py
@@ -1699,10 +1699,13 @@ def assert_no_js_errors(self):
             # If unable to get browser logs, skip the assert and return.
             return
 
+        messenger_library = "//cdnjs.cloudflare.com/ajax/libs/messenger"
         errors = []
         for entry in browser_logs:
             if entry['level'] == 'SEVERE':
-                errors.append(entry)
+                if messenger_library not in entry['message']:
+                    # Add errors if not caused by SeleniumBase dependencies
+                    errors.append(entry)
         if len(errors) > 0:
             current_url = self.get_current_url()
             raise Exception(
@@ -3025,12 +3028,19 @@ def tearDown(self):
                         self.save_screenshot_after_test):
                     test_logpath = self.log_path + "/" + test_id
                     if not os.path.exists(test_logpath):
-                        os.makedirs(test_logpath)
+                        try:
+                            os.makedirs(test_logpath)
+                        except Exception:
+                            pass  # Only reachable during multi-threaded runs
                     log_helper.log_screenshot(test_logpath, self.driver)
+                    self.__add_pytest_html_extra()
                 if self.with_testing_base and has_exception:
                     test_logpath = self.log_path + "/" + test_id
                     if not os.path.exists(test_logpath):
-                        os.makedirs(test_logpath)
+                        try:
+                            os.makedirs(test_logpath)
+                        except Exception:
+                            pass  # Only reachable during multi-threaded runs
                     if ((not self.with_screen_shots) and (
                             not self.with_basic_test_info) and (
                             not self.with_page_source)):
@@ -3096,7 +3106,10 @@ def tearDown(self):
                                         self._testMethodName)
                 test_logpath = "latest_logs/" + test_id
                 if not os.path.exists(test_logpath):
-                    os.makedirs(test_logpath)
+                    try:
+                        os.makedirs(test_logpath)
+                    except Exception:
+                        pass  # Only reachable during multi-threaded runs
                 log_helper.log_test_failure_data(
                     self, test_logpath, self.driver, self.browser)
                 if len(self._drivers_list) > 0:
@@ -3108,7 +3121,10 @@ def tearDown(self):
                                         self._testMethodName)
                 test_logpath = "latest_logs/" + test_id
                 if not os.path.exists(test_logpath):
-                    os.makedirs(test_logpath)
+                    try:
+                        os.makedirs(test_logpath)
+                    except Exception:
+                        pass  # Only reachable during multi-threaded runs
                 log_helper.log_screenshot(test_logpath, self.driver)
             # Finally close all open browser windows
             self.__quit_all_drivers()
diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py
@@ -48,12 +48,12 @@ def pytest_addoption(parser):
                      dest='with_testing_base',
                      default=True,
                      help="""Use to save logs and screenshots when tests fail.
-                          It's no longer needed to add the following arguments:
-                          --with-screen_shots
-                          --with-basic_test_info
+                          The following options are now active by default
+                          with --with-testing_base (which is on by default):
+                          --with-screen_shots ,
+                          --with-basic_test_info ,
                           --with-page_source
-                          (Those modes are all active by default now when
-                          --with-testing_base is active. (Default: active)""")
+                          """)
     parser.addoption('--log_path', dest='log_path',
                      default='latest_logs/',
                      help='Where the log files are saved.')
@@ -81,17 +81,17 @@ def pytest_addoption(parser):
                      dest='with_screen_shots',
                      default=False,
                      help="""Use to save screenshots on test failure.
-                          (When "--with-testing_base" is True, this is on.)""")
+                          (Automatically on when using --with-testing_base)""")
     parser.addoption('--with-basic_test_info', action="store_true",
                      dest='with_basic_test_info',
                      default=False,
                      help="""Use to save basic test info on test failure.
-                          (When "--with-testing_base" is True, this is on.)""")
+                          (Automatically on when using --with-testing_base)""")
     parser.addoption('--with-page_source', action="store_true",
                      dest='with_page_source',
                      default=False,
                      help="""Use to save page source on test failure.
-                          (When "--with-testing_base" is True, this is on.)""")
+                          (Automatically on when using --with-testing_base)""")
     parser.addoption('--server', action='store',
                      dest='servername',
                      default='localhost',
diff --git a/seleniumbase/resources/ReadMe.md b/seleniumbase/resources/ReadMe.md
@@ -1,4 +1,4 @@
-## Resources Help
+## <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> Resources Help
 
 SeleniumBase uses some Javascript libraries for optional advanced features such as website tours, messenger, highlighting elements on a page, and other jQuery actions. In some cases, you may want to host these Javascript and CSS files from your own websites. For simplicity and convenience, these resources have been downloaded into the "resources" folder. If you decide to use your local versions, you may need to update the corresponding URLs in [base_case.py](https://github.com/seleniumbase/SeleniumBase/blob/master/seleniumbase/fixtures/base_case.py).
 
@@ -10,6 +10,12 @@ Here are some of the resource files you'll find here:
 
 **messenger/** - Files in this folder are used for creating Javascript notifications during test runs in Demo Mode.
 
-**shepherd/** - Files in this folder are used for creating website tours using the Shepherd Javascript library.
+**jquery_confirm/** - Files in this folder are used for creating Javascript confirmation prompts during test runs when using MasterQA.
 
-**bootstrap_tour/** - Files in this folder are used for creating website tours using the Bootstrap Tour Javascript library. (SeleniumBase currently defaults to using Shepherd Tours.)
+**shepherd/** - Files in this folder are used for creating website tours using the Shepherd Javascript library. (This is the default tour library.)
+
+**bootstrap_tour/** - Files in this folder are used for creating website tours using the Bootstrap Tour Javascript library.
+
+**introjs/** - Files in this folder are used for creating website tours using the IntroJS Javascript library.
+
+**hopscotch/** - Files in this folder are used for creating website tours using the Hopscotch Javascript library.
diff --git a/setup.py b/setup.py
@@ -17,7 +17,7 @@
 
 setup(
     name='seleniumbase',
-    version='1.21.5',
+    version='1.21.6',
     description='Reliable Browser Automation & Testing Framework',
     long_description=long_description,
     long_description_content_type='text/markdown',
@@ -61,7 +61,7 @@
         'selenium==3.141.0',
         'requests==2.21.0',  # Changing this may effect "urllib3"
         'urllib3==1.24.1',  # Keep this lib in sync with "requests"
-        'pytest>=4.3.0',
+        'pytest>=4.3.1',
         'pytest-cov>=2.6.1',
         'pytest-html>=1.20.0',
         'pytest-rerunfailures>=6.0',

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-### ![http://seleniumbase.com](https://cdn2.hubspot.net/hubfs/100006/images/super_logo_tiny.png "SeleniumBase") SeleniumBase Help Documents:`
	`1`	`+### <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> SeleniumBase Help Documents`
`2`	`2`
`3`	`3`	`> Table of Contents / Navigation:`
`4`	`4`	`> - [SeleniumBase Features List](https://github.com/seleniumbase/SeleniumBase/blob/master/help_docs/features_list.md)`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-## Console Scripts`
	`1`	`+## <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> Console Scripts`
`2`	`2`
`3`	`3`	`SeleniumBase console scripts help you get things done more easily, such as installing web drivers, creating a test directory with necessary configuration files, converting old Webdriver unittest scripts into SeleniumBase code, and using the Selenium Grid.`
`4`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-### SeleniumBase web driver storage`
	`1`	`+### <img src="https://cdn2.hubspot.net/hubfs/100006/images/super_square_logo_3a.png" title="SeleniumBase" height="32"> SeleniumBase web driver storage`
`2`	`2`
`3`	`3`	`#### Usage:`
`4`	`4`