Improve comments on app engine imports and add docs

docs/contributing-code/source_code.md

@@ -38,3 +38,12 @@ permalink: /contributing-code/source-code/
   * **platform_requirements.txt** - platform dependent package list.
 * **bower.json** - component dependencies for Polymer 2.
 * **butler.py** - helper script for various command line tasks (e.g. testing, deployment).
+
+## Pitfalls
+
+* **App Engine imports** - Directories listed in
+  [`.gcloudignore`](https://github.com/google/clusterfuzz/blob/master/src/appengine/.gcloudignore)
+  are not uploaded to App Engine. Shared modules cannot use top-level imports
+  from these directories, as this causes a `ModuleNotFoundError` on App Engine
+  startup. Instead, import them locally inside the functions or methods where
+  they are used.

src/clusterfuzz/_internal/build_management/build_archive.py

@@ -96,7 +96,9 @@ def list_fuzz_targets(self) -> List[str]:
         The list of fuzz targets.
     """
     if self._fuzz_targets is None:
-      # Import here as this path is not available in App Engine context.
+      # `clusterfuzz._internal.bot` has to be imported locally since it is not
+      # uploaded to GCP with App Engine context. See:
+      # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
       from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
       self._fuzz_targets = {
@@ -207,7 +209,9 @@ def get_target_dependencies(
 
   @override
   def find_fuzz_targets(self) -> List[str]:
-    # Import here as this path is not available in App Engine context.
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
     return [
@@ -328,7 +332,9 @@ def __init__(self,
           'archive_schema_version field')
       self._archive_schema_version = default_archive_schema_version
 
-    # Import here as this path is not available in App Engine context.
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
     self._manifest_fuzz_targets = (

src/clusterfuzz/_internal/build_management/build_manager.py

@@ -330,7 +330,9 @@ def delete(self):
 
 def _read_schema_version_from_manifest(build_dir: str) -> int:
   """Reads archive_schema_version from clusterfuzz_manifest.json."""
-  # Import here as this path is not available in App Engine context.
+  # `clusterfuzz._internal.bot` has to be imported locally since it is not
+  # uploaded to GCP with App Engine context. See:
+  # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
   from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
   manifest = fuzzer_utils.read_chrome_manifest(build_dir)
@@ -369,7 +371,9 @@ def _patch_rpaths(build_dir: str, app_path_env: str):
     return
 
   if environment.is_engine_fuzzer_job():
-    # Import here as this path is not available in App Engine context.
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
     for target_path in fuzzer_utils.get_fuzz_targets(build_dir):
@@ -607,7 +611,9 @@ def _unpack_build(self,
 
   def _get_fuzz_targets_from_dir(self, build_dir):
     """Get iterator of fuzz targets from build dir."""
-    # Import here as this path is not available in App Engine context.
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
     for path in fuzzer_utils.get_fuzz_targets(build_dir):
@@ -1357,6 +1363,9 @@ def setup_regular_build(revision,
 
   build_class = RegularBuild
   if environment.is_trusted_host():
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.untrusted_runner import build_setup_host
     build_class = build_setup_host.RemoteRegularBuild
   elif environment.platform() == 'FUCHSIA':
@@ -1380,7 +1389,9 @@ def setup_regular_build(revision,
   # Additional binaries to pull (for fuzzing engines such as Centipede).
   extra_bucket_path = get_bucket_path('EXTRA_BUILD_BUCKET_PATH')
   if extra_bucket_path and not build.is_discovery:
-    # Import here as this path is not available in App Engine context.
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
     extra_build_urls = get_build_urls_list(extra_bucket_path)
     extra_build_url = revisions.find_build_url(extra_bucket_path,
@@ -1426,6 +1437,9 @@ def setup_symbolized_builds(revision):
 
   build_class = SymbolizedBuild
   if environment.is_trusted_host():
+    # `clusterfuzz._internal.bot` has to be imported locally since it is not
+    # uploaded to GCP with App Engine context. See:
+    # https://google.github.io/clusterfuzz/contributing-code/source-code/#pitfalls
     from clusterfuzz._internal.bot.untrusted_runner import build_setup_host
     build_class = build_setup_host.RemoteSymbolizedBuild  # pylint: disable=no-member