Add Kokoro job configuration for Windows

Change-Id: I418e50648fb3e39fbf0fdc3738e9ee4ac34101ce
GitOrigin-RevId: b774261461ab2c8b3b28ea099b0c1bb0a93f94ab

Author: Googler

.kokoro/tests/run_tests.bat

@@ -0,0 +1,44 @@
+@rem  Copyright 2022 Google LLC
+@rem
+@rem  Licensed under the Apache License, Version 2.0 (the "License");
+@rem  you may not use this file except in compliance with the License.
+@rem  You may obtain a copy of the License at
+@rem
+@rem      http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem  Unless required by applicable law or agreed to in writing, software
+@rem  distributed under the License is distributed on an "AS IS" BASIS,
+@rem  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem  See the License for the specific language governing permissions and
+@rem  limitations under the License.
+
+@echo "Install pyenv tool for installing multiple Pythons"
+call C:\Python37\python.exe -m pip install pyenv-win --target %USERPROFILE%\.pyenv
+set "PYENV=%USERPROFILE%\.pyenv\pyenv-win\"
+set "PYENV_ROOT=%USERPROFILE%\.pyenv\pyenv-win\"
+set "PYENV_HOME=%USERPROFILE%\.pyenv\pyenv-win\"
+set "PATH=%USERPROFILE%\.pyenv\pyenv-win\bin;%USERPROFILE%\.pyenv\pyenv-win\shims;%PATH%"
+
+@echo "Install additional Python versions"
+call %USERPROFILE%\.pyenv\pyenv-win\bin\pyenv install --skip-existing --register 3.8.10 3.9.13 3.10.5
+call %USERPROFILE%\.pyenv\pyenv-win\bin\pyenv global 3.8.10 3.9.13 3.10.5
+
+if defined KOKORO_BUILD_ID ( @rem export vars only for Kokoro job
+  @rem  Setup service account credentials
+  set "GOOGLE_APPLICATION_CREDENTIALS=%KOKORO_GFILE_DIR%/kokoro/service-account-key.json"
+
+  @rem Setup project id
+  set /p PROJECT_ID=<%KOKORO_GFILE_DIR%/kokoro/project-id.txt
+  set COMPOSER_TESTS_PROJECT_ID=PROJECT_ID
+)
+
+@rem  install nox for testing
+call C:\Python37\python.exe -m pip install --require-hashes --upgrade --quiet -r .kokoro/tests/requirements.txt
+call C:\Python37\python.exe -m nox --version
+
+@echo "******************** Running unit tests... ********************\n"
+call C:\Python37\python.exe -m nox -s "unit"
+@echo "******************** Unit tests complete.  ********************\n"
+@echo "******************** Running E2E tests... ********************\n"
+@rem call C:\C:\Python37\python.exe\python.exe -m nox -s "e2e"
+@echo "******************** Tests complete.  ********************\n"

.kokoro/tests/run_tests.sh

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set -xe
 
 # add user's pip binary path to PATH
 export PATH="${HOME}/.local/bin:${PATH}"
@@ -32,7 +32,9 @@ if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX
   pyenv global 3.7.13
 else
   pyenv install --skip-existing 3.7.10
-  pyenv global 3.7.10
+  pyenv install --skip-existing 3.8.10
+  pyenv install --skip-existing 3.9.5
+  pyenv global 3.7.10 3.8.10 3.9.5
 fi
 
 # install nox for testing

.kokoro/windows/kokoro_build.bat

@@ -0,0 +1,17 @@
+@rem Copyright 2022 Google LLC
+
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+
+cd "%KOKORO_ARTIFACTS_DIR%\git\composer-local-development"
+call .kokoro\tests\run_tests.bat

.kokoro/windows/presubmit.cfg

@@ -0,0 +1,10 @@
+# -*- protobuffer -*-
+# proto-file: google3/devtools/kokoro/config/proto/build.proto
+# proto-message: BuildConfig
+
+# Resources needed for E2E tests
+gfile_resources: "/bigstore/composer-local-dev-test-resources"
+
+# Location of the bash script. Should have value <github_scm.name>/<path_from_repository_root>.
+# github_scm.name is specified in the job configuration (next section).
+build_file: "composer-local-development/.kokoro/windows/kokoro_build.bat"

noxfile.py

@@ -17,7 +17,7 @@
 import nox
 
 DEFAULT_PYTHON_VERSION = "3.9"
-TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.10"]  # TODO Fix 3.9 on Kokoro
+TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"]
 
 CURRENT_DIRECTORY = Path(__file__).parent.absolute()
 

tests/unit/test_environment.py

@@ -412,8 +412,9 @@ def test_missing_variables_env(self):
             f"Environment variables file '{env_dir / 'variables.env'}' "
             f"not found."
         )
-        with pytest.raises(errors.ComposerCliError, match=exp_error):
+        with pytest.raises(errors.ComposerCliError) as err:
             environment.load_environment_variables(env_dir)
+            assert str(err) == exp_error
 
     def test_load_environment_variables_filter_blocked_env_vars(self):
         env_dir = (TEST_DATA_DIR / "blocked_env_vars").resolve()
@@ -465,10 +466,9 @@ def test_invalid_env_variables(self):
         exp_error = constants.INVALID_ENV_VARIABLES_FILE_ERROR.format(
             env_file_path=env_file_path, line="AIRFLOW_KEYVALUE"
         )
-        with pytest.raises(
-            errors.FailedToParseVariablesError, match=re.escape(exp_error)
-        ):
+        with pytest.raises(errors.FailedToParseVariablesError) as err:
             environment.load_environment_variables(env_dir)
+            assert str(err) == exp_error
 
     @mock.patch("composer_local_dev.environment.docker.from_env")
     @mock.patch(
@@ -484,10 +484,10 @@ def test_missing_requirements(self, mocked_docker, mocked_tag, tmpdir):
             location="location",
             dags_path=str(env_dir),
         )
-        with pytest.raises(
-            errors.ComposerCliError, match=f"Missing '{requirement_file}' file."
-        ):
+        exp_error = f"Missing '{requirement_file}' file."
+        with pytest.raises(errors.ComposerCliError) as err:
             env.assert_requirements_exist()
+            assert str(err) == exp_error
 
     @mock.patch("composer_local_dev.environment.docker.from_env")
     def test_get_container(self, mocked_docker, default_env):
@@ -1059,17 +1059,19 @@ def test_invalid_config(self):
             config_path=config_path, error=""
         )
         with pytest.raises(
-            errors.FailedToParseConfigError, match=exp_error
-        ), working_directory(env_dir):
+            errors.FailedToParseConfigError
+        ) as err, working_directory(env_dir):
             environment.EnvironmentConfig(env_dir, None)
+            assert str(err) == exp_error
 
     def test_missing_config(self):
         env_dir = (TEST_DATA_DIR / "missing_composer").resolve()
         exp_error = f"Configuration file '{env_dir / 'config.json'}' not found."
-        with pytest.raises(
-            errors.ComposerCliError, match=exp_error
-        ), working_directory(env_dir):
+        with pytest.raises(errors.ComposerCliError) as err, working_directory(
+            env_dir
+        ):
             environment.EnvironmentConfig(env_dir, None)
+            assert str(err) == exp_error
 
     @mock.patch(
         "composer_local_dev.environment.EnvironmentConfig.load_configuration_from_file"
@@ -1091,10 +1093,9 @@ def test_param_invalid_int(self, mocked_load_conf, tmp_path, param, value):
         exp_error = constants.INVALID_INT_VALUE_ERROR.format(
             param_name=param, value=value
         )
-        with pytest.raises(
-            errors.FailedToParseConfigParamIntError, match=exp_error
-        ):
+        with pytest.raises(errors.FailedToParseConfigParamIntError) as err:
             environment.EnvironmentConfig(tmp_path, None)
+            assert str(err) == exp_error
 
     @mock.patch(
         "composer_local_dev.environment.EnvironmentConfig.load_configuration_from_file"
@@ -1122,7 +1123,6 @@ def test_param_invalid_int_range(
         exp_error = constants.INVALID_INT_RANGE_VALUE_ERROR.format(
             param_name=param, value=value, allowed_range=allowed_range
         )
-        with pytest.raises(
-            errors.FailedToParseConfigParamIntRangeError, match=exp_error
-        ):
+        with pytest.raises(errors.FailedToParseConfigParamIntRangeError) as err:
             environment.EnvironmentConfig(tmp_path, None)
+            assert str(err) == exp_error

tests/unit/test_files.py

@@ -56,10 +56,11 @@ def test_no_composer_dir(self, env_name):
             )
             + constants.ADD_DEBUG_ON_ERROR_INFO
         )
-        with pytest.raises(
-            errors.ComposerCliError, match=re.escape(exp_error)
-        ), working_directory(env_dir):
+        with pytest.raises(errors.ComposerCliError) as err, working_directory(
+            env_dir
+        ):
             files.resolve_environment_path(env_name)
+            assert str(err) == exp_error
 
     @pytest.mark.parametrize("env_name", [None, "test"])
     def test_no_envs(self, env_name):
@@ -71,10 +72,11 @@ def test_no_envs(self, env_name):
             )
             + constants.ADD_DEBUG_ON_ERROR_INFO
         )
-        with pytest.raises(
-            errors.ComposerCliError, match=re.escape(exp_error)
-        ), working_directory(env_dir):
+        with pytest.raises(errors.ComposerCliError) as err, working_directory(
+            env_dir
+        ):
             files.resolve_environment_path(env_name)
+            assert str(err) == exp_error
 
     @pytest.mark.parametrize("env_name", ["one_env", "two_envs"])
     def test_not_exising_name(self, env_name):
@@ -87,10 +89,11 @@ def test_not_exising_name(self, env_name):
             )
             + constants.ADD_DEBUG_ON_ERROR_INFO
         )
-        with pytest.raises(
-            errors.ComposerCliError, match=re.escape(exp_error)
-        ), working_directory(env_dir):
+        with pytest.raises(errors.ComposerCliError) as err, working_directory(
+            env_dir
+        ):
             files.resolve_environment_path(invalid_env)
+            assert str(err) == exp_error
 
     @pytest.mark.parametrize("env_name", [None, "example_env"])
     def test_one_env_existing_name(self, env_name):
@@ -113,10 +116,11 @@ def test_two_envs_no_default(self):
         error_msg = constants.ENVIRONMENT_NOT_SELECTED_ERROR.format(
             env_dir=composer_dir, env_names=env_names
         )
-        with pytest.raises(
-            errors.ComposerCliError, match=error_msg
-        ), working_directory(env_dir):
+        with pytest.raises(errors.ComposerCliError) as err, working_directory(
+            env_dir
+        ):
             files.resolve_environment_path(None)
+            assert str(err) == error_msg
 
 
 class TestGetEnvironmentDirectories:
@@ -210,8 +214,9 @@ def test_existing_path(self, tmp_path):
     def test_missing_path(self):
         path = "i/dont/exist"
         error_msg = f"Dags path does not exist or is not a directory: {path}"
-        with pytest.raises(errors.DAGPathNotExistError, match=error_msg):
+        with pytest.raises(errors.DAGPathNotExistError) as err:
             files.assert_dag_path_exists(path)
+            assert str(err) == error_msg
 
     def test_path_is_file(self, tmp_path):
         file_path = tmp_path / "file.ext"
@@ -220,8 +225,9 @@ def test_path_is_file(self, tmp_path):
         error_msg = (
             f"Dags path does not exist or is not a directory: {file_path}"
         )
-        with pytest.raises(errors.DAGPathNotExistError, match=error_msg):
+        with pytest.raises(errors.DAGPathNotExistError) as err:
             files.assert_dag_path_exists(str(file_path))
+            assert str(err) == error_msg
 
 
 def test_dos2unix_file(tmp_path):

tests/unit/test_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import os
 import pathlib
 import re
 import subprocess
@@ -107,12 +108,12 @@ def test_resolve_path_on_windows(self, mocked_os_check, tmpdir):
     @mock.patch.object(pathlib.Path, "is_dir")
     @mock.patch.object(pathlib.Path, "expanduser")
     @mock.patch(
-        "composer_local_dev.environment.utils.is_windows_os", return_value=True
+        "composer_local_dev.environment.utils.is_windows_os", return_value=False
     )
     def test_resolve_path_not_on_windows(
         self, mocked_os_check, mocked_expand, mocked_is_dir
     ):
-        config_path = "path/to/config"
+        config_path = os.pathsep.join(["path", "to", "config"])
         mocked_expand.return_value = pathlib.Path(config_path)
         mocked_is_dir.return_value = True
         actual_config_path = utils.resolve_gcloud_config_path()