Merge branch 'wip/python-environment' into 'develop'

Author: BenjaminHamon

Automation/Scripts/automation_scripts/commands/distribution.py

@@ -54,19 +54,16 @@ def check_requirements(self, arguments: argparse.Namespace, **kwargs) -> None:
     def run(self, arguments: argparse.Namespace, simulate: bool, **kwargs) -> None:
         python_executable = sys.executable
         project_configuration: ProjectConfiguration = kwargs["configuration"]
+
         all_python_packages = project_configuration.list_python_packages()
+        python_package_metadata = project_configuration.get_python_package_metadata()
 
         process_runner = ProcessRunner(ProcessSpawner(is_console = True))
         python_package_builder = PythonPackageBuilder(python_executable, process_runner)
 
         logger.info("Generating python package metadata")
         for python_package in all_python_packages:
-            python_package_builder.generate_package_metadata(
-                product_identifier = project_configuration.project_identifier,
-                project_version = project_configuration.project_version,
-                copyright_text = project_configuration.copyright,
-                python_package = python_package,
-                simulate = simulate)
+            python_package_builder.generate_package_metadata(python_package, python_package_metadata, simulate = simulate)
 
 
     async def run_async(self, arguments: argparse.Namespace, simulate: bool, **kwargs) -> None:
@@ -92,7 +89,6 @@ async def run_async(self, arguments: argparse.Namespace, simulate: bool, **kwarg
         python_executable = sys.executable
         project_configuration: ProjectConfiguration = kwargs["configuration"]
 
-        version = project_configuration.project_version.full_identifier
         all_python_packages = project_configuration.list_python_packages()
 
         process_runner = ProcessRunner(ProcessSpawner(is_console = True))
@@ -104,7 +100,7 @@ async def run_async(self, arguments: argparse.Namespace, simulate: bool, **kwarg
             log_file_path = os.path.join("Artifacts", "Logs", "BuildDistributionPackage_%s.log" % python_package.identifier)
 
             await python_package_builder.build_distribution_package(
-                python_package, version, output_directory, log_file_path, simulate = simulate)
+                python_package, output_directory, log_file_path, simulate = simulate)
 
 
 class _UploadCommand(AutomationCommand):

Automation/Scripts/automation_scripts/configuration/project_configuration.py

@@ -3,6 +3,7 @@
 
 from bhamon_development_toolkit.automation.project_version import ProjectVersion
 from bhamon_development_toolkit.python.python_package import PythonPackage
+from bhamon_development_toolkit.python.python_package_metadata import PythonPackageMetadata
 
 
 class ProjectConfiguration:
@@ -36,6 +37,15 @@ def get_artifact_default_parameters(self) -> dict:
         }
 
 
+    def get_python_package_metadata(self) -> PythonPackageMetadata:
+        return PythonPackageMetadata(
+            product_identifier = self.project_identifier,
+            version_identifier = self.project_version.full_identifier,
+            revision_date = self.project_version.revision_date,
+            copyright_text = self.copyright,
+        )
+
+
     def list_automation_packages(self) -> List[PythonPackage]:
         return [
             PythonPackage(

Automation/Setup/python_helpers.py

@@ -41,7 +41,7 @@ def find_system_python_executable(python_versions: List[str]) -> Optional[str]:
     raise ValueError("Unsupported platform: '%s'" % platform.system())
 
 
-def setup_virtual_environment(python_system_executable: str, venv_directory: str, simulate: bool) -> None:
+def setup_virtual_environment(system_python_executable: str, venv_directory: str, simulate: bool) -> None:
     logger.info("Setting up python virtual environment (Path: %s)", venv_directory)
 
     venv_python_executable = get_venv_python_executable(venv_directory)
@@ -54,7 +54,7 @@ def setup_virtual_environment(python_system_executable: str, venv_directory: str
             os.remove(os.path.join(venv_directory, "scripts", "python.exe"))
         shutil.rmtree(venv_directory)
 
-    run_python_command([ python_system_executable, "-m", "venv", venv_directory ], simulate = simulate)
+    run_python_command([ system_python_executable, "-m", "venv", venv_directory ], simulate = simulate)
 
     if platform.system() == "Darwin": # pylint: disable = no-else-raise
         raise NotImplementedError("MacOS is not supported")

Automation/Setup/setup.py

@@ -17,11 +17,6 @@
 
 
 def main() -> None:
-
-    # Prevent active pyvenv from overriding a python executable specified in a command
-    if "__PYVENV_LAUNCHER__" in os.environ:
-        del os.environ["__PYVENV_LAUNCHER__"]
-
     with automation_helpers.execute_in_workspace(__file__):
         arguments = parse_arguments()
         automation_helpers.configure_logging(arguments.verbosity)
@@ -31,11 +26,11 @@ def main() -> None:
 
         logger.info("Setting up local workspace (Path: %s)", os.getcwd())
 
-        python_system_executable = python_helpers.find_and_check_system_python_executable(python_versions)
+        system_python_executable = python_helpers.find_and_check_system_python_executable(python_versions)
         venv_python_executable = python_helpers.get_venv_python_executable(venv_directory)
         python_package_collection = [ "Automation/Scripts[dev]" ] + python_helpers.list_python_packages("Sources")
 
-        python_helpers.setup_virtual_environment(python_system_executable, venv_directory, simulate = arguments.simulate)
+        python_helpers.setup_virtual_environment(system_python_executable, venv_directory, simulate = arguments.simulate)
         python_helpers.install_python_packages(venv_python_executable, python_package_collection, simulate = arguments.simulate)
 
 

Sources/toolkit/bhamon_development_toolkit/processes/process_helpers.py

@@ -1,7 +1,12 @@
+import logging
 import shlex
+import subprocess
 from typing import List, Optional, TextIO
 
 from bhamon_development_toolkit.logging.raw_logger import RawLogger
+from bhamon_development_toolkit.processes.exceptions.process_failure_exception import ProcessFailureException
+from bhamon_development_toolkit.processes.executable_command import ExecutableCommand
+from bhamon_development_toolkit.processes.process_result import ProcessResult
 
 
 def format_executable_command(command: List[str]):
@@ -22,3 +27,40 @@ def create_raw_logger(stream: Optional[TextIO] = None, log_file_path: Optional[s
         raw_logger.configure_log_file(log_file_path, "debug", mode = "w", encoding = "utf-8")
 
     return raw_logger
+
+
+def run_simple(logger: logging.Logger, command: ExecutableCommand,
+        working_directory: Optional[str] = None, check_exit_code: bool = True, simulate: bool = False) -> ProcessResult:
+
+    logger.debug("+ %s", format_executable_command(command.get_command_for_logging()))
+
+    subprocess_options = {
+        "cwd": working_directory,
+        "capture_output": True,
+        "text": True,
+        "encoding": "utf-8",
+        "stdin": subprocess.DEVNULL,
+    }
+
+    if not simulate:
+        result = subprocess.run(command.get_command(), check = False, **subprocess_options)
+        for line in result.stdout.splitlines():
+            logger.debug(line)
+        for line in result.stderr.splitlines():
+            logger.error(line)
+
+        if check_exit_code and result.returncode != 0:
+            exception_message = "Subprocess failed (Executable: '%s', ExitCode: %s)" % (command.executable_path, result.returncode)
+            raise ProcessFailureException(exception_message, command.executable_path, result.returncode)
+
+        return ProcessResult(
+            executable = command.executable_path,
+            exit_code = result.returncode,
+            standard_output = result.stdout,
+            error_output = result.stderr,
+        )
+
+    return ProcessResult(
+        executable = command.executable_path,
+        exit_code = 0,
+    )

Sources/toolkit/bhamon_development_toolkit/python/python_environment.py

@@ -0,0 +1,88 @@
+import logging
+import os
+import platform
+import shutil
+import sys
+from typing import List, Optional
+
+from bhamon_development_toolkit.processes import process_helpers
+from bhamon_development_toolkit.processes.executable_command import ExecutableCommand
+
+
+logger = logging.getLogger("Python")
+
+
+class PythonEnvironment:
+
+
+    def __init__(self, system_python_executable: str, venv_directory: str) -> None:
+        self._system_python_executable = system_python_executable
+        self._venv_directory = venv_directory
+
+
+    def get_venv_directory(self) -> str:
+        return self._venv_directory
+
+
+    def get_venv_python_executable(self) -> str:
+        return self.get_venv_executable("python")
+
+
+    def get_venv_executable(self, executable: str) -> str:
+        if platform.system() == "Windows":
+            return os.path.join(self._venv_directory, "scripts", executable + ".exe")
+        return os.path.join(self._venv_directory, "bin", executable)
+
+
+    def _get_pip_configuration_file_path(self) -> str:
+        if platform.system() == "Windows":
+            return os.path.join(self._venv_directory, "pip.ini")
+        return os.path.join(self._venv_directory, "pip.conf")
+
+
+    def setup_virtual_environment(self, pip_configuration_file_path: Optional[str] = None, simulate: bool = False) -> None:
+        venv_python_executable = self.get_venv_python_executable()
+        if sys.executable.lower() == os.path.abspath(venv_python_executable).lower():
+            raise RuntimeError("Active python is the target virtual environment")
+
+        if os.path.isdir(self._venv_directory) and not simulate:
+            # Try to remove the executable first since it might be in use, otherwise we would be leaving a broken venv
+            if platform.system() == "Windows" and os.path.exists(os.path.join(self._venv_directory, "scripts", "python.exe")):
+                os.remove(os.path.join(self._venv_directory, "scripts", "python.exe"))
+            shutil.rmtree(self._venv_directory)
+
+        venv_command = ExecutableCommand(self._system_python_executable)
+        venv_command.add_arguments([ "-m", "venv", self._venv_directory ])
+
+        process_helpers.run_simple(logger, venv_command, simulate = simulate)
+
+        if pip_configuration_file_path is not None:
+            pip_configuration_file_path_in_venv = self._get_pip_configuration_file_path()
+            if not simulate:
+                shutil.copy(pip_configuration_file_path, pip_configuration_file_path_in_venv)
+
+        self.install_python_packages([ "pip", "wheel" ], simulate = simulate)
+
+
+    def install_python_packages(self,
+            name_or_path_collection: List[str], simulate: bool = False) -> None:
+
+        install_command = ExecutableCommand(self.get_venv_python_executable())
+        install_command.add_arguments([ "-m", "pip", "install", "--upgrade" ] + name_or_path_collection)
+
+        process_helpers.run_simple(logger, install_command, simulate = simulate)
+
+
+    def install_python_packages_for_development(self,
+            name_or_path_collection: List[str], simulate: bool = False) -> None:
+
+        def is_local_package(name_or_path: str) -> bool:
+            return name_or_path.startswith(".") or "/" in name_or_path or "\\" in name_or_path
+
+        install_command = ExecutableCommand(self.get_venv_python_executable())
+        install_command.add_arguments([ "-m", "pip", "install", "--upgrade" ])
+
+        for name_or_path in name_or_path_collection:
+            install_command.add_arguments([ "--editable", name_or_path ] if is_local_package(name_or_path) else [ name_or_path ])
+
+        process_helpers.run_simple(logger, install_command, simulate = simulate)

Sources/toolkit/bhamon_development_toolkit/python/python_helpers.py

@@ -2,17 +2,19 @@
 import os
 import platform
 import shutil
-import subprocess
 import sys
 from typing import List, Optional
 
-from bhamon_development_toolkit.processes import process_helpers
-from bhamon_development_toolkit.processes.executable_command import ExecutableCommand
-
 
 logger = logging.getLogger("Python")
 
 
+def resolve_system_python_executable() -> str:
+    if hasattr(sys, "_base_executable"):
+        return sys._base_executable # type: ignore # pylint: disable = protected-access
+    raise RuntimeError("Unable to resolve the system Python executable")
+
+
 def find_and_check_system_python_executable(python_versions: List[str]) -> str:
     python_executable = find_system_python_executable(python_versions)
     if python_executable is None or not shutil.which(python_executable):
@@ -23,15 +25,11 @@ def find_and_check_system_python_executable(python_versions: List[str]) -> str:
 
 def find_system_python_executable(python_versions: List[str]) -> Optional[str]:
     if platform.system() == "Linux":
-        return "/usr/bin/python3"
-
-    if platform.system() == "Windows":
         possible_paths = []
 
         for version in python_versions:
             possible_paths += [
-                os.path.join(os.environ["SystemDrive"] + "\\", "Python%s" % version.replace(".", ""), "python.exe"),
-                os.path.join(os.environ["ProgramFiles"], "Python%s" % version.replace(".", ""), "python.exe"),
+                "/usr/bin/python" + version,
             ]
 
         for path in possible_paths:
@@ -40,74 +38,19 @@ def find_system_python_executable(python_versions: List[str]) -> Optional[str]:
 
         return None
 
-    raise ValueError("Unsupported platform: '%s'" % platform.system())
-
-
-def setup_virtual_environment(python_system_executable: str, venv_directory: str, simulate: bool) -> None:
-    logger.info("Setting up python virtual environment (Path: %s)", venv_directory)
-
-    venv_python_executable = get_venv_python_executable(venv_directory)
-    if sys.executable.lower() == os.path.abspath(venv_python_executable).lower():
-        raise RuntimeError("Active python is the target virtual environment")
-
-    if os.path.isdir(venv_directory) and not simulate:
-        # Try to remove the executable first since it might be in use, otherwise we would be leaving a broken venv
-        if platform.system() == "Windows" and os.path.exists(os.path.join(venv_directory, "scripts", "python.exe")):
-            os.remove(os.path.join(venv_directory, "scripts", "python.exe"))
-        shutil.rmtree(venv_directory)
-
-    venv_command = ExecutableCommand(python_system_executable)
-    venv_command.add_arguments([ "-m", "venv", venv_directory ])
-
-    run_python_command(venv_command, simulate = simulate)
-
-    if platform.system() in [ "Darwin", "Linux" ] and not os.path.exists(os.path.join(venv_directory, "scripts")) and not simulate:
-        os.symlink("bin", os.path.join(venv_directory, "scripts"))
-
-    install_python_packages(venv_python_executable, [ "pip", "wheel" ], simulate = simulate)
-
-
-def get_venv_python_executable(venv_directory: str) -> str:
     if platform.system() == "Windows":
-        return os.path.join(venv_directory, "scripts", "python.exe")
-    return os.path.join(venv_directory, "bin", "python")
-
-
-def install_python_packages(python_executable: str,
-        name_or_path_collection: List[str], python_package_repository: Optional[str] = None, simulate: bool = False) -> None:
-
-    def is_local_package(name_or_path: str) -> bool:
-        return name_or_path.startswith(".") or "/" in name_or_path or "\\" in name_or_path
-
-    install_command = ExecutableCommand(python_executable)
-    install_command.add_arguments([ "-m", "pip", "install", "--upgrade" ])
-
-    if python_package_repository is not None:
-        install_command.add_arguments([ "--extra-index", python_package_repository ])
-
-    for name_or_path in name_or_path_collection:
-        install_command.add_arguments([ "--editable", name_or_path ] if is_local_package(name_or_path) else [ name_or_path ])
-
-    run_python_command(install_command, simulate = simulate)
-
+        possible_paths = []
 
-def run_python_command(command: ExecutableCommand, working_directory: Optional[str] = None, simulate: bool = False) -> None:
-    logger.info("+ %s", process_helpers.format_executable_command(command.get_command_for_logging()))
+        for version in python_versions:
+            possible_paths += [
+                os.path.join(os.environ["SystemDrive"] + "\\", "Python%s" % version.replace(".", ""), "python.exe"),
+                os.path.join(os.environ["ProgramFiles"], "Python%s" % version.replace(".", ""), "python.exe"),
+            ]
 
-    subprocess_options = {
-        "cwd": working_directory,
-        "capture_output": True,
-        "text": True,
-        "encoding": "utf-8",
-        "stdin": subprocess.DEVNULL,
-    }
+        for path in possible_paths:
+            if os.path.exists(path):
+                return path
 
-    if not simulate:
-        process_result = subprocess.run(command.get_command(), check = False, **subprocess_options)
-        for line in process_result.stdout.splitlines():
-            logger.debug(line)
-        for line in process_result.stderr.splitlines():
-            logger.error(line)
+        return None
 
-        if process_result.returncode != 0:
-            raise RuntimeError("Python command failed (ExitCode: %r)" % process_result.returncode)
+    raise ValueError("Unsupported platform: '%s'" % platform.system())