Tentative rework for raw logger

Author: BenjaminHamon

Sources/toolkit/bhamon_development_toolkit/automation/automation_helpers.py

@@ -46,9 +46,9 @@ def configure_logging(arguments: argparse.Namespace):
     logging.addLevelName(logging.ERROR, "Error")
     logging.addLevelName(logging.CRITICAL, "Critical")
 
-    logging_helpers.configure_log_stream(sys.stdout, log_stream_verbosity, message_format, date_format)
+    logging_helpers.configure_log_stream(logging.root, sys.stdout, log_stream_verbosity, message_format, date_format)
     if log_file_path is not None:
-        logging_helpers.configure_log_file(log_file_path, log_file_verbosity, message_format, date_format, mode = "w", encoding = "utf-8")
+        logging_helpers.configure_log_file(logging.root, log_file_path, log_file_verbosity, message_format, date_format, mode = "w", encoding = "utf-8")
 
 
 def create_argument_parser() -> argparse.ArgumentParser:

Sources/toolkit/bhamon_development_toolkit/logging/logging_helpers.py

@@ -4,6 +4,7 @@
 
 
 all_log_levels = [ "debug", "info", "warning", "error", "critical" ]
+date_format_iso = "%Y-%m-%dT%H:%M:%S"
 
 
 def get_level_as_integer(level_as_string: str) -> int:
@@ -21,18 +22,18 @@ def get_level_as_integer(level_as_string: str) -> int:
     raise ValueError("Unknown logging level '%s'" % level_as_string)
 
 
-def configure_log_stream(stream: TextIO, level: str, message_format: str, date_format: str) -> None:
+def configure_log_stream(logger: logging.Logger, stream: TextIO, level: str, message_format: str, date_format: str) -> None:
     formatter = logging.Formatter(fmt = message_format, datefmt = date_format, style = "{")
 
     stream_handler = logging.StreamHandler(stream)
     stream_handler.setLevel(get_level_as_integer(level))
     stream_handler.formatter = formatter
 
-    logging.root.addHandler(stream_handler)
+    logger.addHandler(stream_handler)
 
 
 def configure_log_file( # pylint: disable = too-many-arguments
-        file_path: str, level: str, message_format: str, date_format: str, mode: str, encoding: str) -> None:
+        logger: logging.Logger, file_path: str, level: str, message_format: str, date_format: str, mode: str, encoding: str) -> None:
 
     if os.path.dirname(file_path):
         os.makedirs(os.path.dirname(file_path), exist_ok = True)
@@ -43,4 +44,4 @@ def configure_log_file( # pylint: disable = too-many-arguments
     file_handler.setLevel(get_level_as_integer(level))
     file_handler.formatter = formatter
 
-    logging.root.addHandler(file_handler)
+    logger.addHandler(file_handler)

Sources/toolkit/bhamon_development_toolkit/logging/raw_logger.py

@@ -0,0 +1,40 @@
+import logging
+from typing import Optional, TextIO
+
+from bhamon_development_toolkit.logging import logging_helpers
+
+
+class RawLogger:
+
+
+    def __init__(self, name: Optional[str] = None) -> None:
+        if name is None:
+            name = "raw"
+
+        self._logger = logging.Logger(name, logging.DEBUG)
+
+
+    def get_actual_logger(self) -> logging.Logger:
+        return self._logger
+
+
+    def __enter__(self) -> "RawLogger":
+        return self
+
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        self.dispose()
+
+
+    def configure_log_stream(self, stream: TextIO, level: str) -> None:
+        logging_helpers.configure_log_stream(self._logger, stream, level, "{message}", logging_helpers.date_format_iso)
+
+
+    def configure_log_file(self, file_path: str, level: str, mode: str, encoding: str) -> None:
+        logging_helpers.configure_log_file(self._logger, file_path, level, "{message}", logging_helpers.date_format_iso, mode, encoding)
+
+
+    def dispose(self) -> None:
+        for handler in list(self._logger.handlers):
+            self._logger.removeHandler(handler)
+            handler.close()

Sources/toolkit/bhamon_development_toolkit/processes/process_helpers.py

@@ -1,8 +1,8 @@
-import logging
-import os
 import shlex
 from typing import List, Optional, TextIO
 
+from bhamon_development_toolkit.logging.raw_logger import RawLogger
+
 
 def format_executable_command(command: List[str]):
     return " ".join(format_executable_command_element(element) for element in command)
@@ -12,24 +12,13 @@ def format_executable_command_element(element: str) -> str:
     return shlex.quote(element)
 
 
-def create_raw_logger(stream: Optional[TextIO] = None, log_file_path: Optional[str] = None) -> logging.Logger:
-    logger = logging.Logger("raw")
-    logger.setLevel(logging.DEBUG)
-
-    formatter = logging.Formatter("{message}", style = "{")
+def create_raw_logger(stream: Optional[TextIO] = None, log_file_path: Optional[str] = None) -> RawLogger:
+    raw_logger = RawLogger()
 
     if stream is not None:
-        stream_handler = logging.StreamHandler(stream)
-        stream_handler.setLevel(logging.INFO)
-        stream_handler.formatter = formatter
-        logger.addHandler(stream_handler)
+        raw_logger.configure_log_stream(stream, "info")
 
     if log_file_path is not None:
-        if os.path.dirname(log_file_path):
-            os.makedirs(os.path.dirname(log_file_path), exist_ok = True)
-        file_handler = logging.FileHandler(log_file_path, mode = "w", encoding = "utf-8")
-        file_handler.setLevel(logging.DEBUG)
-        file_handler.formatter = formatter
-        logger.addHandler(file_handler)
-
-    return logger
+        raw_logger.configure_log_file(log_file_path, "debug", mode = "w", encoding = "utf-8")
+
+    return raw_logger

Sources/toolkit/bhamon_development_toolkit/processes/process_output_logger.py

@@ -6,16 +6,20 @@
 class ProcessOutputLogger(ProcessOutputHandler):
 
 
-    def __init__(self, logger: logging.Logger) -> None:
+    def __init__(self, logger: logging.Logger,
+            stdout_logging_level: int = logging.INFO, stderr_logging_level: int = logging.ERROR) -> None:
+
         self._logger = logger
+        self._stdout_logging_level = stdout_logging_level
+        self._stderr_logging_level = stderr_logging_level
 
 
     def process_stdout_line(self, line: str) -> None:
-        self._logger.debug(line.rstrip())
+        self._logger.log(self._stdout_logging_level, line.rstrip())
 
 
     def process_stderr_line(self, line: str) -> None:
-        self._logger.debug(line.rstrip())
+        self._logger.log(self._stderr_logging_level, line.rstrip())
 
 
     def process_stdout_end(self) -> None:

Sources/toolkit/bhamon_development_toolkit/python/pylint_runner.py

@@ -7,6 +7,7 @@
 from bhamon_development_toolkit.processes.executable_command import ExecutableCommand
 from bhamon_development_toolkit.processes.process_options import ProcessOptions
 from bhamon_development_toolkit.processes.process_output_logger import ProcessOutputLogger
+from bhamon_development_toolkit.processes.process_result import ProcessResult
 from bhamon_development_toolkit.processes.process_runner import ProcessRunner
 from bhamon_development_toolkit.python.pylint_output_handler import PylintOutputHandler
 from bhamon_development_toolkit.python.pylint_scope import PylintScope
@@ -66,7 +67,8 @@ async def _run_with_scope(self,
         command.add_internal_arguments([ "--output-format=text,json:%s" % os.path.abspath(json_report_file_path) ], [])
 
         process_options = ProcessOptions(working_directory = working_directory)
-        raw_output_logger = ProcessOutputLogger(process_helpers.create_raw_logger(log_file_path = log_file_path))
+        raw_logger = process_helpers.create_raw_logger(log_file_path = log_file_path)
+        process_output_logger = ProcessOutputLogger(raw_logger.get_actual_logger())
         pylint_output_handler = PylintOutputHandler()
 
         command.add_internal_arguments([ "--msg-template", pylint_output_handler.get_message_template() ], [])
@@ -75,11 +77,16 @@ async def _run_with_scope(self,
 
         success = True
 
-        if not simulate:
-            result = await self._process_runner.run(command, process_options, [ raw_output_logger, pylint_output_handler ], check_exit_code = False)
+        try:
+            if not simulate:
+                result = await self._process_runner.run(command, process_options, [ process_output_logger, pylint_output_handler ], check_exit_code = False)
+            else:
+                result = ProcessResult(executable = self._python_executable, exit_code = 0)
+        finally:
+            raw_logger.dispose()
 
-            self._check_exit_code(result.exit_code)
-            success = self._get_success_from_exit_code(result.exit_code)
+        self._check_exit_code(result.exit_code)
+        success = self._get_success_from_exit_code(result.exit_code)
 
         return success
 

Sources/toolkit/bhamon_development_toolkit/python/pytest_runner.py

@@ -10,6 +10,7 @@
 from bhamon_development_toolkit.processes.executable_command import ExecutableCommand
 from bhamon_development_toolkit.processes.process_options import ProcessOptions
 from bhamon_development_toolkit.processes.process_output_logger import ProcessOutputLogger
+from bhamon_development_toolkit.processes.process_result import ProcessResult
 from bhamon_development_toolkit.processes.process_runner import ProcessRunner
 from bhamon_development_toolkit.python.pytest_output_handler import PytestOutputHandler
 from bhamon_development_toolkit.python.pytest_scope import PytestScope
@@ -57,7 +58,7 @@ async def run(self, # pylint: disable = too-many-arguments
             raise RuntimeError("Pytest session completed with failures")
 
 
-    async def _run_with_scope(self,
+    async def _run_with_scope(self, # pylint: disable = too-many-locals
             scope: PytestScope, result_directory: str, working_directory: Optional[str] = None, simulate: bool = False) -> bool:
 
         log_file_path = os.path.join(result_directory, scope.identifier + ".log")
@@ -72,19 +73,26 @@ async def _run_with_scope(self,
         command.add_internal_arguments([ "--json", os.path.abspath(json_report_file_path) ] if not simulate else [], [])
 
         process_options = ProcessOptions(working_directory = working_directory)
-        raw_output_logger = ProcessOutputLogger(process_helpers.create_raw_logger(log_file_path = log_file_path))
+        raw_logger = process_helpers.create_raw_logger(log_file_path = log_file_path)
+        process_output_logger = ProcessOutputLogger(raw_logger.get_actual_logger())
         pytest_output_handler = PytestOutputHandler(scope)
 
         logger.info("+ %s", process_helpers.format_executable_command(command.get_command_for_logging()))
 
         success = True
 
-        if not simulate:
-            result = await self._process_runner.run(command, process_options, [ raw_output_logger, pytest_output_handler ], check_exit_code = False)
+        try:
+            if not simulate:
+                result = await self._process_runner.run(command, process_options, [ process_output_logger, pytest_output_handler ], check_exit_code = False)
+            else:
+                result = ProcessResult(executable = self._python_executable, exit_code = 0)
+        finally:
+            raw_logger.dispose()
 
-            self._check_exit_code(result.exit_code)
-            success = self._get_success_from_exit_code(result.exit_code)
+        self._check_exit_code(result.exit_code)
+        success = self._get_success_from_exit_code(result.exit_code)
 
+        if not simulate:
             with open(json_report_file_path, mode = "r", encoding = "utf-8") as json_report_file:
                 json_report = json.load(json_report_file)
             with open(json_report_file_path + ".tmp", mode = "w", encoding = "utf-8") as json_report_file:

Sources/toolkit/bhamon_development_toolkit/python/python_package_builder.py

@@ -49,13 +49,16 @@ async def build_distribution_package(self, # pylint: disable = too-many-argument
         setup_command.add_arguments([ "setup.py", "bdist_wheel" ])
 
         process_options = ProcessOptions(working_directory = python_package.path_to_sources)
-        raw_logger = process_helpers.create_raw_logger(sys.stdout, log_file_path)
-        process_output_logger = ProcessOutputLogger(raw_logger)
+        raw_logger = process_helpers.create_raw_logger(stream = sys.stdout, log_file_path = log_file_path)
+        process_output_logger = ProcessOutputLogger(raw_logger.get_actual_logger())
 
         logger.info("+ %s", process_helpers.format_executable_command(setup_command.get_command_for_logging()))
 
-        if not simulate:
-            await self._process_runner.run(setup_command, process_options, [ process_output_logger ])
+        try:
+            if not simulate:
+                await self._process_runner.run(setup_command, process_options, [ process_output_logger ])
+        finally:
+            raw_logger.dispose()
 
         archive_name = python_package.name_for_file_system + "-" + version
         source_path = os.path.join(python_package.path_to_sources, "dist", archive_name + "-py3-none-any.whl")