Merge pull request #78 from per1234/warnings-delta

libraries/compile-examples: add support for reporting changes in compiler warning count caused by a commit or PR

Author: aentinger

README.md

@@ -114,11 +114,15 @@ Path in which to save a JSON formatted file containing data from the sketch comp
 
 ### `github-token`
 
-GitHub access token used to get information from the GitHub API. Only needed for private repositories with `enable-size-deltas-report` set to `true`. It will be convenient to use [`${{ secrets.GITHUB_TOKEN }}`](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token). Default `""`.
+GitHub access token used to get information from the GitHub API. Only needed for private repositories with `enable-deltas-report` set to `true`. It will be convenient to use [`${{ secrets.GITHUB_TOKEN }}`](https://help.github.com/en/actions/configuring-and-managing-workflows/authenticating-with-the-github_token). Default `""`.
 
-### `enable-size-deltas-report`
+### `enable-deltas-report`
 
-Set to `true` to cause the action to determine the change in memory usage of the compiled sketches. If the workflow is triggered by a `pull_request` event, the comparison is between the pull request branch and the tip of the pull request's base branch. If the workflow is triggered by a `push` event, the comparison is between the pushed commit and its immediate parent. This may be used with the [`arduino/actions/libraries/report-size-deltas` action](https://github.com/arduino/actions/tree/master/libraries/report-size-deltas). Default `false`.
+Set to `true` to cause the action to determine the change in memory usage and compiler warnings of the compiled sketches. If the workflow is triggered by a `pull_request` event, the comparison is between the pull request branch and the tip of the pull request's base branch. If the workflow is triggered by a `push` event, the comparison is between the pushed commit and its immediate parent. The deltas will be displayed in the GitHub Actions build log. This may be used with the [`arduino/actions/libraries/report-size-deltas` action](https://github.com/arduino/actions/tree/master/libraries/report-size-deltas). Default `false`.
+
+### `enable-warnings-report`
+
+Set to `true` to cause the action to record the compiler warning count for each sketch compilation in the sketches report. Default `false`.
 
 ## Example usage
 
@@ -133,11 +137,11 @@ Only compiling examples:
         version: 1.1.3
 ```
 
-Storing the memory usage change report as a [workflow artifact](https://help.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts):
+Storing the sketches compilation report report as a [workflow artifact](https://help.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts):
 ```yaml
 - uses: arduino/actions/libraries/compile-examples@master
   with:
-    enable-size-deltas-report: true
+    enable-deltas-report: true
 - if: github.event_name == 'pull_request'
   uses: actions/upload-artifact@v1
   with:

action.yml

@@ -25,9 +25,13 @@ inputs:
   github-token:
     description: 'GitHub access token used to get information from the GitHub API. Only needed if you are using the size deltas report feature with a private repository.'
     default: ''
-  enable-size-deltas-report:
-    description: 'Set to true to cause the action to determine the change in memory usage of the compiled sketches between the head and base refs of a PR and the immediate parent commit of a push'
+  enable-deltas-report:
+    description: 'Set to true to cause the action to determine the change in memory usage and compiler warnings of the compiled sketches between the head and base refs of a PR and the immediate parent commit of a push'
     default: false
+  enable-warnings-report:
+    description: 'Set to true to cause the action to record the compiler warning count for each sketch compilation in the sketches report'
+    default: false
+
 runs:
   using: 'docker'
   image: 'Dockerfile'

compilesketches/compilesketches.py

@@ -28,6 +28,11 @@ def main():
               "sketches-report-path instead.")
         os.environ["INPUT_SKETCHES-REPORT-PATH"] = os.environ["INPUT_SIZE-DELTAS-REPORT-FOLDER-NAME"]
 
+    if "INPUT_ENABLE-SIZE-DELTAS-REPORT" in os.environ:
+        print("::warning::The enable-size-deltas-report input is deprecated. Use the equivalent input: "
+              "enable-deltas-report instead.")
+        os.environ["INPUT_ENABLE-DELTAS-REPORT"] = os.environ["INPUT_ENABLE-SIZE-DELTAS-REPORT"]
+
     if "INPUT_ENABLE-SIZE-TRENDS-REPORT" in os.environ:
         print("::warning::The size trends report feature has been moved to a dedicated action. See the documentation "
               "at https://github.com/arduino/actions/tree/report-size-trends-action/libraries/report-size-trends")
@@ -40,7 +45,8 @@ def main():
         sketch_paths=os.environ["INPUT_SKETCH-PATHS"],
         verbose=os.environ["INPUT_VERBOSE"],
         github_token=os.environ["INPUT_GITHUB-TOKEN"],
-        enable_size_deltas_report=os.environ["INPUT_ENABLE-SIZE-DELTAS-REPORT"],
+        enable_deltas_report=os.environ["INPUT_ENABLE-DELTAS-REPORT"],
+        enable_warnings_report=os.environ["INPUT_ENABLE-WARNINGS-REPORT"],
         sketches_report_path=os.environ["INPUT_SKETCHES-REPORT-PATH"]
     )
 
@@ -60,7 +66,9 @@ class CompileSketches:
                     recursively for sketches.
     verbose -- set to "true" for verbose output ("true", "false")
     github_token -- GitHub access token
-    enable_size_deltas_report -- set to "true" to cause the action to determine the change in memory usage
+    enable_deltas_report -- set to "true" to cause the action to determine the change in memory usage
+                                 ("true", "false")
+    enable_warnings_report -- set to "true" to cause the action to add compiler warning count to the sketches report
                                  ("true", "false")
     sketches_report_path -- folder to save the sketches report to
     """
@@ -87,6 +95,7 @@ class ReportKeys:
         commit_url = "commit_url"
         compilation_success = "compilation_success"
         sizes = "sizes"
+        warnings = "warnings"
         name = "name"
         absolute = "absolute"
         relative = "relative"
@@ -106,7 +115,7 @@ class ReportKeys:
     latest_release_indicator = "latest"
 
     def __init__(self, cli_version, fqbn_arg, platforms, libraries, sketch_paths, verbose, github_token,
-                 enable_size_deltas_report, sketches_report_path):
+                 enable_deltas_report, enable_warnings_report, sketches_report_path):
         """Process, store, and validate the action's inputs."""
         self.cli_version = cli_version
 
@@ -129,13 +138,19 @@ def __init__(self, cli_version, fqbn_arg, platforms, libraries, sketch_paths, ve
         else:
             self.github_api = github.Github(login_or_token=github_token)
 
-        self.enable_size_deltas_report = parse_boolean_input(boolean_input=enable_size_deltas_report)
-        # The enable-size-deltas-report input has a default value so it should always be either True or False
-        if self.enable_size_deltas_report is None:
-            print("::error::Invalid value for enable-size-deltas-report input")
+        self.enable_deltas_report = parse_boolean_input(boolean_input=enable_deltas_report)
+        # The enable-deltas-report input has a default value so it should always be either True or False
+        if self.enable_deltas_report is None:
+            print("::error::Invalid value for enable-deltas-report input")
+            sys.exit(1)
+
+        self.enable_warnings_report = parse_boolean_input(boolean_input=enable_warnings_report)
+        # The enable-deltas-report input has a default value so it should always be either True or False
+        if self.enable_warnings_report is None:
+            print("::error::Invalid value for enable-warnings-report input")
             sys.exit(1)
 
-        if self.enable_size_deltas_report:
+        if self.enable_deltas_report:
             self.deltas_base_ref = self.get_deltas_base_ref()
         else:
             # If deltas reports are not enabled, there is no use for the base ref and it could result in an GitHub API
@@ -185,7 +200,9 @@ def compile_sketches(self):
 
         sketch_list = self.find_sketches()
         for sketch in sketch_list:
-            compilation_result = self.compile_sketch(sketch_path=sketch)
+            # It's necessary to clear the cache between each compilation to get a true compiler warning count, otherwise
+            # only the first sketch compilation's warning count would reflect warnings from cached code
+            compilation_result = self.compile_sketch(sketch_path=sketch, clean_build_cache=self.enable_warnings_report)
             if not compilation_result.success:
                 all_compilations_successful = False
 
@@ -845,17 +862,22 @@ def find_sketches(self):
 
         return sketch_list
 
-    def compile_sketch(self, sketch_path):
+    def compile_sketch(self, sketch_path, clean_build_cache):
         """Compile the specified sketch and returns an object containing the result:
         sketch -- the sketch path relative to the workspace
         success -- the success of the compilation (True, False)
         output -- stdout from Arduino CLI
 
         Keyword arguments:
         sketch_path -- path of the sketch to compile
+        clean_build_cache -- whether to delete cached compiled from previous compilations before compiling
         """
         compilation_command = ["compile", "--warnings", "all", "--fqbn", self.fqbn, sketch_path]
 
+        if clean_build_cache:
+            for cache_path in pathlib.Path("/tmp").glob(pattern="arduino*"):
+                shutil.rmtree(path=cache_path)
+
         compilation_data = self.run_arduino_cli_command(
             command=compilation_command, enable_output=self.RunCommandOutput.NONE, exit_on_failure=False)
         # Group compilation output to make the log easy to read
@@ -881,8 +903,15 @@ def get_sketch_report(self, compilation_result):
         compilation_result -- object returned by compile_sketch()
         """
         current_sizes = self.get_sizes_from_output(compilation_result=compilation_result)
+        if self.enable_warnings_report:
+            current_warning_count = self.get_warning_count_from_output(compilation_result=compilation_result)
+        else:
+            current_warning_count = None
         previous_sizes = None
-        if self.do_size_deltas_report(compilation_result=compilation_result, current_sizes=current_sizes):
+        previous_warning_count = None
+        if self.do_deltas_report(compilation_result=compilation_result,
+                                 current_sizes=current_sizes,
+                                 current_warnings=current_warning_count):
             # Get data for the sketch at the base ref
             # Get the head ref
             repository = git.Repo(path=os.environ["GITHUB_WORKSPACE"])
@@ -893,20 +922,30 @@ def get_sketch_report(self, compilation_result):
 
             # Compile the sketch again
             print("Compiling previous version of sketch to determine memory usage change")
-            previous_compilation_result = self.compile_sketch(sketch_path=compilation_result.sketch)
+            previous_compilation_result = self.compile_sketch(sketch_path=compilation_result.sketch,
+                                                              clean_build_cache=self.enable_warnings_report)
 
             # git checkout the head ref to return the repository to its previous state
             repository.git.checkout(original_git_ref, recurse_submodules=True)
 
             previous_sizes = self.get_sizes_from_output(compilation_result=previous_compilation_result)
+            if self.enable_warnings_report:
+                previous_warning_count = (
+                    self.get_warning_count_from_output(compilation_result=previous_compilation_result)
+                )
 
         # Add global data for sketch to report
         sketch_report = {
             self.ReportKeys.name: str(path_relative_to_workspace(path=compilation_result.sketch)),
             self.ReportKeys.compilation_success: compilation_result.success,
             self.ReportKeys.sizes: self.get_sizes_report(current_sizes=current_sizes,
-                                                         previous_sizes=previous_sizes)
+                                                         previous_sizes=previous_sizes),
         }
+        if self.enable_warnings_report:
+            sketch_report[self.ReportKeys.warnings] = (
+                self.get_warnings_report(current_warnings=current_warning_count,
+                                         previous_warnings=previous_warning_count)
+            )
 
         return sketch_report
 
@@ -1003,18 +1042,40 @@ def get_size_data_from_output(self, compilation_output, memory_type, size_data_t
 
         return size_data
 
-    def do_size_deltas_report(self, compilation_result, current_sizes):
+    def get_warning_count_from_output(self, compilation_result):
+        """Parse the stdout from the compilation process and return the number of compiler warnings. Since the
+        information is likely not relevant in that case, "N/A" is returned if compilation failed.
+
+        Keyword arguments:
+        compilation_result -- object returned by compile_sketch()
+        """
+        if compilation_result.success is True:
+            compiler_warning_regex = ":[0-9]+:[0-9]+: warning:"
+            warning_count = len(re.findall(pattern=compiler_warning_regex, string=compilation_result.output))
+        else:
+            warning_count = self.not_applicable_indicator
+
+        return warning_count
+
+    def do_deltas_report(self, compilation_result, current_sizes, current_warnings):
         """Return whether size deltas reporting is enabled.
 
         Keyword arguments:
         compilation_result -- object returned by compile_sketch()
         current_sizes -- memory usage data from the compilation
+        current_warnings -- compiler warning count
         """
         return (
-            self.enable_size_deltas_report
+            self.enable_deltas_report
             and compilation_result.success
-            and any(size.get(self.ReportKeys.absolute) != self.not_applicable_indicator for
+            and (
+                any(size.get(self.ReportKeys.absolute) != self.not_applicable_indicator for
                     size in current_sizes)
+                or (
+                    current_warnings is not None
+                    and current_warnings != self.not_applicable_indicator
+                )
+            )
         )
 
     def checkout_deltas_base_ref(self):
@@ -1107,6 +1168,42 @@ def get_size_report(self, current_size, previous_size):
 
         return size_report
 
+    def get_warnings_report(self, current_warnings, previous_warnings):
+        """Return a dictionary containing the compiler warning counts.
+
+        Keyword arguments:
+        current_warnings -- compiler warning count at the head ref
+        previous_warnings -- compiler warning count at the base ref, or None if the size deltas feature is not enabled
+        """
+        warnings_report = {
+            self.ReportKeys.current: {
+                self.ReportKeys.absolute: current_warnings,
+            }
+        }
+
+        if previous_warnings is not None:
+            # Deltas reports are enabled
+            # Calculate the change in the warnings count
+            if (
+                current_warnings == self.not_applicable_indicator
+                or previous_warnings == self.not_applicable_indicator
+            ):
+                warnings_delta = self.not_applicable_indicator
+            else:
+                warnings_delta = current_warnings - previous_warnings
+
+            # Print the warning count change to the log
+            print("Change in compiler warning count:", warnings_delta)
+
+            warnings_report[self.ReportKeys.previous] = {
+                self.ReportKeys.absolute: previous_warnings
+            }
+            warnings_report[self.ReportKeys.delta] = {
+                self.ReportKeys.absolute: warnings_delta
+            }
+
+        return warnings_report
+
     def get_sketches_report(self, sketch_report_list):
         """Return the dictionary containing data on all sketch compilations for each board
 
@@ -1136,6 +1233,10 @@ def get_sketches_report(self, sketch_report_list):
         if sizes_summary_report:
             sketches_report[self.ReportKeys.boards][0][self.ReportKeys.sizes] = sizes_summary_report
 
+        warnings_summary_report = self.get_warnings_summary_report(sketch_report_list=sketch_report_list)
+        if warnings_summary_report:
+            sketches_report[self.ReportKeys.boards][0][self.ReportKeys.warnings] = warnings_summary_report
+
         return sketches_report
 
     def get_sizes_summary_report(self, sketch_report_list):
@@ -1236,6 +1337,51 @@ def get_sizes_summary_report(self, sketch_report_list):
 
         return sizes_summary_report
 
+    def get_warnings_summary_report(self, sketch_report_list):
+        """Return a dictionary containing a summary of the compilation warnings count for all sketch compilations.
+
+        Keyword arguments:
+        sketch_report_list -- list of reports from each sketch compilation
+        """
+        summary_report_minimum = None
+        summary_report_maximum = None
+        for sketch_report in sketch_report_list:
+            if (
+                self.ReportKeys.warnings in sketch_report
+                and self.ReportKeys.delta in sketch_report[self.ReportKeys.warnings]
+            ):
+                sketch_report_delta = (
+                    sketch_report[self.ReportKeys.warnings][self.ReportKeys.delta][self.ReportKeys.absolute]
+                )
+
+                if (
+                    summary_report_minimum is None
+                    or summary_report_minimum == self.not_applicable_indicator
+                    or summary_report_minimum > sketch_report_delta
+                ):
+                    summary_report_minimum = sketch_report_delta
+
+                if (
+                    summary_report_maximum is None
+                    or summary_report_maximum == self.not_applicable_indicator
+                    or summary_report_maximum < sketch_report_delta
+                ):
+                    summary_report_maximum = sketch_report_delta
+
+        if summary_report_minimum is not None:
+            warnings_summary_report = {
+                self.ReportKeys.delta: {
+                    self.ReportKeys.absolute: {
+                        self.ReportKeys.minimum: summary_report_minimum,
+                        self.ReportKeys.maximum: summary_report_maximum
+                    }
+                }
+            }
+        else:
+            warnings_summary_report = {}
+
+        return warnings_summary_report
+
     def create_sketches_report_file(self, sketches_report):
         """Write the report for the report sketch to a file.