diff --git a/.circleci/config.yml b/.circleci/config.yml index 0eee1c5e0abd1..b9383fb520f6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -372,6 +372,7 @@ jobs: # store (uncombined) coverage from acceptance tests - run: name: fetch isolated acceptance coverage + # might need to combine coverage files in future command: | cp target/coverage/.coverage.acceptance.* .coverage.acceptance - store_artifacts: @@ -391,6 +392,11 @@ jobs: coverage report || true coverage html || true coveralls || true + - run: + name: store acceptance parity metrics + command: | + mkdir acceptance_parity_metrics + mv target/metric_reports/metric-report*acceptance* acceptance_parity_metrics/ - run: name: Upload test metrics and implemented coverage data to tinybird command: | @@ -400,13 +406,41 @@ jobs: IMPLEMENTATION_COVERAGE_FILE=scripts/implementation_coverage_full.csv \ SOURCE_TYPE=community \ python -m scripts.tinybird.upload_raw_test_metrics_and_coverage - - store_artifacts: - path: parity_metrics/ - run: - name: store acceptance parity metrics + name: Create Coverage Diff (Code Coverage) + # pycobertura diff will return with exit code 0-3 -> we currently expect 2 (2: the changes worsened the overall coverage), + # but we still want cirecleci to continue with the tasks, so we return 0. + # From the docs: + # Upon exit, the diff command may return various exit codes: + # 0: all changes are covered, no new uncovered statements have been introduced + # 1: some exception occurred (likely due to inappropriate usage or a bug in pycobertura) + # 2: the changes worsened the overall coverage + # 3: the changes introduced uncovered statements but the overall coverage is still better than before command: | - mkdir acceptance_parity_metrics - mv target/metric_reports/metric-report*acceptance* acceptance_parity_metrics/ + source .venv/bin/activate + pip install pycobertura + coverage xml --data-file=.coverage -o all.coverage.report.xml --include="localstack/services/*/**" --omit="*/**/__init__.py" + coverage xml --data-file=.coverage.acceptance -o acceptance.coverage.report.xml --include="localstack/services/*/**" --omit="*/**/__init__.py" + pycobertura show --format html -s localstack/ acceptance.coverage.report.xml -o coverage-acceptance.html + bash -c "pycobertura diff --format html -s1 localstack/ -s2 localstack/ all.coverage.report.xml acceptance.coverage.report.xml -o coverage-diff.html; if [[ \$? -eq 1 ]] ; then exit 1 ; else exit 0 ; fi" + - run: + name: Create Metric Coverage Diff (API Coverage) + environment: + COVERAGE_DIR_ALL: "parity_metrics" + COVERAGE_DIR_ACCEPTANCE: "acceptance_parity_metrics" + OUTPUT_DIR: "api-coverage" + command: | + source .venv/bin/activate + mkdir api-coverage + python -m scripts.metrics_coverage.diff_metrics_coverage + - store_artifacts: + path: api-coverage/ + - store_artifacts: + path: coverage-acceptance.html + - store_artifacts: + path: coverage-diff.html + - store_artifacts: + path: parity_metrics/ - store_artifacts: path: acceptance_parity_metrics/ - store_artifacts: diff --git a/scripts/metrics_coverage/diff_metrics_coverage.py b/scripts/metrics_coverage/diff_metrics_coverage.py new file mode 100644 index 0000000000000..6bb09d200fdd7 --- /dev/null +++ b/scripts/metrics_coverage/diff_metrics_coverage.py @@ -0,0 +1,246 @@ +import csv +import os +from pathlib import Path + + +def print_usage(): + print( + """ + Helper script to an output report for the metrics coverage diff. + + Set the env `COVERAGE_DIR_ALL` which points to a folder containing metrics-raw-data reports for the initial tests. + The env `COVERAGE_DIR_ACCEPTANCE` should point to the folder containing metrics-raw-data reports for the acceptance + test suite (usually a subset of the initial tests). + + Use `OUTPUT_DIR` env to set the path where the report will be stored + """ + ) + + +def sort_dict_helper(d): + if isinstance(d, dict): + return {k: sort_dict_helper(v) for k, v in sorted(d.items())} + else: + return d + + +def create_initial_coverage(path_to_initial_metrics: str) -> dict: + """ + Iterates over all csv files in `path_to_initial_metrics` and creates a dict collecting all status_codes that have been + triggered for each service-operation combination: + + { "service_name": + { + "operation_name_1": { "status_code": False}, + "operation_name2": {"status_code_1": False, "status_code_2": False} + }, + "service_name_2": .... + } + :param path_to_initial_metrics: path to the metrics + :returns: Dict + """ + pathlist = Path(path_to_initial_metrics).rglob("*.csv") + coverage = {} + for path in pathlist: + with open(path, "r") as csv_obj: + print(f"Processing integration test coverage metrics: {path}") + csv_dict_reader = csv.DictReader(csv_obj) + for metric in csv_dict_reader: + service = metric.get("service") + operation = metric.get("operation") + response_code = metric.get("response_code") + + service_details = coverage.setdefault(service, {}) + operation_details = service_details.setdefault(operation, {}) + if response_code not in operation_details: + operation_details[response_code] = False + return coverage + + +def mark_coverage_acceptance_test( + path_to_acceptance_metrics: str, coverage_collection: dict +) -> dict: + """ + Iterates over all csv files in `path_to_acceptance_metrics` and updates the information in the `coverage_collection` + dict about which API call was covered by the acceptance metrics + + { "service_name": + { + "operation_name_1": { "status_code": True}, + "operation_name2": {"status_code_1": False, "status_code_2": True} + }, + "service_name_2": .... + } + + If any API calls are identified, that have not been covered with the initial run, those will be collected separately. + Normally, this should never happen, because acceptance tests should be a subset of integrations tests. + Could, however, be useful to identify issues, or when comparing test runs locally. + + :param path_to_acceptance_metrics: path to the metrics + :param coverage_collection: Dict with the coverage collection about the initial test integration run + + :returns: dict with additional recorded coverage, only covered by the acceptance test suite + """ + pathlist = Path(path_to_acceptance_metrics).rglob("*.csv") + additional_tested = {} + add_to_additional = False + for path in pathlist: + with open(path, "r") as csv_obj: + print(f"Processing acceptance test coverage metrics: {path}") + csv_dict_reader = csv.DictReader(csv_obj) + for metric in csv_dict_reader: + service = metric.get("service") + operation = metric.get("operation") + response_code = metric.get("response_code") + + if service not in coverage_collection: + add_to_additional = True + else: + service_details = coverage_collection[service] + if operation not in service_details: + add_to_additional = True + else: + operation_details = service_details.setdefault(operation, {}) + if response_code not in operation_details: + add_to_additional = True + else: + operation_details[response_code] = True + + if add_to_additional: + service_details = additional_tested.setdefault(service, {}) + operation_details = service_details.setdefault(operation, {}) + if response_code not in operation_details: + operation_details[response_code] = True + add_to_additional = False + + return additional_tested + + +def create_readable_report( + coverage_collection: dict, additional_tested_collection: dict, output_dir: str +) -> None: + """ + Helper function to create a very simple HTML view out of the collected metrics. + The file will be named "report_metric_coverage.html" + + :params coverage_collection: the dict with the coverage collection + :params additional_tested_collection: dict with coverage of APIs only for acceptance tests + :params output_dir: the directory where the outcoming html file should be stored to. + """ + service_overview_coverage = """ + + + + + + """ + coverage_details = """ +
ServiceCoverage of Acceptance Tests Suite
+ + + + + + """ + additional_test_details = "" + coverage_collection = sort_dict_helper(coverage_collection) + additional_tested_collection = sort_dict_helper(additional_tested_collection) + for service, operations in coverage_collection.items(): + # count tested operations vs operations that are somehow covered with acceptance + amount_ops = len(operations) + covered_ops = len([op for op, details in operations.items() if any(details.values())]) + percentage_covered = 100 * covered_ops / amount_ops + service_overview_coverage += " \n" + service_overview_coverage += f" \n" + service_overview_coverage += ( + f""" \n""" + ) + service_overview_coverage += " \n" + + for op_name, details in operations.items(): + for response_code, covered in details.items(): + coverage_details += " \n" + coverage_details += f" \n" + coverage_details += f" \n" + coverage_details += f""" \n""" + coverage_details += ( + f""" \n""" + ) + coverage_details += " \n" + if additional_tested_collection: + additional_test_details = """
ServiceOperationReturn CodeCovered By Acceptance Test
{service}{percentage_covered:.2f}%
{service}{op_name}{response_code}{'✅' if covered else '❌'}
+ + + + + + """ + for service, operations in additional_tested_collection.items(): + for op_name, details in operations.items(): + for response_code, covered in details.items(): + additional_test_details += " \n" + additional_test_details += f" \n" + additional_test_details += f" \n" + additional_test_details += f" \n" + additional_test_details += f" \n" + additional_test_details += " \n" + additional_test_details += "
ServiceOperationReturn CodeCovered By Acceptance Test
{service}{op_name}{response_code}{'✅' if covered else '❌'}

\n" + service_overview_coverage += "
\n" + coverage_details += "
\n" + path = Path(output_dir) + file_name = path.joinpath("report_metric_coverage.html") + with open(file_name, "w") as fd: + fd.write( + """ + + +""" + ) + fd.write("

Diff Report Metrics Coverage

\n") + fd.write("

Service Coverage

\n") + fd.write( + "

Assumption: the initial test suite is considered to have 100% coverage.

\n" + ) + fd.write(f"

{service_overview_coverage}

\n") + fd.write("

Coverage Details

\n") + fd.write(f"
{coverage_details}
") + if additional_test_details: + fd.write("

Additional Test Coverage

\n") + fd.write( + "
Note: this is probalby wrong usage of the script. It includes operations that have been covered with the acceptance tests only" + ) + fd.write(f"

{additional_test_details}

\n") + fd.write("