diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb9d5c9..e83d961 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,3 +3,71 @@ repos: rev: 19.10b0 hooks: - id: black +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-ast + - id: check-yaml + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-merge-conflict + - id: check-toml + +- repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + args: ["--py36-plus"] + +## ## - repo: https://github.com/jumanjihouse/pre-commit-hooks +## ## rev: master # or specific git tag +## ## hooks: +## ## - id: forbid-binary +## ## ## TODO - disable for docs/_build/html +## ## - id: markdownlint # Configure in .mdlrc +## ## # TODO - disable MD012 (consecutive blank lines) +## ## # disable MD013 (line length) +## ## # fix MD031 (fenced code needs blank lines) README.md +## ## - id: shellcheck + +## ## - repo: https://github.com/IamTheFij/docker-pre-commit +## ## rev: v2.0.0 +## ## hooks: +## ## - id: docker-compose-check +## ## - id: hadolint +## ## # TODO - disable DL3005 (apt upgrade) +## ## # disable DL3008 (unpinned packages) +## ## # disable DL3009 (left apt-get lists around) + +- repo: https://github.com/PyCQA/bandit + rev: 1.6.2 + hooks: + - id: bandit + args: + - "--skip" + - "B101" # use of assert + - --quiet + +- repo: https://github.com/myint/docformatter + rev: v1.3.1 + hooks: + - id: docformatter + args: [--in-place] + +- repo: https://gitlab.com/iamlikeme/nbhooks + rev: "" + hooks: + - id: nb-ensure-clean + #args: [--meta, ExecuteTime] # Optional blacklist of metadata keys (you can use regex) + +- repo: https://github.com/Yelp/detect-secrets + rev: "" + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + exclude: .*/tests/.* diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..c996184 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,82 @@ +{ + "custom_plugin_paths": [], + "exclude": { + "files": null, + "lines": null + }, + "generated_at": "2020-09-24T16:39:13Z", + "plugins_used": [ + { + "name": "AWSKeyDetector" + }, + { + "name": "ArtifactoryDetector" + }, + { + "base64_limit": 4.5, + "name": "Base64HighEntropyString" + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "hex_limit": 3, + "name": "HexHighEntropyString" + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "keyword_exclude": null, + "name": "KeywordDetector" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "results": { + "docs/_build/html/.buildinfo": [ + { + "hashed_secret": "4409d7400c419585a64ebdc7632b2f20f7050ca8", + "is_verified": false, + "line_number": 3, + "type": "Hex High Entropy String" + }, + { + "hashed_secret": "63256a38625700f409b7ec5db8fa05ffd2cb88b2", + "is_verified": false, + "line_number": 4, + "type": "Hex High Entropy String" + } + ] + }, + "version": "0.14.2", + "word_list": { + "file": null, + "hash": null + } +} diff --git a/github/branches/conftest.py b/github/branches/conftest.py index 572ec16..4fbdae5 100755 --- a/github/branches/conftest.py +++ b/github/branches/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -14,7 +13,7 @@ import os import pathlib from typing import List -import subprocess +import subprocess # nosec import pytest @@ -56,7 +55,11 @@ def repos_to_check() -> List[str]: # python 3.6 doesn't support capture_output # status = subprocess.run(cmd, capture_output=True) - status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # fmt: off + status = subprocess.run( # nosec + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE # nosec + ) + # fmt:on # return as array of non-empty, unquoted, "lines" return [ x.translate({ord('"'): None, ord("'"): None}) diff --git a/github/orgs/conftest.py b/github/orgs/conftest.py index 4116d4a..a8179d1 100755 --- a/github/orgs/conftest.py +++ b/github/orgs/conftest.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -16,7 +15,7 @@ from typing import List, Set -import subprocess +import subprocess # nosec from sgqlc.endpoint.http import HTTPEndpoint # noqa: I900 @@ -49,7 +48,11 @@ def orgs_to_check() -> Set[str]: # python 3.6 doesn't support capture_output # status = subprocess.run(cmd, capture_output=True) - status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # fmt: off + status = subprocess.run( # nosec + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE # nosec + ) + # fmt:on assert not status.stderr.decode("utf-8") # return as array of non-empty, unquoted, "lines" return { diff --git a/github/orgs/retrieve_github_data.py b/github/orgs/retrieve_github_data.py index f4601b0..b986297 100755 --- a/github/orgs/retrieve_github_data.py +++ b/github/orgs/retrieve_github_data.py @@ -2,11 +2,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -""" -Collect Information about branches sufficient to check for all branch -protection guideline compliance. - -""" +"""Collect Information about branches sufficient to check for all branch +protection guideline compliance.""" import csv from functools import lru_cache @@ -14,7 +11,7 @@ import os from dataclasses import dataclass, field from pathlib import Path -import subprocess +import subprocess # nosec import sys from typing import Any, List, Optional, Set @@ -39,8 +36,7 @@ class OrgInfo: @staticmethod def idfn(val: Any) -> Optional[str]: - """ provide ID for pytest Parametrization - """ + """provide ID for pytest Parametrization.""" if isinstance(val, (OrgInfo,)): return f"{val.id_}-{val.login}" return None @@ -64,7 +60,7 @@ def csv_row(self) -> List[Optional[str]]: def create_operation(owner): - """ Create the default Query operation + """Create the default Query operation. We build the structure for: organization: @@ -109,9 +105,7 @@ def get_org_info(endpoint: Any, org: str) -> OrgInfo: def extract_org_data(orgdata) -> OrgInfo: - """ extract relevant data from sgqlc structure - - """ + """extract relevant data from sgqlc structure.""" org_data = OrgInfo( name=orgdata.name, login=orgdata.login, @@ -209,7 +203,11 @@ def _orgs_to_check() -> Set[str]: # python 3.6 doesn't support capture_output # status = subprocess.run(cmd, capture_output=True) - status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # fmt: off + status = subprocess.run( # nosec + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE # nosec + ) + # fmt:on assert not status.stderr.decode("utf-8") # return as array of non-empty, unquoted, "lines" return { @@ -220,7 +218,7 @@ def _orgs_to_check() -> Set[str]: def get_all_org_data(endpoint: Any = None, orgs: List[str] = None) -> List[OrgInfo]: - """ Generator of org data """ + """Generator of org data.""" if not endpoint: # if we're creating the endpoint, then arguments must already be # in environment variables. @@ -245,14 +243,14 @@ def _compact_fmt(d): lst.append(_compact_fmt(e)) else: lst.append(repr(e)) - s.append("%s=[%s]" % (k, ", ".join(lst))) + s.append("{}=[{}]".format(k, ", ".join(lst))) continue - s.append("%s=%r" % (k, v)) + s.append(f"{k}={v!r}") return "(" + ", ".join(s) + ")" def _report_download_errors(errors): - """ error handling for graphql comms """ + """error handling for graphql comms.""" logger.error("Document contain %d errors", len(errors)) for i, e in enumerate(errors): msg = e.pop("message") diff --git a/github/run.sh b/github/run.sh index 091b937..1661fc3 100755 --- a/github/run.sh +++ b/github/run.sh @@ -20,13 +20,13 @@ pytest_json=results-$PROFILE-$TODAY.json pytest --continue-on-collection-errors \ --quiet --tb=no \ $PROFILE \ - --json=$pytest_json \ + --json="$pytest_json" \ --config "${PATH_TO_EXEMPTIONS}" || true # post processing works directly with the output from pytest -$PATH_TO_SCRIPTS/manage_issues.py $pytest_json -$PATH_TO_SCRIPTS/create_metrics.py $pytest_json +"$PATH_TO_SCRIPTS/manage_issues.py" "$pytest_json" +"$PATH_TO_SCRIPTS/create_metrics.py" "$pytest_json" @@ -38,6 +38,3 @@ $PATH_TO_SCRIPTS/create_metrics.py $pytest_json # }' \ # $pytest_json \ # | create_issue - - -