Add pre-commit hooks

And some minor code tweaks to pass black & dectect-secrets

Also fixed some potential shell issues

.pre-commit-config.yaml

@@ -3,3 +3,71 @@ repos:
     rev: 19.10b0
     hooks:
     - id: black
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-ast
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: check-case-conflict
+    -   id: check-docstring-first
+    -   id: check-merge-conflict
+    -   id: check-toml
+
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
+    hooks:
+    -   id: pyupgrade
+        args: ["--py36-plus"]
+
+## ## - repo: https://github.com/jumanjihouse/pre-commit-hooks
+## ##   rev: master  # or specific git tag
+## ##   hooks:
+## ##     - id: forbid-binary
+## ##       ## TODO - disable for docs/_build/html
+## ##     - id: markdownlint # Configure in .mdlrc
+## ##       # TODO - disable MD012 (consecutive blank lines)
+## ##       #        disable MD013 (line length)
+## ##       #        fix MD031 (fenced code needs blank lines) README.md
+## ##     - id: shellcheck
+
+## ## - repo: https://github.com/IamTheFij/docker-pre-commit
+## ##   rev: v2.0.0
+## ##   hooks:
+## ##         - id: docker-compose-check
+## ##         - id: hadolint
+## ##           # TODO - disable DL3005 (apt upgrade)
+## ##           #        disable DL3008 (unpinned packages)
+## ##           #        disable DL3009 (left apt-get lists around)
+
+- repo: https://github.com/PyCQA/bandit
+  rev: 1.6.2
+  hooks:
+    - id: bandit
+      args:
+      - "--skip"
+      - "B101"          # use of assert
+      - --quiet
+
+- repo: https://github.com/myint/docformatter
+  rev: v1.3.1
+  hooks:
+    - id: docformatter
+      args: [--in-place]
+
+- repo: https://gitlab.com/iamlikeme/nbhooks
+  rev: ""
+  hooks:
+  - id: nb-ensure-clean
+    #args: [--meta, ExecuteTime]  # Optional blacklist of metadata keys (you can use regex)
+
+- repo: https://github.com/Yelp/detect-secrets
+  rev: ""
+  hooks:
+    - id: detect-secrets
+      args: ['--baseline', '.secrets.baseline']
+      exclude: .*/tests/.*

.secrets.baseline

@@ -0,0 +1,82 @@
+{
+  "custom_plugin_paths": [],
+  "exclude": {
+    "files": null,
+    "lines": null
+  },
+  "generated_at": "2020-09-24T16:39:13Z",
+  "plugins_used": [
+    {
+      "name": "AWSKeyDetector"
+    },
+    {
+      "name": "ArtifactoryDetector"
+    },
+    {
+      "base64_limit": 4.5,
+      "name": "Base64HighEntropyString"
+    },
+    {
+      "name": "BasicAuthDetector"
+    },
+    {
+      "name": "CloudantDetector"
+    },
+    {
+      "hex_limit": 3,
+      "name": "HexHighEntropyString"
+    },
+    {
+      "name": "IbmCloudIamDetector"
+    },
+    {
+      "name": "IbmCosHmacDetector"
+    },
+    {
+      "name": "JwtTokenDetector"
+    },
+    {
+      "keyword_exclude": null,
+      "name": "KeywordDetector"
+    },
+    {
+      "name": "MailchimpDetector"
+    },
+    {
+      "name": "PrivateKeyDetector"
+    },
+    {
+      "name": "SlackDetector"
+    },
+    {
+      "name": "SoftlayerDetector"
+    },
+    {
+      "name": "StripeDetector"
+    },
+    {
+      "name": "TwilioKeyDetector"
+    }
+  ],
+  "results": {
+    "docs/_build/html/.buildinfo": [
+      {
+        "hashed_secret": "4409d7400c419585a64ebdc7632b2f20f7050ca8",
+        "is_verified": false,
+        "line_number": 3,
+        "type": "Hex High Entropy String"
+      },
+      {
+        "hashed_secret": "63256a38625700f409b7ec5db8fa05ffd2cb88b2",
+        "is_verified": false,
+        "line_number": 4,
+        "type": "Hex High Entropy String"
+      }
+    ]
+  },
+  "version": "0.14.2",
+  "word_list": {
+    "file": null,
+    "hash": null
+  }
+}

github/branches/conftest.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -14,7 +13,7 @@
 import os
 import pathlib
 from typing import List
-import subprocess
+import subprocess  # nosec
 
 import pytest
 
@@ -56,7 +55,11 @@ def repos_to_check() -> List[str]:
 
     # python 3.6 doesn't support capture_output
     # status = subprocess.run(cmd, capture_output=True)
-    status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # fmt: off
+    status = subprocess.run(  # nosec
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE  # nosec
+    )
+    # fmt:on
     # return as array of non-empty, unquoted, "lines"
     return [
         x.translate({ord('"'): None, ord("'"): None})

github/orgs/conftest.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -16,7 +15,7 @@
 
 from typing import List, Set
 
-import subprocess
+import subprocess  # nosec
 
 from sgqlc.endpoint.http import HTTPEndpoint  # noqa: I900
 
@@ -49,7 +48,11 @@ def orgs_to_check() -> Set[str]:
 
     # python 3.6 doesn't support capture_output
     # status = subprocess.run(cmd, capture_output=True)
-    status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # fmt: off
+    status = subprocess.run(  # nosec
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE  # nosec
+    )
+    # fmt:on
     assert not status.stderr.decode("utf-8")
     # return as array of non-empty, unquoted, "lines"
     return {

github/orgs/retrieve_github_data.py

@@ -2,19 +2,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
-"""
-Collect Information about branches sufficient to check for all branch
-protection guideline compliance.
-
-"""
+"""Collect Information about branches sufficient to check for all branch
+protection guideline compliance."""
 
 import csv
 from functools import lru_cache
 import logging
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
-import subprocess
+import subprocess  # nosec
 import sys
 from typing import Any, List, Optional, Set
 
@@ -39,8 +36,7 @@ class OrgInfo:
 
     @staticmethod
     def idfn(val: Any) -> Optional[str]:
-        """ provide ID for pytest Parametrization
-        """
+        """provide ID for pytest Parametrization."""
         if isinstance(val, (OrgInfo,)):
             return f"{val.id_}-{val.login}"
         return None
@@ -64,7 +60,7 @@ def csv_row(self) -> List[Optional[str]]:
 
 
 def create_operation(owner):
-    """ Create the default Query operation
+    """Create the default Query operation.
 
     We build the structure for:
       organization:
@@ -109,9 +105,7 @@ def get_org_info(endpoint: Any, org: str) -> OrgInfo:
 
 
 def extract_org_data(orgdata) -> OrgInfo:
-    """ extract relevant data from sgqlc structure
-
-    """
+    """extract relevant data from sgqlc structure."""
     org_data = OrgInfo(
         name=orgdata.name,
         login=orgdata.login,
@@ -209,7 +203,11 @@ def _orgs_to_check() -> Set[str]:
 
     # python 3.6 doesn't support capture_output
     # status = subprocess.run(cmd, capture_output=True)
-    status = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    # fmt: off
+    status = subprocess.run(  # nosec
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE  # nosec
+    )
+    # fmt:on
     assert not status.stderr.decode("utf-8")
     # return as array of non-empty, unquoted, "lines"
     return {
@@ -220,7 +218,7 @@ def _orgs_to_check() -> Set[str]:
 
 
 def get_all_org_data(endpoint: Any = None, orgs: List[str] = None) -> List[OrgInfo]:
-    """ Generator of org data """
+    """Generator of org data."""
     if not endpoint:
         # if we're creating the endpoint, then arguments must already be
         # in environment variables.
@@ -245,14 +243,14 @@ def _compact_fmt(d):
                     lst.append(_compact_fmt(e))
                 else:
                     lst.append(repr(e))
-            s.append("%s=[%s]" % (k, ", ".join(lst)))
+            s.append("{}=[{}]".format(k, ", ".join(lst)))
             continue
-        s.append("%s=%r" % (k, v))
+        s.append(f"{k}={v!r}")
     return "(" + ", ".join(s) + ")"
 
 
 def _report_download_errors(errors):
-    """ error handling for graphql comms """
+    """error handling for graphql comms."""
     logger.error("Document contain %d errors", len(errors))
     for i, e in enumerate(errors):
         msg = e.pop("message")

github/run.sh

@@ -20,13 +20,13 @@ pytest_json=results-$PROFILE-$TODAY.json
 pytest --continue-on-collection-errors \
     --quiet --tb=no \
     $PROFILE \
-    --json=$pytest_json \
+    --json="$pytest_json" \
     --config "${PATH_TO_EXEMPTIONS}" || true
 
 
 # post processing works directly with the output from pytest
-$PATH_TO_SCRIPTS/manage_issues.py $pytest_json
-$PATH_TO_SCRIPTS/create_metrics.py $pytest_json
+"$PATH_TO_SCRIPTS/manage_issues.py" "$pytest_json"
+"$PATH_TO_SCRIPTS/create_metrics.py" "$pytest_json"
 
 
 
@@ -38,6 +38,3 @@ $PATH_TO_SCRIPTS/create_metrics.py $pytest_json
 #     }' \
 #     $pytest_json \
 #     | create_issue
-
-
-