Skip to content

Commit aade898

Browse files
RKuttruffRiley K. Kuttruff
RKuttruff
and
Riley K. Kuttruff
authored
Add secrets scan into build pipeline (+ precommit hook config) (#481)
* Add secrets baseline * Updated secrets baseline * Added config for pre-commit hook * CI work - No longer need compare script to source * Turn off shell command echoing * Move secret scan & compare to script files * Chmod script files * Add suggested exclude patterns * Removed old code from jenkinsfile * Fix incorrect cl arg for secrets scan --------- Co-authored-by: Riley K. Kuttruff <[email protected]>
1 parent f235fe3 commit aade898

File tree

5 files changed

+417
-0
lines changed

5 files changed

+417
-0
lines changed

Diff for: ‎.ci/jenkins/build-test/Jenkinsfile

+13
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,18 @@ pipeline {
2121
required: true)
2222
}
2323
stages {
24+
stage('Scan for possible secrets'){
25+
steps{
26+
sh label: "Secrets scan",
27+
script: ".ci/scripts/util/secrets_scan.sh"
28+
29+
archiveArtifacts artifacts: ".secrets.new", allowEmptyArchive: true, fingerprint: true
30+
archiveArtifacts artifacts: ".secrets.diff", allowEmptyArchive: true, fingerprint: true
31+
32+
sh label: "Compare scan result to baseline",
33+
script: ".ci/scripts/util/secrets_scan_compare.sh"
34+
}
35+
}
2436
stage('Build OPERA PGE Docker image(s)') {
2537
steps {
2638
script {
@@ -93,6 +105,7 @@ pipeline {
93105
echo 'Unstable :/'
94106
}
95107
failure {
108+
archiveArtifacts artifacts: ".secrets.diff", allowEmptyArchive: true, fingerprint: true
96109
echo 'Failed :('
97110
}
98111
changed {

Diff for: ‎.ci/scripts/util/secrets_scan.sh

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
set +xe
4+
5+
echo '
6+
===============================================
7+
8+
Scanning workspace for potential leaked secrets
9+
10+
===============================================
11+
'
12+
13+
14+
# defaults
15+
[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
16+
17+
echo "WORKSPACE: $WORKSPACE"
18+
19+
if [ ! -f ${WORKSPACE}/.secrets.baseline ] ;
20+
then
21+
# This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files.
22+
# Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results.
23+
echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file."
24+
mkdir -p ${WORKSPACE}/empty-dir
25+
detect-secrets -C ${WORKSPACE}/empty-dir scan > ${WORKSPACE}/.secrets.baseline
26+
echo "✅ Blank .secrets.baseline file created successfully."
27+
rm -r ${WORKSPACE}/empty-dir
28+
else
29+
echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created."
30+
fi
31+
# scripts scan repository for new secrets
32+
# backup list of known secrets
33+
cp -pr ${WORKSPACE}/.secrets.baseline ${WORKSPACE}/.secrets.new
34+
# find secrets in the repository
35+
detect-secrets -C ${WORKSPACE} scan \
36+
--disable-plugin AbsolutePathDetectorExperimental \
37+
--all-files \
38+
--baseline ${WORKSPACE}/.secrets.new \
39+
--exclude-files '\.secrets..*' \
40+
--exclude-files '\.git.*' \
41+
--exclude-files 'test_results' \
42+
--exclude-files '\.pytest_cache' \
43+
--exclude-files '\.venv' \
44+
--exclude-files 'venv' \
45+
--exclude-files 'dist' \
46+
--exclude-files 'build' \
47+
--exclude-files '.*\.egg-info'
48+
# break build when new secrets discovered
49+
# function compares baseline/new secrets w/o listing results -- success(0) when new secret found
50+
51+
jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "${WORKSPACE}/.secrets.baseline" | sort > /data/tmp/.secrets.1
52+
jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "${WORKSPACE}/.secrets.new" | sort > /data/tmp/.secrets.2
53+
54+
diff /data/tmp/.secrets.1 /data/tmp/.secrets.2 > ${WORKSPACE}/.secrets.diff || true
55+
56+
rm -f /data/tmp/.secrets.1 /data/tmp/.secrets.2

Diff for: ‎.ci/scripts/util/secrets_scan_compare.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
3+
set +xe
4+
5+
echo '
6+
============================================
7+
8+
Comparing secrets scan results with baseline
9+
10+
============================================
11+
'
12+
13+
14+
# defaults
15+
[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
16+
17+
echo "WORKSPACE: $WORKSPACE"
18+
19+
if grep -q '>' < ${WORKSPACE}/.secrets.diff;
20+
then
21+
echo "⚠️ Attention Required! ⚠️" >&2
22+
echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
23+
echo "" >&2
24+
echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
25+
echo "" >&2
26+
echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
27+
echo "" >&2
28+
echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
29+
echo "" >&2
30+
echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
31+
exit 1
32+
else
33+
echo "🟢 Secrets tests PASSED! 🟢" >&1
34+
echo "No new secrets were detected in comparison to any baseline configurations." >&1
35+
exit 0
36+
fi

Diff for: ‎.pre-commit-config.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
repos:
2+
- repo: https://github.com/NASA-AMMOS/slim-detect-secrets
3+
# using commit id for now, will change to tag when official version is released
4+
rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe
5+
hooks:
6+
- id: detect-secrets
7+
args:
8+
- '--disable-plugin'
9+
- 'AbsolutePathDetectorExperimental'
10+
- '--baseline'
11+
- '.secrets.baseline'
12+
- '--exclude-files'
13+
- '\.git*'
14+
- '--exclude-files'
15+
- '\.secrets.*'
16+
- '--exclude-files'
17+
- 'test_results'
18+
- '--exclude-files'
19+
- '\.pytest_cache'
20+
- '--exclude-files'
21+
- '\.venv'
22+
- '--exclude-files'
23+
- 'venv'
24+
- '--exclude-files'
25+
- 'dist'
26+
- '--exclude-files'
27+
- 'build'
28+
- '--exclude-files'
29+
- '.*\.egg-info'

0 commit comments

Comments
 (0)