Add support for Slack notifications

We need to add more monitoring on the pipeline. Otherwise it's going to
be very easy to go red for a few days without noticing.

My initial goal was to hook up to IRC, but the reality is that I think
we'll need both Slack and IRC because many folks mostly live in Slack
nowadays.

(That said, I still fully intend to add IRC support via fedmsgs as
discussed in coreos#41).

Now, how this patch works is that we add two new Jenkins plugins:
- configuration-as-code
- slack

The first one is used to configure the second one. More broadly, it's
able to configure almost all of Jenkins and its plugins via YAML instead
of dropping to XML, so we'll likely be leveraging it some more in the
future.

The actual configuration bit here is more or less the same as RHCOS,
with some minor variations (we use secrets instead of defining the token
at template generation time, and we just set a default channel instead
of passing it through an env var)

Of course, this is all still optional. The local developer workflow
should still work fine.

Author: jlebon

HACKING.md

@@ -163,7 +163,7 @@ If you're planning to test changes, it would be best to fork
 this repo so that you do your work there. The workflow
 requires a remote repo to which to push changes.
 
-### Creating AWS credentials configs
+### [OPTIONAL] Creating AWS credentials configs
 
 If you are in production where we upload builds to S3 OR you want to
 test uploading to S3 as part of your pipeline development, you need to
@@ -217,6 +217,16 @@ $ aws s3 mb my-fcos-bucket
 
 And provide it to `--bucket` below.
 
+### [OPTIONAL] Slack integration
+
+If you want to be able to have build status messages appear in Slack,
+create a `slack-api-token` secret:
+
+```
+$ echo -n "$TOKEN" > slack-token
+$ oc create secret generic slack-api-token --from-file=token=slack-token
+```
+
 ### Create a Jenkins instance with a persistent volume backing store
 
 ```

Jenkinsfile

@@ -125,6 +125,11 @@ node {
 podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultContainer: 'jnlp') {
     node('coreos-assembler') { container('coreos-assembler') {
 
+        // declare this early so we can use it in Slack
+        def newBuildID
+
+        try {
+
         // this is defined IFF we *should* and we *can* upload to S3
         def s3_stream_dir
 
@@ -219,12 +224,13 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
             """)
         }
 
-        def newBuildID = utils.shwrap_capture("readlink builds/latest")
-        if (prevBuildID == newBuildID) {
+        def buildID = utils.shwrap_capture("readlink builds/latest")
+        if (prevBuildID == buildId) {
             currentBuild.result = 'SUCCESS'
             currentBuild.description = "[${params.STREAM}] 💤 (no new build)"
             return
         } else {
+            newBuildID = buildID
             currentBuild.description = "[${params.STREAM}] ⚡ ${newBuildID}"
 
             // and insert the parent info into meta.json so we can display it in
@@ -423,5 +429,37 @@ podTemplate(cloud: 'openshift', label: 'coreos-assembler', yaml: pod, defaultCon
                 }
             }
         }
+
+        } catch (e) {
+            currentBuild.result = 'FAILURE'
+            throw e
+        } finally {
+            def color
+            def message = "[${params.STREAM}] <${env.BUILD_URL}|${env.BUILD_NUMBER}>"
+
+            if (currentBuild.result == 'SUCCESS') {
+                if (!newBuildID) {
+                    // SUCCESS, but no new builds? Must've been a no-op
+                    return
+                }
+                message = ":fcos: :sparkles: ${message} - SUCCESS"
+                color = 'good';
+            } else {
+                message = ":fcos: :trashfire: ${message} - FAILURE"
+                color = 'danger';
+            }
+
+            if (newBuildID) {
+                message = "${message} (${newBuildID})"
+            }
+
+            try {
+                if (official) {
+                    slackSend(color: color, message: message)
+                }
+            } finally {
+                echo message
+            }
+        }
     }}
 }

jenkins/master/plugins.txt

@@ -1 +1,3 @@
 github-oauth:0.33
+configuration-as-code:1.33
+slack:2.34

manifests/jenkins.yaml

@@ -74,6 +74,9 @@ objects:
             value: "true"
           - name: JNLP_SERVICE_NAME
             value: ${JNLP_SERVICE_NAME}
+          # DELTA: point c-as-c plugin to config map files; see below
+          - name: CASC_JENKINS_CONFIG
+            value: /var/lib/jenkins/jcasc
           image: ' '
           imagePullPolicy: IfNotPresent
           livenessProbe:
@@ -100,13 +103,30 @@ objects:
           volumeMounts:
           - mountPath: /var/lib/jenkins
             name: ${JENKINS_SERVICE_NAME}-data
+          # DELTA: mount c-as-c config map
+          - name: ${JENKINS_SERVICE_NAME}-cfg
+            mountPath: /var/lib/jenkins/jcasc
+            readOnly: true
+          # DELTA: mount Slack token; see below
+          - name: slack-token
+            mountPath: /var/run/secrets/slack-api-token
+            readOnly: true
         dnsPolicy: ClusterFirst
         restartPolicy: Always
         serviceAccountName: ${JENKINS_SERVICE_NAME}
         volumes:
         - name: ${JENKINS_SERVICE_NAME}-data
           persistentVolumeClaim:
             claimName: ${JENKINS_SERVICE_NAME}
+        # DELTA: add a configmap -- it's defined in pipeline.yaml
+        - name: ${JENKINS_SERVICE_NAME}-cfg
+          configMap:
+            name: jenkins-cfg
+        # DELTA: add the Slack token
+        - name: slack-token
+          secret:
+            secretName: slack-api-token
+            optional: true
     triggers:
     - imageChangeParams:
         automatic: true

manifests/pipeline.yaml

@@ -44,6 +44,12 @@ parameters:
   - description: Whether to use KVM device plugin or legacy OCI KVM hook
     name: KVM_SELECTOR
     value: kvm-device-plugin
+  - description: Slack domain on which to post build status updates
+    name: SLACK_DOMAIN
+    value: coreos
+  - description: Slack channel on which to post build status updates
+    name: SLACK_CHANNEL
+    value: jenkins-coreos
 
 objects:
 
@@ -123,6 +129,34 @@ objects:
           importPolicy:
             scheduled: true
 
+  ### JENKINS CONFIGURATION ###
+
+  # this uses the configuration-as-code plugin:
+  # https://github.com/jenkinsci/configuration-as-code-plugin
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: jenkins-cfg
+      annotations:
+        coreos.com/deploy-default: "true"
+    data:
+      jenkins.yaml: |
+          credentials:
+            system:
+              domainCredentials:
+                - credentials:
+                  - string:
+                      scope: GLOBAL
+                      id: slack-token
+                      secret: ${slack-api-token/token}
+                      description: Slack API token
+          unclassified:
+            slackNotifier:
+              teamDomain: ${SLACK_DOMAIN}
+              tokenCredentialId: slack-token
+              room: "#${SLACK_CHANNEL}"
+
+
   ### COREOS-ASSEMBLER ###
 
   # keep a local copy of coreos-assembler so we're not constantly pulling it