Rework cf-deployment pipeline for 3.0 changes

[#159758276]

Author: valeriap

ci/pipelines/acceptance-tests.yml

@@ -18,14 +18,16 @@ jobs:
     input_mapping: {dev-release: postgres-release}
     output_mapping: {dev-release-tarball: postgres-tarball}
     params: &bosh1_params
-      BOSH_DIRECTOR: {{bosh1_director}}
+      BOSH_DIRECTOR_IP: {{bosh1_director}}
+      BOSH_DIRECTOR_NAME: {{bosh1_director}}
       BOSH_CLIENT: {{bosh1_user}}
       BOSH_CLIENT_SECRET: {{bosh1_password}}
       BOSH_CA_CERT: {{bosh1_ca_cert}}
       REL_NAME: postgres
       REL_VERSION: v99+dev.1
       BBR_VERSION: 1.2.2
       STEMCELL_VERSION: {{stemcell_version}}
+      STEMCELL_TYPE: bosh-softlayer-xen-ubuntu-trusty
   - task: upload-postgres-dev-release
     file: postgres-release/ci/scripts/upload-dev-release-tarball/task.yml
     input_mapping: {dev-release-tarball: postgres-tarball}

ci/pipelines/cf-deployment.yml

@@ -1,4 +1,10 @@
 resources:
+- name: bosh-deployment
+  type: git
+  source:
+    uri: https://github.com/cloudfoundry/bosh-deployment.git
+    branch: master
+
 - name: cf-deployment-rc
   type: git
   source:
@@ -23,76 +29,58 @@ resources:
     uri: https://github.com/cloudfoundry/cf-acceptance-tests.git
     branch: master
 
-- name: postgres-ci-env
+- name: cf-deployment-concourse-tasks
   type: git
   source:
-    uri: [email protected]:cloudfoundry/postgres-ci-env
     branch: master
-    private_key: {{postgres_ci_env_private_key}}
-
-- name: cats-concourse-task
-  type: git
-  source:
-    branch: master
-    uri: https://github.com/cloudfoundry/cats-concourse-task.git
+    uri: https://github.com/cloudfoundry/cf-deployment-concourse-tasks.git
 
 jobs:
 - name: upload-stemcells-releases
-  serial_groups: [cf-fresh,cf-old]
+  serial_groups: [cf]
   plan:
   - aggregate:
     - get: postgres-release
       resource: postgres-release-develop
-  - task: upload-stemcell-fresh
-    file: postgres-release/ci/scripts/upload-stemcell/task.yml
-    params: &fresh_params
-      BOSH_DIRECTOR: {{fresh_bosh_director}}
-      BOSH_CLIENT: {{fresh_bosh_user}}
-      BOSH_CLIENT_SECRET: {{fresh_bosh_password}}
-      BOSH_CA_CERT: {{fresh_ca_cert}}
-      BOSH_PUBLIC_IP: {{fresh_bosh_public_ip}}
-      STEMCELL_VERSION: {{stemcell_version}}
-      CF_DEPLOYMENT: {{fresh_cf_deployment}}
-      API_PASSWORD: {{cf_api_password}}
-      S3_ACCESS_KEY: {{s3_access_key}}
-      S3_SECRET_KEY: {{s3_secret_key}}
-      S3_HOST: {{s3_host}}
-  - task: upload-stemcell-old
+    - get: cf-deployment
+      resource: cf-deployment-rc
+    - get: bosh-deployment
+      resource: bosh-deployment
+  - task: upload-stemcell-lite
     file: postgres-release/ci/scripts/upload-stemcell/task.yml
-    params: &old_params
-      BOSH_DIRECTOR: {{old_bosh_director}}
-      BOSH_CLIENT: {{old_bosh_user}}
-      BOSH_CLIENT_SECRET: {{old_bosh_password}}
-      BOSH_CA_CERT: {{old_ca_cert}}
-      BOSH_PUBLIC_IP: {{old_bosh_public_ip}}
+    params: &lite_params
+      BOSH_DIRECTOR_IP: {{lite_bosh_director}}
+      BOSH_DIRECTOR_NAME: {{lite_bosh_director_name}}
+      BOSH_CLIENT: {{lite_bosh_user}}
+      BOSH_CLIENT_SECRET: {{lite_bosh_password}}
+      BOSH_CA_CERT: {{lite_bosh_ca_cert}}
       STEMCELL_VERSION: {{stemcell_version}}
-      CF_DEPLOYMENT: {{old_cf_deployment}}
-      API_USER: {{cf_api_user}}
+      API_USER: admin
       API_PASSWORD: {{cf_api_password}}
-      S3_ACCESS_KEY: {{s3_access_key}}
-      S3_SECRET_KEY: {{s3_secret_key}}
-      S3_HOST: {{s3_host}}
   - task: create-postgres-dev-release-tarball
     file: postgres-release/ci/scripts/create-dev-release-tarball/task.yml
     input_mapping: {dev-release: postgres-release}
     output_mapping: {dev-release-tarball: postgres-tarball}
     params:
-      <<: *fresh_params
+      <<: *lite_params
       REL_NAME: postgres
       REL_VERSION: v999+dev.1
-  - task: upload-postgres-dev-release-fresh
+  - task: upload-postgres-dev-release-lite
     file: postgres-release/ci/scripts/upload-dev-release-tarball/task.yml
     input_mapping: {dev-release-tarball: postgres-tarball}
     params:
-      <<: *fresh_params
-  - task: upload-postgres-dev-release-old
-    file: postgres-release/ci/scripts/upload-dev-release-tarball/task.yml
-    input_mapping: {dev-release-tarball: postgres-tarball}
+      <<: *lite_params
+  - task: update-cloud-config
+    file: postgres-release/ci/scripts/update-cloud-config/task.yml
+    params:
+      <<: *lite_params
+  - task: update-runtime-config
+    file: postgres-release/ci/scripts/update-runtime-config/task.yml
     params:
-      <<: *old_params
+      <<: *lite_params
 
-- name: test-fresh-deployment
-  serial_groups: [cf-fresh]
+- name: deploy-fresh-cf
+  serial_groups: [cf]
   plan:
   - aggregate:
     - get: cf-deployment
@@ -104,74 +92,80 @@ jobs:
   - task: deploy-cf
     file: postgres-release/ci/scripts/deploy-cf/task.yml
     params:
-      <<: *fresh_params
+      <<: *lite_params
       USE_LATEST_PGREL: true
 
 - name: test-fresh-run-cats
-  serial_groups: [cf-fresh]
+  serial_groups: [cf]
   plan:
   - aggregate:
-    - get: postgres-ci-env
     - get: cf-acceptance-tests
-    - get: cats-concourse-task
+    - get: cf-deployment-concourse-tasks
     - get: postgres-release
       resource: postgres-release-develop
-      passed: [test-fresh-deployment]
+      passed: [deploy-fresh-cf]
       trigger: true
+  - task: prepare-cats-config
+    file: postgres-release/ci/scripts/prepare-cats-config/task.yml
+    output_mapping: {cats-config: integration-config}
+    params:
+      <<: *lite_params
+      CONFIG_FILE_NAME: cats_config.json
   - task: run-cats
-    file: cats-concourse-task/task.yml
-    input_mapping: {integration-config: postgres-ci-env}
+    file: cf-deployment-concourse-tasks/run-cats/task.yml
+    input_mapping: {integration-config: integration-config}
     params:
-      CONFIG_FILE_PATH: deployments/cf/cats_config.json
+      CONFIG_FILE_PATH: cats_config.json
       NODES: 4
       CAPTURE_LOGS: true
 
+- name: delete-deployment-fresh
+  serial_groups: [cf]
+  plan:
+  - aggregate:
+    - get: postgres-release
+      resource: postgres-release-develop
+      passed: [test-fresh-run-cats]
+      trigger: true
+  - task: delete-cf-deployment
+    file: postgres-release/ci/scripts/run-bosh-delete/task.yml
+    params:
+      <<: *lite_params
+
 - name: test-old-deployment
-  serial_groups: [cf-old]
+  serial_groups: [cf]
   plan:
   - aggregate:
     - get: cf-deployment-old
     - get: cf-deployment-rc
     - get: postgres-release
       resource: postgres-release-develop
-      passed: [upload-stemcells-releases]
+      passed: [delete-deployment-fresh]
       trigger: true
     - get: cf-acceptance-tests
   - task: deploy-cf-old
     input_mapping: {cf-deployment: cf-deployment-old}
     file: postgres-release/ci/scripts/deploy-cf/task.yml
     params:
-      <<: *old_params
+      <<: *lite_params
+      USE_LATEST_PGREL: false
   - task: populate-db
     file: postgres-release/ci/scripts/populate-db/task.yml
     params:
-      <<: *old_params
+      <<: *lite_params
   - task: deploy-cf
-    file: postgres-release/ci/scripts/deploy-cf/task_upg.yml
+    file: postgres-release/ci/scripts/deploy-cf/task.yml
     input_mapping: {cf-deployment: cf-deployment-rc}
     params:
-      <<: *old_params
+      <<: *lite_params
+      USE_LATEST_PGREL: true
   - task: check-db
     file: postgres-release/ci/scripts/check-db/task.yml
     params:
-      <<: *old_params
+      <<: *lite_params
 
-- name: delete-deployments-fresh
-  serial_groups: [cf-fresh]
-  plan:
-  - aggregate:
-    - get: postgres-release
-      resource: postgres-release-develop
-      passed: [test-fresh-run-cats]
-      trigger: true
-  - task: delete-cf-deployment
-    file: postgres-release/ci/scripts/run-bosh-delete/task.yml
-    params:
-      <<: *fresh_params
-      DEPLOYMENT_NAME: {{fresh_cf_deployment}}
-
-- name: delete-deployments-old
-  serial_groups: [cf-old]
+- name: delete-deployment-old
+  serial_groups: [cf]
   plan:
   - aggregate:
     - get: postgres-release
@@ -181,22 +175,17 @@ jobs:
   - task: delete-cf-deployment
     file: postgres-release/ci/scripts/run-bosh-delete/task.yml
     params:
-      <<: *old_params
-      DEPLOYMENT_NAME: {{old_cf_deployment}}
+      <<: *lite_params
 
 - name: cleanup-releases
-  serial_groups: [cf-fresh,cf-old]
+  serial_groups: [cf]
   plan:
   - aggregate:
     - get: postgres-release
       resource: postgres-release-develop
-      passed: [delete-deployments-fresh, delete-deployments-old]
+      passed: [delete-deployment-old]
       trigger: true
-  - task: cleanup-releases-fresh
-    file: postgres-release/ci/scripts/run-bosh-cleanup/task.yml
-    params:
-      <<: *fresh_params
-  - task: cleanup-releases-old
+  - task: cleanup-releases
     file: postgres-release/ci/scripts/run-bosh-cleanup/task.yml
     params:
-      <<: *old_params
+      <<: *lite_params

ci/pipelines/cleanup.yml

@@ -6,42 +6,30 @@ resources:
     branch: develop
 
 jobs:
-- name: delete-cf-deployments
+- name: delete-cf-deployment
   serial_groups: [cleanup]
   plan:
   - aggregate:
     - get: postgres-release
       resource: postgres-release
-  - task: delete-cf-fresh-deployment
+  - task: delete-cf-deployment
     file: postgres-release/ci/scripts/run-bosh-delete/task.yml
-    params: &base_params_fresh
-      BOSH_DIRECTOR: {{fresh_bosh_director}}
-      BOSH_CLIENT: {{fresh_bosh_user}}
-      BOSH_CLIENT_SECRET: {{fresh_bosh_password}}
-      BOSH_CA_CERT: {{fresh_ca_cert}}
-      DEPLOYMENT_NAME: {{fresh_cf_deployment}}
-  - task: delete-cf-old-deployment
-    file: postgres-release/ci/scripts/run-bosh-delete/task.yml
-    params: &base_params_old
-      BOSH_DIRECTOR: {{old_bosh_director}}
-      BOSH_CLIENT: {{old_bosh_user}}
-      BOSH_CLIENT_SECRET: {{old_bosh_password}}
-      BOSH_CA_CERT: {{old_ca_cert}}
-      DEPLOYMENT_NAME: {{old_cf_deployment}}
+    params: &lite_params
+      BOSH_DIRECTOR_IP: {{lite_bosh_director}}
+      BOSH_DIRECTOR_NAME: {{lite_bosh_director_name}}
+      BOSH_CLIENT: {{lite_bosh_user}}
+      BOSH_CLIENT_SECRET: {{lite_bosh_password}}
+      BOSH_CA_CERT: {{lite_bosh_ca_cert}}
 
 - name: cleanup-resources
   serial_groups: [cleanup]
   plan:
   - aggregate:
     - get: postgres-release
       resource: postgres-release
-      passed: [delete-cf-deployments]
+      passed: [delete-cf-deployment]
       trigger: true
-  - task: cleanup-releases-fresh
-    file: postgres-release/ci/scripts/run-bosh-cleanup/task.yml
-    params:
-      <<: *base_params_fresh
-  - task: cleanup-releases-old
+  - task: cleanup-releases
     file: postgres-release/ci/scripts/run-bosh-cleanup/task.yml
     params:
-      <<: *base_params_old
+      <<: *lite_params

ci/scripts/check-db/task.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -exu
 preflight_check() {
   set +x
+  test -n "${BOSH_DIRECTOR_IP}"
   test -n "${CF_DEPLOYMENT}"
   test -n "${API_USER}"
   test -n "${API_PASSWORD}"
@@ -11,7 +12,7 @@ function main() {
 
   local root="${1}"
   preflight_check
-  local api_endpoint="api.apps.${CF_DEPLOYMENT}.microbosh"
+  local api_endpoint="api.${BOSH_DIRECTOR_IP}.nip.io"
 
   cf api ${api_endpoint} --skip-ssl-validation
   set +x
@@ -20,7 +21,7 @@ function main() {
   cf target -o ${CF_DEPLOYMENT} -s ${CF_DEPLOYMENT}
 
   cf apps
-  curl --fail dora.apps.${CF_DEPLOYMENT}.microbosh
+  curl --fail dora.${BOSH_DIRECTOR_IP}.nip.io
 
 }
 

ci/scripts/check-db/task.yml

@@ -13,6 +13,7 @@ run:
   path: postgres-release/ci/scripts/check-db/task.sh
 
 params:
-  CF_DEPLOYMENT:
-  API_USER:
+  BOSH_DIRECTOR_IP:
+  CF_DEPLOYMENT: cf
+  API_USER: admin
   API_PASSWORD:

ci/scripts/configure_for_bosh.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set +x
+test -n "${BOSH_DIRECTOR_IP}"
+test -n "${BOSH_DIRECTOR_NAME}"
+set -x
+
+echo "${BOSH_DIRECTOR_IP} ${BOSH_DIRECTOR_NAME}" >> /etc/hosts
+export BOSH_ENVIRONMENT="https://${BOSH_DIRECTOR_NAME}:25555"

ci/scripts/create-dev-release-tarball/task.sh

@@ -2,7 +2,8 @@
 
 preflight_check() {
   set +x
-  test -n "${BOSH_DIRECTOR}"
+  test -n "${BOSH_DIRECTOR_IP}"
+  test -n "${BOSH_DIRECTOR_NAME}"
   test -n "${BOSH_CLIENT}"
   test -n "${BOSH_CLIENT_SECRET}"
   test -n "${BOSH_CA_CERT}"
@@ -14,7 +15,7 @@ preflight_check() {
 function main(){
   local root="${1}"
   preflight_check
-  export BOSH_ENVIRONMENT="https://${BOSH_DIRECTOR}:25555"
+  source ${root}/postgres-release/ci/scripts/configure_for_bosh.sh
 
   pushd ${root}/dev-release
   bosh create-release --force --tarball=${root}/dev-release-tarball/${REL_NAME}-${REL_VERSION}.tgz --version "${REL_VERSION}" --name "${REL_NAME}"