Run periodic task in janitor

[#155438732]

Signed-off-by: Steffen Uhlig <[email protected]>

Author: valeriap

README.md

@@ -100,6 +100,9 @@ databases.hooks.pre-start | Script to run before starting PostgreSQL.
 databases.hooks.post-start | Script to run after PostgreSQL has started.
 databases.hooks.pre-stop | Script to run before stopping PostgreSQL.
 databases.hooks.post-stop | Script to run after PostgreSQL has stopped.
+janitor.script | If specified, this script would be run periodically. This would be useful for running house-keeping tasks.
+janitor.interval | Interval in seconds between two invocations of the janitor script. By default it's set to `1` day.
+janitor.timeout | Time limit in seconds for the janitor script. By default it's set to `0` that means no time limit.
 
 *Note*
 - Removing a database from `databases.databases` list and deploying again does not trigger a physical deletion of the database in PostgreSQL.
@@ -157,6 +160,8 @@ If you plan to use this feature, you have to take into consideration that:
   If for example you want to use psql in your hook, you can specify:
   `${PACKAGE_DIR}/bin/psql -p ${PORT} -U vcap postgres -c "\l"`
 
+If you are interested in running something periodically, see the `janitor` configuration.
+
 ## Contributing
 
 ### Contributor License Agreement

jobs/postgres/spec

@@ -7,6 +7,7 @@ templates:
   pre-start.sh.erb: bin/pre-start
   postgres_ctl.sh.erb: bin/postgres_ctl
   pg_janitor_ctl.sh.erb: bin/pg_janitor_ctl
+  pg_janitor.sh.erb: bin/pg_janitor.sh
   postgres_start.sh.erb: bin/postgres_start.sh
   pgconfig.sh.erb: bin/pgconfig.sh
   utils.sh.erb: bin/utils.sh
@@ -19,6 +20,7 @@ templates:
   server.ca_cert.erb: config/certificates/server.ca_cert
   hooks/call-hooks.sh.erb: bin/hooks/call-hooks.sh
   hooks/postgres-pre-start.sh.erb: bin/hooks/postgres-pre-start.sh
+  hooks/janitor.sh.erb: bin/hooks/janitor.sh
   hooks/postgres-pre-stop.sh.erb: bin/hooks/postgres-pre-stop.sh
   hooks/postgres-post-start.sh.erb: bin/hooks/postgres-post-start.sh
   hooks/postgres-post-stop.sh.erb: bin/hooks/postgres-post-stop.sh
@@ -112,3 +114,16 @@ properties:
   databases.hooks.post_stop:
     description: "Script to run after PostgreSQL has stopped"
     default: ''
+  janitor.script:
+    description: "If specified, janitor would periodically run this script"
+    default: ''
+    example: |
+      #!/bin/bash
+      echo "Run VACUUM"
+      ${PACKAGE_DIR}/bin/psql -p ${PORT} -U vcap sandbox -c "VACUUM ANALYZE"
+  janitor.interval:
+    description: "Interval in seconds between two invocations of the janitor script. By default it's set to 1 day."
+    default: 86400
+  janitor.timeout:
+    description: "Time limit in seconds for the janitor script. By default it's set to 0 that means no time limit"
+    default: 0

jobs/postgres/templates/hooks/janitor.sh.erb

@@ -0,0 +1 @@
+<%= p("janitor.script", "") %>

jobs/postgres/templates/pg_janitor.sh.erb

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+
+exec > >(tee -a >(logger -p user.info -t vcap.$(basename $0).stdout) | awk -W interactive '{ system("echo -n [$(date +\"%Y-%m-%d %H:%M:%S%z\")]"); print " " $0 }' >> /var/vcap/sys/log/postgres/janitor.log)
+exec 2> >(tee -a >(logger -p user.error -t vcap.$(basename $0).stderr) | awk -W interactive '{ system("echo -n [$(date +\"%Y-%m-%d %H:%M:%S%z\")]"); print " " $0 }' >> /var/vcap/sys/log/postgres/janitor.err.log)
+
+source /var/vcap/jobs/postgres/bin/pgconfig.sh
+<% if p("janitor.script").empty? %>
+sleep infinity
+<% else %>
+while :
+do
+  echo "Invoking janitor script"
+  env -i PACKAGE_DIR=${PACKAGE_DIR} PORT=${PORT} DATA_DIR=${DATA_DIR} timeout <%=p("janitor.timeout") %> ${JOB_DIR}/bin/hooks/janitor.sh
+  retcode=$?
+  echo "Janitor script invoked with return code $retcode"
+  sleep <%= p("janitor.interval") %>
+done
+<% end %>

jobs/postgres/templates/pg_janitor_ctl.sh.erb

@@ -25,15 +25,17 @@ function main() {
         fi
         sleep 1
       done"
-      # update/create roles and databases
       if [ "$?" != "0" ]; then
         echo "Failed while waiting for postgres to start"
         exit 1
       fi
+
+      # update/create roles and databases
       create_roles
       create_databases
+
       echo $$ > "${PGJANITOR_PIDFILE}"
-      sleep infinity
+      exec ${JOB_DIR}/bin/pg_janitor.sh
       ;;
 
     "stop")

src/acceptance-tests/deploy/deploy_single_node_test.go

@@ -1,6 +1,7 @@
 package deploy_test
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
@@ -204,7 +205,18 @@ var _ = Describe("Deploy single instance", func() {
 			pre_stop_value := fmt.Sprintf(psql_command, pre_stop_role_name)
 			post_stop_value := fmt.Sprintf("echo %s", post_stop_uuid)
 
-			err = createOrUpdateDeployment(version, manifestPath, envName, variables, helpers.DefineHooks("0", pre_start_value, post_start_value, pre_stop_value, post_stop_value))
+			jan := helpers.Janitor{
+				Script: `${PACKAGE_DIR}/bin/psql -U vcap -p ${PORT} -d postgres << EOF
+CREATE TABLE IF NOT EXISTS test_hook(name VARCHAR(10) NOT NULL UNIQUE,total INTEGER NOT NULL);
+INSERT INTO test_hook (name, total) VALUES ('test', 1) ON CONFLICT (name) DO NOTHING;
+UPDATE test_hook SET total = total + 1 WHERE name = 'test';
+EOF
+`,
+				Timeout:  60,
+				Interval: 1,
+			}
+
+			err = createOrUpdateDeployment(version, manifestPath, envName, variables, append(jan.GetOpDefinitions(), helpers.DefineHooks("0", pre_start_value, post_start_value, pre_stop_value, post_stop_value)...))
 			Expect(err).NotTo(HaveOccurred())
 
 			sshKeyFile, err := writeSSHKey(envName)
@@ -235,6 +247,52 @@ var _ = Describe("Deploy single instance", func() {
 			cmd = exec.Command("ssh", "-i", sshKeyFile, "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("%s@%s", variables["testuser_name"], pgHost), fmt.Sprintf(bosh_ssh_command, post_stop_uuid))
 			err = cmd.Run()
 			Expect(err).NotTo(HaveOccurred())
+			err = os.Remove(sshKeyFile)
+
+			By("Testing the frequency-based hook")
+			conn, err := DB.GetSuperUserConnection()
+			Eventually(func() int {
+				rows, err := conn.Run("select total from test_hook where name = 'test'")
+				var counter struct {
+					Total int `json:"total"`
+				}
+				Expect(err).NotTo(HaveOccurred())
+				err = json.Unmarshal([]byte(rows[0]), &counter)
+				Expect(err).NotTo(HaveOccurred())
+				return counter.Total
+			}, "15s", "2s").Should(BeNumerically(">", 10))
+
+			By("Verifying that janitor script failure causes monit to restart janitor")
+			jan = helpers.Janitor{
+				Script: `#!/bin/bash
+STATEFILE=/tmp/statefile
+if [ -f $STATEFILE ]; then
+  echo second start >> $STATEFILE
+else
+  touch $STATEFILE
+  chmod 777 $STATEFILE
+  exit 1
+fi`,
+				Timeout:  60,
+				Interval: 86400,
+			}
+
+			err = createOrUpdateDeployment(version, manifestPath, envName, variables, jan.GetOpDefinitions())
+			Expect(err).NotTo(HaveOccurred())
+
+			sshKeyFile, err = writeSSHKey(envName)
+			Expect(err).NotTo(HaveOccurred())
+
+			Eventually(func() string {
+				bosh_ssh_command = "grep second /tmp/statefile"
+				cmd = exec.Command("ssh", "-i", sshKeyFile, "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", fmt.Sprintf("%s@%s", variables["testuser_name"], pgHost), bosh_ssh_command)
+				err = cmd.Run()
+				if err != nil {
+					return err.Error()
+				}
+				return ""
+			}, "10s", "2s").Should(BeEmpty())
+			err = os.Remove(sshKeyFile)
 
 			By("Verifying that hooks failure does not prevent postgres to start")
 			pre_start_uuid = helpers.GetUUID()
@@ -248,6 +306,7 @@ var _ = Describe("Deploy single instance", func() {
 			Expect(err).NotTo(HaveOccurred())
 			_, err = DB.GetPostgreSQLVersion()
 			Expect(err).NotTo(HaveOccurred())
+			err = os.Remove(sshKeyFile)
 		})
 
 		It("Successfully deploys a fresh env", func() {
@@ -457,6 +516,7 @@ var _ = Describe("Deploy single instance", func() {
 				variables = make(map[string]interface{})
 				variables["defuser_name"] = "pgadmin"
 				variables["defuser_password"] = "admin"
+				opDefs = nil
 			})
 			It("Successfully upgrades from older", AssertUpgradeSuccessful())
 		})
@@ -467,6 +527,7 @@ var _ = Describe("Deploy single instance", func() {
 				variables = make(map[string]interface{})
 				variables["defuser_name"] = "pgadmin"
 				variables["defuser_password"] = "admin"
+				opDefs = nil
 			})
 			It("Successfully upgrades from old", AssertUpgradeSuccessful())
 		})
@@ -488,6 +549,7 @@ var _ = Describe("Deploy single instance", func() {
 				variables = make(map[string]interface{})
 				variables["defuser_name"] = "pgadmin"
 				variables["defuser_password"] = "admin"
+				opDefs = nil
 			})
 			It("Successfully upgrades from master", AssertUpgradeSuccessful())
 		})

src/acceptance-tests/testing/helpers/op_defs_utilities.go

@@ -1,5 +1,11 @@
 package helpers
 
+type Janitor struct {
+	Timeout  int
+	Interval int
+	Script   string
+}
+
 func AddOpDefinition(ods *[]OpDefinition, defType string, defPath string, defValue interface{}) {
 	od := OpDefinition{
 		Type:  defType,
@@ -182,3 +188,19 @@ func DefineHooks(hooks_timeout string, pre_start string, post_start string, pre_
 
 	return ops
 }
+
+func (j Janitor) GetOpDefinitions() []OpDefinition {
+	var ops []OpDefinition
+	var path string
+
+	path = "/instance_groups/name=postgres/jobs/name=postgres/properties/janitor?/timeout?"
+	AddOpDefinition(&ops, "replace", path, j.Timeout)
+
+	path = "/instance_groups/name=postgres/jobs/name=postgres/properties/janitor?/interval?"
+	AddOpDefinition(&ops, "replace", path, j.Interval)
+
+	path = "/instance_groups/name=postgres/jobs/name=postgres/properties/janitor?/script?"
+	AddOpDefinition(&ops, "replace", path, j.Script)
+
+	return ops
+}

src/acceptance-tests/testing/templates/postgres_simple.yml

@@ -63,4 +63,4 @@ update:
   canary_watch_time: 30000-600000
   max_in_flight: 1
   serial: true
-  update_watch_time: 5000-600000
+  update_watch_time: 15000-300000