Be more aggressive about killing an unresponsive service

Author: tonyg

on-ping-service-failure.sh

@@ -5,7 +5,8 @@
 #
 
 # Request a dump of running threads:
-touch $HOME/racket-pkg-website/signals/.dumpinfo
+dumprequestfile=$HOME/racket-pkg-website/signals/.dumpinfo
+touch $dumprequestfile
 
 # Wait a few seconds for the dump to complete:
 sleep 10
@@ -17,8 +18,20 @@ logarchive=$HOME/ping-failure-logs-$(date +%Y%m%d%H%M%S).tar.gz
     ls -tr | tail -n 10 | xargs tar -zcf $logarchive \
 )
 
-# Restart the service using daemontools:
-svc -du $HOME/service/racket-pkg-website
+# Restart the service using daemontools.
+if [ -f $dumprequestfile ]
+then
+    # If the `.dumpinfo` signal is still there after our sleep, then
+    # the process is so far off the rails we shouldn't bother waiting
+    # for it, so kill it hard.
+    echo "Killing service hard and restarting it."
+    svc -dku $HOME/service/racket-pkg-website
+else
+    # Otherwise, it's at least partially awake, so try asking it
+    # nicely.
+    echo "Politely requesting service termination before restart."
+    svc -du $HOME/service/racket-pkg-website
+fi
 
 # Finally, complain out loud. We expect to be running in some kind of
 # cron-ish context, so the output we produce here will likely find its