Skip to content

Commit e38912f

Browse files
rjeberhardrjeberhard
authored
Proper shut down for operator and webhook; Have webhook delete VWC (#3761)
* Proper shut down for operator and webhook; Have webhook delete VWC
1 parent 4c9c0b2 commit e38912f

File tree

13 files changed

+250
-30
lines changed

13 files changed

+250
-30
lines changed

‎deployment/scripts/operator.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,7 @@ HEAP="-XshowSettings:vm"
7070
java $HEAP $MOCKING_WLS $DEBUG $LOGGING -jar /operator/weblogic-kubernetes-operator.jar &
7171
PID=$!
7272
wait $PID
73+
74+
SHUTDOWN_COMPLETE_MARKER_FILE="${DEPLOYMENT_DIR}/marker.shutdown-complete"
75+
76+
touch ${SHUTDOWN_COMPLETE_MARKER_FILE}

‎deployment/scripts/stop.sh

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
# Copyright (c) 2022, Oracle and/or its affiliates.
3+
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
4+
5+
echo "Setting stop signal"
6+
7+
DEPLOYMENT_DIR="/deployment"
8+
SHUTDOWN_MARKER_FILE="${DEPLOYMENT_DIR}/marker.shutdown"
9+
SHUTDOWN_COMPLETE_MARKER_FILE="${DEPLOYMENT_DIR}/marker.shutdown-complete"
10+
11+
touch ${SHUTDOWN_MARKER_FILE}
12+
13+
while true; do
14+
if [ -e ${SHUTDOWN_COMPLETE_MARKER_FILE} ] ; then
15+
exit 0
16+
fi
17+
sleep 1
18+
done

‎deployment/scripts/webhook.sh

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,3 +63,8 @@ HEAP="-XshowSettings:vm"
6363
java -cp /operator/weblogic-kubernetes-operator.jar $HEAP $MOCKING_WLS $DEBUG $LOGGING oracle.kubernetes.operator.WebhookMain &
6464
PID=$!
6565
wait $PID
66+
67+
DEPLOYMENT_DIR="/deployment"
68+
SHUTDOWN_COMPLETE_MARKER_FILE="${DEPLOYMENT_DIR}/marker.shutdown-complete"
69+
70+
touch ${SHUTDOWN_COMPLETE_MARKER_FILE}

‎kubernetes/charts/weblogic-operator/templates/_operator-clusterrole-general.tpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,5 +38,5 @@ rules:
3838
verbs: ["create"]
3939
- apiGroups: ["admissionregistration.k8s.io"]
4040
resources: ["validatingwebhookconfigurations"]
41-
verbs: ["get", "create", "update", "patch"]
41+
verbs: ["get", "create", "update", "patch", "delete"]
4242
{{- end }}

‎kubernetes/charts/weblogic-operator/templates/_operator-dep.tpl

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@ spec:
5454
- name: "weblogic-operator"
5555
image: {{ .image | quote }}
5656
imagePullPolicy: {{ .imagePullPolicy | quote }}
57-
command: ["bash"]
58-
args: ["/deployment/operator.sh"]
57+
command: ["/deployment/operator.sh"]
58+
lifecycle:
59+
preStop:
60+
exec:
61+
command: ["/deployment/stop.sh"]
5962
env:
6063
- name: "OPERATOR_NAMESPACE"
6164
valueFrom:
@@ -131,17 +134,13 @@ spec:
131134
{{- if not .remoteDebugNodePortEnabled }}
132135
livenessProbe:
133136
exec:
134-
command:
135-
- "bash"
136-
- "/probes/livenessProbe.sh"
137+
command: ["/probes/livenessProbe.sh"]
137138
initialDelaySeconds: 40
138139
periodSeconds: 10
139140
failureThreshold: 5
140141
readinessProbe:
141142
exec:
142-
command:
143-
- "bash"
144-
- "/probes/readinessProbe.sh"
143+
command: ["/probes/readinessProbe.sh"]
145144
initialDelaySeconds: 2
146145
periodSeconds: 10
147146
{{- end }}
@@ -276,8 +275,11 @@ spec:
276275
- name: "weblogic-operator-webhook"
277276
image: {{ .image | quote }}
278277
imagePullPolicy: {{ .imagePullPolicy | quote }}
279-
command: ["bash"]
280-
args: ["/deployment/webhook.sh"]
278+
command: ["/deployment/webhook.sh"]
279+
lifecycle:
280+
preStop:
281+
exec:
282+
command: ["/deployment/stop.sh"]
281283
env:
282284
- name: "WEBHOOK_NAMESPACE"
283285
valueFrom:
@@ -341,16 +343,12 @@ spec:
341343
{{- if not .remoteDebugNodePortEnabled }}
342344
livenessProbe:
343345
exec:
344-
command:
345-
- "bash"
346-
- "/probes/livenessProbe.sh"
346+
command: ["/probes/livenessProbe.sh"]
347347
initialDelaySeconds: 40
348348
periodSeconds: 5
349349
readinessProbe:
350350
exec:
351-
command:
352-
- "bash"
353-
- "/probes/readinessProbe.sh"
351+
command: ["/probes/readinessProbe.sh"]
354352
initialDelaySeconds: 2
355353
periodSeconds: 10
356354
{{- end }}

‎kubernetes/charts/weblogic-operator/templates/_operator-role.tpl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2021, Oracle and/or its affiliates.
1+
# Copyright (c) 2018, 2022, Oracle and/or its affiliates.
22
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
{{- define "operator.operatorRole" }}
@@ -16,5 +16,5 @@ rules:
1616
verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "deletecollection"]
1717
- apiGroups: ["admissionregistration.k8s.io"]
1818
resources: ["validatingwebhookconfigurations"]
19-
verbs: ["get", "create", "update", "patch"]
19+
verbs: ["get", "create", "update", "patch", "delete"]
2020
{{- end }}

‎kubernetes/src/test/java/oracle/kubernetes/operator/create/CreateOperatorGeneratedFilesTestBase.java

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313
import io.kubernetes.client.openapi.models.V1Deployment;
1414
import io.kubernetes.client.openapi.models.V1DeploymentStrategy;
1515
import io.kubernetes.client.openapi.models.V1EnvVarSource;
16+
import io.kubernetes.client.openapi.models.V1ExecAction;
1617
import io.kubernetes.client.openapi.models.V1LabelSelector;
18+
import io.kubernetes.client.openapi.models.V1Lifecycle;
19+
import io.kubernetes.client.openapi.models.V1LifecycleHandler;
1720
import io.kubernetes.client.openapi.models.V1Namespace;
1821
import io.kubernetes.client.openapi.models.V1PolicyRule;
1922
import io.kubernetes.client.openapi.models.V1Probe;
@@ -218,8 +221,10 @@ protected V1Deployment getExpectedWeblogicOperatorDeployment() {
218221
.image(getInputs().getWeblogicOperatorImage())
219222
.imagePullPolicy(
220223
getInputs().getWeblogicOperatorImagePullPolicy())
221-
.addCommandItem("bash")
222-
.addArgsItem("/deployment/operator.sh")
224+
.addCommandItem("/deployment/operator.sh")
225+
.lifecycle(
226+
new V1Lifecycle().preStop(new V1LifecycleHandler().exec(
227+
new V1ExecAction().addCommandItem("/deployment/stop.sh"))))
223228
.addEnvItem(
224229
newEnvVar()
225230
.name("OPERATOR_NAMESPACE")
@@ -305,7 +310,7 @@ private V1Probe createProbe(Integer initialDelaySeconds, Integer periodSeconds,
305310
.initialDelaySeconds(initialDelaySeconds)
306311
.periodSeconds(periodSeconds)
307312
.failureThreshold(failureThreshold)
308-
.exec(newExecAction().addCommandItem("bash").addCommandItem(shellScript));
313+
.exec(newExecAction().addCommandItem(shellScript));
309314
}
310315

311316
@Test
@@ -758,7 +763,8 @@ private V1PolicyRule newPolicyRuleForValidatingWebhookConfiguration() {
758763
"get",
759764
"create",
760765
"update",
761-
"patch"));
766+
"patch",
767+
"delete"));
762768
}
763769

764770
@Test

‎operator/src/main/java/oracle/kubernetes/operator/BaseMain.java

Lines changed: 72 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.io.InputStream;
1010
import java.io.OutputStream;
1111
import java.io.PrintStream;
12+
import java.nio.file.Files;
1213
import java.security.KeyManagementException;
1314
import java.security.KeyStoreException;
1415
import java.security.NoSuchAlgorithmException;
@@ -35,6 +36,7 @@
3536
import oracle.kubernetes.operator.logging.LoggingFacade;
3637
import oracle.kubernetes.operator.logging.LoggingFactory;
3738
import oracle.kubernetes.operator.tuning.TuningParameters;
39+
import oracle.kubernetes.operator.utils.PathSupport;
3840
import oracle.kubernetes.operator.work.Component;
3941
import oracle.kubernetes.operator.work.Container;
4042
import oracle.kubernetes.operator.work.ContainerResolver;
@@ -130,6 +132,36 @@ void startDeployment(Runnable completionAction) {
130132
}
131133
}
132134

135+
void stopDeployment(Runnable completionAction) {
136+
Step shutdownSteps = createShutdownSteps();
137+
if (shutdownSteps != null) {
138+
try {
139+
delegate.runSteps(new Packet(), shutdownSteps, new ReleaseShutdownSignalRunnable(completionAction));
140+
acquireShutdownSignal();
141+
} catch (Throwable e) {
142+
LOGGER.warning(MessageKeys.EXCEPTION, e);
143+
}
144+
} else if (completionAction != null) {
145+
completionAction.run();
146+
}
147+
}
148+
149+
private class ReleaseShutdownSignalRunnable implements Runnable {
150+
final Runnable inner;
151+
152+
ReleaseShutdownSignalRunnable(Runnable inner) {
153+
this.inner = inner;
154+
}
155+
156+
@Override
157+
public void run() {
158+
if (inner != null) {
159+
inner.run();
160+
}
161+
releaseShutdownSignal();
162+
}
163+
}
164+
133165
void markReadyAndStartLivenessThread() {
134166
try {
135167
new DeploymentReady(delegate).create();
@@ -153,6 +185,11 @@ void startRestServer(Container container)
153185

154186
abstract BaseRestServer createRestServer();
155187

188+
// For test
189+
AtomicReference<BaseServer> getRestServer() {
190+
return restServer;
191+
}
192+
156193
void stopRestServer() {
157194
Optional.ofNullable(restServer.getAndSet(null)).ifPresent(BaseServer::stop);
158195
}
@@ -181,24 +218,56 @@ void stopMetricsServer() {
181218

182219
abstract Step createStartupSteps();
183220

221+
Step createShutdownSteps() {
222+
return null;
223+
}
224+
184225
abstract void logStartingLivenessMessage();
185226

186227
void stopAllWatchers() {
187228
// no-op
188229
}
189230

190-
void waitForDeath() {
191-
Runtime.getRuntime().addShutdownHook(new Thread(shutdownSignal::release));
192-
231+
private void acquireShutdownSignal() {
193232
try {
194233
shutdownSignal.acquire();
195234
} catch (InterruptedException ignore) {
196235
Thread.currentThread().interrupt();
197236
}
237+
}
238+
239+
private void releaseShutdownSignal() {
240+
shutdownSignal.release();
241+
}
242+
243+
// For test
244+
int getShutdownSignalAvailablePermits() {
245+
return shutdownSignal.availablePermits();
246+
}
247+
248+
void waitForDeath() {
249+
Runtime.getRuntime().addShutdownHook(new Thread(this::releaseShutdownSignal));
250+
scheduleCheckForShutdownMarker();
251+
252+
acquireShutdownSignal();
198253

199254
stopAllWatchers();
200255
}
201256

257+
void scheduleCheckForShutdownMarker() {
258+
wrappedExecutorService.scheduleWithFixedDelay(
259+
() -> {
260+
File marker = new File(delegate.getDeploymentHome(), "marker.shutdown");
261+
if (isFileExists(marker)) {
262+
releaseShutdownSignal();
263+
}
264+
}, 5, 2, TimeUnit.SECONDS);
265+
}
266+
267+
private static boolean isFileExists(File file) {
268+
return Files.isRegularFile(PathSupport.getPath(file));
269+
}
270+
202271
static Packet createPacketWithLoggingContext(String ns) {
203272
Packet packet = new Packet();
204273
packet.getComponents().put(

‎operator/src/main/java/oracle/kubernetes/operator/OperatorMain.java

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,7 @@ public static void main(String[] args) {
172172
// now we just wait until the pod is terminated
173173
operatorMain.waitForDeath();
174174

175-
operatorMain.stopRestServer();
176-
operatorMain.stopMetricsServer();
175+
operatorMain.stopDeployment(operatorMain::completeStop);
177176
} finally {
178177
LOGGER.info(MessageKeys.OPERATOR_SHUTTING_DOWN);
179178
}
@@ -286,6 +285,11 @@ private void completeBegin() {
286285
}
287286
}
288287

288+
private void completeStop() {
289+
stopRestServer();
290+
stopMetricsServer();
291+
}
292+
289293
NamespaceWatcher getNamespaceWatcher() {
290294
return namespaceWatcher;
291295
}

‎operator/src/main/java/oracle/kubernetes/operator/WebhookMain.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,7 @@ public static void main(String[] args) {
8484
// now we just wait until the pod is terminated
8585
main.waitForDeath();
8686

87-
main.stopRestServer();
88-
main.stopMetricsServer();
87+
main.stopDeployment(main::completeStop);
8988
} finally {
9089
LOGGER.info(MessageKeys.WEBHOOK_SHUTTING_DOWN);
9190
}
@@ -115,6 +114,11 @@ protected Step createStartupSteps() {
115114
WebhookHelper.createValidatingWebhookConfigurationStep(certs)));
116115
}
117116

117+
@Override
118+
protected Step createShutdownSteps() {
119+
return WebhookHelper.deleteValidatingWebhookConfigurationStep();
120+
}
121+
118122
private static Step createInitializeWebhookIdentityStep(WebhookMainDelegate delegate, Step next) {
119123
return new InitializeWebhookIdentityStep(delegate, next);
120124
}
@@ -139,6 +143,11 @@ void completeBegin() {
139143
}
140144
}
141145

146+
void completeStop() {
147+
stopRestServer();
148+
stopMetricsServer();
149+
}
150+
142151
Runnable recheckCrd() {
143152
return () -> delegate.runSteps(createCRDRecheckSteps());
144153
}

‎operator/src/main/java/oracle/kubernetes/operator/helpers/WebhookHelper.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
import io.kubernetes.client.openapi.models.AdmissionregistrationV1ServiceReference;
1616
import io.kubernetes.client.openapi.models.AdmissionregistrationV1WebhookClientConfig;
17+
import io.kubernetes.client.openapi.models.V1DeleteOptions;
1718
import io.kubernetes.client.openapi.models.V1ObjectMeta;
1819
import io.kubernetes.client.openapi.models.V1RuleWithOperations;
1920
import io.kubernetes.client.openapi.models.V1ValidatingWebhook;
@@ -23,6 +24,7 @@
2324
import oracle.kubernetes.operator.calls.UnrecoverableErrorBuilder;
2425
import oracle.kubernetes.operator.logging.LoggingFacade;
2526
import oracle.kubernetes.operator.logging.LoggingFactory;
27+
import oracle.kubernetes.operator.steps.DefaultResponseStep;
2628
import oracle.kubernetes.operator.utils.Certificates;
2729
import oracle.kubernetes.operator.work.NextAction;
2830
import oracle.kubernetes.operator.work.Packet;
@@ -339,4 +341,22 @@ protected NextAction onFailureNoRetry(Packet packet,
339341
}
340342
}
341343
}
344+
345+
public static Step deleteValidatingWebhookConfigurationStep() {
346+
return new DeleteValidatingWebhookConfigurationStep();
347+
}
348+
349+
private static class DeleteValidatingWebhookConfigurationStep extends Step {
350+
@Override
351+
public NextAction apply(Packet packet) {
352+
return doNext(createActionStep(), packet);
353+
}
354+
355+
private Step createActionStep() {
356+
V1DeleteOptions deleteOptions = new V1DeleteOptions();
357+
return new CallBuilder()
358+
.deleteValidatingWebhookConfigurationAsync(VALIDATING_WEBHOOK_NAME, deleteOptions,
359+
new DefaultResponseStep<>(getNext()));
360+
}
361+
}
342362
}

0 commit comments

Comments
 (0)