Skip to content

Commit 4df9615

Browse files
rjeberhardrjeberhard
committed
Merge remote-tracking branch 'origin/main' into release/4.0
2 parents bed40dc + cf9bb16 commit 4df9615

File tree

6 files changed

+126
-17
lines changed

6 files changed

+126
-17
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItHorizontalPodAutoscalerCustomMetrics.java

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,17 @@
88
import java.nio.file.Path;
99
import java.nio.file.Paths;
1010
import java.nio.file.StandardCopyOption;
11+
import java.time.OffsetDateTime;
1112
import java.util.HashMap;
1213
import java.util.List;
1314
import java.util.Map;
1415

1516
import io.kubernetes.client.custom.Quantity;
1617
import io.kubernetes.client.openapi.ApiException;
18+
import io.kubernetes.client.openapi.models.CoreV1Event;
1719
import io.kubernetes.client.openapi.models.V1LocalObjectReference;
1820
import io.kubernetes.client.openapi.models.V1ResourceRequirements;
21+
import io.kubernetes.client.util.Yaml;
1922
import oracle.weblogic.domain.ClusterResource;
2023
import oracle.weblogic.domain.DomainResource;
2124
import oracle.weblogic.domain.ServerPod;
@@ -56,11 +59,13 @@
5659
import static oracle.weblogic.kubernetes.actions.TestActions.deletePersistentVolumeClaim;
5760
import static oracle.weblogic.kubernetes.actions.TestActions.deletePod;
5861
import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
62+
import static oracle.weblogic.kubernetes.actions.TestActions.now;
5963
import static oracle.weblogic.kubernetes.actions.TestActions.uninstallNginx;
6064
import static oracle.weblogic.kubernetes.actions.impl.primitive.Kubernetes.deleteNamespace;
6165
import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterAndVerify;
6266
import static oracle.weblogic.kubernetes.utils.ClusterUtils.createClusterResource;
6367
import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.createDomainResource;
68+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkClusterReplicaCountMatches;
6469
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.checkPodReadyAndServiceExists;
6570
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
6671
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
@@ -69,6 +74,7 @@
6974
import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile;
7075
import static oracle.weblogic.kubernetes.utils.ImageUtils.createBaseRepoSecret;
7176
import static oracle.weblogic.kubernetes.utils.ImageUtils.createTestRepoSecret;
77+
import static oracle.weblogic.kubernetes.utils.K8sEvents.getEvents;
7278
import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.createIngressForDomainAndVerify;
7379
import static oracle.weblogic.kubernetes.utils.LoadBalancerUtils.installAndVerifyNginx;
7480
import static oracle.weblogic.kubernetes.utils.MonitoringUtils.cleanupPromGrafanaClusterRoles;
@@ -80,11 +86,12 @@
8086
import static oracle.weblogic.kubernetes.utils.MonitoringUtils.installMonitoringExporter;
8187
import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
8288
import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPvAndPvc;
83-
import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDeleted;
89+
import static oracle.weblogic.kubernetes.utils.PodUtils.isPodDeleted;
8490
import static oracle.weblogic.kubernetes.utils.SecretUtils.createSecretWithUsernamePassword;
8591
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
8692
import static org.assertj.core.api.Assertions.assertThat;
8793
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
94+
import static org.junit.jupiter.api.Assertions.assertEquals;
8895
import static org.junit.jupiter.api.Assertions.assertNotNull;
8996
import static org.junit.jupiter.api.Assertions.assertTrue;
9097

@@ -296,6 +303,7 @@ void testHPAWithCustomMetrics() {
296303
//reboot server1 and server2 to kill open sessions
297304
assertDoesNotThrow(() -> deletePod(managedServerPrefix + 1, domainNamespace));
298305
assertDoesNotThrow(() -> deletePod(managedServerPrefix + 2, domainNamespace));
306+
OffsetDateTime timestamp = now();
299307
// wait until reboot
300308
for (int i = 1; i < 3; i++) {
301309
checkPodReadyAndServiceExists(managedServerPrefix + i, domainUid, domainNamespace);
@@ -312,21 +320,42 @@ void testHPAWithCustomMetrics() {
312320
logger,
313321
"Checking if replica switched to 2");
314322

315-
try {
316-
checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
317-
} catch (Exception ex) {
323+
if (!isPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace)) {
324+
318325
//check if different server was scaled down
319-
try {
320-
if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 1)) {
326+
assertDoesNotThrow(() -> {
327+
logger.info("Checking if HPA scaled down managed server 1 or managed server 2");
328+
String command = KUBERNETES_CLI + " get pods -n" + domainNamespace;
329+
330+
logger.info("Executing command " + command);
331+
ExecResult result = ExecCommand.exec(command);
332+
logger.info(" Result output: " + result.stdout());
333+
command = KUBERNETES_CLI + " describe pod " + managedServerPrefix + 3 + " -n" + domainNamespace;
334+
335+
logger.info("Executing command " + command);
336+
result = ExecCommand.exec(command);
337+
logger.info(" Result output: " + result.stdout());
338+
List<CoreV1Event> events = getEvents(domainNamespace,timestamp);
339+
for (CoreV1Event event : events) {
340+
logger.info("Generated events after HPA scaling " + Yaml.dump(event));
341+
}
342+
int numberOfManagedSvs = 3;
343+
if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 1)
344+
|| Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 1)) {
321345
logger.info("HPA scaled down managed server 1");
322-
} else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 2)) {
346+
--numberOfManagedSvs;
347+
} else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 2)
348+
|| Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 2)) {
323349
logger.info("HPA scaled down managed server 2");
324-
} else {
325-
checkPodDeleted(managedServerPrefix + 3, domainUid, domainNamespace);
350+
--numberOfManagedSvs;
351+
} else if (!Kubernetes.doesPodExist(domainNamespace, domainUid, managedServerPrefix + 3)
352+
|| Kubernetes.isPodTerminating(domainNamespace, domainUid, managedServerPrefix + 2)) {
353+
logger.info("HPA scaled down managed server 3");
354+
--numberOfManagedSvs;
326355
}
327-
} catch (Exception ex1) {
328-
throw ex;
329-
}
356+
assertTrue(checkClusterReplicaCountMatches(clusterResName, domainNamespace, 2));
357+
assertEquals(2, numberOfManagedSvs);
358+
});
330359
}
331360
}
332361

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/CommonTestUtils.java

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes.utils;
@@ -189,6 +189,27 @@ public static void testUntil(ConditionFactory conditionFactory, Callable<Boolean
189189
}
190190
}
191191

192+
/**
193+
* Test assertion over time until it passes or the timeout expires.
194+
* @param conditionFactory Configuration for Awaitility condition factory
195+
* @param conditionEvaluator Condition evaluator
196+
* @param logger Logger
197+
* @param msg Message for logging
198+
* @param params Parameter to message for logging
199+
* @return false if timeout, true for success
200+
*/
201+
public static boolean testUntilNoException(ConditionFactory conditionFactory, Callable<Boolean> conditionEvaluator,
202+
LoggingFacade logger, String msg, Object... params) {
203+
try {
204+
conditionFactory
205+
.conditionEvaluationListener(createConditionEvaluationListener(logger, msg, params))
206+
.until(conditionEvaluator);
207+
return true;
208+
} catch (ConditionTimeoutException timeout) {
209+
return false;
210+
}
211+
}
212+
192213
private static <T> ConditionEvaluationListener<T> createConditionEvaluationListener(
193214
LoggingFacade logger, String msg, Object... params) {
194215
return new ConditionEvaluationListener<T>() {

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/K8sEvents.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,30 @@ public static List<CoreV1Event> getEvents(String domainNamespace,
509509
return events;
510510
}
511511

512+
/**
513+
* Get matching event objects after specific timestamp.
514+
* @param namespace namespace in which the event is logged
515+
* @param timestamp the timestamp after which to see events
516+
* @return CoreV1Event matching event object
517+
*/
518+
public static List<CoreV1Event> getEvents(String namespace,
519+
OffsetDateTime timestamp) {
520+
521+
List<CoreV1Event> events = new ArrayList<>();
522+
523+
try {
524+
List<CoreV1Event> allEvents = Kubernetes.listNamespacedEvents(namespace);
525+
for (CoreV1Event event : allEvents) {
526+
if ((isEqualOrAfter(timestamp, event))) {
527+
events.add(event);
528+
}
529+
}
530+
} catch (ApiException ex) {
531+
Logger.getLogger(K8sEvents.class.getName()).log(Level.SEVERE, null, ex);
532+
}
533+
return events;
534+
}
535+
512536
private static boolean isEqualOrAfter(OffsetDateTime timestamp, CoreV1Event event) {
513537
return event.getLastTimestamp().isEqual(timestamp)
514538
|| event.getLastTimestamp().isAfter(timestamp);

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/utils/PodUtils.java

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2021, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2021, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes.utils;
@@ -42,6 +42,7 @@
4242
import static oracle.weblogic.kubernetes.assertions.TestAssertions.podInitialized;
4343
import static oracle.weblogic.kubernetes.assertions.TestAssertions.podReady;
4444
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
45+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntilNoException;
4546
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withLongRetryPolicy;
4647
import static oracle.weblogic.kubernetes.utils.JobUtils.getIntrospectJobName;
4748
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
@@ -343,6 +344,25 @@ public static void checkPodDeleted(String podName, String domainUid, String domN
343344
domNamespace);
344345
}
345346

347+
/**
348+
* Check if the pods are deleted.
349+
* @param podName pod name
350+
* @param domainUid unique id of the domain
351+
* @param domNamespace namespace where domain exists
352+
* @return true if pod is deleted
353+
*/
354+
public static boolean isPodDeleted(String podName, String domainUid, String domNamespace) {
355+
final LoggingFacade logger = getLogger();
356+
return testUntilNoException(withLongRetryPolicy,
357+
assertDoesNotThrow(() -> podDoesNotExist(podName, domainUid, domNamespace),
358+
String.format("podDoesNotExist failed with ApiException for %s in namespace in %s",
359+
podName, domNamespace)),
360+
logger,
361+
"pod {0} to be deleted in namespace {1}",
362+
podName,
363+
domNamespace);
364+
}
365+
346366
public static String getExternalServicePodName(String adminServerPodName) {
347367
return getExternalServicePodName(adminServerPodName, TestConstants.DEFAULT_EXTERNAL_SERVICE_NAME_SUFFIX);
348368
}

‎operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,6 @@ private Step deleteIntrospectorJob() {
498498
}
499499

500500
class DeleteDomainIntrospectorJobStep extends Step {
501-
502501
@Override
503502
public NextAction apply(Packet packet) {
504503
logJobDeleted(getDomainUid(), getNamespace(), getJobName(), packet);
@@ -885,14 +884,18 @@ static void logJobDeleted(String domainUid, String namespace, String jobName, Pa
885884

886885
packet.remove(ProcessingConstants.INTROSPECTOR_JOB_FAILURE_LOGGED);
887886
if (domainIntrospectorJob != null
888-
&& !JobWatcher.isComplete(domainIntrospectorJob)) {
887+
&& hasStatusAndCondition(domainIntrospectorJob) && !JobWatcher.isComplete(domainIntrospectorJob)) {
889888
logIntrospectorFailure(packet, domainIntrospectorJob);
890889
}
891890
packet.remove(ProcessingConstants.JOB_POD_NAME);
892891

893892
LOGGER.fine(getJobDeletedMessageKey(), domainUid, namespace, jobName);
894893
}
895894

895+
private static boolean hasStatusAndCondition(V1Job job) {
896+
return job.getStatus() != null && job.getStatus().getConditions() != null;
897+
}
898+
896899
static String getJobDeletedMessageKey() {
897900
return MessageKeys.JOB_DELETED;
898901
}

‎operator/src/test/java/oracle/kubernetes/operator/helpers/DomainIntrospectorJobTest.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1317,13 +1317,25 @@ void whenIntrospectorJobNotNeeded_validateDomainAgainstPreviousTopology() throws
13171317
}
13181318

13191319
@Test
1320-
void whenJobLogContainsSevereError_logJobInfosOnDelete() {
1320+
void whenJobStatusContainsNoConditions_dontLogJobFailedAndInfosOnDelete() {
13211321
testSupport.defineResources(createIntrospectorJob());
13221322
IntrospectionTestUtils.defineIntrospectionPodLog(testSupport, SEVERE_MESSAGE);
13231323
testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
13241324

13251325
testSupport.runSteps(JobHelper.deleteDomainIntrospectorJobStep(null));
13261326

1327+
assertThat(logRecords, not(containsInfo(getJobFailedMessageKey())));
1328+
assertThat(logRecords, not(containsFine(getJobFailedDetailMessageKey())));
1329+
assertThat(logRecords, containsFine(getJobDeletedMessageKey()));
1330+
}
1331+
1332+
@Test
1333+
void whenJobStatusHasFailedCondition_logJobInfosOnDelete() {
1334+
testSupport.defineResources(asFailedJob(createIntrospectorJob()));
1335+
testSupport.addToPacket(DOMAIN_INTROSPECTOR_JOB, testSupport.getResourceWithName(JOB, getJobName()));
1336+
1337+
testSupport.runSteps(JobHelper.deleteDomainIntrospectorJobStep(null));
1338+
13271339
assertThat(logRecords, containsInfo(getJobFailedMessageKey()));
13281340
assertThat(logRecords, containsFine(getJobFailedDetailMessageKey()));
13291341
assertThat(logRecords, containsFine(getJobDeletedMessageKey()));

0 commit comments

Comments
 (0)