Merge pull request #716 from oracle/sit-cfg-fixes3

rjeberhard · web-flow · commit bfca5ba3a8bc · 2019-01-02T13:24:28.000-05:00
POST-2.0-rc1: Sit-cfg override fixes:
diff --git a/README.md b/README.md
@@ -63,7 +63,7 @@ please consult this table of contents:
   various tasks related to the operator.
 * The [Developer guide](site/developer.md) provides details for people who want to understand how the operator is built, tested, and so on. Those who wish to contribute to the operator code will find useful information here.  This section also includes
   API documentation (Javadoc) and Swagger/OpenAPI documentation for the REST APIs.
-* The [Contributing](#contributing-to-the-operator) section provides information about conribution requirements.
+* The [Contributing](#contributing-to-the-operator) section provides information about contribution requirements.
 
 
 # User guide
diff --git a/kubernetes/internal/utility.sh b/kubernetes/internal/utility.sh
@@ -184,15 +184,35 @@ function toDNS1123Legal {
 
 #
 # Function to check if a value is lowercase
-# $1 - value to check
-# $2 - name of object being checked
+# $1 - name of object being checked
+# $2 - value to check
 function validateLowerCase {
   local lcVal=$(toLower $2)
   if [ "$lcVal" != "$2" ]; then
     validationError "The value of $1 must be lowercase: $2"
   fi
 }
 
+# 
+# Function to lowercase a value and make it a legal DNS1123 name 
+# $1 - value to convert to DNS legal name
+function toDNS1123Legal { 
+  local val=`echo $1 | tr "[:upper:]" "[:lower:]"` 
+  val=${val//"_"/"-"} 
+  echo "$val" 
+}
+
+# 
+# Function to check if a value is lowercase and legal DNS name 
+# $1 - name of object being checked 
+# $2 - value to check 
+function validateDNS1123LegalName { 
+  local val=$(toDNS1123Legal $2) 
+  if [ "$val" != "$2" ]; then 
+    validationError "The value of $1 contains invalid charaters (uppercase letters or "_"): $2" 
+  fi 
+}
+
 #
 # Function to check if a value is lowercase and legal DNS name
 # $1 - value to check
diff --git a/kubernetes/src/test/java/oracle/kubernetes/operator/utils/DomainValues.java b/kubernetes/src/test/java/oracle/kubernetes/operator/utils/DomainValues.java
@@ -87,8 +87,8 @@ protected String convertNullToEmptyString(String val) {
   public DomainValues withTestDefaults() {
     return this.adminNodePort("30702")
         .adminPort("7002")
-        .adminServerName("TestAdminServer")
-        .clusterName("TestCluster")
+        .adminServerName("test-admin-server")
+        .clusterName("test-cluster")
         .clusterType(CLUSTER_TYPE_DYNAMIC)
         .domainName("TestDomain")
         .weblogicImage("store/oracle/weblogic:19.1.0.0")
@@ -97,7 +97,7 @@ public DomainValues withTestDefaults() {
         .loadBalancerDashboardPort("31315")
         .loadBalancerWebPort("31305")
         .configuredManagedServerCount("4")
-        .managedServerNameBase("TestManagedServer")
+        .managedServerNameBase("test-managed-server")
         .managedServerPort("8002")
         .initialManagedServerReplicas("3")
         .weblogicDomainStorageNFSServer("TestDomainStorageNFSServer")
diff --git a/operator/src/main/resources/scripts/introspectDomain.py b/operator/src/main/resources/scripts/introspectDomain.py
@@ -603,6 +603,14 @@ def addServerTemplates(self):
     self.undent()
 
   def addServerTemplate(self, serverTemplate):
+    # TBD This looks wrong.  It should match addClusteredServer.
+    #     It adds default port & listen address. But:
+    #       It's missing adding admin port, SSL, and NAPs.
+    #       It reports the wrong listen address (original rather than overridden).
+    #     Probably tests are passing fine, as it's likely (A)
+    #     the Operator.java is only reading the port, and (B) cluster tests
+    #     are only looking at the cluster default port.
+    #     So:  Follow up and fix this, and file test JIRAs for tests.
     name=self.name(serverTemplate)
     self.writeln("- name: " + name)
     if serverTemplate.isListenPortEnabled():
@@ -803,28 +811,34 @@ def customizeServers(self):
     for server in self.env.getDomain().getServers():
       self.customizeServer(server)
 
+  def writeListenAddress(self, originalValue, newValue):
+    repVerb="\"replace\""
+    if originalValue is None or len(originalValue)==0:
+      repVerb="\"add\""
+    self.writeln("<d:listen-address f:combine-mode=" + repVerb + ">" + newValue + "</d:listen-address>")
+
   def customizeServer(self, server):
     name=server.getName()
     listen_address=self.env.toDNS1123Legal(self.env.getDomainUID() + "-" + name)
     self.writeln("<d:server>")
     self.indent()
     self.writeln("<d:name>" + name + "</d:name>")
-    self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-    if server.getSSL():
-      self.writeln("<d:ssl>")
-      self.indent()
-      self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-      self.undent()
-      self.writeln("</d:ssl>")
+    self.customizeLog(name, server, false)
+    self.writeListenAddress(server.getListenAddress(),listen_address)
     for nap in server.getNetworkAccessPoints():
+      # Don't bother 'add' a nap listen-address, only do a 'replace'.
+      # If we try 'add' this appears to mess up an attempt to 
+      #   'add' PublicAddress/Port via custom sit-cfg.
+      # FWIW there's theoretically no need to 'add' or 'replace' when empty
+      #   since the runtime default is the server listen-address.
       nap_name=nap.getName()
-      self.writeln("<d:network-access-point>")
-      self.indent()
-      self.writeln("<d:name>" + nap_name + "</d:name>")
-      self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
-      self.undent()
-      self.writeln("</d:network-access-point>")
-    self.customizeLog(name, server, false)
+      if not (nap.getListenAddress() is None) and len(nap.getListenAddress()) > 0:
+        self.writeln("<d:network-access-point>")
+        self.indent()
+        self.writeln("<d:name>" + nap_name + "</d:name>")
+        self.writeListenAddress("force a replace",listen_address)
+        self.undent()
+        self.writeln("</d:network-access-point>")
     self.undent()
     self.writeln("</d:server>")
 
@@ -839,8 +853,8 @@ def customizeServerTemplate(self, template):
     self.writeln("<d:server-template>")
     self.indent()
     self.writeln("<d:name>" + name + "</d:name>")
-    self.writeln("<d:listen-address f:combine-mode=\"replace\">" + listen_address + "</d:listen-address>")
     self.customizeLog(server_name_prefix + "${id}", template, false)
+    self.writeListenAddress(template.getListenAddress(),listen_address)
     self.undent()
     self.writeln("</d:server-template>")
 
diff --git a/operator/src/main/resources/scripts/startNodeManager.sh b/operator/src/main/resources/scripts/startNodeManager.sh
@@ -111,6 +111,12 @@ createFolder ${NODEMGR_HOME}
 
 NODEMGR_LOG_HOME=${NODEMGR_LOG_HOME:-${LOG_HOME:-${NODEMGR_HOME}/${DOMAIN_UID}}}
 
+trace "Info: NODEMGR_HOME='${NODEMGR_HOME}'"
+trace "Info: LOG_HOME='${LOG_HOME}'"
+trace "Info: SERVER_NAME='${SERVER_NAME}'"
+trace "Info: DOMAIN_UID='${DOMAIN_UID}'"
+trace "Info: NODEMGR_LOG_HOME='${NODEMGR_LOG_HOME}'"
+
 createFolder ${NODEMGR_LOG_HOME}
 
 nodemgr_log_file=${NODEMGR_LOG_HOME}/${SERVER_NAME}_nodemanager.log
diff --git a/operator/src/test/java/oracle/kubernetes/operator/steps/ReadHealthStepTest.java b/operator/src/test/java/oracle/kubernetes/operator/steps/ReadHealthStepTest.java
@@ -20,6 +20,7 @@
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.http.HttpClient;
 import oracle.kubernetes.operator.steps.ReadHealthStep.ReadHealthWithHttpClientStep;
+import oracle.kubernetes.operator.work.Component;
 import oracle.kubernetes.operator.work.NextAction;
 import oracle.kubernetes.operator.work.Packet;
 import oracle.kubernetes.operator.work.Step;
@@ -79,6 +80,12 @@ PacketStub withServerName(String serverName) {
       return this;
     }
 
+    PacketStub addSpi(Class clazz, Object spiObject) {
+      Component component = Component.createFor(spiObject);
+      this.getComponents().put(clazz.getName(), component);
+      return this;
+    }
+
     @Override
     public Object get(Object key) {
       if (HttpClient.KEY.equals(key)) {
diff --git a/site/architecture.md b/site/architecture.md
@@ -7,7 +7,15 @@ The operator consists of the following two main parts:
 *	The operator itself, a process that runs in a Docker container deployed into a Kubernetes pod and which performs the actual management tasks.
 *	A Kubernetes job which can be used to create a WebLogic domain.
 
-The operator is packaged in a Docker image `container-registry.oracle.com/middleware/weblogic-operator:latest`.  This image can be deployed to a Kubernetes cluster.  It is recommended that the operator be deployed in its own namespace.  Only one operator is permitted in a namespace; however, multiple operators may be deployed in a Kubernetes cluster provided they are each in their own namespace and the list of namespaces they manage do not overlap.
+The operator is packaged in a [Docker image](https://hub.docker.com/r/oracle/weblogic-kubernetes-operator/) which you can access using the following `docker pull` command:  
+
+```
+
+docker pull oracle/weblogic-kubernetes-operator:1.1
+
+```
+
+This image can be deployed to a Kubernetes cluster.  It is recommended that the operator be deployed in its own namespace.  Only one operator is permitted in a namespace; however, multiple operators may be deployed in a Kubernetes cluster provided they are each in their own namespace and the list of namespaces they manage do not overlap.
 
 Scripts are provided to deploy the operator to a Kubernetes cluster.  These scripts also provide options to install and configure a load balancer and Elastic Stack integration.
 
diff --git a/site/helm-charts.md b/site/helm-charts.md
@@ -4,6 +4,15 @@
 
 The WebLogic Kubernetes Operator uses Helm to create and deploy any necessary resources and then run the operator in a Kubernetes cluster. Helm helps you manage Kubernetes applications. Helm charts help you define and install applications into the Kubernetes cluster. The operator's Helm chart is located in the `kubernetes/charts/weblogic-operator` directory.
 
+> If you have an old version of the operator installed on your cluster you must remove
+  it before installing this version.  You should remove the deployment (for example `kubectl delete deploy weblogic-operator -n your-namespace`) and the custom
+  resource definition (for example `kubectl delete crd domain`).  If you do not remove
+  the custom resource definition you may see errors like this: 
+  
+    `Error from server (BadRequest): error when creating "/scratch/output/uidomain/weblogic-domains/uidomain/domain.yaml": 
+    the API version in the data (weblogic.oracle/v2) does not match the expected API version (weblogic.oracle/v1`
+
+
 ## Install Helm and Tiller
 
 Helm has two parts: a client (helm) and a server (tiller). Tiller runs inside of your Kubernetes cluster, and manages releases (installations) of your charts.  See https://github.com/kubernetes/helm/blob/master/docs/install.md for detailed instructions on installing helm and tiller.
diff --git a/site/quickstart.md b/site/quickstart.md
@@ -4,6 +4,14 @@ Use this quick start guide to create a WebLogic deployment in a Kubernetes clust
 These instructions assume that you are already familiar with Kubernetes.  If you need more detailed instructions, please
 refer to the [User guide](user-guide.md).
 
+> If you have an old version of the operator installed on your cluster you must remove
+  it before installing this version.  You should remove the deployment (for example `kubectl delete deploy weblogic-operator -n your-namespace`) and the custom
+  resource definition (for example `kubectl delete crd domain`).  If you do not remove
+  the custom resource definition you may see errors like this: 
+  
+    `Error from server (BadRequest): error when creating "/scratch/output/uidomain/weblogic-domains/uidomain/domain.yaml": 
+    the API version in the data (weblogic.oracle/v2) does not match the expected API version (weblogic.oracle/v1`
+
 ## Prerequisites
 For this exercise, you’ll need a Kubernetes cluster. If you need help setting one up, check out our [cheat sheet](k8s_setup.md).
 
@@ -164,9 +172,14 @@ domain namespace (`sample-domain1-ns`) and the `domainHomeImageBase` (`oracle/we
 For example, assuming you named your copy `my-inputs.yaml`:
 ```
 $ cd kubernetes/samples/scripts/create-weblogic-domain/domain-home-in-image
-$ ./create-domain.sh -i my-inputs.yaml -o /some/output/directory -e -v
+$ ./create-domain.sh -i my-inputs.yaml -o /some/output/directory -u username -p password -e
 ```
 
+You need to provide the WebLogic administration username and password in the `-u` and `-p` options
+respectively, as shown in the example.  If you specify the `-e` option, the script will generate the 
+Kubernetes YAML files *and* apply them to your cluster.  If you omit the `-e` option, the 
+script will just generate the YAML files, but will not take any action on your cluster.
+
 c.	Confirm that the operator started the servers for the domain:
 * Use `kubectl` to show that the domain resource was created:
 ```
diff --git a/site/v1.0/creating-domain.md b/site/v1.0/creating-domain.md
@@ -109,7 +109,7 @@ The following parameters must be provided in the input file.
 | `exposeAdminNodePort` | Boolean indicating if the Administration Server is exposed outside of the Kubernetes cluster. | `false` |
 | `exposeAdminT3Channel` | Boolean indicating if the T3 administrative channel is exposed outside the Kubernetes cluster. | `false` |
 | `initialManagedServerReplicas` | Number of Managed Servers to initially start for the domain. | `2` |
-| `javaOptions` | Java options for starting the Administration and Managed Servers. | `-Dweblogic.StdoutDebugEnabled=false` |
+| `javaOptions` | Java options for starting the Administration and Managed Servers. A Java option can have references to one or more of the following pre-defined variables to obtain WebLogic domain information: $(DOMAIN_NAME), $(DOMAIN_HOME), $(ADMIN_NAME), $(ADMIN_PORT), and $(SERVER_NAME). | `-Dweblogic.StdoutDebugEnabled=false` |
 | `loadBalancer` | Type of load balancer to create.  Legal values are `NONE` and `TRAEFIK`. | `TRAEFIK` |
 | `loadBalancerDashboardPort` | Node port for the load balancer to accept dashboard traffic. | `30315` |
 | `loadBalancerWebPort` | Node port for the load balancer to accept user traffic. | `30305` |
diff --git a/site/v1.1/apache.md b/site/v1.1/apache.md
@@ -18,7 +18,7 @@ You need to build the Docker image for the Apache HTTP Server that embeds the Or
 
 ```
 
-For more information about the Apache plugin, see [Apache HTTP Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/12213/webtier/develop-plugin/apache.htm#PLGWL395).
+For more information about the Apache plugin, see [Configuring the Plug-In for Apache HTTP Server](https://docs.oracle.com/middleware/12213/webtier/develop-plugin/apache.htm#PLGWL395).
 
 After you have access to the Docker image of the Apache HTTP Server, you can follow the instructions below to set up and start the Kubernetes resources for the Apache HTTP Server.
 
@@ -250,7 +250,7 @@ You can fine tune the behavior of the Apache plugin by providing your own Apache
 loadBalancerVolumePath: <host-config-dir>
 ```
 
-After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the `custom_mod_wl_apache.conf` file in `<host-config-dir>` directory to replace what is in the Docker image.
+After the `loadBalancerVolumePath` property is specified, the `create-weblogic-domain.sh` script will use the `custom_mod_wl_apache.conf` file in the `<host-config-dir>` directory to replace what is in the Docker image.
 
 The generated YAML files will look similar except with un-commented entries like below:
 
@@ -270,7 +270,7 @@ The generated YAML files will look similar except with un-commented entries like
 
 ## Use the Apache load balancer with a manually created WebLogic Domain
 
-If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes' resources for the Apache HTTP Server.
+If your WebLogic domain is not created by the WebLogic Operator, you need to manually create and start all Kubernetes resources for the Apache HTTP Server.
 
   1. Create your own `custom_mod_wl_apache.conf` file, and put it in a local directory, for example, `<host-conf-dir>`. See the instructions in [Apache Web Server with Oracle WebLogic Server Proxy Plugin on Docker](https://docs.oracle.com/middleware/1213/webtier/develop-plugin/apache.htm#PLGWL395).
 
diff --git a/src/integration-tests/introspector/checkBeans.inputt b/src/integration-tests/introspector/checkBeans.inputt
@@ -1,4 +1,7 @@
+/Log/base_domain,FileName,logs/base_domain.log,/shared/logs/base_domain.log
+
 /Servers/${ADMIN_NAME},ListenAddress,,${DOMAIN_UID}-${ADMIN_NAME}
+
 /Servers/${ADMIN_NAME}/Log/${ADMIN_NAME},FileName,logs/${ADMIN_NAME}.log,/shared/logs/${ADMIN_NAME}.log
 
 /Servers/${ADMIN_NAME}/NetworkAccessPoints/T3Channel1,ListenAddress,unresolvable-dns-name,${DOMAIN_UID}-${ADMIN_NAME}
@@ -17,6 +20,9 @@
 /Servers/standalone1,MaxMessageSize,7777777,1111111
 /Servers/standalone2,MaxMessageSize,10000000,2222222
 
+/ServerTemplates/mycluster-template/Log/mycluster-template,FileName,logs/mycluster-template.log,/shared/logs/managed-server${id}.log
+/ServerTemplates/mycluster-template,ListenAddress,None,domain1-managed-server${id}
+
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS,Url,jdbc:oracle:thin:@myoriginalhostname:1521:myoriginalsid,jdbc:oracle:thin:@mynewhost:1521:mynewsid
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS,PasswordEncrypted,*,!
 /JDBCSystemResources/testDS/JDBCResource/testDS/JDBCDriverParams/testDS/Properties/testDS/Properties/user,Value,scott,supersecret
diff --git a/src/integration-tests/introspector/introspectTest.sh b/src/integration-tests/introspector/introspectTest.sh
@@ -179,14 +179,20 @@ function cleanupMajor() {
 function cleanupMinor() {
   trace "Info: RERUN_INTROSPECT_ONLY==true, skipping cleanup.sh and domain home setup, and only deleting wl pods + introspector job."
 
-  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${ADMIN_NAME} > /dev/null 2>&1
-  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${MANAGED_SERVER_NAME_BASE}1 > /dev/null 2>&1
-  kubectl -n $NAMESPACE delete job ${DOMAIN_UID}-introspect-domain-job > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${ADMIN_NAME}                --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}-${MANAGED_SERVER_NAME_BASE}1 --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete job ${DOMAIN_UID}-introspect-domain-job        --grace-period=2 > /dev/null 2>&1
+  kubectl -n $NAMESPACE delete pod ${DOMAIN_UID}--introspect-domain-pod       --grace-period=2 > /dev/null 2>&1
   rm -fr ${test_home}/jobfiles
   tracen "Info: Waiting for wl pods to completely go away before continuing."
   while [ 1 -eq 1 ]; do
     echo -n "."
-    [ "`kubectl -n ${NAMESPACE} get pods | grep '${DOMAIN_UID}.*server'`" = "" ] && break
+    # echo
+    # echo "Waiting for: '`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*server\"`'"
+    # echo "Waiting for: '`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*introspect\"`'"
+    [ "`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*server\"`" = "" ] \
+     && [ "`kubectl -n ${NAMESPACE} get pods | grep \"${DOMAIN_UID}.*introspect\"`" = "" ] \
+     && break
     sleep 1
   done
   echo
@@ -339,7 +345,7 @@ function deployTestScriptConfigMap() {
   cp ${SCRIPTPATH}/introspectDomainProxy.sh ${test_home}/test-scripts || exit 1
 
   if [ "$CREATE_DOMAIN" = "true" ]; then
-    rm -f ${test_home}/scripts/createDomain.py
+    rm -f ${test_home}/test-scripts/createDomain.py
     ${SCRIPTPATH}/util_subst.sh -g createDomain.pyt ${test_home}/test-scripts/createDomain.py || exit 1
   fi
   
@@ -498,19 +504,15 @@ function deployIntrospectJobPod() {
 
   trace "Info: Run introspection job, parse its output to files, and put files in configmap '$introspect_output_cm_name'."
 
-  # delete anything left over from a previous invocation of this function
+  # delete anything left over from a previous invocation of this function, assume all pods
+  # have already been cleaned up
+
+  rm -f ${target_yaml}
 
   kubectl -n $NAMESPACE delete cm $introspect_output_cm_name \
     --ignore-not-found  \
     2>&1 | tracePipe "Info: kubectl output: "
 
-  if [ -f "${target_yaml}" ]; then
-    kubectl -n $NAMESPACE delete -f ${target_yaml} \
-      --ignore-not-found \
-      2>&1 | tracePipe "Info: kubectl output: "
-    rm -f ${target_yaml}
-  fi
-
   trace "Info: Deploying job pod '$pod_name' and waiting for it to be ready."
 
   (
@@ -755,11 +757,13 @@ function checkOverrides() {
 #     matches
 #
 
+
 kubectl -n $NAMESPACE delete secret my-secret > /dev/null 2>&1
 kubectl -n $NAMESPACE create secret generic my-secret \
         --from-literal=key1=supersecret  \
         --from-literal=key2=topsecret 2>&1 | tracePipe "Info: kubectl output: "
 
+
 if [ ! "$RERUN_INTROSPECT_ONLY" = "true" ]; then
 
   cleanupMajor
@@ -785,9 +789,13 @@ else
 
   cleanupMinor
 
+  deployDomainConfigMap
+  deployTestScriptConfigMap
   deployCustomOverridesConfigMap
-  deployIntrospectJob
+  #deployIntrospectJob
+  deployIntrospectJobPod
   deployPod ${ADMIN_NAME?}
+  deployPod ${MANAGED_SERVER_NAME_BASE?}1
 
 fi
 
diff --git a/src/integration-tests/introspector/util_subst.sh b/src/integration-tests/introspector/util_subst.sh
