Merge branch 'develop'

Author: Steve Salas

codedx/sample-values/values-external-db.yaml

@@ -4,9 +4,9 @@
 # Here are the steps required to use Code Dx with an external database:
 #
 # 1) Create a database user for Code Dx. You can customize the following statement to create
-#    a Code Dx database user named codedx.
+#    a Code Dx database user named codedx (remove REQUIRE SSL if SSL/TLS is not required).
 #
-#    CREATE USER 'codedx'@'%' IDENTIFIED BY 'enter-a-password-here';
+#    CREATE USER 'codedx'@'%' IDENTIFIED BY 'enter-a-password-here' REQUIRE SSL;
 #
 # 2) Apply any database configuration changes necessary to allow remote database connections. 
 #
@@ -20,16 +20,25 @@
 #    GRANT SELECT, INSERT, UPDATE, DELETE,  CREATE, ALTER, REFERENCES, INDEX, DROP ON codedxdb.* to 'codedx'@'%';
 #    FLUSH PRIVILEGES;
 #
-# 5) Set the optimizer_search_depth database variable to 0 with the below configuration. Failure to
-#    complete this step will negatively affect Code Dx performance.
+# 5) Set the optimizer_search_depth database variable to 0, the character set to utf8mb4, and the collation to
+#    utf8mb4_general_ci with the below configuration. Failure to complete this step will negatively affect Code Dx
+#    performance or functionality.
 #
 #    [mysqld]
 #    optimizer_search_depth=0
+#    character-set-server=utf8mb4
+#    collation-server=utf8mb4_general_ci
+#
+#    Also, make sure that sql_mode contains neither ONLY_FULL_GROUP_BY nor PAD_CHAR_TO_FULL_LENGTH.
 #
 # 6) When you install Code Dx, specify existingSecret (values.yaml) with fields
 #    codedx-mariadb-username and codedx.mariadb.password for the username and 
 #    password that Code Dx will use to access the Code Dx database.
 #
+#    If you're using One-Way TLS for MariaDB clients, do not set externalDatabaseSkipTls and specify 
+#    a path to the MariaDB server CA as the externalDatabaseServerCert parameter when running the 
+#    setup script.
+#
 # 7) Edit the externalDbUrl connection string by replacing host.docker.internal with your database
 #    hostname and codedxdb with the name of the Code Dx database you created.
 #

codedx/values.yaml

@@ -573,15 +573,15 @@ mariadb:
       bind-address=0.0.0.0
       pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid
       log-error=/opt/bitnami/mariadb/logs/mysqld.log
-      character-set-server=UTF8
+      character-set-server=utf8mb4
       collation-server=utf8_general_ci
       optimizer_search_depth=0
       innodb_flush_log_at_trx_commit=0
 
       [client]
       port=3306
       socket=/opt/bitnami/mariadb/tmp/mysql.sock
-      default-character-set=UTF8
+      default-character-set=utf8mb4
 
       [manager]
       port=3306
@@ -607,15 +607,15 @@ mariadb:
       bind-address=0.0.0.0
       pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid
       log-error=/opt/bitnami/mariadb/logs/mysqld.log
-      character-set-server=UTF8
+      character-set-server=utf8mb4
       collation-server=utf8_general_ci
       optimizer_search_depth=0
       innodb_flush_log_at_trx_commit=0
 
       [client]
       port=3306
       socket=/opt/bitnami/mariadb/tmp/mysql.sock
-      default-character-set=UTF8
+      default-character-set=utf8mb4
 
       [manager]
       port=3306

setup/aws/setup.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.0
+.VERSION 1.0.1
 .GUID 7324446b-ac6b-4870-846d-bef7547de642
 .AUTHOR Code Dx
 #>
@@ -36,5 +36,3 @@ $provisionNetworkPolicy = {
 & (join-path $PSScriptRoot '../setup.ps1') `
   -storageClassName $storageClassName `
   -provisionNetworkPolicy $provisionNetworkPolicy @args
-
-Write-Verbose 'Deployment complete!'

setup/azure/setup.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.1
+.VERSION 1.0.2
 .GUID 01d0c54b-ce7a-4462-b4cd-fb27a4f847bc
 .AUTHOR Code Dx
 #>
@@ -36,5 +36,3 @@ if ($nginxIngressControllerLoadBalancerIP -eq '') {
   -storageClassName $storageClassName `
   -nginxIngressControllerLoadBalancerIP $nginxIngressControllerLoadBalancerIP `
   @args
-
-Write-Verbose 'Deployment complete!'

setup/common/codedx.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.2
+.VERSION 1.0.3
 .GUID 6b1307f7-7098-4c65-9a86-8478840ad4cd
 .AUTHOR Code Dx
 #>
@@ -64,6 +64,7 @@ function New-CodeDxDeployment([string] $codeDxDnsName,
 	[string[]] $serviceAnnotationsCodeDx,
 	[string]   $ingressControllerNamespace,
 	[string[]] $ingressAnnotations,
+	[string]   $caCertsFilename,
 	[string]   $caCertsFilePwd,
 	[string]   $externalDbUrl,
 	[string]   $externalDbUser,
@@ -73,7 +74,8 @@ function New-CodeDxDeployment([string] $codeDxDnsName,
 	[switch]   $enablePSPs,
 	[switch]   $enableNetworkPolicies,
 	[switch]   $configureTls,
-	[switch]   $skipDatabase) {
+	[switch]   $skipDatabase,
+	[switch]   $offlineMode) {
 
 	if (-not (Test-Namespace $namespace)) {
 		New-Namespace  $namespace
@@ -197,14 +199,14 @@ mariadb:
       backup:
         size: {14}Gi
 {22}
-cacertsFile: ''
+cacertsFile: '{30}'
 cacertsFilePwd: '{21}'
 codedxProps:
   internalExtra:
   - type: values
     key: codedx-offline-props
     values:
-    - "codedx.offline-mode = true"
+    - "codedx.offline-mode = {31}"
 {29}
 '@ -f (Get-CodeDxPdSecretName $releaseName), $tomcatImage, $imagePullSecretYaml, `
 $psp, $networkPolicy, `
@@ -220,7 +222,7 @@ $defaultKeyStorePwd, `
 $codeDxTomcatPortNumber, $codeDxTlsTomcatPortNumber, `
 $serviceTypeCodeDx, (ConvertTo-YamlMap $serviceAnnotationsCodeDx), `
 $enableDb, $ingressNginxAssumption, `
-$externalDb
+$externalDb, $caCertsFilename, $offlineMode.ToString().ToLower()
 
 	$valuesFile = 'codedx-values.yaml'
 	$values | out-file $valuesFile -Encoding ascii -Force
@@ -428,54 +430,32 @@ function Get-RunningCodeDxPodName([string] $codedxNamespace) {
 	$name
 }
 
-function Set-TrustedCerts([string] $workDir, 
+function Get-RunningCodeDxKeystore([string] $codedxNamespace, [string] $outPath) {
+
+	$podName = Get-RunningCodeDxPodName $codedxNamespace
+	$podFile = "$podName`:/usr/local/openjdk-8/jre/lib/security/cacerts"
+
+	kubectl -n $codedxNamespace cp $podFile $outPath
+	if ($LASTEXITCODE -ne 0) {
+		throw "Unable to copy out cacerts file from '$podFile', kubectl exited with code $LASTEXITCODE."
+	}
+}
+
+function Set-TrustedCerts([string] $workDir,
 	[string]   $waitSeconds,
 	[string]   $codedxNamespace,
 	[string]   $codedxReleaseName,
 	[string[]] $extraValuesPaths,
 	[string]   $adminPwd,
-	[string]   $caCertsFilePwd,
-	[string]   $caCertsFileNewPwd,
+	[string]   $keystorePwd,
 	[string]   $externalDbUser,
 	[string]   $externalDbPwd,
-	[string[]] $trustedCertPaths,
 	[switch]   $offlineMode) {
 
-	$caCertsFilePath = './cacerts'
-	if (test-path $caCertsFilePath) {
-		remove-item $caCertsFilePath -force
-	}
-
-	$chartFolder = (join-path $workDir codedx-kubernetes/codedx)
-	$chartFolderCaCertsFilePath = join-path $chartFolder $caCertsFilePath
-
-	# if cacerts already exists in the chart folder via -extraCodeDxChartFilesPaths, use 
-	# that copy; otherwise, pull a copy from the running Code Dx pod
-	if (test-path $chartFolderCaCertsFilePath) {
-		copy-item $chartFolderCaCertsFilePath $caCertsFilePath
-	} else {
-		$podName = Get-RunningCodeDxPodName $codedxNamespace
-		$podFile = "$podName`:/usr/local/openjdk-8/jre/lib/security/cacerts"
-
-		kubectl -n $codedxNamespace cp $podFile $caCertsFilePath
-		if ($LASTEXITCODE -ne 0) {
-			throw "Unable to copy out cacerts file, kubectl exited with code $LASTEXITCODE."
-		}
-	}
-
-	# set cacerts password
-	$keystorePwd = $caCertsFilePwd
-	if ('' -ne $caCertsFileNewPwd -and $caCertsFilePwd -ne $caCertsFileNewPwd) {
-		$keystorePwd = $caCertsFileNewPwd
-	}
-	Set-KeystorePassword $caCertsFilePath $caCertsFilePwd $keystorePwd 
 	New-CodeDxPdSecret $codedxNamespace $codedxReleaseName $adminPwd $keystorePwd $externalDbUser $externalDbPwd
-
-	Import-TrustedCaCerts $caCertsFilePath $keystorePwd $trustedCertPaths
-
-	# move edited cacerts file to chart directory where it can be found during chart install
-	copy-item $caCertsFilePath $chartFolder -Force
-
+	
+	$chartFolder = (join-path $workDir codedx-kubernetes/codedx)
+	
 	$values = @'
 cacertsFile: cacerts
 codedxProps:
@@ -752,3 +732,30 @@ function Get-CommonName([string] $name) {
 	}
 	$name.TrimEnd('-')
 }
+
+function Get-TrustedCaCertsFilePwd([string] $currentPwd, [string] $newPwd) {
+
+	$pwd = $currentPwd
+	if ('' -ne $newPwd -and $pwd -ne $newPwd) {
+		$pwd = $newPwd
+	}
+	$pwd
+}
+
+function New-TrustedCaCertsFile([string] $basePath,
+	[string]   $currentPwd, [string] $newPwd,
+	[string[]] $certPathsToImport,
+	[string]   $destinationDirectory) {
+
+	$filePath = "./cacerts"
+	if (Test-Path $filePath) {
+		Remove-Item $filePath -force
+	}
+	Copy-Item $basePath $filePath
+
+	$pwd = (Get-TrustedCaCertsFilePwd $currentPwd $newPwd)
+	Set-KeystorePassword $filePath $currentPwd $pwd
+
+	Import-TrustedCaCerts $filePath $pwd $certPathsToImport
+	Copy-Item $filePath $destinationDirectory -Force
+}

setup/common/k8s.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.1
+.VERSION 1.0.2
 .GUID 5614d5a5-d33b-4a86-a7bb-ccc91c3f9bb3
 .AUTHOR Code Dx
 #>
@@ -376,7 +376,8 @@ function Wait-ReplicasReady([string] $message, [int] $waitSeconds, [string] $nam
 	}
 
 	$sleepSeconds = [math]::min(60, ($waitSeconds * .05))
-	$timeoutTime = [datetime]::Now.AddSeconds($waitSeconds)
+	$startTime = [datetime]::Now
+	$timeoutTime = $startTime.AddSeconds($waitSeconds)
 	while ($true) {
 
 		$resourceExists = ($resourceType -eq 'deployment' -and (Test-Deployment $namespace $resourceName)) -or 
@@ -408,7 +409,14 @@ function Wait-ReplicasReady([string] $message, [int] $waitSeconds, [string] $nam
 			throw "Unable to continue because the $resourceType $resourceName is not ready ($message)"
 		}
 
-		Write-Verbose "Current replica count does not yet match desired count. Another check will occur in $sleepSeconds seconds ($message)."
+		kubectl -n $namespace get pod
+		
+		$now = [datetime]::Now
+		Write-Verbose $message
+		Write-Verbose "  Current replica count does not yet match desired count."
+		Write-Verbose "    Elapsed time is $($now.Subtract($startTime).TotalSeconds) seconds."
+		Write-Verbose "    Wait will timeout in $($timeoutTime.Subtract($now).TotalSeconds) seconds."
+		Write-Verbose "    Another replica count check will occur in $sleepSeconds seconds."
 		start-sleep -seconds $sleepSeconds
 	}
 }

setup/common/keytool.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.0
+.VERSION 1.0.1
 .GUID a0b1e49c-0f56-43fa-bd1d-ae211ac63c2a
 .AUTHOR Code Dx
 #>

setup/common/mariadb.ps1

@@ -1,5 +1,5 @@
 <#PSScriptInfo
-.VERSION 1.0.1
+.VERSION 1.0.2
 .GUID d7edc525-a26e-4f80-b65b-262a0e56422e
 .AUTHOR Code Dx
 #>
@@ -137,3 +137,25 @@ function Start-SlaveDB([string] $namespace,
 		Write-Error "Unable to start DB slave, kubectl exited with exit code $LASTEXITCODE."
 	}
 }
+
+function Get-DatabaseUrl([string] $databaseHost, [int] $databasePort,
+	[string] $databaseName,
+	[string] $databaseCertPath,
+	[switch] $databaseSkipTls) {
+
+	$url = "jdbc:mysql://$databaseHost"
+	if ($databasePort -ne 3306) {
+		$url = "$url`:$databasePort"
+	}
+
+	$url = "$url/$databaseName"
+
+	if (-not $databaseSkipTls) {
+
+		if ($databaseCertPath -eq '' -or -not (Test-Path $databaseCertPath -PathType Leaf)) {
+			throw "Using a One-Way SSL/TLS configuration requires a server certificate"
+		}
+		$url = "$url`?useSSL=true&requireSSL=true"
+	}
+	$url
+}