Sdn 1185 new

contrib/kind.sh

@@ -100,6 +100,7 @@ usage() {
     echo "                 [-ric | --run-in-container |"
     echo "                 [-cn | --cluster-name |"
     echo "                 [-ehp|--egress-ip-healthcheck-port <num>]"
+    echo "                 [-is | --ipsec]"
     echo "                 [-h]]"
     echo ""
     echo "-cf  | --config-file                Name of the KIND J2 configuration file."
@@ -143,6 +144,7 @@ usage() {
     echo "-cn  | --cluster-name               Configure the kind cluster's name"
     echo "-ric | --run-in-container           Configure the script to be run from a docker container, allowing it to still communicate with the kind controlplane" 
     echo "-ehp | --egress-ip-healthcheck-port TCP port used for gRPC session by egress IP node check. DEFAULT: 9107 (Use "0" for legacy dial to port 9)."
+    echo "-is  | --ipsec                      Enable IPsec encryption (spawns ovn-ipsec pods)"
     echo "--delete                      	    Delete current cluster"
     echo ""
 }
@@ -199,6 +201,8 @@ parse_args() {
                                                 ;;
             -i6 | --ipv6 )                      KIND_IPV6_SUPPORT=true
                                                 ;;
+            -is | --ipsec )                     ENABLE_IPSEC=true
+                                                ;;
             -wk | --num-workers )               shift
                                                 if ! [[ "$1" =~ ^[0-9]+$ ]]; then
                                                     echo "Invalid num-workers: $1"
@@ -309,6 +313,7 @@ print_params() {
      echo "KIND_REMOVE_TAINT = $KIND_REMOVE_TAINT"
      echo "KIND_IPV4_SUPPORT = $KIND_IPV4_SUPPORT"
      echo "KIND_IPV6_SUPPORT = $KIND_IPV6_SUPPORT"
+     echo "ENABLE_IPSEC = $ENABLE_IPSEC"
      echo "KIND_NUM_WORKER = $KIND_NUM_WORKER"
      echo "KIND_ALLOW_SYSTEM_WRITES = $KIND_ALLOW_SYSTEM_WRITES"
      echo "KIND_EXPERIMENTAL_PROVIDER = $KIND_EXPERIMENTAL_PROVIDER"
@@ -373,6 +378,11 @@ check_dependencies() {
     exit 1
   fi
 
+  if ! command_exists awk ; then
+    echo "Dependency not met: Command not found 'awk'"
+    exit 1
+  fi
+
   if ! command_exists j2 ; then
     if ! command_exists pip ; then
       echo "Dependency not met: 'j2' not installed and cannot install with 'pip'"
@@ -388,6 +398,24 @@ check_dependencies() {
   fi
 }
 
+OPENSSL=""
+set_openssl_binary() {
+  for s in openssl openssl3; do
+      if ! command_exists "${s}" ; then
+          continue
+      fi
+      if [ "$(${s} version | awk -F '[ |.]' '{print $2}')" == "3" ]; then
+          OPENSSL="${s}"
+          echo "Found OpenSSL version 3 in binary ${OPENSSL}"
+          break
+      fi
+  done
+  if [ "${OPENSSL}" == "" ] ; then
+    echo "Dependency not met: Cannot find openssl version 3 (searched for openssl and openssl3)"
+    exit 1
+  fi
+}
+
 set_default_params() {
   # Set default values
   # Used for multi cluster setups
@@ -412,6 +440,7 @@ set_default_params() {
   KIND_REMOVE_TAINT=${KIND_REMOVE_TAINT:-true}
   KIND_IPV4_SUPPORT=${KIND_IPV4_SUPPORT:-true}
   KIND_IPV6_SUPPORT=${KIND_IPV6_SUPPORT:-false}
+  ENABLE_IPSEC=${ENABLE_IPSEC:-false}
   OVN_HYBRID_OVERLAY_ENABLE=${OVN_HYBRID_OVERLAY_ENABLE:-false}
   OVN_DISABLE_SNAT_MULTIPLE_GWS=${OVN_DISABLE_SNAT_MULTIPLE_GWS:-false}
   OVN_DISABLE_PKT_MTU_CHECK=${OVN_DISABLE_PKT_MTU_CHECK:-false}
@@ -633,6 +662,7 @@ create_ovn_kube_manifests() {
     --net-cidr="${NET_CIDR}" \
     --svc-cidr="${SVC_CIDR}" \
     --gateway-mode="${OVN_GATEWAY_MODE}" \
+    --enable-ipsec="${ENABLE_IPSEC}" \
     --hybrid-enabled="${OVN_HYBRID_OVERLAY_ENABLE}" \
     --disable-snat-multiple-gws="${OVN_DISABLE_SNAT_MULTIPLE_GWS}" \
     --disable-pkt-mtu-check="${OVN_DISABLE_PKT_MTU_CHECK}" \
@@ -712,40 +742,97 @@ install_ingress() {
   run_kubectl apply -f ingress/service-nodeport.yaml
 }
 
+# kubectl_wait_pods will set a total timeout of 300s for IPv4 and 480s for IPv6. It will first wait for all
+# DaemonSets to complete with kubectl rollout. This command will block until all pods of the DS are actually up.
+# Next, it iterates over all pods with name=ovnkube-db and ovnkube-master and waits for them to post "Ready".
+# Last, it will do the same with all pods in the kube-system namespace.
 kubectl_wait_pods() {
-  echo "Waiting for k8s to create ovn-kubernetes pod resources..."
-  local PODS_CREATED=false
-  for i in {1..10}; do
-    local NUM_PODS=$(kubectl -n ovn-kubernetes get pods -o json 2> /dev/null | jq '.items | length')
-    if [[ "${NUM_PODS}" -ne 0 ]]; then
-      echo "ovn-kubernetes pods created."
-      PODS_CREATED=true
-      break
-    fi
-    sleep 1
+  # IPv6 cluster seems to take a little longer to come up, so extend the wait time.
+  OVN_TIMEOUT=300
+  if [ "$KIND_IPV6_SUPPORT" == true ]; then
+    OVN_TIMEOUT=480
+  fi
+
+  # We will make sure that we timeout all commands at current seconds + the desired timeout.
+  endtime=$(( SECONDS + OVN_TIMEOUT ))
+
+  for ds in ovnkube-node ovs-node; do
+    timeout=$(calculate_timeout ${endtime})
+    echo "Waiting for k8s to launch all ${ds} pods (timeout ${timeout})..."
+    kubectl rollout status daemonset -n ovn-kubernetes ${ds} --timeout ${timeout}s
+  done
+  for name in ovnkube-db ovnkube-master; do
+    timeout=$(calculate_timeout ${endtime})
+    echo "Waiting for k8s to create ${name} pods (timeout ${timeout})..."
+    kubectl wait pods -n ovn-kubernetes -l name=${name} --for condition=Ready --timeout=${timeout}s
   done
 
-  if [[ "$PODS_CREATED" == false ]]; then
-    echo "ovn-kubernetes pods were not created."
+  timeout=$(calculate_timeout ${endtime})
+  if ! kubectl wait -n kube-system --for=condition=ready pods --all --timeout=${timeout}s ; then
+    echo "some pods in the system are not running"
+    kubectl get pods -A -o wide || true
     exit 1
   fi
-  echo "ovn-kubernetes pods created."
+}
 
-  # Check that everything is fine and running. IPv6 cluster seems to take a little
-  # longer to come up, so extend the wait time.
-  OVN_TIMEOUT=300s
-  if [ "$KIND_IPV6_SUPPORT" == true ]; then
-    OVN_TIMEOUT=480s
-  fi
-  if ! kubectl wait -n ovn-kubernetes --for=condition=ready pods --all --timeout=${OVN_TIMEOUT} ; then
-    echo "some pods in OVN Kubernetes are not running"
-    kubectl get pods -A -o wide || true
-    exit 1
+# calculate_timeout takes an absolute endtime in seconds (based on bash script runtime, see
+# variable $SECONDS) and calculates a relative timeout value. Should the calculated timeout
+# be <= 0, return one second.
+calculate_timeout() {
+  endtime=$1
+  timeout=$(( endtime - SECONDS ))
+  if [ ${timeout} -le 0 ]; then
+      timeout=1
   fi
-  if ! kubectl wait -n kube-system --for=condition=ready pods --all --timeout=300s ; then
-    echo "some pods in the system are not running"
-    kubectl get pods -A -o wide || true
-    exit 1
+  echo ${timeout}
+}
+
+# install_ipsec will apply the IPsec DaemonSet, create a CA that can be used by the IPsec pods. It will then add it to
+# configmap -n ovn-kubernetes signer-ca. After that, it will monitor all CSRs that are pending and it will sign those
+# with the CA cert. After each iteration, it will check if the ovn-ipsec DaemonSet pods rolled out successfully.
+# Make sure to run this at the very end of the setup process.
+install_ipsec() {
+  pushd "${MANIFEST_OUTPUT_DIR}"
+  run_kubectl apply -f ovn-ipsec.yaml
+  popd
+
+  # Create the CA (stored inside the signer-ca ConfigMap) that the IPsec pods use to sign their certificates
+  ca_dir=$(mktemp -d)
+  pushd "${ca_dir}"
+  ${OPENSSL} genrsa -out ca-bundle.key 4096
+  ${OPENSSL} req -x509 -new -nodes -key ca-bundle.key -sha256 -days 10240 -out ca-bundle.crt \
+      -subj "/C=CA/ST=Arctica/L=Northpole/O=Acme Inc/OU=DevOps/CN=www.example.com/[email protected]"
+  kubectl create configmap -n ovn-kubernetes signer-ca --from-file ca-bundle.crt
+
+  # For ca. 5 minutes max (60 * 5 seconds + overhead) ...
+  success=false
+  for i in {1..60}; do
+    # ... try to get all CSRs and sign them
+    csrs=$(oc get csr -o go-template='{{range .items}}{{if not .status}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')
+    for csr in ${csrs}; do
+      kubectl get csr "${csr}" -o jsonpath='{.spec.request}' | base64 --decode | \
+          sed -n '/BEGIN CERTIFICATE REQUEST/,$p' > "${csr}"
+      ${OPENSSL} x509 -req -in ${csr} -CA ca-bundle.crt -CAkey ca-bundle.key -CAcreateserial -out "${csr}.crt" -days 3650  \
+          -sha256 -extensions v3_req -copy_extensions copy
+      kubectl get csr "${csr}" -o json | \
+          jq '.status.certificate = "'$(base64 "${csr}.crt" | tr -d '\n')'"' | \
+          kubectl replace --raw /apis/certificates.k8s.io/v1/certificatesigningrequests/${csr}/status -f -
+    done
+
+    # ... and then check if the ovn-ipsec DaemonSet rolled out completely (wait for 5 seconds)
+    if kubectl rollout status daemonset -n ovn-kubernetes ovn-ipsec --timeout 5s; then
+        echo "All IPsec pods rolled out successfully"
+        success=true
+        break
+    fi
+    echo "IPsec pods did not roll out successfully yet"
+  done
+  popd
+  rm -Rf "${ca_dir}"
+
+  if ! ${success}; then
+      echo "IPsec pods did not roll out successfully"
+      exit 1
   fi
 }
 
@@ -790,6 +877,9 @@ check_dependencies
 parse_args "$@"
 set_default_params
 print_params
+if [ "${ENABLE_IPSEC}" == true ]; then
+  set_openssl_binary
+fi
 
 set -euxo pipefail
 check_ipv6
@@ -817,3 +907,9 @@ if [ "$KIND_INSTALL_INGRESS" == true ]; then
 fi
 kubectl_wait_pods
 sleep_until_pods_settle
+# Launch IPsec pods last to make sure that CSR signing logic works
+# Launch csr_signer in background
+# Wait for DaemonSet to rollout
+if [ "${ENABLE_IPSEC}" == true ]; then
+  install_ipsec
+fi

dist/images/Dockerfile.fedora

@@ -15,7 +15,7 @@ USER root
 
 ENV PYTHONDONTWRITEBYTECODE yes
 
-ARG ovnver=ovn-22.09.0-4.fc36
+ARG ovnver=ovn-22.09.0-25.fc36
 # Automatically populated when using docker buildx
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
@@ -28,9 +28,11 @@ RUN INSTALL_PKGS=" \
 	libpcap hostname kubernetes-client util-linux \
         ovn ovn-central ovn-host python3-openvswitch tcpdump openvswitch-test python3-pyOpenSSL \
 	iptables iproute iputils strace socat koji \
+        libreswan openvswitch-ipsec \
         " && \
 	dnf install --best --refresh -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
 	dnf clean all && rm -rf /var/cache/dnf/*
+RUN ln -s /usr/bin/python3 /usr/libexec/platform-python
 
 RUN mkdir -p /var/run/openvswitch
 

dist/images/daemonset.sh

@@ -100,6 +100,9 @@ while [ "$1" != "" ]; do
   --gateway-options)
     OVN_GATEWAY_OPTS=$VALUE
     ;;
+  --enable-ipsec)
+    ENABLE_IPSEC=$VALUE
+    ;;
   --ovn-monitor-all)
     OVN_MONITOR_ALL=$VALUE
     ;;
@@ -293,6 +296,9 @@ echo "ovn_gateway_mode: ${ovn_gateway_mode}"
 ovn_gateway_opts=${OVN_GATEWAY_OPTS}
 echo "ovn_gateway_opts: ${ovn_gateway_opts}"
 
+enable_ipsec=${ENABLE_IPSEC:-false}
+echo "enable_ipsec: ${enable_ipsec}"
+
 ovn_db_replicas=${OVN_DB_REPLICAS:-3}
 echo "ovn_db_replicas: ${ovn_db_replicas}"
 ovn_db_minAvailable=$(((${ovn_db_replicas} + 1) / 2))
@@ -499,6 +505,7 @@ ovn_image=${image} \
   ovn_ssl_en=${ovn_ssl_en} \
   ovn_nb_port=${ovn_nb_port} \
   ovn_sb_port=${ovn_sb_port} \
+  enable_ipsec=${enable_ipsec} \
   j2 ../templates/ovnkube-db.yaml.j2 -o ${output_dir}/ovnkube-db.yaml
 
 ovn_image=${image} \
@@ -517,13 +524,19 @@ ovn_image=${image} \
   ovn_sb_port=${ovn_sb_port} \
   ovn_nb_raft_port=${ovn_nb_raft_port} \
   ovn_sb_raft_port=${ovn_sb_raft_port} \
+  enable_ipsec=${enable_ipsec} \
   j2 ../templates/ovnkube-db-raft.yaml.j2 -o ${output_dir}/ovnkube-db-raft.yaml
 
 ovn_image=${image} \
   ovn_image_pull_policy=${image_pull_policy} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   j2 ../templates/ovs-node.yaml.j2 -o ${output_dir}/ovs-node.yaml
 
+if ${enable_ipsec}; then
+  ovn_image=${image} \
+    j2 ../templates/ovn-ipsec.yaml.j2 -o ${output_dir}/ovn-ipsec.yaml
+fi
+
 # ovn-setup.yaml
 net_cidr=${OVN_NET_CIDR:-"10.128.0.0/14/23"}
 svc_cidr=${OVN_SVC_CIDR:-"172.30.0.0/16"}

dist/images/ovndb-raft-functions.sh

@@ -370,6 +370,9 @@ ovsdb-raft() {
       set_election_timer ${db} ${election_timer}
       if [[ ${db} == "nb" ]]; then
         set_northd_probe_interval
+        [[ "true" == "${ENABLE_IPSEC}" ]] && {
+          ovn-nbctl set nb_global . ipsec=true
+        }
       fi
       # set the connection and disable inactivity probe, this deletes the old connection if any
       # this will unblock pod-1 and pod-2 waiters

dist/images/ovnkube.sh

@@ -726,6 +726,10 @@ nb-ovsdb() {
     ovn-nbctl set-ssl ${ovn_nb_pk} ${ovn_nb_cert} ${ovn_ca_cert}
     echo "=============== nb-ovsdb ========== reconfigured for SSL"
   }
+ [[ "true" == "${ENABLE_IPSEC}" ]] && {
+    ovn-nbctl set nb_global . ipsec=true
+    echo "=============== nb-ovsdb ========== reconfigured for ipsec"
+  }
   ovn-nbctl --inactivity-probe=0 set-connection p${transport}:${ovn_nb_port}:$(bracketify ${ovn_db_host})
   if memory_trim_on_compaction_supported "nbdb"
   then