Merge pull request #5384 from input-output-hk/bench-nomad-ssh

Bench nomad ssh

Author: mgmeier

nix/workbench/backend/backend.sh

@@ -38,21 +38,29 @@ backend() {
 local op=${1:-$(usage_backend)} # No need to shift -- backends will use the op.
 
 case "${op}" in
-    is-running )                 backend_$WB_BACKEND "$@";;
+    # Prepare functions
     setenv-defaults )            backend_$WB_BACKEND "$@";;
     allocate-run )               backend_$WB_BACKEND "$@";;
     describe-run )               backend_$WB_BACKEND "$@";;
+    # Start functions
+    is-running )                 backend_$WB_BACKEND "$@";;
+    start-cluster )              backend_$WB_BACKEND "$@";;
     deploy-genesis )             backend_$WB_BACKEND "$@";;
-    start )                      backend_$WB_BACKEND "$@";;
+    # Sceneario functions
+    start-tracers )              backend_$WB_BACKEND "$@";;
     start-nodes )                backend_$WB_BACKEND "$@";;
+    start-generator )            backend_$WB_BACKEND "$@";;
+    start-healthchecks )         backend_$WB_BACKEND "$@";;
+    # Fine grained
     start-node )                 backend_$WB_BACKEND "$@";;
     stop-node )                  backend_$WB_BACKEND "$@";;
     wait-node )                  backend_$WB_BACKEND "$@";;
     wait-node-stopped )          backend_$WB_BACKEND "$@";;
     get-node-socket-path )       backend_$WB_BACKEND "$@";;
-    start-generator )            backend_$WB_BACKEND "$@";;
-    start-healthchecks )         backend_$WB_BACKEND "$@";;
     wait-pools-stopped )         backend_$WB_BACKEND "$@";;
+    # Stop functions
+    stop-all )                   backend_$WB_BACKEND "$@";;
+    fetch-logs )                 backend_$WB_BACKEND "$@";;
     stop-cluster )               backend_$WB_BACKEND "$@";;
     cleanup-cluster )            backend_$WB_BACKEND "$@";;
 

nix/workbench/backend/nomad-job.nix

@@ -11,6 +11,7 @@
 , execTaskDriver
 , generatorTaskName
 , oneTracerPerNode ? false
+, withSsh ? false
 }:
 
 let
@@ -90,9 +91,10 @@ let
       # The SUPERVISORD_CONFIG variable must be set
       [ -z "''${SUPERVISORD_CONFIG:-}" ] && echo "SUPERVISORD_CONFIG env var must be set -- aborting" && exit 1
 
-      # Create symlink to 'supervisor' Nix folder so we can call it from 'ssh'
-      #  or 'nomad exec' without having to know the currently version running.
-      # First check if already exists to be able to restart containers.
+      # Create a symlink to 'supervisor' Nix Store folder so we can call it from
+      # 'ssh' or 'nomad exec' without having it in PATH or knowing the currently
+      # running version. But first check if it already exists to be able to
+      # restart containers without errors.
       if ! test -e "''${SUPERVISOR_NIX}"
       then
         ${coreutils}/bin/ln -s "${supervisor}" "''${SUPERVISOR_NIX}"
@@ -162,7 +164,7 @@ let
     # namespace can be specified either with the flag -namespace or read from
     # the NOMAD_NAMESPACE environment variable."
     # https://developer.hashicorp.com/nomad/tutorials/manage-clusters/namespaces
-    namespace = "perf";
+    namespace = "perf"; # Default to "perf" to avoid errors were possible.
 
     # The region in which to execute the job.
     region = "global"; # SRE: They are actually using global.
@@ -269,10 +271,13 @@ let
         constraint = {
           attribute = "\${node.class}";
           operator = "=";
-          # For testing we avoid using "infra" node class as HA jobs runs there
-          # For benchmarking dedicated static machines in the "perf"
-          # class are used and this value should be updated accordingly.
-          value = "qa";
+          # For cloud benchmarking, dedicated static machines in the "perf"
+          # class are used. We mimic that for local/test runs.
+          # This default is just a precaution, like the top level namespace,
+          # because there are also available "qa" Class nodes but usage of these
+          # must be limited to short test and "infra" Class nodes are used for
+          # HA jobs and must be avoided entirely.
+          value = "perf";
         };
 
         # The network stanza specifies the networking requirements for the task
@@ -387,8 +392,22 @@ let
 
           # Specifies environment variables that will be passed to the running
           # process.
-          # `null` because we are using a "template" (see below).
-          env = {};
+          env = {
+            # The "old Nomad" setup somehow included the CA certs inside the
+            # task namespace but apparently not "new Nomad". We are adding the
+            # necessary Nix package ("cacert") and making sure `wget` finds it.
+            #
+            # "All nix-docker images set environment variables which point to
+            # cacert.", see:
+            # - https://github.com/NixOS/nixpkgs/issues/48211#issuecomment-434102565
+            # - https://github.com/LnL7/nix-docker/blob/8dcfb3aff1f87cdafeecb0d27964b27c3fb8b1d2/default.nix#L70-L71
+            #
+            # Error when using `wget` to deploy the genesis tar file was:
+            # ERROR: cannot verify iog-cardano-perf.s3.eu-central-1.amazonaws.com's certificate, issued by 'CN=Amazon RSA 2048 M01,O=Amazon,C=US':
+            # Unable to locally verify the issuer's authority.
+            # To connect to iog-cardano-perf.s3.eu-central-1.amazonaws.com insecurely, use `--no-check-certificate'.
+            SSL_CERT_FILE = "${containerSpecs.containerPkgs.cacert.nix-store-path}/etc/ssl/certs/ca-bundle.crt";
+          };
 
           # Sensible defaults to run cloud version of "default", "ci-test" and
           # "ci-bench" in cardano-world qa class Nomad nodes.
@@ -432,7 +451,7 @@ let
               # When using Cardano World (nomad.world.dev.cardano.org) "perf"
               # class nodes we use public IPs/routing, all the other cloud runs
               # are behind a VPC/firewall. Local runs just use 12.0.0.1.
-              if lib.strings.hasPrefix "cw-perf" profileData.profileName
+              if lib.strings.hasInfix "cw-perf" profileData.profileName
               then "\${attr.unique.platform.aws.public-ipv4}"
               else ""
             ;
@@ -592,6 +611,7 @@ let
                     withGenerator = taskName == generatorTaskName;
                     # Only for the tracer task or also nodes if oneTracerPerNode
                     withTracer = oneTracerPerNode || taskName == "tracer";
+                    inherit withSsh;
                     # ''{{ env "NOMAD_TASK_DIR" }}/supervisor.sock''
                     inherit unixHttpServerPort;
                   };
@@ -728,6 +748,43 @@ let
               perms = "744"; # Only for every "start.sh" script. Default: "644"
             }
           ]
+          ++
+          # ssh
+          (lib.optionals withSsh (
+            let
+              ssh-service = import
+                  ../service/ssh.nix
+                  {
+                    inherit pkgs;
+                    bashInteractive = containerSpecs.containerPkgs.bashInteractive.nix-store-path;
+                    coreutils = containerSpecs.containerPkgs.coreutils.nix-store-path;
+                    openssh_hacks = containerSpecs.containerPkgs.openssh_hacks.nix-store-path;
+                  }
+              ;
+            in [
+              ## ssh start.sh script.
+              {
+                env = false;
+                destination = "${task_statedir}/ssh/start.sh";
+                data = escapeTemplate ssh-service.start.value;
+                change_mode = "noop";
+                error_on_missing_key = true;
+                perms = "744"; # Only for every "start.sh" script. Default: "644"
+              }
+              ## ssh config file.
+              {
+                env = false;
+                destination = "${task_statedir}/ssh/sshd_config";
+                data = escapeTemplate ssh-service.config.value;
+                change_mode = "noop";
+                error_on_missing_key = true;
+                perms = "744"; # Only for every "start.sh" script. Default: "644"
+              }
+              # The deployer script must add the templates for the private keys:
+              # - ${task_statedir}/ssh/sshd.id_ed25519
+              # - ${task_statedir}/ssh/nobody.id_ed25519.pub
+            ]
+          ))
           ;
 
           # Specifies logging configuration for the stdout and stderr of the
@@ -1121,17 +1178,19 @@ let
         # Port string from
         ''--port ${toString nodeSpec.port}''
       ]
-      # On cloud deployments with cardano world, that uses AWS, the hosts at the
-      # Linux level aren't aware of the EIP public address they have, so the
-      # private IP is the only network interface address to bind to.
-      # An alternative is to bind to 0.0.0.0 but I prefer being more specific.
+      # On cloud deployments to SRE-managed Nomad, that uses AWS, the hosts at
+      # Linux level may not be aware of the EIP public address they have so we
+      # can't bind to the public IP (that we can resolve to using templates).
+      # I prefer being more specific but the "all-weather" alternative is to
+      # bind to 0.0.0.0 instead of the private IP, just in case the Nomad Client
+      # was not started with the correct `-network-interface XX` parameter.
       [
         # Address string to
-        ''--host-addr {{ env "NOMAD_HOST_IP_${servicePortName}" }}''
+        ''--host-addr 0.0.0.0''
         # Alternatives (may not work):
+        #''--host-addr {{ env "NOMAD_HOST_IP_${servicePortName}" }}''
         #''--host-addr {{ env "NOMAD_IP_${servicePortName}" }}''
         #''--host-addr {{range nomadService "${servicePortName}"}}{{.Address}}{{end}}''
-        #''--host-addr 0.0.0.0''
 
         # Port string to
         ''--port {{ env "NOMAD_HOST_PORT_${servicePortName}" }}''