CDRIVER-5985 remove username derivation for MONGODB-X509 (#2000)

* add x509 Atlas auth test
* remove no-longer-needed internal `mongoc_ssl_extract_subject`
* add x509 tests with test certs
* set up Kerberos config and keytab files in script
** Avoid persisting keytab file in `/tmp` directory. Avoid modifying system-shared `/etc/krb5.conf`
* remove `prepare-kerberos` Evergreen function
* update user and password in docs to match what is used in Evergreen

Author: kevinAlbs

.evergreen/config_generator/components/funcs/prepare_kerberos.py

@@ -327,27 +327,6 @@ functions:
         args:
           - -c
           - .evergreen/scripts/compile-openssl-static.sh
-  prepare-kerberos:
-    - command: subprocess.exec
-      type: setup
-      params:
-        binary: bash
-        working_dir: mongoc
-        silent: true
-        args:
-          - -c
-          - |
-            if test "${keytab|}" && command -v kinit >/dev/null; then
-                echo "${keytab}" > /tmp/drivers.keytab.base64
-                cat /tmp/drivers.keytab.base64 | base64 -d > /tmp/drivers.keytab
-                if touch /etc/krb5.conf 2>/dev/null; then
-                    cat .evergreen/etc/kerberos.realm | tee -a /etc/krb5.conf
-                elif command sudo true 2>/dev/null; then
-                    cat .evergreen/etc/kerberos.realm | sudo tee -a /etc/krb5.conf
-                else
-                    echo "Cannot append kerberos.realm to /etc/krb5.conf; skipping." 1>&2
-                fi
-            fi
   restore-instance-profile:
     - command: subprocess.exec
       params:

.evergreen/generated_configs/functions.yml

@@ -1193,7 +1193,6 @@ tasks:
   - func: fetch-build
     vars:
       BUILD_NAME: debug-compile-sasl-openssl
-  - func: prepare-kerberos
   - func: run auth tests
 - name: authentication-tests-darwinssl
   tags:
@@ -1206,7 +1205,6 @@ tasks:
   - func: fetch-build
     vars:
       BUILD_NAME: debug-compile-sasl-darwinssl
-  - func: prepare-kerberos
   - func: run auth tests
 - name: authentication-tests-winssl
   tags:
@@ -1219,7 +1217,6 @@ tasks:
   - func: fetch-build
     vars:
       BUILD_NAME: debug-compile-sspi-winssl
-  - func: prepare-kerberos
   - func: run auth tests
 - name: authentication-tests-openssl-nosasl
   tags:
@@ -1232,7 +1229,6 @@ tasks:
   - func: fetch-build
     vars:
       BUILD_NAME: debug-compile-nosasl-openssl
-  - func: prepare-kerberos
   - func: run auth tests
 - name: test-mongohouse
   depends_on:
@@ -1260,7 +1256,6 @@ tasks:
       script: |-
         set -o errexit
         env SANITIZE=address SASL=AUTO SSL=OPENSSL .evergreen/scripts/compile.sh
-  - func: prepare-kerberos
   - func: run auth tests
     vars:
       ASAN: 'on'

.evergreen/generated_configs/legacy-config.yml

@@ -612,7 +612,6 @@ def additional_dependencies(self) -> Iterable[DependencySpec]:
 
     def post_commands(self) -> Iterable[Value]:
         yield func("fetch-build", BUILD_NAME=self.build_task_name)
-        yield func("prepare-kerberos")
         yield func("run auth tests")
 
     @property
@@ -664,7 +663,6 @@ def pre_commands(self) -> Iterable[Value]:
             """,
                     add_expansions_to_env=True,
                 ),
-                func("prepare-kerberos"),
                 func("run auth tests", ASAN="on"),
             ],
         )

.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py

@@ -18,6 +18,40 @@ mongoc_dir="$(to_absolute "${script_dir}/../..")"
 declare install_dir="${mongoc_dir}/install-dir"
 declare openssl_install_dir="${mongoc_dir}/openssl-install-dir"
 
+# Create directory for secrets within Evergreen task directory. Task directory is cleaned up between tasks.
+declare secrets_dir
+secrets_dir="$(to_absolute "${mongoc_dir}/../secrets")"
+mkdir -p "${secrets_dir}"
+chmod 700 "${secrets_dir}"
+
+# Create certificate to test X509 auth with Atlas:
+atlas_x509_path="${secrets_dir:?}/atlas_x509.pem"
+echo "${atlas_x509_cert_base64:?}" | base64 --decode > "${secrets_dir:?}/atlas_x509.pem"
+# On Windows, convert certificate to PKCS#1 to work around CDRIVER-4269:
+if $IS_WINDOWS; then
+    openssl pkey -in "${secrets_dir:?}/atlas_x509.pem" -traditional > "${secrets_dir:?}/atlas_x509_pkcs1.pem"
+    openssl x509 -in "${secrets_dir:?}/atlas_x509.pem" >> "${secrets_dir:?}/atlas_x509_pkcs1.pem"
+    atlas_x509_path="$(cygpath -m "${secrets_dir:?}/atlas_x509_pkcs1.pem")"
+fi
+
+# Create Kerberos config and keytab files.
+echo "Setting up Kerberos ... begin"
+if command -v kinit >/dev/null; then
+    # Copy host config and append realm:
+    if [ -e /etc/krb5.conf ]; then
+      cat /etc/krb5.conf > "${secrets_dir:?}/krb5.conf"
+    fi
+    cat "${mongoc_dir}/.evergreen/etc/kerberos.realm" >> "${secrets_dir:?}/krb5.conf"
+    # Set up keytab:
+    echo "${keytab:?}" | base64 --decode > "${secrets_dir:?}/drivers.keytab"
+    # Initialize kerberos:
+    KRB5_CONFIG="${secrets_dir:?}/krb5.conf" kinit -k -t "${secrets_dir:?}/drivers.keytab" -p [email protected]
+    echo "Setting up Kerberos ... done"
+else
+    echo "No 'kinit' detected"
+    echo "Setting up Kerberos ... skipping"
+fi
+
 declare c_timeout="connectTimeoutMS=30000&serverSelectionTryOnce=false"
 
 declare sasl="OFF"
@@ -62,10 +96,6 @@ esac
 : "${test_gssapi:?}"
 : "${ip_addr:?}"
 
-if command -v kinit >/dev/null && [[ -f /tmp/drivers.keytab ]]; then
-  kinit -k -t /tmp/drivers.keytab -p [email protected] || true
-fi
-
 # Archlinux (which we use for testing various self-installed OpenSSL versions)
 # stores their trust list under /etc/ca-certificates/extracted/.
 # We need to copy it to our custom installed OpenSSL trust store.
@@ -158,6 +188,10 @@ if [[ "${ssl}" != "OFF" ]]; then
     echo "Connecting to Atlas Serverless"
     LD_LIBRARY_PATH="${openssl_lib_prefix}" "${ping}" "${atlas_serverless:?}&${c_timeout}"
   fi
+
+  echo "Connecting to Atlas with X509"
+  LD_LIBRARY_PATH="${openssl_lib_prefix}" "${ping}" "${atlas_x509:?}&tlsCertificateKeyFile=${atlas_x509_path}&${c_timeout}"
+
 fi
 
 echo "Authenticating using PLAIN"

.evergreen/scripts/run-auth-tests.sh

@@ -148,7 +148,7 @@ $ mongod --auth --setParameter enableTestCommands=1 --dbpath db/
 In another terminal, use the `mongosh` shell to create a user:
 
 ```
-$ mongosh --eval "db.createUser({user: 'admin', pwd: 'pass', roles: ['root']})" admin
+$ mongosh --eval "db.createUser({user: 'bob', pwd: 'pwd123', roles: ['root']})" admin
 ```
 
 Authentication in MongoDB 3.0 and later uses SCRAM-SHA-1, which in turn
@@ -157,8 +157,8 @@ requires a driver built with SSL.
 Set the user and password environment variables, then build and run the tests:
 
 ```
-$ export MONGOC_TEST_USER=admin
-$ export MONGOC_TEST_PASSWORD=pass
+$ export MONGOC_TEST_USER=bob
+$ export MONGOC_TEST_PASSWORD=pwd123
 $ ./test-libmongoc
 ```
 

CONTRIBUTING.md

@@ -235,10 +235,7 @@ _mongoc_cluster_create_server_stream (const mongoc_topology_description_t *td,
                                       mongoc_stream_t *stream);
 
 bool
-_mongoc_cluster_get_auth_cmd_x509 (const mongoc_uri_t *uri,
-                                   const mongoc_ssl_opt_t *ssl_opts,
-                                   bson_t *cmd /* OUT */,
-                                   bson_error_t *error /* OUT */);
+_mongoc_cluster_get_auth_cmd_x509 (const mongoc_uri_t *uri, bson_t *cmd /* OUT */, bson_error_t *error /* OUT */);
 
 /* Returns true if a versioned server API has been selected, otherwise returns
  * false. */

src/libmongoc/src/mongoc/mongoc-cluster-private.h

@@ -804,12 +804,7 @@ _stream_run_hello (mongoc_cluster_t *cluster,
    _mongoc_topology_dup_handshake_cmd (cluster->client->topology, &handshake_command);
 
    if (cluster->requires_auth && speculative_auth_response) {
-      mongoc_ssl_opt_t *ssl_opts = NULL;
-#ifdef MONGOC_ENABLE_SSL
-      ssl_opts = &cluster->client->ssl_opts;
-#endif
-
-      _mongoc_topology_scanner_add_speculative_authentication (&handshake_command, cluster->uri, ssl_opts, scram);
+      _mongoc_topology_scanner_add_speculative_authentication (&handshake_command, cluster->uri, scram);
    }
 
    if (negotiate_sasl_supported_mechs) {
@@ -1053,10 +1048,7 @@ _mongoc_cluster_auth_node_plain (mongoc_cluster_t *cluster,
 }
 
 bool
-_mongoc_cluster_get_auth_cmd_x509 (const mongoc_uri_t *uri,
-                                   const mongoc_ssl_opt_t *ssl_opts,
-                                   bson_t *cmd /* OUT */,
-                                   bson_error_t *error /* OUT */)
+_mongoc_cluster_get_auth_cmd_x509 (const mongoc_uri_t *uri, bson_t *cmd /* OUT */, bson_error_t *error /* OUT */)
 {
 #ifndef MONGOC_ENABLE_SSL
    _mongoc_set_error (error,
@@ -1067,41 +1059,21 @@ _mongoc_cluster_get_auth_cmd_x509 (const mongoc_uri_t *uri,
    return false;
 #else
    const char *username_from_uri = NULL;
-   char *username_from_subject = NULL;
 
    BSON_ASSERT (uri);
+   BSON_UNUSED (error);
 
    username_from_uri = mongoc_uri_get_username (uri);
    if (username_from_uri) {
       TRACE ("%s", "X509: got username from URI");
-   } else {
-      if (!ssl_opts || !ssl_opts->pem_file) {
-         _mongoc_set_error (error,
-                            MONGOC_ERROR_CLIENT,
-                            MONGOC_ERROR_CLIENT_AUTHENTICATE,
-                            "cannot determine username for "
-                            "X-509 authentication.");
-         return false;
-      }
-
-      username_from_subject = mongoc_ssl_extract_subject (ssl_opts->pem_file, ssl_opts->pem_pwd);
-      if (!username_from_subject) {
-         _mongoc_set_error (error,
-                            MONGOC_ERROR_CLIENT,
-                            MONGOC_ERROR_CLIENT_AUTHENTICATE,
-                            "No username provided for X509 authentication.");
-         return false;
-      }
-
-      TRACE ("%s", "X509: got username from certificate");
    }
 
    bson_init (cmd);
    BSON_APPEND_INT32 (cmd, "authenticate", 1);
    BSON_APPEND_UTF8 (cmd, "mechanism", "MONGODB-X509");
-   BSON_APPEND_UTF8 (cmd, "user", username_from_uri ? username_from_uri : username_from_subject);
-
-   bson_free (username_from_subject);
+   if (username_from_uri) {
+      BSON_APPEND_UTF8 (cmd, "user", username_from_uri);
+   }
 
    return true;
 #endif
@@ -1132,7 +1104,7 @@ _mongoc_cluster_auth_node_x509 (mongoc_cluster_t *cluster,
    BSON_ASSERT (cluster);
    BSON_ASSERT (stream);
 
-   if (!_mongoc_cluster_get_auth_cmd_x509 (cluster->uri, &cluster->client->ssl_opts, &cmd, error)) {
+   if (!_mongoc_cluster_get_auth_cmd_x509 (cluster->uri, &cmd, error)) {
       return false;
    }
 

src/libmongoc/src/mongoc/mongoc-cluster.c

@@ -38,8 +38,6 @@ bool
 _mongoc_openssl_check_peer_hostname (SSL *ssl, const char *host, bool allow_invalid_hostname);
 SSL_CTX *
 _mongoc_openssl_ctx_new (mongoc_ssl_opt_t *opt);
-char *
-_mongoc_openssl_extract_subject (const char *filename, const char *passphrase);
 void
 _mongoc_openssl_init (void);
 void

src/libmongoc/src/mongoc/mongoc-openssl-private.h

@@ -1012,57 +1012,6 @@ _mongoc_openssl_ctx_new (mongoc_ssl_opt_t *opt)
    return ctx;
 }
 
-
-char *
-_mongoc_openssl_extract_subject (const char *filename, const char *passphrase)
-{
-   X509_NAME *subject = NULL;
-   X509 *cert = NULL;
-   BIO *certbio = NULL;
-   BIO *strbio = NULL;
-   char *str = NULL;
-   int ret;
-
-   BSON_UNUSED (passphrase);
-
-   if (!filename) {
-      return NULL;
-   }
-
-   certbio = BIO_new (BIO_s_file ());
-   strbio = BIO_new (BIO_s_mem ());
-
-   BSON_ASSERT (certbio);
-   BSON_ASSERT (strbio);
-
-
-   if (BIO_read_filename (certbio, filename) && (cert = PEM_read_bio_X509 (certbio, NULL, 0, NULL))) {
-      if ((subject = X509_get_subject_name (cert))) {
-         ret = X509_NAME_print_ex (strbio, subject, 0, XN_FLAG_RFC2253);
-
-         if ((ret > 0) && (ret < INT_MAX)) {
-            str = (char *) bson_malloc (ret + 2);
-            BIO_gets (strbio, str, ret + 1);
-            str[ret] = '\0';
-         }
-      }
-   }
-
-   if (cert) {
-      X509_free (cert);
-   }
-
-   if (certbio) {
-      BIO_free (certbio);
-   }
-
-   if (strbio) {
-      BIO_free (strbio);
-   }
-
-   return str;
-}
-
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 #ifdef _WIN32
 

src/libmongoc/src/mongoc/mongoc-openssl.c

@@ -32,10 +32,6 @@
 
 BSON_BEGIN_DECLS
 
-
-char *
-_mongoc_secure_channel_extract_subject (const char *filename, const char *passphrase);
-
 bool
 mongoc_secure_channel_setup_ca (mongoc_stream_tls_secure_channel_t *secure_channel, mongoc_ssl_opt_t *opt);