pkey: track whether pkey is private key or not

There are multiple places where it's necessary to know whether a pkey
is a private key, a public key, or just key parameters. Unfortunately,
OpenSSL doesn't expose an API for this purpose (even though it has one
for its internal use).

Currently, we drill down into the backing object, such as RSA, and see
if the corresponding fields are set or not to determine it. This doesn't
work on OpenSSL 3.0 because of the architecture changes.

Let's manually track this information in an instance variable for now.
This has been partly done for ENGINE-backed pkeys. Now all pkeys get
this flag.

PKeys are immutable on OpenSSL 3.0, so it just needs to be stored once
on initialization. On OpenSSL 1.1 or before (including LibreSSL), it
must be updated whenever a modification is made to the object.

This comes with a slight behavior change. PKey returned by following
method will be explicitly marked as "public", even if it happens to
point at an EVP_PKEY struct containing private key components. I expect
the effect is minimum since these methods explicitly say "public key".

 - OpenSSL::X509::Certificate#public_key
 - OpenSSL::X509::Request#public_key
 - OpenSSL::Netscape::SPKI#public_key

Author: rhenium

ext/openssl/openssl_missing.h

@@ -140,6 +140,7 @@ IMPL_KEY_ACCESSOR3(RSA, crt_params, dmp1, dmq1, iqmp, (dmp1 == obj->dmp1 || dmq1
 IMPL_PKEY_GETTER(DSA, dsa)
 IMPL_KEY_ACCESSOR2(DSA, key, pub_key, priv_key, (pub_key == obj->pub_key || (obj->priv_key && priv_key == obj->priv_key)))
 IMPL_KEY_ACCESSOR3(DSA, pqg, p, q, g, (p == obj->p || q == obj->q || g == obj->g))
+static inline ENGINE *DSA_get0_engine(DSA *dsa) { return dsa->engine; }
 #endif
 
 #if !defined(OPENSSL_NO_DH)

ext/openssl/ossl_engine.c

@@ -373,8 +373,7 @@ ossl_engine_load_privkey(int argc, VALUE *argv, VALUE self)
     GetEngine(self, e);
     pkey = ENGINE_load_private_key(e, sid, NULL, sdata);
     if (!pkey) ossl_raise(eEngineError, NULL);
-    obj = ossl_pkey_new(pkey);
-    OSSL_PKEY_SET_PRIVATE(obj);
+    obj = ossl_pkey_new(pkey, OSSL_PKEY_HAS_PRIVATE);
 
     return obj;
 }
@@ -403,7 +402,7 @@ ossl_engine_load_pubkey(int argc, VALUE *argv, VALUE self)
     pkey = ENGINE_load_public_key(e, sid, NULL, sdata);
     if (!pkey) ossl_raise(eEngineError, NULL);
 
-    return ossl_pkey_new(pkey);
+    return ossl_pkey_new(pkey, OSSL_PKEY_HAS_PUBLIC);
 }
 
 /*

ext/openssl/ossl_ns_spki.c

@@ -190,7 +190,7 @@ ossl_spki_get_public_key(VALUE self)
 	ossl_raise(eSPKIError, NULL);
     }
 
-    return ossl_pkey_new(pkey); /* NO DUP - OK */
+    return ossl_pkey_new(pkey, OSSL_PKEY_HAS_PUBLIC); /* NO DUP - OK */
 }
 
 /*

ext/openssl/ossl_pkcs12.c

@@ -152,7 +152,7 @@ ossl_pkcs12_s_create(int argc, VALUE *argv, VALUE self)
 static VALUE
 ossl_pkey_new_i(VALUE arg)
 {
-    return ossl_pkey_new((EVP_PKEY *)arg);
+    return ossl_pkey_new((EVP_PKEY *)arg, OSSL_PKEY_HAS_PRIVATE);
 }
 
 static VALUE

ext/openssl/ossl_pkey.c

@@ -19,7 +19,7 @@
 VALUE mPKey;
 VALUE cPKey;
 VALUE ePKeyError;
-static ID id_private_q;
+ID ossl_pkey_feature_id;
 
 static void
 ossl_evp_pkey_free(void *ptr)
@@ -65,7 +65,7 @@ pkey_new0(VALUE arg)
 }
 
 VALUE
-ossl_pkey_new(EVP_PKEY *pkey)
+ossl_pkey_new(EVP_PKEY *pkey, enum ossl_pkey_feature ps)
 {
     VALUE obj;
     int status;
@@ -75,6 +75,7 @@ ossl_pkey_new(EVP_PKEY *pkey)
 	EVP_PKEY_free(pkey);
 	rb_jump_tag(status);
     }
+    ossl_pkey_set(obj, ps);
 
     return obj;
 }
@@ -83,34 +84,61 @@ ossl_pkey_new(EVP_PKEY *pkey)
 # include <openssl/decoder.h>
 
 EVP_PKEY *
-ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type)
+ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type, enum ossl_pkey_feature *ps)
 {
     void *ppass = (void *)pass;
     OSSL_DECODER_CTX *dctx;
     EVP_PKEY *pkey = NULL;
     int pos = 0, pos2;
+    size_t i;
 
     dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, input_type, 0, NULL, NULL);
     if (!dctx)
         goto out;
     if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
         goto out;
 
+    /*
+     * This is inefficient as it will parse the same DER/PEM encoding
+     * repeatedly, but OpenSSL 3.0 doesn't provide an API to return what
+     * information an EVP_PKEY is holding.
+     * OpenSSL issue: https://github.com/openssl/openssl/issues/9467
+     */
+    struct { int selection; enum ossl_pkey_feature ps; } selections[] = {
+        { EVP_PKEY_KEYPAIR, OSSL_PKEY_HAS_PRIVATE },
+        { EVP_PKEY_PUBLIC_KEY, OSSL_PKEY_HAS_PUBLIC },
+        { EVP_PKEY_KEY_PARAMETERS, OSSL_PKEY_HAS_NONE },
+    };
+
     /* First check DER */
-    if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-        goto out;
+    for (i = 0; i < sizeof(selections)/sizeof(selections[0]); i++) {
+        OSSL_DECODER_CTX_set_selection(dctx, selections[i].selection);
+        *ps = selections[i].ps;
+        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+            goto out;
+        OSSL_BIO_reset(bio);
+    }
 
-    /* Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed */
-    OSSL_BIO_reset(bio);
+    /*
+     * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed for
+     * each selection in case of stacked PEM structures
+     */
     if (OSSL_DECODER_CTX_set_input_type(dctx, "PEM") != 1)
         goto out;
-    while (OSSL_DECODER_from_bio(dctx, bio) != 1) {
-        if (BIO_eof(bio))
-            goto out;
-        pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
-            goto out;
-        pos = pos2;
+    for (i = 0; i < sizeof(selections)/sizeof(selections[0]); i++) {
+        OSSL_DECODER_CTX_set_selection(dctx, selections[i].selection);
+        *ps = selections[i].ps;
+        while (true) {
+            if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+                goto out;
+            if (BIO_eof(bio))
+                break;
+            pos2 = BIO_tell(bio);
+            if (pos2 < 0 || pos2 <= pos)
+                break;
+            pos = pos2;
+        }
+        OSSL_BIO_reset(bio);
     }
 
   out:
@@ -119,26 +147,35 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type)
 }
 #else
 EVP_PKEY *
-ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type)
+ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type, enum ossl_pkey_feature *ps)
 {
     void *ppass = (void *)pass;
     EVP_PKEY *pkey;
 
+    *ps = OSSL_PKEY_HAS_PRIVATE;
     if ((pkey = d2i_PrivateKey_bio(bio, NULL)))
 	goto out;
     OSSL_BIO_reset(bio);
     if ((pkey = d2i_PKCS8PrivateKey_bio(bio, NULL, ossl_pem_passwd_cb, ppass)))
 	goto out;
+
+    *ps = OSSL_PKEY_HAS_PUBLIC;
     OSSL_BIO_reset(bio);
     if ((pkey = d2i_PUBKEY_bio(bio, NULL)))
 	goto out;
-    OSSL_BIO_reset(bio);
+
     /* PEM_read_bio_PrivateKey() also parses PKCS #8 formats */
+    *ps = OSSL_PKEY_HAS_PRIVATE;
+    OSSL_BIO_reset(bio);
     if ((pkey = PEM_read_bio_PrivateKey(bio, NULL, ossl_pem_passwd_cb, ppass)))
 	goto out;
+
+    *ps = OSSL_PKEY_HAS_PUBLIC;
     OSSL_BIO_reset(bio);
     if ((pkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL)))
 	goto out;
+
+    *ps = OSSL_PKEY_HAS_NONE;
     OSSL_BIO_reset(bio);
     if ((pkey = PEM_read_bio_Parameters(bio, NULL)))
 	goto out;
@@ -195,14 +232,15 @@ ossl_pkey_new_from_data(int argc, VALUE *argv, VALUE self)
     EVP_PKEY *pkey;
     BIO *bio;
     VALUE data, pass;
+    enum ossl_pkey_feature ps;
 
     rb_scan_args(argc, argv, "11", &data, &pass);
     bio = ossl_obj2bio(&data);
-    pkey = ossl_pkey_read_generic(bio, ossl_pem_passwd_value(pass), NULL);
+    pkey = ossl_pkey_read_generic(bio, ossl_pem_passwd_value(pass), NULL, &ps);
     BIO_free(bio);
     if (!pkey)
 	ossl_raise(ePKeyError, "Could not parse PKey");
-    return ossl_pkey_new(pkey);
+    return ossl_pkey_new(pkey, ps);
 }
 
 static VALUE
@@ -405,7 +443,7 @@ pkey_generate(int argc, VALUE *argv, VALUE self, int genparam)
         }
     }
 
-    return ossl_pkey_new(gen_arg.pkey);
+    return ossl_pkey_new(gen_arg.pkey, OSSL_PKEY_HAS_PRIVATE);
 }
 
 /*
@@ -527,18 +565,9 @@ GetPrivPKeyPtr(VALUE obj)
     EVP_PKEY *pkey;
 
     GetPKey(obj, pkey);
-    if (OSSL_PKEY_IS_PRIVATE(obj))
-        return pkey;
-    /*
-     * The EVP API does not provide a way to check if the EVP_PKEY has private
-     * components. Assuming it does...
-     */
-    if (!rb_respond_to(obj, id_private_q))
-        return pkey;
-    if (RTEST(rb_funcallv(obj, id_private_q, 0, NULL)))
-        return pkey;
-
-    rb_raise(rb_eArgError, "private key is needed");
+    if (!ossl_pkey_has(obj, OSSL_PKEY_HAS_PRIVATE))
+        rb_raise(rb_eArgError, "private key is needed");
+    return pkey;
 }
 
 EVP_PKEY *
@@ -614,6 +643,33 @@ ossl_pkey_oid(VALUE self)
     return rb_str_new_cstr(OBJ_nid2sn(nid));
 }
 
+/*
+ * call-seq:
+ *    pkey.public? -> true | false
+ *
+ * Indicates whether this PKey instance has a public key associated with it or
+ * not.
+ */
+static VALUE
+ossl_pkey_is_public(VALUE self)
+{
+    return ossl_pkey_has(self, OSSL_PKEY_HAS_PUBLIC) ? Qtrue : Qfalse;
+}
+
+/*
+ * call-seq:
+ *    pkey.private? -> true | false
+ *
+ * Indicates whether this PKey instance has a private key associated with it or
+ * not.
+ */
+static VALUE
+ossl_pkey_is_private(VALUE self)
+{
+    return ossl_pkey_has(self, OSSL_PKEY_HAS_PRIVATE) ? Qtrue : Qfalse;
+}
+
+
 /*
  * call-seq:
  *    pkey.inspect -> string
@@ -1580,6 +1636,8 @@ Init_ossl_pkey(void)
     rb_undef_method(cPKey, "initialize_copy");
 #endif
     rb_define_method(cPKey, "oid", ossl_pkey_oid, 0);
+    rb_define_method(cPKey, "public?", ossl_pkey_is_public, 0);
+    rb_define_method(cPKey, "private?", ossl_pkey_is_private, 0);
     rb_define_method(cPKey, "inspect", ossl_pkey_inspect, 0);
     rb_define_method(cPKey, "to_text", ossl_pkey_to_text, 0);
     rb_define_method(cPKey, "private_to_der", ossl_pkey_private_to_der, -1);
@@ -1597,7 +1655,7 @@ Init_ossl_pkey(void)
     rb_define_method(cPKey, "encrypt", ossl_pkey_encrypt, -1);
     rb_define_method(cPKey, "decrypt", ossl_pkey_decrypt, -1);
 
-    id_private_q = rb_intern("private?");
+    ossl_pkey_feature_id = rb_intern_const("state");
 
     /*
      * INIT rsa, dsa, dh, ec

ext/openssl/ossl_pkey.h

@@ -14,10 +14,7 @@ extern VALUE mPKey;
 extern VALUE cPKey;
 extern VALUE ePKeyError;
 extern const rb_data_type_t ossl_evp_pkey_type;
-
-/* For ENGINE */
-#define OSSL_PKEY_SET_PRIVATE(obj) rb_ivar_set((obj), rb_intern("private"), Qtrue)
-#define OSSL_PKEY_IS_PRIVATE(obj)  (rb_attr_get((obj), rb_intern("private")) == Qtrue)
+extern ID ossl_pkey_feature_id;
 
 #define GetPKey(obj, pkey) do {\
     TypedData_Get_Struct((obj), EVP_PKEY, &ossl_evp_pkey_type, (pkey)); \
@@ -26,10 +23,30 @@ extern const rb_data_type_t ossl_evp_pkey_type;
     } \
 } while (0)
 
+/*
+ * Store whether pkey contains public key, private key, or neither of them.
+ * This is ugly, but OpenSSL currently (3.0) doesn't provide a public API for
+ * this purpose.
+ */
+enum ossl_pkey_feature {
+    OSSL_PKEY_HAS_NONE, /* or parameters only */
+    OSSL_PKEY_HAS_PUBLIC,
+    OSSL_PKEY_HAS_PRIVATE,
+};
+static inline void ossl_pkey_set(VALUE self, enum ossl_pkey_feature state)
+{
+    rb_ivar_set(self, ossl_pkey_feature_id, INT2FIX(state));
+}
+static inline int ossl_pkey_has(VALUE self, enum ossl_pkey_feature state)
+{
+    return FIX2INT(rb_attr_get(self, ossl_pkey_feature_id)) >= (int)state;
+}
+
 /* Takes ownership of the EVP_PKEY */
-VALUE ossl_pkey_new(EVP_PKEY *);
+VALUE ossl_pkey_new(EVP_PKEY *pkey, enum ossl_pkey_feature ps);
 void ossl_pkey_check_public_key(const EVP_PKEY *);
-EVP_PKEY *ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type);
+EVP_PKEY *ossl_pkey_read_generic(BIO *bio, VALUE pass, const char *input_type,
+                                 enum ossl_pkey_feature *ps);
 EVP_PKEY *GetPKeyPtr(VALUE);
 EVP_PKEY *DupPKeyPtr(VALUE);
 EVP_PKEY *GetPrivPKeyPtr(VALUE);
@@ -145,6 +162,7 @@ static VALUE ossl_##_keytype##_set_##_group(VALUE self, VALUE v1, VALUE v2, VALU
 		BN_clear_free(bn3);					\
 		ossl_raise(ePKeyError, #_type"_set0_"#_group);		\
 	}								\
+	_keytype##_fix_selection(self, obj);				\
 	return self;							\
 }
 
@@ -172,6 +190,7 @@ static VALUE ossl_##_keytype##_set_##_group(VALUE self, VALUE v1, VALUE v2) \
 		BN_clear_free(bn2);					\
 		ossl_raise(ePKeyError, #_type"_set0_"#_group);		\
 	}								\
+	_keytype##_fix_selection(self, obj);				\
 	return self;							\
 }
 #else /* OSSL_HAVE_PROVIDER */