pkey/rsa: port RSA#{private,public}_{encrypt,decrypt} to the EVP API

Implement these methods using the new OpenSSL::PKey::PKey#{encrypt,sign}
family. The definitions are now in lib/openssl/pkey.rb.

Also, recommend using those generic methods in the documentation.

Author: rhenium

ext/openssl/ossl_pkey_rsa.c

@@ -328,138 +328,6 @@ ossl_rsa_to_der(VALUE self)
         return ossl_pkey_export_spki(self, 1);
 }
 
-/*
- * call-seq:
- *   rsa.public_encrypt(string)          => String
- *   rsa.public_encrypt(string, padding) => String
- *
- * Encrypt _string_ with the public key.  _padding_ defaults to PKCS1_PADDING.
- * The encrypted string output can be decrypted using #private_decrypt.
- */
-static VALUE
-ossl_rsa_public_encrypt(int argc, VALUE *argv, VALUE self)
-{
-    RSA *rsa;
-    const BIGNUM *rsa_n;
-    int buf_len, pad;
-    VALUE str, buffer, padding;
-
-    GetRSA(self, rsa);
-    RSA_get0_key(rsa, &rsa_n, NULL, NULL);
-    if (!rsa_n)
-	ossl_raise(eRSAError, "incomplete RSA");
-    rb_scan_args(argc, argv, "11", &buffer, &padding);
-    pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
-    StringValue(buffer);
-    str = rb_str_new(0, RSA_size(rsa));
-    buf_len = RSA_public_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
-				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
-    if (buf_len < 0) ossl_raise(eRSAError, NULL);
-    rb_str_set_len(str, buf_len);
-
-    return str;
-}
-
-/*
- * call-seq:
- *   rsa.public_decrypt(string)          => String
- *   rsa.public_decrypt(string, padding) => String
- *
- * Decrypt _string_, which has been encrypted with the private key, with the
- * public key.  _padding_ defaults to PKCS1_PADDING.
- */
-static VALUE
-ossl_rsa_public_decrypt(int argc, VALUE *argv, VALUE self)
-{
-    RSA *rsa;
-    const BIGNUM *rsa_n;
-    int buf_len, pad;
-    VALUE str, buffer, padding;
-
-    GetRSA(self, rsa);
-    RSA_get0_key(rsa, &rsa_n, NULL, NULL);
-    if (!rsa_n)
-	ossl_raise(eRSAError, "incomplete RSA");
-    rb_scan_args(argc, argv, "11", &buffer, &padding);
-    pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
-    StringValue(buffer);
-    str = rb_str_new(0, RSA_size(rsa));
-    buf_len = RSA_public_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
-				 (unsigned char *)RSTRING_PTR(str), rsa, pad);
-    if (buf_len < 0) ossl_raise(eRSAError, NULL);
-    rb_str_set_len(str, buf_len);
-
-    return str;
-}
-
-/*
- * call-seq:
- *   rsa.private_encrypt(string)          => String
- *   rsa.private_encrypt(string, padding) => String
- *
- * Encrypt _string_ with the private key.  _padding_ defaults to PKCS1_PADDING.
- * The encrypted string output can be decrypted using #public_decrypt.
- */
-static VALUE
-ossl_rsa_private_encrypt(int argc, VALUE *argv, VALUE self)
-{
-    RSA *rsa;
-    const BIGNUM *rsa_n;
-    int buf_len, pad;
-    VALUE str, buffer, padding;
-
-    GetRSA(self, rsa);
-    RSA_get0_key(rsa, &rsa_n, NULL, NULL);
-    if (!rsa_n)
-	ossl_raise(eRSAError, "incomplete RSA");
-    if (!RSA_PRIVATE(self, rsa))
-	ossl_raise(eRSAError, "private key needed.");
-    rb_scan_args(argc, argv, "11", &buffer, &padding);
-    pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
-    StringValue(buffer);
-    str = rb_str_new(0, RSA_size(rsa));
-    buf_len = RSA_private_encrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
-				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
-    if (buf_len < 0) ossl_raise(eRSAError, NULL);
-    rb_str_set_len(str, buf_len);
-
-    return str;
-}
-
-/*
- * call-seq:
- *   rsa.private_decrypt(string)          => String
- *   rsa.private_decrypt(string, padding) => String
- *
- * Decrypt _string_, which has been encrypted with the public key, with the
- * private key.  _padding_ defaults to PKCS1_PADDING.
- */
-static VALUE
-ossl_rsa_private_decrypt(int argc, VALUE *argv, VALUE self)
-{
-    RSA *rsa;
-    const BIGNUM *rsa_n;
-    int buf_len, pad;
-    VALUE str, buffer, padding;
-
-    GetRSA(self, rsa);
-    RSA_get0_key(rsa, &rsa_n, NULL, NULL);
-    if (!rsa_n)
-	ossl_raise(eRSAError, "incomplete RSA");
-    if (!RSA_PRIVATE(self, rsa))
-	ossl_raise(eRSAError, "private key needed.");
-    rb_scan_args(argc, argv, "11", &buffer, &padding);
-    pad = (argc == 1) ? RSA_PKCS1_PADDING : NUM2INT(padding);
-    StringValue(buffer);
-    str = rb_str_new(0, RSA_size(rsa));
-    buf_len = RSA_private_decrypt(RSTRING_LENINT(buffer), (unsigned char *)RSTRING_PTR(buffer),
-				  (unsigned char *)RSTRING_PTR(str), rsa, pad);
-    if (buf_len < 0) ossl_raise(eRSAError, NULL);
-    rb_str_set_len(str, buf_len);
-
-    return str;
-}
-
 /*
  * call-seq:
  *    rsa.sign_pss(digest, data, salt_length:, mgf1_hash:) -> String
@@ -844,10 +712,6 @@ Init_ossl_rsa(void)
     rb_define_alias(cRSA, "to_s", "export");
     rb_define_method(cRSA, "to_der", ossl_rsa_to_der, 0);
     rb_define_method(cRSA, "public_key", ossl_rsa_to_public_key, 0);
-    rb_define_method(cRSA, "public_encrypt", ossl_rsa_public_encrypt, -1);
-    rb_define_method(cRSA, "public_decrypt", ossl_rsa_public_decrypt, -1);
-    rb_define_method(cRSA, "private_encrypt", ossl_rsa_private_encrypt, -1);
-    rb_define_method(cRSA, "private_decrypt", ossl_rsa_private_decrypt, -1);
     rb_define_method(cRSA, "sign_pss", ossl_rsa_sign_pss, -1);
     rb_define_method(cRSA, "verify_pss", ossl_rsa_verify_pss, -1);
 
@@ -865,11 +729,6 @@ Init_ossl_rsa(void)
 
     rb_define_method(cRSA, "params", ossl_rsa_get_params, 0);
 
-    DefRSAConst(PKCS1_PADDING);
-    DefRSAConst(SSLV23_PADDING);
-    DefRSAConst(NO_PADDING);
-    DefRSAConst(PKCS1_OAEP_PADDING);
-
 /*
  * TODO: Test it
     rb_define_method(cRSA, "blinding_on!", ossl_rsa_blinding_on, 0);

lib/openssl/pkey.rb

@@ -71,5 +71,111 @@ def to_bn(conversion_form = group.point_conversion_form)
 
   class RSA
     include OpenSSL::Marshal
+
+    # :call-seq:
+    #    rsa.private_encrypt(string)          -> String
+    #    rsa.private_encrypt(string, padding) -> String
+    #
+    # Encrypt +string+ with the private key.  +padding+ defaults to
+    # PKCS1_PADDING. The encrypted string output can be decrypted using
+    # #public_decrypt.
+    #
+    # <b>Deprecated in version 2.3</b>.
+    # Consider using PKey::PKey#sign_raw and PKey::PKey#verify_raw, and
+    # PKey::PKey#verify_recover instead.
+    def private_encrypt(string, padding = PKCS1_PADDING)
+      n or raise OpenSSL::PKey::RSAError, "incomplete RSA"
+      private? or raise OpenSSL::PKey::RSAError, "private key needed."
+      begin
+        sign_raw(nil, string, {
+          "rsa_padding_mode" => translate_padding_mode(padding),
+        })
+      rescue OpenSSL::PKey::PKeyError
+        raise OpenSSL::PKey::RSAError, $!.message
+      end
+    end
+
+    # :call-seq:
+    #    rsa.public_decrypt(string)          -> String
+    #    rsa.public_decrypt(string, padding) -> String
+    #
+    # Decrypt +string+, which has been encrypted with the private key, with the
+    # public key.  +padding+ defaults to PKCS1_PADDING.
+    #
+    # <b>Deprecated in version 2.3</b>.
+    # Consider using PKey::PKey#sign_raw and PKey::PKey#verify_raw, and
+    # PKey::PKey#verify_recover instead.
+    def public_decrypt(string, padding = PKCS1_PADDING)
+      n or raise OpenSSL::PKey::RSAError, "incomplete RSA"
+      begin
+        verify_recover(nil, string, {
+          "rsa_padding_mode" => translate_padding_mode(padding),
+        })
+      rescue OpenSSL::PKey::PKeyError
+        raise OpenSSL::PKey::RSAError, $!.message
+      end
+    end
+
+    # :call-seq:
+    #    rsa.public_encrypt(string)          -> String
+    #    rsa.public_encrypt(string, padding) -> String
+    #
+    # Encrypt +string+ with the public key.  +padding+ defaults to
+    # PKCS1_PADDING. The encrypted string output can be decrypted using
+    # #private_decrypt.
+    #
+    # <b>Deprecated in version 2.3</b>.
+    # Consider using PKey::PKey#encrypt and PKey::PKey#decrypt instead.
+    def public_encrypt(data, padding = PKCS1_PADDING)
+      n or raise OpenSSL::PKey::RSAError, "incomplete RSA"
+      begin
+        encrypt(data, {
+          "rsa_padding_mode" => translate_padding_mode(padding),
+        })
+      rescue OpenSSL::PKey::PKeyError
+        raise OpenSSL::PKey::RSAError, $!.message
+      end
+    end
+
+    # :call-seq:
+    #    rsa.private_decrypt(string)          -> String
+    #    rsa.private_decrypt(string, padding) -> String
+    #
+    # Decrypt +string+, which has been encrypted with the public key, with the
+    # private key. +padding+ defaults to PKCS1_PADDING.
+    #
+    # <b>Deprecated in version 2.3</b>.
+    # Consider using PKey::PKey#encrypt and PKey::PKey#decrypt instead.
+    def private_decrypt(data, padding = PKCS1_PADDING)
+      n or raise OpenSSL::PKey::RSAError, "incomplete RSA"
+      private? or raise OpenSSL::PKey::RSAError, "private key needed."
+      begin
+        decrypt(data, {
+          "rsa_padding_mode" => translate_padding_mode(padding),
+        })
+      rescue OpenSSL::PKey::PKeyError
+        raise OpenSSL::PKey::RSAError, $!.message
+      end
+    end
+
+    PKCS1_PADDING = 1
+    SSLV23_PADDING = 2
+    NO_PADDING = 3
+    PKCS1_OAEP_PADDING = 4
+
+    private def translate_padding_mode(num)
+      case num
+      when PKCS1_PADDING
+        "pkcs1"
+      when SSLV23_PADDING
+        "sslv23"
+      when NO_PADDING
+        "none"
+      when PKCS1_OAEP_PADDING
+        "oaep"
+      else
+        raise OpenSSL::PKey::PKeyError, "unsupported padding mode"
+      end
+    end
   end
 end