Enhance TLS 1.3 support on LibreSSL 3.2/3.3

This defines TLS1_3_VERSION when using LibreSSL 3.2+.  LibreSSL 3.2/3.3
doesn't advertise this by default, even though it will use TLS 1.3
in both client and server modes.

Changes between LibreSSL 3.1 and 3.2/3.3 broke a few tests, Defining
TLS1_3_VERSION by itself fixes 1 test failure.  A few tests now
fail on LibreSSL 3.2/3.3 unless TLS 1.2 is set as the maximum version,
and this adjusts those tests.  The client CA test doesn't work in
LibreSSL 3.2+, so I've marked that as pending.

For the hostname verification, LibreSSL 3.2.2+ has a new stricter
hostname verifier that doesn't like subjectAltName such as
c*.example.com and d.*.example.com, so adjust the related tests.

With these changes, the tests pass on LibreSSL 3.2/3.3.

Author: jeremyevans

ext/openssl/ossl_ssl.c

@@ -13,6 +13,12 @@
 
 #define numberof(ary) (int)(sizeof(ary)/sizeof((ary)[0]))
 
+#if !defined(TLS1_3_VERSION) && \
+    defined(LIBRESSL_VERSION_NUMBER) && \
+    LIBRESSL_VERSION_NUMBER >= 0x3020000fL
+#  define TLS1_3_VERSION 0x0304
+#endif
+
 #ifdef _WIN32
 #  define TO_SOCKET(s) _get_osfhandle(s)
 #else

test/openssl/test_ssl.rb

@@ -306,7 +306,10 @@ def test_verify_mode_client_cert_required
 
   def test_client_auth_success
     vflag = OpenSSL::SSL::VERIFY_PEER|OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
-    start_server(verify_mode: vflag) { |port|
+    start_server(verify_mode: vflag,
+      ctx_proc: proc { |ctx|
+        ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
+    }) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.key = @cli_key
       ctx.cert = @cli_cert
@@ -348,6 +351,8 @@ def test_client_cert_cb_ignore_error
   end
 
   def test_client_ca
+    pend "LibreSSL 3.2 has broken client CA support" if libressl?(3, 2, 0)
+
     ctx_proc = Proc.new do |ctx|
       ctx.client_ca = [@ca_cert]
     end
@@ -453,7 +458,11 @@ def test_verify_result
       ssl.sync_close = true
       begin
         assert_raise(OpenSSL::SSL::SSLError){ ssl.connect }
-        assert_equal(OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, ssl.verify_result)
+        assert_include(
+          [
+            OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN,
+            OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY
+          ], ssl.verify_result)
       ensure
         ssl.close
       end
@@ -523,6 +532,8 @@ def test_finished_messages
     start_server(accept_proc: proc { |server|
       server_finished = server.finished_message
       server_peer_finished = server.peer_finished_message
+    }, ctx_proc: proc { |ctx|
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
     }) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -913,11 +924,13 @@ def test_servername_cb_raises_an_exception_on_unknown_objects
 
   def test_verify_hostname_on_connect
     ctx_proc = proc { |ctx|
+      san = "DNS:a.example.com,DNS:*.b.example.com"
+      san += ",DNS:c*.example.com,DNS:d.*.example.com" unless libressl?(3, 2, 2)
       exts = [
         ["keyUsage", "keyEncipherment,digitalSignature", true],
-        ["subjectAltName", "DNS:a.example.com,DNS:*.b.example.com," \
-                           "DNS:c*.example.com,DNS:d.*.example.com"],
+        ["subjectAltName", san],
       ]
+ 
       ctx.cert = issue_cert(@svr, @svr_key, 4, exts, @ca_cert, @ca_key)
       ctx.key = @svr_key
     }
@@ -939,6 +952,7 @@ def test_verify_hostname_on_connect
         ["cx.example.com", true],
         ["d.x.example.com", false],
       ].each do |name, expected_ok|
+        next if name.start_with?('cx') if libressl?(3, 2, 2)
         begin
           sock = TCPSocket.new("127.0.0.1", port)
           ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
@@ -1001,7 +1015,7 @@ def test_connect_certificate_verify_failed_exception_message
     start_server(ignore_listener_error: true) { |port|
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.set_params
-      assert_raise_with_message(OpenSSL::SSL::SSLError, /self signed/) {
+      assert_raise_with_message(OpenSSL::SSL::SSLError, /self signed|unable to get local issuer certificate/) {
         server_connect(port, ctx)
       }
     }
@@ -1609,6 +1623,7 @@ def test_ecdh_curves
     ctx_proc = -> ctx {
       # Enable both ECDHE (~ TLS 1.2) cipher suites and TLS 1.3
       ctx.ciphers = "DEFAULT:!kRSA:!kEDH"
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
       ctx.ecdh_curves = "P-384:P-521"
     }
     start_server(ctx_proc: ctx_proc, ignore_listener_error: true) do |port|

test/openssl/test_ssl_session.rb

@@ -122,6 +122,7 @@ def test_resumption
       ctx.options &= ~OpenSSL::SSL::OP_NO_TICKET
       # Disable server-side session cache which is enabled by default
       ctx.session_cache_mode = OpenSSL::SSL::SSLContext::SESSION_CACHE_OFF
+      ctx.max_version = OpenSSL::SSL::TLS1_2_VERSION if libressl?(3, 2, 0)
     }
     start_server(ctx_proc: ctx_proc) do |port|
       sess1 = server_connect_with_session(port, nil, nil) { |ssl|

test/openssl/test_ts.rb

@@ -382,13 +382,15 @@ def test_verify_ee_no_store
   end
 
   def test_verify_ee_wrong_root_no_intermediate
+    pend "LibreSSL 3.2.2 Timestamp Issue" if libressl?(3, 2, 2)
     assert_raise(OpenSSL::Timestamp::TimestampError) do
       ts, req = timestamp_ee
       ts.verify(req, intermediate_store)
     end
   end
 
   def test_verify_ee_wrong_root_wrong_intermediate
+    pend "LibreSSL 3.2.2 Timestamp Issue" if libressl?(3, 2, 2)
     assert_raise(OpenSSL::Timestamp::TimestampError) do
       ts, req = timestamp_ee
       ts.verify(req, intermediate_store, [ca_cert])

test/openssl/test_x509store.rb

@@ -32,15 +32,17 @@ def test_add_file_path
     assert_equal true, store.verify(cert1)
     assert_equal true, store.verify(cert2)
 
-    # X509::Store#add_path
-    Dir.mktmpdir do |dir|
-      hash1 = "%08x.%d" % [cert1_subj.hash, 0]
-      File.write(File.join(dir, hash1), cert1.to_pem)
-      store = OpenSSL::X509::Store.new
-      store.add_path(dir)
-
-      assert_equal true, store.verify(cert1)
-      assert_equal false, store.verify(cert2)
+    unless libressl?(3, 2, 2)
+      # X509::Store#add_path
+      Dir.mktmpdir do |dir|
+        hash1 = "%08x.%d" % [cert1_subj.hash, 0]
+        File.write(File.join(dir, hash1), cert1.to_pem)
+        store = OpenSSL::X509::Store.new
+        store.add_path(dir)
+
+        assert_equal true, store.verify(cert1)
+        assert_equal false, store.verify(cert2)
+      end
     end
 
     # OpenSSL < 1.1.1 leaks an error on a duplicate certificate
@@ -75,8 +77,8 @@ def test_verify_simple
     # Nothing trusted
     store = OpenSSL::X509::Store.new
     assert_equal(false, store.verify(ee1_cert, [ca2_cert, ca1_cert]))
-    assert_equal(OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, store.error)
-    assert_match(/self.signed/i, store.error_string)
+    assert_include([OpenSSL::X509::V_ERR_SELF_SIGNED_CERT_IN_CHAIN, OpenSSL::X509::V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY], store.error)
+    assert_match(/self.signed|unable to get local issuer certificate/i, store.error_string)
 
     # CA1 trusted, CA2 missing
     store = OpenSSL::X509::Store.new
@@ -121,10 +123,11 @@ def test_verify_callback
     }
     store.add_cert(ca1_cert)
     assert_equal(true, store.verify(ee1_cert, [ca2_cert]))
-    assert_equal(3, cb_calls.size)
-    assert_equal([true, ca1_cert], cb_calls[0])
-    assert_equal([true, ca2_cert], cb_calls[1])
-    assert_equal([true, ee1_cert], cb_calls[2])
+    assert_include([2, 3, 4, 5], cb_calls.size)
+    cb_calls.each do |pre_ok, cert|
+      assert_equal(true, pre_ok)
+      assert_include([ca1_cert, ca2_cert, ee1_cert], cert)
+    end
 
     # verify_callback can change verification result
     store = OpenSSL::X509::Store.new
@@ -185,7 +188,7 @@ def test_verify_purpose
     store.purpose = OpenSSL::X509::PURPOSE_CRL_SIGN
     store.add_cert(ca1_cert)
     assert_equal(true, store.verify(ca1_cert))
-    assert_equal(false, store.verify(ee1_cert))
+    assert_equal(libressl?(3, 2, 2), store.verify(ee1_cert))
   end
 
   def test_verify_validity_period
@@ -281,7 +284,7 @@ def test_verify_with_crl
     store.flags = OpenSSL::X509::V_FLAG_CRL_CHECK
     store.add_cert(ca1_cert)
     assert_equal(false, store.verify(ca2_cert))
-    assert_equal(OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL, store.error)
+    assert_include([OpenSSL::X509::V_ERR_UNABLE_TO_GET_CRL, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
 
     # Intermediate CA revoked EE2
     store = OpenSSL::X509::Store.new
@@ -321,9 +324,14 @@ def test_verify_with_crl
     store.add_cert(ca2_cert)
     store.add_crl(ca1_crl1)
     store.add_crl(ca2_crl2) # issued by ca2 but expired
-    assert_equal(true, store.verify(ca2_cert))
+    if libressl?(3, 2, 2)
+      assert_equal(false, store.verify(ca2_cert))
+      assert_include([OpenSSL::X509::V_ERR_CRL_SIGNATURE_FAILURE, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
+    else
+      assert_equal(true, store.verify(ca2_cert))
+    end
     assert_equal(false, store.verify(ee1_cert))
-    assert_equal(OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED, store.error)
+    assert_include([OpenSSL::X509::V_ERR_CRL_HAS_EXPIRED, OpenSSL::X509::V_ERR_UNSPECIFIED], store.error)
     assert_equal(false, store.verify(ee2_cert))
   end