avformat/tls_openssl: fix serial create

Signed-off-by: Jack Lau <[email protected]>

Author: JackLau1222

libavformat/tls_openssl.c

@@ -314,7 +314,8 @@ static int openssl_gen_private_key(EVP_PKEY **pkey)
 
 static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerprint)
 {
-    int ret = 0, serial, expire_day;
+    int ret = 0, expire_day;
+    uint64_t serial;
     const char *aor = "lavf";
     X509_NAME* subject = NULL;
 
@@ -329,9 +330,8 @@ static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerpri
         goto enomem_end;
     }
 
-    // According to RFC5280 4.1.2.2, The serial number MUST be a positive integer
-    serial = (int)(av_get_random_seed() & 0x7FFFFFFF);
-    if (ASN1_INTEGER_set(X509_get_serialNumber(*cert), serial) != 1) {
+    serial = av_get_random_seed();
+    if (ASN1_INTEGER_set_uint64(X509_get_serialNumber(*cert), serial) != 1) {
         av_log(NULL, AV_LOG_ERROR, "TLS: Failed to set serial, %s\n", ERR_error_string(ERR_get_error(), NULL));
         goto einval_end;
     }