Skip to content

tests/runner.rs: use pretty-assertions crate #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 22, 2025
Merged

Conversation

ctz
Copy link
Member

@ctz ctz commented May 22, 2025

The workflow here involves looking for differences in output from the test programs between us and openssl. That is not very friendly because rust's default Debug format escapes newlines. Add a newtype for Output which doesn't do that.

This is before and after for artificial test failures:

Before:


thread 'ciphers' panicked at tests/runner.rs:317:5:
assertion `left == right` failed
  left: Output { status: ExitStatus(unix_wait_status(0)), stdout: "Found for 0xc02b\nopenssl_id=0x0300c02b protocol_id=0x0000c02b auth=1047 kx=1038 bits=128 alg_bits=128\nname='ECDHE-ECDSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'\ndesc='ECDHE-ECDSA-AES128-GCM-SHA256  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0xc02c\nopenssl_id=0x0300c02c protocol_id=0x0000c02c auth=1047 kx=1038 bits=256 alg_bits=256\nname='ECDHE-ECDSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'\ndesc='ECDHE-ECDSA-AES256-GCM-SHA384  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0xcca9\nopenssl_id=0x0300cca9 protocol_id=0x0000cca9 auth=1047 kx=1038 bits=256 alg_bits=256\nname='ECDHE-ECDSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'\ndesc='ECDHE-ECDSA-CHACHA20-POLY1305  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nFound for 0xc02f\nopenssl_id=0x0300c02f protocol_id=0x0000c02f auth=1046 kx=1038 bits=128 alg_bits=128\nname='ECDHE-RSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'\ndesc='ECDHE-RSA-AES128-GCM-SHA256    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0xc030\nopenssl_id=0x0300c030 protocol_id=0x0000c030 auth=1046 kx=1038 bits=256 alg_bits=256\nname='ECDHE-RSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'\ndesc='ECDHE-RSA-AES256-GCM-SHA384    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0xcca8\nopenssl_id=0x0300cca8 protocol_id=0x0000cca8 auth=1046 kx=1038 bits=256 alg_bits=256\nname='ECDHE-RSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'\ndesc='ECDHE-RSA-CHACHA20-POLY1305    TLSv1.2 Kx=ECDH     Au=RSA   Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nFound for 0x1301\nopenssl_id=0x03001301 protocol_id=0x00001301 auth=1064 kx=1063 bits=128 alg_bits=128\nname='TLS_AES_128_GCM_SHA256' standard_name='TLS_AES_128_GCM_SHA256' version='TLSv1.3'\ndesc='TLS_AES_128_GCM_SHA256         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0x1302\nopenssl_id=0x03001302 protocol_id=0x00001302 auth=1064 kx=1063 bits=256 alg_bits=256\nname='TLS_AES_256_GCM_SHA384' standard_name='TLS_AES_256_GCM_SHA384' version='TLSv1.3'\ndesc='TLS_AES_256_GCM_SHA384         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0x1303\nopenssl_id=0x03001303 protocol_id=0x00001303 auth=1064 kx=1063 bits=256 alg_bits=256\nname='TLS_CHACHA20_POLY1305_SHA256' standard_name='TLS_CHACHA20_POLY1305_SHA256' version='TLSv1.3'\ndesc='TLS_CHACHA20_POLY1305_SHA256   TLSv1.3 Kx=any      Au=any   Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nopenssl_id=undef protocol_id=undef auth=undef kx=undef bits=0 alg_bits=-1\nname='(NONE)' standard_name='(NONE)' version='(NONE)'\ndesc=undef\n", stderr: "" }
 right: Output { status: ExitStatus(unix_wait_status(0)), stdout: "Found for 0xc02b\nopenssl_id=0x0300c02b protocol_id=0x0000c02b auth=1047 kx=9999 bits=128 alg_bits=128\nname='ECDHE-ECDSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'\ndesc='ECDHE-ECDSA-AES128-GCM-SHA256  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0xc02c\nopenssl_id=0x0300c02c protocol_id=0x0000c02c auth=1047 kx=9999 bits=256 alg_bits=256\nname='ECDHE-ECDSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'\ndesc='ECDHE-ECDSA-AES256-GCM-SHA384  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0xcca9\nopenssl_id=0x0300cca9 protocol_id=0x0000cca9 auth=1047 kx=9999 bits=256 alg_bits=256\nname='ECDHE-ECDSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'\ndesc='ECDHE-ECDSA-CHACHA20-POLY1305  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nFound for 0xc02f\nopenssl_id=0x0300c02f protocol_id=0x0000c02f auth=1046 kx=9999 bits=128 alg_bits=128\nname='ECDHE-RSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'\ndesc='ECDHE-RSA-AES128-GCM-SHA256    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0xc030\nopenssl_id=0x0300c030 protocol_id=0x0000c030 auth=1046 kx=9999 bits=256 alg_bits=256\nname='ECDHE-RSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'\ndesc='ECDHE-RSA-AES256-GCM-SHA384    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0xcca8\nopenssl_id=0x0300cca8 protocol_id=0x0000cca8 auth=1046 kx=9999 bits=256 alg_bits=256\nname='ECDHE-RSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'\ndesc='ECDHE-RSA-CHACHA20-POLY1305    TLSv1.2 Kx=ECDH     Au=RSA   Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nFound for 0x1301\nopenssl_id=0x03001301 protocol_id=0x00001301 auth=1064 kx=1063 bits=128 alg_bits=128\nname='TLS_AES_128_GCM_SHA256' standard_name='TLS_AES_128_GCM_SHA256' version='TLSv1.3'\ndesc='TLS_AES_128_GCM_SHA256         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(128)            Mac=AEAD\n'\nFound for 0x1302\nopenssl_id=0x03001302 protocol_id=0x00001302 auth=1064 kx=1063 bits=256 alg_bits=256\nname='TLS_AES_256_GCM_SHA384' standard_name='TLS_AES_256_GCM_SHA384' version='TLSv1.3'\ndesc='TLS_AES_256_GCM_SHA384         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(256)            Mac=AEAD\n'\nFound for 0x1303\nopenssl_id=0x03001303 protocol_id=0x00001303 auth=1064 kx=1063 bits=256 alg_bits=256\nname='TLS_CHACHA20_POLY1305_SHA256' standard_name='TLS_CHACHA20_POLY1305_SHA256' version='TLSv1.3'\ndesc='TLS_CHACHA20_POLY1305_SHA256   TLSv1.3 Kx=any      Au=any   Enc=CHACHA20/POLY1305(256) Mac=AEAD\n'\nopenssl_id=undef protocol_id=undef auth=undef kx=undef bits=0 alg_bits=-1\nname='(NONE)' standard_name='(NONE)' version='(NONE)'\ndesc=undef\n", stderr: "" }
note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

After:

thread 'ciphers' panicked at tests/runner.rs:319:5:
assertion failed: `(left == right)`

Diff < left / right > :
 Output {
     status: ExitStatus(
         unix_wait_status(
             0,
         ),
     ),
     stdout: "<follows>",
     stderr: "<follows>",
 }
 stdout => Found for 0xc02b
<stdout => openssl_id=0x0300c02b protocol_id=0x0000c02b auth=1047 kx=1038 bits=128 alg_bits=128
>stdout => openssl_id=0x0300c02b protocol_id=0x0000c02b auth=1047 kx=9999 bits=128 alg_bits=128
 stdout => name='ECDHE-ECDSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'
 stdout => desc='ECDHE-ECDSA-AES128-GCM-SHA256  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(128)            Mac=AEAD
 stdout => '
 stdout => Found for 0xc02c
<stdout => openssl_id=0x0300c02c protocol_id=0x0000c02c auth=1047 kx=1038 bits=256 alg_bits=256
>stdout => openssl_id=0x0300c02c protocol_id=0x0000c02c auth=1047 kx=9999 bits=256 alg_bits=256
 stdout => name='ECDHE-ECDSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'
 stdout => desc='ECDHE-ECDSA-AES256-GCM-SHA384  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=AESGCM(256)            Mac=AEAD
 stdout => '
 stdout => Found for 0xcca9
<stdout => openssl_id=0x0300cca9 protocol_id=0x0000cca9 auth=1047 kx=1038 bits=256 alg_bits=256
>stdout => openssl_id=0x0300cca9 protocol_id=0x0000cca9 auth=1047 kx=9999 bits=256 alg_bits=256
 stdout => name='ECDHE-ECDSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'
 stdout => desc='ECDHE-ECDSA-CHACHA20-POLY1305  TLSv1.2 Kx=ECDH     Au=ECDSA Enc=CHACHA20/POLY1305(256) Mac=AEAD
 stdout => '
 stdout => Found for 0xc02f
<stdout => openssl_id=0x0300c02f protocol_id=0x0000c02f auth=1046 kx=1038 bits=128 alg_bits=128
>stdout => openssl_id=0x0300c02f protocol_id=0x0000c02f auth=1046 kx=9999 bits=128 alg_bits=128
 stdout => name='ECDHE-RSA-AES128-GCM-SHA256' standard_name='TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256' version='TLSv1.2'
 stdout => desc='ECDHE-RSA-AES128-GCM-SHA256    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(128)            Mac=AEAD
 stdout => '
 stdout => Found for 0xc030
<stdout => openssl_id=0x0300c030 protocol_id=0x0000c030 auth=1046 kx=1038 bits=256 alg_bits=256
>stdout => openssl_id=0x0300c030 protocol_id=0x0000c030 auth=1046 kx=9999 bits=256 alg_bits=256
 stdout => name='ECDHE-RSA-AES256-GCM-SHA384' standard_name='TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384' version='TLSv1.2'
 stdout => desc='ECDHE-RSA-AES256-GCM-SHA384    TLSv1.2 Kx=ECDH     Au=RSA   Enc=AESGCM(256)            Mac=AEAD
 stdout => '
 stdout => Found for 0xcca8
<stdout => openssl_id=0x0300cca8 protocol_id=0x0000cca8 auth=1046 kx=1038 bits=256 alg_bits=256
>stdout => openssl_id=0x0300cca8 protocol_id=0x0000cca8 auth=1046 kx=9999 bits=256 alg_bits=256
 stdout => name='ECDHE-RSA-CHACHA20-POLY1305' standard_name='TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256' version='TLSv1.2'
 stdout => desc='ECDHE-RSA-CHACHA20-POLY1305    TLSv1.2 Kx=ECDH     Au=RSA   Enc=CHACHA20/POLY1305(256) Mac=AEAD
 stdout => '
 stdout => Found for 0x1301
 stdout => openssl_id=0x03001301 protocol_id=0x00001301 auth=1064 kx=1063 bits=128 alg_bits=128
 stdout => name='TLS_AES_128_GCM_SHA256' standard_name='TLS_AES_128_GCM_SHA256' version='TLSv1.3'
 stdout => desc='TLS_AES_128_GCM_SHA256         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(128)            Mac=AEAD
 stdout => '
 stdout => Found for 0x1302
 stdout => openssl_id=0x03001302 protocol_id=0x00001302 auth=1064 kx=1063 bits=256 alg_bits=256
 stdout => name='TLS_AES_256_GCM_SHA384' standard_name='TLS_AES_256_GCM_SHA384' version='TLSv1.3'
 stdout => desc='TLS_AES_256_GCM_SHA384         TLSv1.3 Kx=any      Au=any   Enc=AESGCM(256)            Mac=AEAD
 stdout => '
 stdout => Found for 0x1303
 stdout => openssl_id=0x03001303 protocol_id=0x00001303 auth=1064 kx=1063 bits=256 alg_bits=256
 stdout => name='TLS_CHACHA20_POLY1305_SHA256' standard_name='TLS_CHACHA20_POLY1305_SHA256' version='TLSv1.3'
 stdout => desc='TLS_CHACHA20_POLY1305_SHA256   TLSv1.3 Kx=any      Au=any   Enc=CHACHA20/POLY1305(256) Mac=AEAD
 stdout => '
 stdout => openssl_id=undef protocol_id=undef auth=undef kx=undef bits=0 alg_bits=-1
 stdout => name='(NONE)' standard_name='(NONE)' version='(NONE)'
 stdout => desc=undef



note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

It's also colour highlighted --

image

The workflow here involves looking for differences in output from
the test programs between us and openssl.  That is not very
friendly because rust's default Debug format escapes newlines.  Add
a newtype for `Output` which doesn't do that.
Copy link
Member

@cpu cpu left a comment

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's a really nice quality of life improvement 🤩

@ctz ctz merged commit 21c3aa5 into main May 22, 2025
16 checks passed
@ctz ctz deleted the jbp-pretty-assertions branch May 22, 2025 21:50
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

Successfully merging this pull request may close these issues.

2 participants