diff --git a/VERSION b/VERSION index cf43527f..5cdd6f64 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5.30 +2.5.31 diff --git a/doc/CHANGELOG.md b/doc/CHANGELOG.md index 38f370b7..fb45effd 100644 --- a/doc/CHANGELOG.md +++ b/doc/CHANGELOG.md @@ -1,5 +1,17 @@ # CHANGELOG for Mercury +## Version 2.5.31 +* Mercury now outputs `tls.client.certs` and `tls.undetermined.certs` + as well as `tls.server.certs`, for TLS version 1.2 and earlier. + Client and server certificate chains are distinguished by the + handshake type of the key exchange data that follows the + `Certificate` data. If no key exchange data is present, then the + certificate is reported as `tls.undetermined.certs`. +* Timestamp counter reading support for ARM added in + [tsc_clock.hpp](src/libmerc/tsc_clock.hpp). +* If a timestamp of `0` is passed to `libmerc`, a timestamp is + computed in order to improve the reassembly of TCP messages, as needed. + ## Version 2.5.30 * Dramatic improvements to TCP reassembly for output, performance and TCP segments handling. * Improved error handling for malformed UTF-8 strings encountered in protocol fields. diff --git a/src/libmerc/bench.h b/src/libmerc/bench.h index 269f2c5e..2f06e3f3 100644 --- a/src/libmerc/bench.h +++ b/src/libmerc/bench.h @@ -5,46 +5,13 @@ #ifndef BENCH_H #define BENCH_H -// The cycle_counter class will compile anywhere, but will only work -// correctly on platforms that provide a function to read the -// timestamp counter. The following preprocessor conditionals -// identify the appropriate function, if one is present, and set -// benchmark_is_valid to true or false. That value can be accessed -// through the constexpr static boolean benchmark::is_valid. -// -#ifdef HAVE_X86INTRIN_H - #include - #define read_timestamp_counter() __rdtsc() - #define benchmark_is_valid true -#else - #define read_timestamp_counter() 0 - #define benchmark_is_valid false -#endif -// -// TODO: add the corresponding ARM equivalent function - #include // for sqrt() +#include "tsc_clock.hpp" namespace benchmark { - static constexpr bool is_valid = benchmark_is_valid; - - // An object of class cycle_counter counts the number of clock - // cycles between its construction and the invocation of the - // delta() function - // - class cycle_counter { - uint64_t value; - - public: - - cycle_counter() : value{read_timestamp_counter()} { } - - uint64_t delta() const { return read_timestamp_counter() - value; } - - }; + static bool is_valid = tsc_clock::is_valid(); - // An object of class count_and_mean maintains a count and mean of // all of the observed numbers. Each observation is reported with // the member function +=, e.g. 's += x' observes x. // diff --git a/src/libmerc/libmerc.cc b/src/libmerc/libmerc.cc index f7eb8b6c..45c3d585 100644 --- a/src/libmerc/libmerc.cc +++ b/src/libmerc/libmerc.cc @@ -14,6 +14,7 @@ #include "pkt_proc.h" #include "config_generator.h" #include "global_config.h" +#include "tsc_clock.hpp" #ifndef MERCURY_SEMANTIC_VERSION #warning MERCURY_SEMANTIC_VERSION is not defined @@ -89,7 +90,9 @@ mercury_context mercury_init(const struct libmerc_config *vars, int verbosity) { // taking place // assert(printf_err(log_info, "libmerc is running assert() tests\n") != 0); - + // Calculate the cpu ticks per sec during initialization time + // to avoid delay during packet processing path + tsc_clock::init(); try { m = new mercury{vars, verbosity}; return m; diff --git a/src/libmerc/pkt_proc.cc b/src/libmerc/pkt_proc.cc index 6719d1a0..900e3f1a 100644 --- a/src/libmerc/pkt_proc.cc +++ b/src/libmerc/pkt_proc.cc @@ -53,6 +53,7 @@ #include "openvpn.h" #include "mysql.hpp" #include "geneve.hpp" +#include "tsc_clock.hpp" // double malware_prob_threshold = -1.0; // TODO: document hidden option @@ -217,9 +218,11 @@ void stateful_pkt_proc::set_tcp_protocol(protocol &x, break; } case tcp_msg_type_tls_server_hello: - case tcp_msg_type_tls_certificate: x.emplace(pkt, tcp_pkt); break; + case tcp_msg_type_tls_certificate: + x.emplace(pkt, tcp_pkt); + break; case tcp_msg_type_ssh: x.emplace(pkt); break; @@ -508,6 +511,11 @@ size_t stateful_pkt_proc::ip_write_json(void *buffer, } } + if (ts->tv_sec == 0) { + tsc_clock time_now; + ts->tv_sec = time_now.time_in_seconds(); + } + // process transport/application protocols // protocol x; @@ -785,6 +793,12 @@ bool stateful_pkt_proc::analyze_ip_packet(const uint8_t *packet, if (reassembler) { reassembler->dump_pkt = false; } + + if (ts->tv_sec == 0) { + tsc_clock time_now; + ts->tv_sec = time_now.time_in_seconds(); + } + if (transport_proto == ip::protocol::tcp) { tcp_packet tcp_pkt{pkt, &ip_pkt}; if (!tcp_pkt.is_valid()) { diff --git a/src/libmerc/pkt_proc_util.h b/src/libmerc/pkt_proc_util.h index 03d38c8a..28279473 100644 --- a/src/libmerc/pkt_proc_util.h +++ b/src/libmerc/pkt_proc_util.h @@ -38,6 +38,7 @@ struct http_request; // start of tcp protocols struct http_response; struct tls_client_hello; class tls_server_hello_and_certificate; +class tls_certificate; struct ssh_init_packet; struct ssh_kex_init; class smtp_client; @@ -72,6 +73,7 @@ using protocol = std::variantreassembly_needed(certificate.additional_bytes_needed); + } + } + + bool is_not_empty() { + return certificate.is_not_empty(); + } + + void write_json(struct json_object &record, bool metadata_output, bool certs_json_output) { + (void)metadata_output; + + bool have_certificate = certificate.is_not_empty(); + if (have_certificate) { + + // output certificate + // + const char *role = "undetermined"; + if (entity == client) { + role = "client"; + } else if (entity == server) { + role = "server"; + } + struct json_object tls{record, "tls"}; + json_object client_or_server{tls, role}; + struct json_array certs{client_or_server, "certs"}; + certificate.write_json(certs, certs_json_output); + certs.close(); + client_or_server.close(); + tls.close(); + + } + } + +}; + + static uint16_t degrease_uint16(uint16_t x) { switch(x) { case 0x0a0a: diff --git a/src/libmerc/tsc_clock.hpp b/src/libmerc/tsc_clock.hpp new file mode 100644 index 00000000..010a11e7 --- /dev/null +++ b/src/libmerc/tsc_clock.hpp @@ -0,0 +1,119 @@ +/* + * tsc_clock.hpp + * + * Methods to measure time elapsed using CPU ticks + * + * Copyright (c) 2021 Cisco Systems, Inc. All rights reserved. License at + * https://github.com/cisco/mercury/blob/master/LICENSE + */ + +#ifndef TSC_CLOCK_HPP +#define TSC_CLOCK_HPP + +#include +#include +#include + +using namespace std::chrono_literals; + +class tsc_clock +{ + uint64_t start_tick; + +public: + + static uint64_t get_ticks_per_sec() { + if (!is_valid()) { + return 0; + } + static uint64_t ticks_per_second = 0; + /* + * Calculating the number of cpu ticks per second + * by counting the number of cpu ticks in 1/100th of second + * and then upscaling to get the number of cpu ticks per second + */ + + if (ticks_per_second == 0) { + tsc_clock start; + std::this_thread::sleep_for(10ms); + tsc_clock end; + ticks_per_second = (end.get_start_tick() - start.get_start_tick()) * 100; + } + return ticks_per_second; + } + + static void init() { + get_ticks_per_sec(); + } + + tsc_clock() { + start_tick = read_timestamp_counter(); + } + + uint64_t time_in_seconds() const { + if (!is_valid()) { + return 0; + } + + return (std::round(static_cast(get_start_tick()) / get_ticks_per_sec())); + } + + uint64_t elapsed_tick() const { + if (!is_valid()) { + return 0; + } + + return(read_timestamp_counter() - get_start_tick()); + } + + time_t elapsed_time_in_sec() const { + if (!is_valid()) { + return 0; + } + + return (std::round(static_cast(elapsed_tick()) / get_ticks_per_sec())); + } + + uint64_t get_start_tick() const { + return start_tick; + } + + static inline bool is_valid() { + static bool tsc_counter = read_timestamp_counter(); + if (tsc_counter == 0) { + return false; + } + + return true; + } + + static inline uint64_t read_timestamp_counter() { +#if defined(__i386__) || defined(__x86_64__) + uint32_t lo, hi; + asm volatile("rdtsc" : "=a" (lo), "=d" (hi)); + return ((uint64_t)hi << 32) | lo; +#elif defined(__aarch64__) + uint64_t ticks; + asm volatile("mrs %0, CNTVCT_EL0" : "=r" (ticks)); + return ticks; +#else + return 0; +#endif + } + + static bool unit_test() { + tsc_clock start; + // Pass this test for unsupported platform + if (!is_valid()) { + return true; + } + std::this_thread::sleep_for(1s); + uint64_t elapsed_time = start.elapsed_time_in_sec(); + if (elapsed_time == 1) { + return true; + } + return false; + } +}; + +#endif //TSC_CLOCK_HPP diff --git a/src/pcap_file_io.c b/src/pcap_file_io.c index 127e6826..d8cda5f6 100644 --- a/src/pcap_file_io.c +++ b/src/pcap_file_io.c @@ -484,9 +484,9 @@ enum status pcap_file_dispatch_pkt_processor(struct pcap_file *f, packet_info_init_from_pkthdr(&pi, &pkthdr); pi.linktype = f->linktype; // process the packet that was read - benchmark::cycle_counter cc; + tsc_clock cc; pkt_processor->apply(&pi, packet_data); - s += cc.delta(); + s += cc.elapsed_tick(); num_packets++; total_length += pkthdr.caplen + sizeof(struct pcap_packet_hdr); } diff --git a/unit_tests/functional_unit_test.cc b/unit_tests/functional_unit_test.cc index 97594864..000e9f15 100644 --- a/unit_tests/functional_unit_test.cc +++ b/unit_tests/functional_unit_test.cc @@ -11,6 +11,7 @@ #include "snmp.h" #include "tofsee.hpp" #include "utf8.hpp" +#include "tsc_clock.hpp" /* * The unit_test() functions defined in header files @@ -24,4 +25,5 @@ TEST_CASE("Testing unit_test() defined in class") { CHECK(tofsee_initial_message::unit_test() == true); CHECK(tls_extensions::unit_test() == true); CHECK(utf8_string::unit_test() == true); + CHECK(tsc_clock::unit_test() == true); }