netstacklat: Exclude TCP reads for HOL blocked segments

The 'tcp-socket-read' currently reports the latency for the skb
containing the last TCP segment read from the socket. However, this
segment might have been head of line (HOL) blocked by a previous
segment missing. In this case, netstacklat's reported latency will
include HOL blocking periods that is dependent on external
factors (such as network packet loss, and network latency impacts
retransmission time). As netstacklat is primarily intended to identify
issues within the local host (in the network stack or receiving
applications), by default filter out any socket reads were the last
read SKB might have experienced HOL-blocking.

Add the new -y/--include-tcp-hol-delay option to retain the old
behavior of reporting latency for all reads, including those that are
HOL-blocked. This may be useful in some scenarios when you still want
to be aware of latency issues caused by HOL-blocking, even though it
is caused by external components. For example, in a data center
context were you have full control over the network, it may still be
relevant to monitor HOL-based caused by the network.

To exclude HOL-blocked reads, detect if any new ooo-segments have
arrived by checking for differences in the number of ooo-packets in
tcp_sock->rcv_ooopack. If any new ooo-segments have arrived, exclude
the latency sample from the current read and set a limit for the next
safe sequence number to read where the current ooo-packets must have
been passed so segments can no longer be HOL-blocked. If there are
skbs in the ooo-queue, set the limit to the end of the
ooo-queue. Otherwise, set the limit to the current rcv_nxt (as if the
ooo-queue is empty the detected ooo-segments must already have been
merged into the receive queue and rcv_nxt must have advanced past
them). If the read is past the safe sequence limit and no new
ooo-segments have arrived, it's safe to start including the latency
samples again.

For sockets were some ooo-segments have been observed, keep the
ooo-range state in socket storage (BPF_MAP_TYPE_SK_STORAGE). Skip
protecting this state with a spin-lock, as it should only be
concurrently accessed if there are concurrent reads on the same TCP
socket, which is assumed to be very rare as applications attempting
that cannot know which part of the data each of their concurrent reads
will get.

There are some scenarios that may cause this ooo-filtering to fail.
- If multiple reads are done to the socket concurrently, we may not
  correctly track the last read byte. The kernel does not keep a lock
  on the TCP socket at the time our hooked function
  tcp_recv_timestamp() runs. If two reads are done in parallel, it's
  therefore possible that for both reads we will check the last read
  byte (tcp_sock.copied_seq) after the second read has updated it. We
  may then incorrectly conclude that the first read was ahead of the
  ooo-range when it was not, and record its latency when we should
  have excluded it. In practice I belive this issue should be quite
  rare, as most applications will probably not attempt to perform
  multiple concurrent reads to a single connected TCP socket in
  parallel (as then you cannot know which part of the payload the
  parallel reads will return).

- As tcp_recv_timestamp() runs outside of the socket lock, the various
  state members we access may concurrently be updated as we're
  attempting to read them. An especially problematic one is
  tcp_sock.ooo_last_skb, which keeps a pointer to an SKB that is only
  valid while the ooo-queue is non-empty. It is possible that between
  our check for if the ooo-queue is non-empty and following the
  ooo_last_skb pointer, the ooo-queue is cleared and the ooo_last_skb
  pointer may end up pointing towards a freed SKB. If the socket
  members we access are updated before or while we read them, it can
  break the filtering in numerous ways, e.g. result in includes
  samples that should have been excluded (due to e.g. copied_seq being
  updated before our read) or excluding a large amount of valid
  samples (due to e.g. setting a sequence limit based on garbage in a
  freed SKB).

Signed-off-by: Simon Sundberg <[email protected]>

Author: simosund

headers/vmlinux/vmlinux_common.h

@@ -13,6 +13,10 @@ struct list_head {
 	struct list_head *prev;
 };
 
+struct rb_root {
+	struct rb_node *rb_node;
+};
+
 struct rb_node {
 	long unsigned int __rb_parent_color;
 	struct rb_node *rb_right;

headers/vmlinux/vmlinux_net.h

@@ -161,6 +161,35 @@ struct sk_buff {
 	struct skb_ext *extensions;
 };
 
+struct tcp_skb_cb {
+	__u32 seq;
+	__u32 end_seq;
+	union {
+		struct {
+			u16 tcp_gso_segs;
+			u16 tcp_gso_size;
+		};
+	};
+	__u8 tcp_flags;
+	__u8 sacked;
+	__u8 ip_dsfield;
+	__u8 txstamp_ack : 1;
+	__u8 eor : 1;
+	__u8 has_rxtstamp : 1;
+	__u8 unused : 5;
+	__u32 ack_seq;
+	union {
+		struct {
+			__u32 is_app_limited : 1;
+			__u32 delivered_ce : 20;
+			__u32 unused : 11;
+			__u32 delivered;
+			u64 first_tx_mstamp;
+			u64 delivered_mstamp;
+		} tx;
+	};
+};
+
 struct nf_conn {
 	unsigned long status;
 };
@@ -202,4 +231,183 @@ struct sock {
 	u32 sk_rx_dst_cookie;
 };
 
+struct inet_sock {
+	struct sock sk;
+};
+
+struct inet_connection_sock {
+	struct inet_sock icsk_inet;
+};
+
+struct tcp_sock {
+	struct inet_connection_sock inet_conn;
+	__u8 __cacheline_group_begin__tcp_sock_read_tx[0];
+	u32 max_window;
+	u32 rcv_ssthresh;
+	u32 reordering;
+	u32 notsent_lowat;
+	u16 gso_segs;
+	struct sk_buff *lost_skb_hint;
+	struct sk_buff *retransmit_skb_hint;
+	__u8 __cacheline_group_end__tcp_sock_read_tx[0];
+	__u8 __cacheline_group_begin__tcp_sock_read_txrx[0];
+	u32 tsoffset;
+	u32 snd_wnd;
+	u32 mss_cache;
+	u32 snd_cwnd;
+	u32 prr_out;
+	u32 lost_out;
+	u32 sacked_out;
+	u16 tcp_header_len;
+	u8 scaling_ratio;
+	u8 chrono_type: 2;
+	u8 repair: 1;
+	u8 tcp_usec_ts: 1;
+	u8 is_sack_reneg: 1;
+	u8 is_cwnd_limited: 1;
+	__u8 __cacheline_group_end__tcp_sock_read_txrx[0];
+	__u8 __cacheline_group_begin__tcp_sock_read_rx[0];
+	u32 copied_seq;
+	u32 rcv_tstamp;
+	u32 snd_wl1;
+	u32 tlp_high_seq;
+	u32 rttvar_us;
+	u32 retrans_out;
+	u16 advmss;
+	u16 urg_data;
+	u32 lost;
+	/* struct minmax rtt_min; */
+	struct rb_root out_of_order_queue;
+	u32 snd_ssthresh;
+	u8 recvmsg_inq: 1;
+	__u8 __cacheline_group_end__tcp_sock_read_rx[0];
+	long: 0;
+	__u8 __cacheline_group_begin__tcp_sock_write_tx[0];
+	u32 segs_out;
+	u32 data_segs_out;
+	u64 bytes_sent;
+	u32 snd_sml;
+	u32 chrono_start;
+	u32 chrono_stat[3];
+	u32 write_seq;
+	u32 pushed_seq;
+	u32 lsndtime;
+	u32 mdev_us;
+	u32 rtt_seq;
+	u64 tcp_wstamp_ns;
+	struct list_head tsorted_sent_queue;
+	struct sk_buff *highest_sack;
+	u8 ecn_flags;
+	__u8 __cacheline_group_end__tcp_sock_write_tx[0];
+	__u8 __cacheline_group_begin__tcp_sock_write_txrx[0];
+	__be32 pred_flags;
+	u64 tcp_clock_cache;
+	u64 tcp_mstamp;
+	u32 rcv_nxt;
+	u32 snd_nxt;
+	u32 snd_una;
+	u32 window_clamp;
+	u32 srtt_us;
+	u32 packets_out;
+	u32 snd_up;
+	u32 delivered;
+	u32 delivered_ce;
+	u32 app_limited;
+	u32 rcv_wnd;
+	/* struct tcp_options_received rx_opt; */
+	u8 nonagle: 4;
+	u8 rate_app_limited: 1;
+	__u8 __cacheline_group_end__tcp_sock_write_txrx[0];
+	long: 0;
+	__u8 __cacheline_group_begin__tcp_sock_write_rx[0];
+	u64 bytes_received;
+	u32 segs_in;
+	u32 data_segs_in;
+	u32 rcv_wup;
+	u32 max_packets_out;
+	u32 cwnd_usage_seq;
+	u32 rate_delivered;
+	u32 rate_interval_us;
+	u32 rcv_rtt_last_tsecr;
+	u64 first_tx_mstamp;
+	u64 delivered_mstamp;
+	u64 bytes_acked;
+	struct {
+		u32 rtt_us;
+		u32 seq;
+		u64 time;
+	} rcv_rtt_est;
+	struct {
+		u32 space;
+		u32 seq;
+		u64 time;
+	} rcvq_space;
+	__u8 __cacheline_group_end__tcp_sock_write_rx[0];
+	u32 dsack_dups;
+	u32 compressed_ack_rcv_nxt;
+	struct list_head tsq_node;
+	/* struct tcp_rack rack; */
+	u8 compressed_ack;
+	u8 dup_ack_counter: 2;
+	u8 tlp_retrans: 1;
+	u8 unused: 5;
+	u8 thin_lto: 1;
+	u8 fastopen_connect: 1;
+	u8 fastopen_no_cookie: 1;
+	u8 fastopen_client_fail: 2;
+	u8 frto: 1;
+	u8 repair_queue;
+	u8 save_syn: 2;
+	u8 syn_data: 1;
+	u8 syn_fastopen: 1;
+	u8 syn_fastopen_exp: 1;
+	u8 syn_fastopen_ch: 1;
+	u8 syn_data_acked: 1;
+	u8 keepalive_probes;
+	u32 tcp_tx_delay;
+	u32 mdev_max_us;
+	u32 reord_seen;
+	u32 snd_cwnd_cnt;
+	u32 snd_cwnd_clamp;
+	u32 snd_cwnd_used;
+	u32 snd_cwnd_stamp;
+	u32 prior_cwnd;
+	u32 prr_delivered;
+	u32 last_oow_ack_time;
+	/* struct hrtimer pacing_timer; */
+	/* struct hrtimer compressed_ack_timer; */
+	struct sk_buff *ooo_last_skb;
+	/* struct tcp_sack_block duplicate_sack[1]; */
+	/* struct tcp_sack_block selective_acks[4]; */
+	/* struct tcp_sack_block recv_sack_cache[4]; */
+	int lost_cnt_hint;
+	u32 prior_ssthresh;
+	u32 high_seq;
+	u32 retrans_stamp;
+	u32 undo_marker;
+	int undo_retrans;
+	u64 bytes_retrans;
+	u32 total_retrans;
+	u32 rto_stamp;
+	u16 total_rto;
+	u16 total_rto_recoveries;
+	u32 total_rto_time;
+	u32 urg_seq;
+	unsigned int keepalive_time;
+	unsigned int keepalive_intvl;
+	int linger2;
+	u8 bpf_sock_ops_cb_flags;
+	u8 bpf_chg_cc_inprogress: 1;
+	u16 timeout_rehash;
+	u32 rcv_ooopack;
+	struct {
+		u32 probe_seq_start;
+		u32 probe_seq_end;
+	} mtu_probe;
+	u32 plb_rehash;
+	u32 mtu_info;
+	bool is_mptcp;
+};
+
+
 #endif /* __VMLINUX_NET_H__ */