Collaplse skbs in socket write queue

- According our investigation collapsing skb in socket
write queue significantly increases perfomance for small
responses.
- Fix `tcp_can_coalesce_send_queue_head`. We should make
Tempesta FW checks before checking `len <= skb->len`.
- We should also check `SKBTX_SHARED_FRAG` flag because
collapsing skbs with defferent value of this flag is
prohibited.

Author: EvgeniiMekhanik

linux-5.10.35.patch

@@ -2550,7 +2550,7 @@ index f0f67b25c..58fbfb071 100644
  		return NULL;
  	}
 diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
-index f99494637..042f9e38d 100644
+index f99494637..522c265a5 100644
 --- a/net/ipv4/tcp_output.c
 +++ b/net/ipv4/tcp_output.c
 @@ -39,6 +39,9 @@
@@ -2744,27 +2744,108 @@ index f99494637..042f9e38d 100644
 
  	/* Correct the sequence numbers. */
  	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
-@@ -2303,6 +2349,14 @@ static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
+@@ -2292,24 +2338,186 @@ static inline void tcp_mtu_check_reprobe(struct sock *sk)
+ 	}
+ }
 
- 		if (unlikely(TCP_SKB_CB(skb)->eor) || tcp_has_tx_tstamp(skb))
- 			return false;
++#ifdef CONFIG_SECURITY_TEMPESTA
++
++static bool tfw_tcp_skb_can_collapse(struct sock *sk, struct sk_buff *skb,
++				     struct sk_buff *next)
++{
++	BUG_ON(!sock_flag(sk, SOCK_TEMPESTA));
++
++	if (!tcp_skb_is_last(sk, skb)
++	    && ((skb_tfw_tls_type(skb) != skb_tfw_tls_type(next))
++		|| (skb->mark != next->mark)
++		|| (((skb_shinfo(skb)->tx_flags & SKBTX_SHARED_FRAG)
++		    != (skb_shinfo(next)->tx_flags & SKBTX_SHARED_FRAG)))))
++		return false;
++	return true;
++}
++
++#endif
++
+ static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
+ {
+ 	struct sk_buff *skb, *next;
+ 
+ 	skb = tcp_send_head(sk);
+ 	tcp_for_write_queue_from_safe(skb, next, sk) {
 +#ifdef CONFIG_SECURITY_TEMPESTA
 +		/* Do not coalesce tempesta skbs with tls type or set mark. */
-+		if ((next != ((struct sk_buff *)&(sk)->sk_write_queue))
-+		    && ((skb_tfw_tls_type(skb) != skb_tfw_tls_type(next))
-+			|| (sock_flag(sk, SOCK_TEMPESTA)
-+			    && (skb->mark != next->mark))))
++		if (sock_flag(sk, SOCK_TEMPESTA)
++		    && !tfw_tcp_skb_can_collapse(sk, skb, next))
 +			return false;
 +#endif
+ 		if (len <= skb->len)
+ 			break;
 
+ 		if (unlikely(TCP_SKB_CB(skb)->eor) || tcp_has_tx_tstamp(skb))
+ 			return false;
+-
  		len -= skb->len;
  	}
-@@ -2310,6 +2364,76 @@ static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
+ 
  	return true;
  }
 
 +#ifdef CONFIG_SECURITY_TEMPESTA
 +
++/* First skb in the write queue is smaller than ideal packet size.
++ * Check if we can move payload from the second skb in the queue.
++ */
++static unsigned int tfw_tcp_grow_skb(struct sock *sk, struct sk_buff *skb,
++				     unsigned int amount)
++{
++	struct sk_buff *next = skb->next;
++	unsigned int nlen;
++	bool stolen;
++	int delta;
++
++	if (tcp_skb_is_last(sk, skb))
++		return 0;
++
++	if (!tfw_tcp_skb_can_collapse(sk, skb, next))
++		return 0;
++
++	if (!tcp_skb_can_collapse(skb, next))
++		return 0;
++
++	nlen = next->len;
++	BUG_ON(!nlen);
++
++	if (amount < nlen)
++		return 0;
++
++	if (amount > nlen
++	    && (unlikely(TCP_SKB_CB(next)->eor) || tcp_has_tx_tstamp(next)))
++		return 0;
++
++	if (!skb_try_coalesce(skb, next, &stolen, &delta))
++		return 0;
++
++	TCP_SKB_CB(skb)->end_seq += nlen;
++	TCP_SKB_CB(next)->seq += nlen;
++
++	if (TCP_SKB_CB(next)->tcp_flags &TCPHDR_FIN)
++		TCP_SKB_CB(skb)->end_seq++;
++	/* We've eaten all the data from this skb.
++	* Throw it away. */
++	TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next)->tcp_flags;
++	/* If this is the last SKB we copy and eor is set
++	 * we need to propagate it to the new skb.
++	 */
++	TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next)->eor;
++	tcp_skb_collapse_tstamp(skb, next);
++	tcp_unlink_write_queue(next, sk);
++	sk_wmem_queued_add(sk, delta - next->truesize);
++	sk_mem_charge(sk, delta - next->truesize);
++	kfree_skb_partial(next, stolen);
++
++	return nlen;
++}
++
 +/**
 + * The next funtion is called from places: from `tcp_write_xmit`
 + * (a usual case) and from `tcp_write_wakeup`. In other places where
@@ -2831,12 +2912,27 @@ index f99494637..042f9e38d 100644
 +		max_size -= TLS_MAX_OVERHEAD;			\
 +} while(0)
 +
++static void tfw_coalesce_send_queue(struct sock *sk, struct sk_buff *skb,
++				    unsigned int amount, unsigned int mss_now)
++{
++	unsigned int nlen;
++
++	if (!sock_flag(sk, SOCK_TEMPESTA))
++		return;
++
++	amount = amount > skb->len ? amount - skb->len : 0;
++	while (amount && (nlen = tfw_tcp_grow_skb(sk, skb, amount)))
++		amount -= nlen;
++
++	tcp_set_skb_tso_segs(skb, mss_now);
++}
++
 +#endif
 +
  /* Create a new MTU probe if we are ready.
   * MTU probe is regularly attempting to increase the path MTU by
   * deliberately sending larger packets.  This discovers routing
-@@ -2330,6 +2454,9 @@ static int tcp_mtu_probe(struct sock *sk)
+@@ -2330,6 +2538,9 @@ static int tcp_mtu_probe(struct sock *sk)
  	int copy, len;
  	int mss_now;
  	int interval;
@@ -2846,15 +2942,15 @@ index f99494637..042f9e38d 100644
 
  	/* Not currently probing/verifying,
  	 * not in recovery,
-@@ -2382,6 +2509,7 @@ static int tcp_mtu_probe(struct sock *sk)
+@@ -2382,6 +2593,7 @@ static int tcp_mtu_probe(struct sock *sk)
  			return 0;
  	}
 
 +	TFW_ADJUST_TLS_OVERHEAD(probe_size);
  	if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
  		return -1;
 
-@@ -2402,6 +2530,10 @@ static int tcp_mtu_probe(struct sock *sk)
+@@ -2402,6 +2614,10 @@ static int tcp_mtu_probe(struct sock *sk)
  	nskb->csum = 0;
  	nskb->ip_summed = CHECKSUM_PARTIAL;
 
@@ -2865,7 +2961,7 @@ index f99494637..042f9e38d 100644
  	tcp_insert_write_queue_before(nskb, skb, sk);
  	tcp_highest_sack_replace(sk, skb, nskb);
 
-@@ -2440,6 +2572,24 @@ static int tcp_mtu_probe(struct sock *sk)
+@@ -2440,6 +2656,24 @@ static int tcp_mtu_probe(struct sock *sk)
  	}
  	tcp_init_tso_segs(nskb, nskb->len);
 
@@ -2890,7 +2986,7 @@ index f99494637..042f9e38d 100644
  	/* We're ready to send.  If this fails, the probe will
  	 * be resegmented into mss-sized pieces by tcp_write_xmit().
  	 */
-@@ -2666,7 +2816,17 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
+@@ -2666,11 +2900,23 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
  							  cwnd_quota,
  							  max_segs),
  						    nonagle);
@@ -2909,7 +3005,13 @@ index f99494637..042f9e38d 100644
  		if (skb->len > limit &&
  		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
  			break;
-@@ -2681,7 +2841,13 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
+ 
++		tfw_coalesce_send_queue(sk, skb, limit, mss_now);
++
+ 		if (tcp_small_queue_check(sk, skb, 0))
+ 			break;
+ 
+@@ -2681,7 +2927,13 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
  		 */
  		if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
  			break;
@@ -2924,15 +3026,15 @@ index f99494637..042f9e38d 100644
  		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
  			break;
 
-@@ -2866,6 +3032,7 @@ void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
+@@ -2866,6 +3118,7 @@ void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
  			   sk_gfp_mask(sk, GFP_ATOMIC)))
  		tcp_check_probe_timer(sk);
  }
 +EXPORT_SYMBOL(__tcp_push_pending_frames);
 
  /* Send _single_ skb sitting at the send head. This function requires
   * true push pending frames to setup probe timer etc.
-@@ -3183,7 +3350,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
+@@ -3183,7 +3436,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
  				 cur_mss, GFP_ATOMIC))
  			return -ENOMEM; /* We'll try again later. */
  	} else {
@@ -2941,31 +3043,31 @@ index f99494637..042f9e38d 100644
  			return -ENOMEM;
 
  		diff = tcp_skb_pcount(skb);
-@@ -3374,6 +3541,7 @@ void sk_forced_mem_schedule(struct sock *sk, int size)
+@@ -3374,6 +3627,7 @@ void sk_forced_mem_schedule(struct sock *sk, int size)
  	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
  		mem_cgroup_charge_skmem(sk->sk_memcg, amt);
  }
 +EXPORT_SYMBOL(sk_forced_mem_schedule);
 
  /* Send a FIN. The caller locks the socket for us.
   * We should try to send a FIN packet really hard, but eventually give up.
-@@ -3421,6 +3589,7 @@ void tcp_send_fin(struct sock *sk)
+@@ -3421,6 +3675,7 @@ void tcp_send_fin(struct sock *sk)
  	}
  	__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
  }
 +EXPORT_SYMBOL(tcp_send_fin);
 
  /* We get here when a process closes a file descriptor (either due to
   * an explicit close() or as a byproduct of exit()'ing) and there
-@@ -3454,6 +3623,7 @@ void tcp_send_active_reset(struct sock *sk, gfp_t priority)
+@@ -3454,6 +3709,7 @@ void tcp_send_active_reset(struct sock *sk, gfp_t priority)
  	 */
  	trace_tcp_send_reset(sk, NULL);
  }
 +EXPORT_SYMBOL(tcp_send_active_reset);
 
  /* Send a crossed SYN-ACK during socket establishment.
   * WARNING: This routine must only be called when we have already sent
-@@ -4044,6 +4214,17 @@ int tcp_write_wakeup(struct sock *sk, int mib)
+@@ -4044,6 +4300,17 @@ int tcp_write_wakeup(struct sock *sk, int mib)
  		if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
  		    skb->len > mss) {
  			seg_size = min(seg_size, mss);
@@ -2983,7 +3085,7 @@ index f99494637..042f9e38d 100644
  			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
  			if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
  					 skb, seg_size, mss, GFP_ATOMIC))
-@@ -4052,6 +4233,15 @@ int tcp_write_wakeup(struct sock *sk, int mib)
+@@ -4052,6 +4319,15 @@ int tcp_write_wakeup(struct sock *sk, int mib)
  			tcp_set_skb_tso_segs(skb, mss);
 
  		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;