Merge branch 'master' into ag_2104-implement-ability-to-send-js-challenge-according-custom-rules

Author: biathlon3

Makefile

@@ -73,6 +73,7 @@ DBG_WS ?= 0
 DBG_APM ?= 0
 DBG_GFSM ?= 0
 DBG_HTTP ?= 0
+DBG_HTTP2 ?= 0
 DBG_HTTP_FRAME ?= 0
 DBG_HTTP_SESS ?= 0
 DBG_HTTP_STREAM ?= 0
@@ -89,6 +90,7 @@ TFW_CFLAGS += -DDBG_HTTP_SESS=$(DBG_HTTP_SESS)
 TFW_CFLAGS += -DDBG_HTTP_STREAM=$(DBG_HTTP_STREAM)
 TFW_CFLAGS += -DDBG_HPACK=$(DBG_HPACK) -DDBG_CACHE=$(DBG_CACHE)
 TFW_CFLAGS += -DDBG_SRV=$(DBG_SRV) -DDBG_VHOST=$(DBG_VHOST) -DDBG_TEST=$(DBG_TEST)
+TFW_CFLAGS += -DDBG_HTTP2=$(DBG_HTTP2)
 
 # By default Tempesta TLS randomizes elliptic curve points using RDRAND
 # instruction, which provides a high speed random numbers generator.

db/core/htrie.c

@@ -11,7 +11,7 @@
  * and shutdown are performed in process context.
  *
  * Copyright (C) 2014 NatSys Lab. (info@natsys-lab.com).
- * Copyright (C) 2015-2021 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -918,7 +918,7 @@ tdb_htrie_init(void *p, size_t db_size, unsigned int rec_len)
 		TDB_ERR("cannot allocate per-cpu data\n");
 		return NULL;
 	}
-	for_each_possible_cpu(cpu) {
+	for_each_online_cpu(cpu) {
 		TdbPerCpu *p = per_cpu_ptr(hdr->pcpu, cpu);
 		p->i_wcl = tdb_alloc_blk(hdr);
 		p->d_wcl = tdb_alloc_blk(hdr);

fw/cache.c

@@ -26,6 +26,7 @@
 #include <linux/kthread.h>
 #include <linux/tcp.h>
 #include <linux/topology.h>
+#include <linux/nodemask.h>
 
 #undef DEBUG
 #if DBG_CACHE > 0
@@ -234,13 +235,13 @@ enum {
 };
 
 typedef struct {
-	int		cpu[NR_CPUS];
+	int 		*cpu;
 	atomic_t	cpu_idx;
 	unsigned int	nr_cpus;
 	TDB		*db;
 } CaNode;
 
-static CaNode c_nodes[MAX_NUMNODES];
+static CaNode *c_nodes;
 
 typedef int tfw_cache_write_actor_t(TDB *, TdbVRec **, TfwHttpResp *, char **,
 				    size_t, TfwDecodeCacheIter *);
@@ -333,18 +334,59 @@ tfw_cache_key_node(unsigned long key)
 }
 
 /**
- * Just choose any CPU for each node to use queue_work_on() for
- * nodes scheduling. Reserve 0th CPU for other tasks.
+ * Release node-cpu map.
  */
 static void
+tfw_release_node_cpus(void)
+{
+	int node;
+
+	if(!c_nodes)
+		return;
+
+	for(node = 0; node < nr_online_nodes; node++) {
+		if(c_nodes[node].cpu)
+			kfree(c_nodes[node].cpu);
+	}
+	kfree(c_nodes);
+}
+
+/**
+ * Create node-cpu map to use queue_work_on() for nodes scheduling.
+ * 0th CPU is reserved for other tasks.
+ * At the moment we doesn't support CPU hotplug, so enumerate only online CPUs.
+ */
+static int
 tfw_init_node_cpus(void)
 {
-	int cpu, node;
+	int nr_cpus, cpu, node;
+
+	T_DBG2("nr_online_nodes: %d", nr_online_nodes);
+
+	c_nodes = kzalloc(nr_online_nodes * sizeof(CaNode), GFP_KERNEL);
+	if(!c_nodes) {
+		T_ERR("Failed to allocate nodes map for cache work scheduler");
+		return -ENOMEM;
+	}
+
+	for_each_node_with_cpus(node) {
+		nr_cpus = nr_cpus_node(node);
+		T_DBG2("node: %d  nr_cpus: %d",node, nr_cpus);
+		c_nodes[node].cpu = kmalloc(nr_cpus * sizeof(int), GFP_KERNEL);
+		if(!c_nodes[node].cpu) {
+			T_ERR("Failed to allocate CPU array for node %d for cache work scheduler",
+				node);
+			return -ENOMEM;
+		}
+	}
 
 	for_each_online_cpu(cpu) {
 		node = cpu_to_node(cpu);
+		T_DBG2("node: %d  cpu: %d",node, cpu);
 		c_nodes[node].cpu[c_nodes[node].nr_cpus++] = cpu;
 	}
+
+	return 0;
 }
 
 static TDB *
@@ -976,8 +1018,7 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce)
 
 		resp->mit.start_off = FRAME_HEADER_SIZE;
 
-		r = tfw_h2_resp_status_write(resp, 304, false, true,
-					     stream_id);
+		r = tfw_h2_resp_status_write(resp, 304, false, true);
 		if (unlikely(r))
 			goto err_setup;
 		/* account for :status field itself */
@@ -1018,7 +1059,7 @@ tfw_cache_send_304(TfwHttpReq *req, TfwCacheEntry *ce)
 		return;
 	}
 
-	if (tfw_h2_frame_local_resp(resp, stream_id, h_len, NULL))
+	if (tfw_h2_frame_local_resp(resp, h_len, NULL))
 		goto err_setup;
 
 	tfw_h2_req_unlink_stream(req);
@@ -2624,7 +2665,7 @@ tfw_cache_add_body_page(TfwMsgIter *it, char *p, int sz, bool h2,
  */
 static int
 tfw_cache_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p,
-			  unsigned long body_sz, bool h2, unsigned int stream_id)
+			  unsigned long body_sz, bool h2)
 {
 	int r;
 	bool sh_frag = h2 ? false : true;
@@ -2660,10 +2701,6 @@ tfw_cache_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p,
 						    !body_sz);
 			if (r)
 				return r;
-			if (stream_id) {
-				skb_set_tfw_flags(it->skb, SS_F_HTTT2_FRAME_DATA);
-				skb_set_tfw_cb(it->skb, stream_id);
-			}
 		}
 		if (!body_sz || !(trec = tdb_next_rec_chunk(db, trec)))
 			break;
@@ -2686,8 +2723,7 @@ tfw_cache_build_resp_body(TDB *db, TdbVRec *trec, TfwMsgIter *it, char *p,
 }
 
 static int
-tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce,
-		      unsigned int stream_id)
+tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce)
 {
 	int r;
 	size_t digs;
@@ -2718,8 +2754,7 @@ tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce,
 
 	if (to_h2) {
 		h_age.hpack_idx = 21;
-		if ((r = tfw_hpack_encode(resp, &h_age, false, false,
-					  stream_id)))
+		if ((r = tfw_hpack_encode(resp, &h_age, false, false)))
 			goto err;
 	} else {
 		if ((r = tfw_http_msg_expand_data(&mit->iter, skb_head,
@@ -2761,8 +2796,7 @@ tfw_cache_set_hdr_age(TfwHttpResp *resp, TfwCacheEntry *ce,
  * TODO use iterator and passed skbs to be called from net_tx_action.
  */
 static TfwHttpResp *
-tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime,
-		     unsigned int stream_id)
+tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime)
 {
 	int h;
 	TfwStr dummy_body = { 0 };
@@ -2821,14 +2855,14 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime,
 	 * Set 'set-cookie' header if needed, for HTTP/2 or HTTP/1.1
 	 * response.
 	 */
-	if (tfw_http_sess_resp_process(resp, true, stream_id))
+	if (tfw_http_sess_resp_process(resp, true))
 		goto free;
 	/*
 	 * RFC 7234 p.4 Constructing Responses from Caches:
 	 * When a stored response is used to satisfy a request without
 	 * validation, a cache MUST generate an Age header field.
 	 */
-	if (tfw_cache_set_hdr_age(resp, ce, stream_id))
+	if (tfw_cache_set_hdr_age(resp, ce))
 		goto free;
 
 	if (!TFW_MSG_H2(req)) {
@@ -2856,11 +2890,11 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime,
 	}
 
 	/* Set additional headers for HTTP/2 response. */
-	if (tfw_h2_resp_add_loc_hdrs(resp, h_mods, true, stream_id)
+	if (tfw_h2_resp_add_loc_hdrs(resp, h_mods, true)
 	    || (lifetime > ce->lifetime
-		&& tfw_h2_set_stale_warn(resp, stream_id))
+		&& tfw_h2_set_stale_warn(resp))
 	    || (!test_bit(TFW_HTTP_B_HDR_DATE, resp->flags)
-		&& tfw_h2_add_hdr_date(resp, true, stream_id)))
+		&& tfw_h2_add_hdr_date(resp, true)))
 		goto free;
 
 	h_len += mit->acc_len;
@@ -2881,7 +2915,7 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime,
 	 * send content in the response.
 	 */
 	dummy_body.len = req->method != TFW_HTTP_METH_HEAD ? ce->body_len : 0;
-	if (tfw_h2_frame_local_resp(resp, stream_id, h_len, &dummy_body))
+	if (tfw_h2_frame_local_resp(resp, h_len, &dummy_body))
 		goto free;
 	it->skb = ss_skb_peek_tail(&it->skb_head);
 	it->frag = skb_shinfo(it->skb)->nr_frags - 1;
@@ -2891,7 +2925,7 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long lifetime,
 	BUG_ON(p != TDB_PTR(db->hdr, ce->body));
 	if (ce->body_len && req->method != TFW_HTTP_METH_HEAD) {
 		if (tfw_cache_build_resp_body(db, trec, it, p, ce->body_len,
-					      TFW_MSG_H2(req), stream_id))
+					      TFW_MSG_H2(req)))
 			goto free;
 	}
 	resp->content_length = ce->body_len;
@@ -2952,8 +2986,7 @@ cache_req_process_node(TfwHttpReq *req, tfw_http_cache_cb_t action)
 		}
 	}
 
-	resp = tfw_cache_build_resp(req, ce, lifetime, id);
-
+	resp = tfw_cache_build_resp(req, ce, lifetime);
 	/*
 	 * The stream of HTTP/2-request should be closed here since we have
 	 * successfully created the resulting response from cache and will
@@ -3160,6 +3193,31 @@ tfw_cache_mgr(void *arg)
 }
 #endif
 
+static inline int
+tfw_cache_wq_init(int cpu)
+{
+	TfwWorkTasklet *ct = &per_cpu(cache_wq, cpu);
+	int r;
+
+	r = tfw_wq_init(&ct->wq, TFW_DFLT_QSZ, cpu_to_node(cpu));
+	if (unlikely(r))
+		return r;
+	init_irq_work(&ct->ipi_work, tfw_cache_ipi);
+	tasklet_init(&ct->tasklet, tfw_wq_tasklet, (unsigned long)ct);
+
+	return 0;
+}
+
+static inline void
+tfw_cache_wq_clear(int cpu)
+{
+	TfwWorkTasklet *ct = &per_cpu(cache_wq, cpu);
+
+	tasklet_kill(&ct->tasklet);
+	irq_work_sync(&ct->ipi_work);
+	tfw_wq_destroy(&ct->wq);
+}
+
 static int
 tfw_cache_start(void)
 {
@@ -3171,11 +3229,16 @@ tfw_cache_start(void)
 	if (!(cache_cfg.cache || g_vhost->cache_purge))
 		return 0;
 
-	for_each_node_with_cpus(i) {
+	if ((r = tfw_init_node_cpus()))
+		goto node_cpus_alloc_err;
+
+	for(i = 0; i < nr_online_nodes; i++) {
 		c_nodes[i].db = tdb_open(cache_cfg.db_path,
 					 cache_cfg.db_size, 0, i);
-		if (!c_nodes[i].db)
+		if (!c_nodes[i].db) {
+			r = -ENOMEM;
 			goto close_db;
+		}
 	}
 #if 0
 	cache_mgr_thr = kthread_run(tfw_cache_mgr, NULL, "tfw_cache_mgr");
@@ -3185,19 +3248,14 @@ tfw_cache_start(void)
 		goto close_db;
 	}
 #endif
-	tfw_init_node_cpus();
 
 	TFW_WQ_CHECKSZ(TfwCWork);
 	for_each_online_cpu(i) {
-		TfwWorkTasklet *ct = &per_cpu(cache_wq, i);
-		r = tfw_wq_init(&ct->wq, TFW_DFLT_QSZ, cpu_to_node(i));
-		if (r) {
-			T_ERR_NL("%s: Can't initialize cache work queue for CPU #%d\n",
-				 __func__, i);
-			goto close_db;
+		if (unlikely(r = tfw_cache_wq_init(i))) {
+			T_ERR_NL("%s: Can't initialize cache work"
+				 " queue for CPU #%d\n", __func__, i);
+			goto free_tasklet;
 		}
-		init_irq_work(&ct->ipi_work, tfw_cache_ipi);
-		tasklet_init(&ct->tasklet, tfw_wq_tasklet, (unsigned long)ct);
 	}
 
 #if defined(DEBUG)
@@ -3219,9 +3277,15 @@ tfw_cache_start(void)
 	for_each_online_cpu(i)
 		kfree(per_cpu(ce_dbg_buf, i));
 #endif
+free_tasklet:
+	for_each_online_cpu(i)
+		tfw_cache_wq_clear(i);
 close_db:
 	for_each_node_with_cpus(i)
 		tdb_close(c_nodes[i].db);
+
+node_cpus_alloc_err:
+	tfw_release_node_cpus();
 	return r;
 }
 
@@ -3235,12 +3299,8 @@ tfw_cache_stop(void)
 	if (!cache_cfg.cache)
 		return;
 
-	for_each_online_cpu(i) {
-		TfwWorkTasklet *ct = &per_cpu(cache_wq, i);
-		tasklet_kill(&ct->tasklet);
-		irq_work_sync(&ct->ipi_work);
-		tfw_wq_destroy(&ct->wq);
-	}
+	for_each_online_cpu(i)
+		tfw_cache_wq_clear(i);
 #if 0
 	kthread_stop(cache_mgr_thr);
 #endif
@@ -3252,6 +3312,8 @@ tfw_cache_stop(void)
 
 	for_each_node_with_cpus(i)
 		tdb_close(c_nodes[i].db);
+
+	tfw_release_node_cpus();
 }
 
 static const TfwCfgEnum cache_http_methods_enum[] = {

fw/cfg.h

@@ -183,13 +183,16 @@ typedef struct {
  */
 #define TFW_CFG_ENTRY_FOR_EACH_ATTR(e, idx, k, v)	\
 	for ((idx) = 0, (k) = (e)->attrs[0].key, (v) = (e)->attrs[0].val; \
-	     (idx++) < (e)->attr_n; \
-	     (k) = (e)->attrs[(idx)].key, (v) = (e)->attrs[(idx)].val)
+	     (idx) < (e)->attr_n; \
+	     (idx)++,  \
+	     (k) = (idx < (e)->attr_n ? (e)->attrs[(idx)].key : NULL), \
+	     (v) = (idx < (e)->attr_n ? (e)->attrs[(idx)].val : NULL))
 
 #define TFW_CFG_ENTRY_FOR_EACH_VAL(e, idx, v)	\
 	for ((idx) = 0, (v) = (e)->vals[0];	\
 	     (idx) < (e)->val_n;		\
-	     (v) = (e)->vals[++(idx)])
+	     (idx)++,				\
+	     (v) = (idx < (e)->val_n ? (e)->vals[(idx)] : NULL))
 
 #define TFW_CFG_CHECK_NO_ATTRS(spec, entry)				\
 	if ((entry)->attr_n) {						\