Merge pull request #2457 from tempesta-tech/MorganaFuture/optimize_tfw_current_timestamp

Optimize tfw_current_timestamp() with per-CPU caching

Author: krizhanovsky

fw/access_log.c

@@ -298,9 +298,8 @@ do_access_log_req_mmap(TfwHttpReq *req, u16 resp_status,
 		room_size -= sizeof(val);		\
 	} while (0)
 
-	ktime_get_real_ts64(&ts);
-
-	event->timestamp = ts.tv_sec * 1000 + ts.tv_nsec/1000000;
+	tfw_current_timestamp_ts64(&ts);
+	event->timestamp = ts.tv_sec * 1000 + ts.tv_nsec / 1000000;
 	event->type = TFW_MMAP_LOG_TYPE_ACCESS;
 	event->fields = TFW_MMAP_LOG_ALL_FIELDS_MASK; /* Enable all the fields */
 

lib/common.h

@@ -1,7 +1,7 @@
 /**
  *		Tempesta kernel library
  *
- * Copyright (C) 2019 Tempesta Technologies, Inc.
+ * Copyright (C) 2019-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by
@@ -20,13 +20,64 @@
 #ifndef __LIB_COMMON_H__
 #define __LIB_COMMON_H__
 
-/* Get current timestamp in secs. */
+#include <linux/percpu.h>
+#include <linux/time.h>
+
+#define TFW_TS_REFRESH_INTERVAL (HZ)
+
+/**
+ * Get current timestamp with ktime_get_real_ts64 interface.
+ * Uses per-CPU cached timestamps in softirq context.
+ * WARNING: Must be called from softirq context only.
+ */
+static inline void
+tfw_current_timestamp_ts64(struct timespec64 *ts)
+{
+	static DEFINE_PER_CPU(struct timespec64, tfw_cached_ts);
+	static DEFINE_PER_CPU(unsigned long, tfw_ts_last_update);
+	struct timespec64 *cached_ts;
+	unsigned long *last_update;
+	unsigned long now;
+
+	WARN_ON_ONCE(!in_softirq());
+
+	cached_ts = this_cpu_ptr(&tfw_cached_ts);
+	last_update = this_cpu_ptr(&tfw_ts_last_update);
+	now = jiffies;
+
+	if (unlikely(time_after(now,
+				*last_update + TFW_TS_REFRESH_INTERVAL)))
+	{
+		ktime_get_real_ts64(cached_ts);
+		*last_update = now;
+	}
+
+	*ts = *cached_ts;
+}
+
+/**
+ * Get current timestamp in seconds.
+ * Uses per-CPU cached timestamps in softirq context.
+ * WARNING: Must be called from softirq context only.
+ */
 static inline long
 tfw_current_timestamp(void)
 {
 	struct timespec64 ts;
-	ktime_get_real_ts64(&ts);
+
+	tfw_current_timestamp_ts64(&ts);
 	return ts.tv_sec;
 }
 
+/**
+ * Get current timestamp - real-time version.
+ * For use when precise real-time is needed without caching.
+ */
+static inline void
+tfw_current_timestamp_real(struct timespec64 *ts)
+{
+	WARN_ON_ONCE(!in_task());
+	ktime_get_real_ts64(ts);
+}
+
 #endif /* __LIB_COMMON_H__ */

tls/tls_cli.c

@@ -6,7 +6,7 @@
  * Based on mbed TLS, https://tls.mbed.org.
  *
  * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
- * Copyright (C) 2015-2018 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@
 #if 0 /* TODO #769 Full TLS proxying */
 
 #include "debug.h"
+#include "lib/common.h"
 #include "ttls.h"
 #include "tls_internal.h"
 
@@ -418,7 +419,7 @@ static int ssl_generate_random(TlsCtx *ssl)
 	unsigned char *p = ssl->handshake->randbytes;
 	long t;
 
-	t = ttls_time();
+	t = tfw_current_timestamp();
 	*p++ = (unsigned char)(t >> 24);
 	*p++ = (unsigned char)(t >> 16);
 	*p++ = (unsigned char)(t >>  8);
@@ -965,7 +966,7 @@ static int ssl_parse_server_hello(TlsCtx *ssl)
 	{
 		ssl->state++;
 		ssl->handshake->resume = 0;
-		ssl->session_negotiate->start = ttls_time();
+		ssl->session_negotiate->start = tfw_current_timestamp();
 		ssl->session_negotiate->ciphersuite = i;
 		ssl->session_negotiate->compression = comp;
 		ssl->session_negotiate->id_len = n;

tls/tls_internal.h

@@ -6,7 +6,7 @@
  * Based on mbed TLS, https://tls.mbed.org.
  *
  * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
- * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -397,12 +397,8 @@ ttls_rnd(void *buf, size_t len)
 	memset(buf, 0x55, len);
 }
 
-unsigned long ttls_time_debug(void);
-
-#define ttls_time()		ttls_time_debug()
 
 #else
-#define ttls_time()		get_seconds()
 /*
  * CPUs since Intel Ice Lake are safe against SRBDS attack, so we're good
  * with the hardware random generator.

tls/tls_srv.c

@@ -24,6 +24,7 @@
  */
 #include "debug.h"
 #include "lib/str.h"
+#include "lib/common.h"
 #include "ecp.h"
 #include "mpool.h"
 #include "tls_internal.h"
@@ -1311,7 +1312,7 @@ ttls_write_server_hello(TlsCtx *tls, struct sg_table *sgt,
 	T_DBG("server hello, chosen version %d:%d, buf=%pK\n",
 	      buf[4], buf[5], buf);
 
-	*(unsigned int *)p = htonl(ttls_time());
+	*(unsigned int *)p = htonl(tfw_current_timestamp());
 	p += 4;
 	ttls_rnd(p, 28);
 	p += 28;
@@ -1325,7 +1326,7 @@ ttls_write_server_hello(TlsCtx *tls, struct sg_table *sgt,
 		 */
 		tls->state = TTLS_SERVER_CERTIFICATE;
 
-		tls->sess.start = ttls_time();
+		tls->sess.start = tfw_current_timestamp();
 
 		if (tls->hs->new_session_ticket) {
 			tls->sess.id_len = n = 0;

tls/tls_ticket.c

@@ -6,7 +6,7 @@
  * Based on mbed TLS, https://tls.mbed.org.
  *
  * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
- * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -71,11 +71,23 @@ const char *ticket_key_name_iv =
 static inline unsigned long
 ttls_ticket_get_time(unsigned long lifetime)
 {
-	unsigned long ts = tfw_current_timestamp();
+	struct timespec64 ts;
+	unsigned long ts_sec;
 
-	ts -= ts % lifetime;
+	/*
+	 * This function is called from both process context
+	 * (ttls_tickets_configure) and softirq context
+	 * (ttls_ticket_rotate_keys timer callback).
+	 */
+	if (in_softirq()) {
+		tfw_current_timestamp_ts64(&ts);
+	} else {
+		tfw_current_timestamp_real(&ts);
+	}
+	ts_sec = ts.tv_sec;
+	ts_sec -= ts_sec % lifetime;
 
-	return ts;
+	return ts_sec;
 }
 
 /**
@@ -177,6 +189,7 @@ static void
 ttls_ticket_rotate_keys(struct timer_list *t)
 {
 	TlsTicketPeerCfg *tcfg = from_timer(tcfg, t, timer);
+	struct timespec64 ts;
 	unsigned long secs;
 
 	T_DBG("TLS: Rotate keys for ticket configuration [%pK]\n", tcfg);
@@ -192,7 +205,8 @@ ttls_ticket_rotate_keys(struct timer_list *t)
 	 * and callback will fire at different time on different Tempesta
 	 * nodes. To avoid it need to recalculate timer every time.
 	 */
-	secs = tcfg->lifetime - (tfw_current_timestamp() % tcfg->lifetime);
+	tfw_current_timestamp_ts64(&ts);
+	secs = tcfg->lifetime - (ts.tv_sec % tcfg->lifetime);
 	mod_timer(&tcfg->timer, jiffies + msecs_to_jiffies(secs * 1000));
 }
 
@@ -280,6 +294,7 @@ ttls_tickets_configure(TlsPeerCfg *cfg, unsigned long lifetime,
 	const char *md_ctx_key = secret_str;
 	size_t md_ctx_key_len = len;
 	TlsMdCtx md_ctx;
+	struct timespec64 ts;
 	unsigned long secs;
 
 	tcfg->active_key = 0;
@@ -351,7 +366,8 @@ ttls_tickets_configure(TlsPeerCfg *cfg, unsigned long lifetime,
 	}
 
 	timer_setup(&tcfg->timer, ttls_ticket_rotate_keys, 0);
-	secs = tcfg->lifetime - (tfw_current_timestamp() % tcfg->lifetime);
+	tfw_current_timestamp_real(&ts);
+	secs = tcfg->lifetime - (ts.tv_sec % tcfg->lifetime);
 	mod_timer(&tcfg->timer, jiffies + msecs_to_jiffies(secs * 1000));
 
 err:
@@ -479,7 +495,7 @@ ttls_ticket_sess_save(const TlsSess *sess, TlsState *state, size_t buf_len)
 static int
 ttls_ticket_sess_load(TlsState *state, size_t len, unsigned long lifetime)
 {
-	long time_pass = ttls_time() - state->sess.start;
+	long time_pass = tfw_current_timestamp() - state->sess.start;
 
 	if ((time_pass < 0) || (unsigned long)time_pass > lifetime)
 		return TTLS_ERR_SESSION_TICKET_EXPIRED;

tls/ttls.c

@@ -8,7 +8,7 @@
  * Based on mbed TLS, https://tls.mbed.org.
  *
  * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
- * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -2849,15 +2849,6 @@ ttls_alpn_ext_eq(const ttls_alpn_proto *proto, const unsigned char *buf,
 	return !memcmp_fast(proto->name, buf, len);
 }
 
-#if DBG_TLS >= 3
-unsigned long
-ttls_time_debug(void)
-{
-	static atomic64_t curr_time = ATOMIC_INIT(0);
-
-	return atomic64_inc_return(&curr_time);
-}
-#endif
 
 static void
 ttls_exit(void)

tls/x509.c

@@ -15,7 +15,7 @@
  * Based on mbed TLS, https://tls.mbed.org.
  *
  * Copyright (C) 2006-2015, ARM Limited, All Rights Reserved
- * Copyright (C) 2015-2024 Tempesta Technologies, Inc.
+ * Copyright (C) 2015-2025 Tempesta Technologies, Inc.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -32,6 +32,7 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 #include "debug.h"
+#include "lib/common.h"
 #include "ttls.h"
 #include "asn1.h"
 #include "oid.h"
@@ -708,8 +709,10 @@ static void
 x509_get_current_time(ttls_x509_time *now)
 {
 	struct tm t;
+	struct timespec64 ts;
 
-	time64_to_tm(ttls_time(), 0, &t);
+	tfw_current_timestamp_real(&ts);
+	time64_to_tm(ts.tv_sec, 0, &t);
 
 	now->year = t.tm_year + 1900;
 	now->mon  = t.tm_mon  + 1;