[HPA] Add pool of empty pages to hpa_central

Makefile.in

@@ -231,9 +231,10 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \
 	$(srcroot)test/unit/hpa_thp_always.c \
+	$(srcroot)test/unit/hpa_background_thread.c \
+	$(srcroot)test/unit/hpa_central_pool.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
-	$(srcroot)test/unit/hpa_background_thread.c \
 	$(srcroot)test/unit/hpdata.c \
 	$(srcroot)test/unit/huge.c \
 	$(srcroot)test/unit/inspect.c \

include/jemalloc/internal/arena_externs.h

@@ -47,7 +47,11 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
     hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
-void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
+void arena_stats_global_central_read(tsdn_t *tsdn, hpa_central_stats_t *stats);
+void arena_stats_global_central_mutex_read(
+    tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data);
+
+void     arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);
 void arena_extent_dalloc_large_prep(
@@ -125,7 +129,10 @@ void   arena_prefork5(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork6(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork7(tsdn_t *tsdn, arena_t *arena);
 void   arena_prefork8(tsdn_t *tsdn, arena_t *arena);
+void   arena_global_prefork(tsdn_t *tsdn, bool use_hpa);
 void   arena_postfork_parent(tsdn_t *tsdn, arena_t *arena);
+void   arena_global_postfork_parent(tsdn_t *tsdn, bool use_hpa);
 void   arena_postfork_child(tsdn_t *tsdn, arena_t *arena);
+void   arena_global_postfork_child(tsdn_t *tsdn, bool use_hpa);
 
 #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */

include/jemalloc/internal/ctl.h

@@ -5,6 +5,7 @@
 #include "jemalloc/internal/arena_stats.h"
 #include "jemalloc/internal/background_thread_structs.h"
 #include "jemalloc/internal/bin_stats.h"
+#include "jemalloc/internal/hpa_central.h"
 #include "jemalloc/internal/jemalloc_internal_types.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex_prof.h"
@@ -66,6 +67,7 @@ typedef struct ctl_stats_s {
 	size_t retained;
 
 	background_thread_stats_t background_thread;
+	hpa_central_stats_t       hpa_central;
 	mutex_prof_data_t mutex_prof_data[mutex_prof_num_global_mutexes];
 } ctl_stats_t;
 

include/jemalloc/internal/hpa.h

@@ -50,6 +50,20 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t ndehugifies;
+
+	/*
+	 * The number of times we donated pageslab to central pool
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t ndonated_ps;
+
+	/*
+	 * The number of times we borrowed pageslab from a central pool
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t nborrowed_ps;
 };
 
 /* Completely derived; only used by CTL. */

include/jemalloc/internal/hpa_central.h

@@ -8,8 +8,35 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/tsd_types.h"
 
+typedef struct hpa_pool_s hpa_pool_t;
+struct hpa_pool_s {
+	/*
+	 * Pool of empty huge pages to be shared between shards that are
+	 * participating.
+	 *
+	 * Page is owned by the  pool if it lives in one of these two lists.
+	 * This means that it should not be part of any hpa_shard's psset at the
+	 * same time.
+	 */
+	hpdata_empty_list_t nonpurged;
+	hpdata_empty_list_t purged;
+};
+
+typedef struct hpa_central_stats_s hpa_central_stats_t;
+struct hpa_central_stats_s {
+	/* Number of pages purged while they were in the central pool */
+	uint64_t npurged_pool;
+
+	/* Total number of dirty base pages in the pool */
+	size_t ndirty_pool;
+};
+
 typedef struct hpa_central_s hpa_central_t;
 struct hpa_central_s {
+	/* Guards the access to central pool of empty hugepages */
+	malloc_mutex_t pool_mtx;
+	hpa_pool_t     pool;
+
 	/*
 	 * Guards expansion of eden.  We separate this from the regular mutex so
 	 * that cheaper operations can still continue while we're doing the OS
@@ -30,6 +57,9 @@ struct hpa_central_s {
 
 	/* The HPA hooks. */
 	hpa_hooks_t hooks;
+
+	/* Stats */
+	hpa_central_stats_t stats;
 };
 
 bool hpa_central_init(
@@ -38,4 +68,21 @@ bool hpa_central_init(
 hpdata_t *hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
     uint64_t age, bool hugify_eager, bool *oom);
 
+/* Donate empty pages to central */
+void hpa_central_ps_insert(tsdn_t *tsdn, hpa_central_t *central,
+    hpdata_empty_list_t *pages, const nstime_t *now);
+/* Get empty page from central without growing it */
+hpdata_t *hpa_central_ps_pop(tsdn_t *tsdn, hpa_central_t *central);
+
+/* Purge up to max_ps empty pages in the central */
+size_t hpa_central_purge(
+    tsdn_t *tsdn, hpa_central_t *central, const nstime_t *now, size_t max_ps);
+
+void hpa_central_prefork(tsdn_t *tsdn, hpa_central_t *central);
+void hpa_central_postfork_parent(tsdn_t *tsdn, hpa_central_t *central);
+void hpa_central_postfork_child(tsdn_t *tsdn, hpa_central_t *central);
+
+void hpa_central_stats_read(
+    tsdn_t *tsdn, hpa_central_t *central, hpa_central_stats_t *stats);
+
 #endif /* JEMALLOC_INTERNAL_HPA_CENTRAL_H */

include/jemalloc/internal/hpa_opts.h

@@ -152,6 +152,12 @@ struct hpa_shard_opts_s {
 	 * hpa_hugify_style_t for options).
 	 */
 	hpa_hugify_style_t hugify_style;
+
+	/*
+	 * If use_pool is true this shard will donate empty pages to the pool
+	 * and borrow from the pool before using central allocator.
+	 */
+	bool use_pool;
 };
 
 /* clang-format off */
@@ -183,7 +189,9 @@ struct hpa_shard_opts_s {
 	/* min_purge_delay_ms */             				\
 	0,  								\
 	/* hugify_style */                				\
-	hpa_hugify_style_lazy						\
+	hpa_hugify_style_lazy,  					\
+	/* use_pool */  						\
+	false	        						\
 }
 /* clang-format on */
 

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -17,6 +17,7 @@ extern bool             opt_abort_conf;
 extern bool             opt_trust_madvise;
 extern bool             opt_experimental_hpa_start_huge_if_thp_always;
 extern bool             opt_experimental_hpa_enforce_hugify;
+extern uint64_t         opt_hpa_pool_purge_delay_ms;
 extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;

include/jemalloc/internal/mutex_prof.h

@@ -36,7 +36,8 @@ typedef enum {
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(hpa_central_pool)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/witness.h

@@ -56,6 +56,7 @@ enum witness_rank_e {
 	WITNESS_RANK_HPA_SHARD = WITNESS_RANK_EXTENTS,
 
 	WITNESS_RANK_HPA_CENTRAL_GROW,
+	WITNESS_RANK_HPA_CENTRAL_POOL,
 	WITNESS_RANK_HPA_CENTRAL,
 
 	WITNESS_RANK_EDATA_CACHE,

src/arena.c

@@ -213,6 +213,21 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 }
 
+void
+arena_stats_global_central_read(tsdn_t *tsdn, hpa_central_stats_t *stats) {
+	hpa_central_stats_read(tsdn, &arena_pa_central_global.hpa, stats);
+}
+
+void
+arena_stats_global_central_mutex_read(
+    tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data) {
+	malloc_mutex_lock(tsdn, &arena_pa_central_global.hpa.pool_mtx);
+	malloc_mutex_prof_read(
+	    tsdn, mutex_prof_data, &arena_pa_central_global.hpa.pool_mtx);
+	malloc_mutex_unlock(tsdn, &arena_pa_central_global.hpa.pool_mtx);
+}
+
+
 static void
 arena_background_thread_inactivity_check(
     tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
@@ -2321,6 +2336,13 @@ arena_prefork8(tsdn_t *tsdn, arena_t *arena) {
 	}
 }
 
+void
+arena_global_prefork(tsdn_t *tsdn, bool use_hpa) {
+	if (use_hpa) {
+		hpa_central_prefork(tsdn, &arena_pa_central_global.hpa);
+	}
+}
+
 void
 arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
 	for (unsigned i = 0; i < nbins_total; i++) {
@@ -2336,6 +2358,13 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
 	}
 }
 
+void
+arena_global_postfork_parent(tsdn_t *tsdn, bool use_hpa) {
+	if (use_hpa) {
+		hpa_central_postfork_parent(tsdn, &arena_pa_central_global.hpa);
+	}
+}
+
 void
 arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 	atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
@@ -2374,3 +2403,10 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
 		malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
 	}
 }
+
+void
+arena_global_postfork_child(tsdn_t *tsdn, bool use_hpa) {
+	if (use_hpa) {
+		hpa_central_postfork_child(tsdn, &arena_pa_central_global.hpa);
+	}
+}