x86/mm: Randomize per-cpu entry area

jira VULN-3958
cve CVE-2023-0597
commit-author Peter Zijlstra <peterz@infradead.org>
commit 97e3d26
upstream-diff |
  1. Ignored changes in `arch/x86/kernel/hw_breakpoint.c'. The modified
     function `within_cpu_entry()' doesn't exist in `ciqlts8_6'
     revision. The conflict might have been resolved by pure cherry
     picking of 24ae0c9,
     d390e6d,
     97417cb, but would result in
     introducing dead code: `within_area()' and `within_cpu_entry()'
     functions.
  2. Moved the `arch/x86/include/asm/pgtable_areas.h' changes to
     `arch/x86/include/asm/cpu_entry_area.h'. This must have been done
     because of the 186525b commit
     missing from `ciqlts8_6' history, which factored out the relevant
     #defines from `cpu_entry_area.h' to `pgtable_areas.h'. It was decided
     not to backport this commit as prerequisite since it's too extensive
     and making changes not related to the patch.
  3. Made small adaptation of changes relating to `cea_offset()'
     definitions in `arch/x86/mm/cpu_entry_area.c' which was necessary
     because of the dc4e002 commit
     missing from `ciqlts8_6' history. It was too functionality-intrusive
     to backport as prerequisite for auto resolution of just this single
     conflict.

Seth found that the CPU-entry-area; the piece of per-cpu data that is
mapped into the userspace page-tables for kPTI is not subject to any
randomization -- irrespective of kASLR settings.

On x86_64 a whole P4D (512 GB) of virtual address space is reserved for
this structure, which is plenty large enough to randomize things a
little.

As such, use a straight forward randomization scheme that avoids
duplicates to spread the existing CPUs over the available space.

  [ bp: Fix le build. ]

	Reported-by: Seth Jenkins <sethjenkins@google.com>
	Reviewed-by: Kees Cook <keescook@chromium.org>
	Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
	Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
	Signed-off-by: Borislav Petkov <bp@suse.de>
(cherry picked from commit 97e3d26)
	Signed-off-by: Marcin Wcisło <marcin.wcislo@conclusive.pl>

Author: pvts-mat

arch/x86/include/asm/cpu_entry_area.h

@@ -107,10 +107,6 @@ struct cpu_entry_area {
 };
 
 #define CPU_ENTRY_AREA_SIZE		(sizeof(struct cpu_entry_area))
-#define CPU_ENTRY_AREA_ARRAY_SIZE	(CPU_ENTRY_AREA_SIZE * NR_CPUS)
-
-/* Total size includes the readonly IDT mapping page as well: */
-#define CPU_ENTRY_AREA_TOTAL_SIZE	(CPU_ENTRY_AREA_ARRAY_SIZE + PAGE_SIZE)
 
 DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area);
 DECLARE_PER_CPU(struct cea_exception_stacks *, cea_exception_stacks);
@@ -124,8 +120,14 @@ extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags);
 
 #define CPU_ENTRY_AREA_RO_IDT_VADDR	((void *)CPU_ENTRY_AREA_RO_IDT)
 
-#define CPU_ENTRY_AREA_MAP_SIZE			\
-	(CPU_ENTRY_AREA_PER_CPU + CPU_ENTRY_AREA_ARRAY_SIZE - CPU_ENTRY_AREA_BASE)
+#ifdef CONFIG_X86_32
+#define CPU_ENTRY_AREA_MAP_SIZE		(CPU_ENTRY_AREA_PER_CPU +		\
+					 (CPU_ENTRY_AREA_SIZE * NR_CPUS) -	\
+					 CPU_ENTRY_AREA_BASE)
+#else
+#define CPU_ENTRY_AREA_MAP_SIZE		P4D_SIZE
+#endif
+
 
 extern struct cpu_entry_area *get_cpu_entry_area(int cpu);
 

arch/x86/mm/cpu_entry_area.c

@@ -4,6 +4,7 @@
 #include <linux/percpu.h>
 #include <linux/kallsyms.h>
 #include <linux/kcore.h>
+#include <linux/prandom.h>
 
 #include <asm/cpu_entry_area.h>
 #include <asm/pgtable.h>
@@ -16,12 +17,52 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(struct entry_stack_page, entry_stack_storage)
 #ifdef CONFIG_X86_64
 static DEFINE_PER_CPU_PAGE_ALIGNED(struct exception_stacks, exception_stacks);
 DEFINE_PER_CPU(struct cea_exception_stacks*, cea_exception_stacks);
+
+static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, _cea_offset);
+
+static __always_inline unsigned int cea_offset(unsigned int cpu)
+{
+	return per_cpu(_cea_offset, cpu);
+}
+
+static __init void init_cea_offsets(void)
+{
+	unsigned int max_cea;
+	unsigned int i, j;
+
+	max_cea = (CPU_ENTRY_AREA_MAP_SIZE - PAGE_SIZE) / CPU_ENTRY_AREA_SIZE;
+
+	/* O(sodding terrible) */
+	for_each_possible_cpu(i) {
+		unsigned int cea;
+
+again:
+		cea = prandom_u32_max(max_cea);
+
+		for_each_possible_cpu(j) {
+			if (cea_offset(j) == cea)
+				goto again;
+
+			if (i == j)
+				break;
+		}
+
+		per_cpu(_cea_offset, i) = cea;
+	}
+}
+#else /* !X86_64 */
+
+static __always_inline unsigned int cea_offset(unsigned int cpu)
+{
+	return cpu;
+}
+static inline void init_cea_offsets(void) { }
 #endif
 
 /* Is called from entry code, so must be noinstr */
 noinstr struct cpu_entry_area *get_cpu_entry_area(int cpu)
 {
-	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE;
+	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cea_offset(cpu) * CPU_ENTRY_AREA_SIZE;
 	BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0);
 
 	return (struct cpu_entry_area *) va;
@@ -194,7 +235,6 @@ static __init void setup_cpu_entry_area_ptes(void)
 
 	/* The +1 is for the readonly IDT: */
 	BUILD_BUG_ON((CPU_ENTRY_AREA_PAGES+1)*PAGE_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
-	BUILD_BUG_ON(CPU_ENTRY_AREA_TOTAL_SIZE != CPU_ENTRY_AREA_MAP_SIZE);
 	BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK);
 
 	start = CPU_ENTRY_AREA_BASE;
@@ -210,6 +250,8 @@ void __init setup_cpu_entry_areas(void)
 {
 	unsigned int cpu;
 
+	init_cea_offsets();
+
 	setup_cpu_entry_area_ptes();
 
 	for_each_possible_cpu(cpu)