Merge branch 'x86-pti-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git...
[muen/linux.git] / arch / x86 / mm / kasan_init_64.c
1 // SPDX-License-Identifier: GPL-2.0
2 #define DISABLE_BRANCH_PROFILING
3 #define pr_fmt(fmt) "kasan: " fmt
4
5 #ifdef CONFIG_X86_5LEVEL
6 /* Too early to use cpu_feature_enabled() */
7 #define pgtable_l5_enabled __pgtable_l5_enabled
8 #endif
9
10 #include <linux/bootmem.h>
11 #include <linux/kasan.h>
12 #include <linux/kdebug.h>
13 #include <linux/memblock.h>
14 #include <linux/mm.h>
15 #include <linux/sched.h>
16 #include <linux/sched/task.h>
17 #include <linux/vmalloc.h>
18
19 #include <asm/e820/types.h>
20 #include <asm/pgalloc.h>
21 #include <asm/tlbflush.h>
22 #include <asm/sections.h>
23 #include <asm/pgtable.h>
24 #include <asm/cpu_entry_area.h>
25
26 extern struct range pfn_mapped[E820_MAX_ENTRIES];
27
28 static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
29
30 static __init void *early_alloc(size_t size, int nid, bool panic)
31 {
32         if (panic)
33                 return memblock_virt_alloc_try_nid(size, size,
34                         __pa(MAX_DMA_ADDRESS), BOOTMEM_ALLOC_ACCESSIBLE, nid);
35         else
36                 return memblock_virt_alloc_try_nid_nopanic(size, size,
37                         __pa(MAX_DMA_ADDRESS), BOOTMEM_ALLOC_ACCESSIBLE, nid);
38 }
39
40 static void __init kasan_populate_pmd(pmd_t *pmd, unsigned long addr,
41                                       unsigned long end, int nid)
42 {
43         pte_t *pte;
44
45         if (pmd_none(*pmd)) {
46                 void *p;
47
48                 if (boot_cpu_has(X86_FEATURE_PSE) &&
49                     ((end - addr) == PMD_SIZE) &&
50                     IS_ALIGNED(addr, PMD_SIZE)) {
51                         p = early_alloc(PMD_SIZE, nid, false);
52                         if (p && pmd_set_huge(pmd, __pa(p), PAGE_KERNEL))
53                                 return;
54                         else if (p)
55                                 memblock_free(__pa(p), PMD_SIZE);
56                 }
57
58                 p = early_alloc(PAGE_SIZE, nid, true);
59                 pmd_populate_kernel(&init_mm, pmd, p);
60         }
61
62         pte = pte_offset_kernel(pmd, addr);
63         do {
64                 pte_t entry;
65                 void *p;
66
67                 if (!pte_none(*pte))
68                         continue;
69
70                 p = early_alloc(PAGE_SIZE, nid, true);
71                 entry = pfn_pte(PFN_DOWN(__pa(p)), PAGE_KERNEL);
72                 set_pte_at(&init_mm, addr, pte, entry);
73         } while (pte++, addr += PAGE_SIZE, addr != end);
74 }
75
76 static void __init kasan_populate_pud(pud_t *pud, unsigned long addr,
77                                       unsigned long end, int nid)
78 {
79         pmd_t *pmd;
80         unsigned long next;
81
82         if (pud_none(*pud)) {
83                 void *p;
84
85                 if (boot_cpu_has(X86_FEATURE_GBPAGES) &&
86                     ((end - addr) == PUD_SIZE) &&
87                     IS_ALIGNED(addr, PUD_SIZE)) {
88                         p = early_alloc(PUD_SIZE, nid, false);
89                         if (p && pud_set_huge(pud, __pa(p), PAGE_KERNEL))
90                                 return;
91                         else if (p)
92                                 memblock_free(__pa(p), PUD_SIZE);
93                 }
94
95                 p = early_alloc(PAGE_SIZE, nid, true);
96                 pud_populate(&init_mm, pud, p);
97         }
98
99         pmd = pmd_offset(pud, addr);
100         do {
101                 next = pmd_addr_end(addr, end);
102                 if (!pmd_large(*pmd))
103                         kasan_populate_pmd(pmd, addr, next, nid);
104         } while (pmd++, addr = next, addr != end);
105 }
106
107 static void __init kasan_populate_p4d(p4d_t *p4d, unsigned long addr,
108                                       unsigned long end, int nid)
109 {
110         pud_t *pud;
111         unsigned long next;
112
113         if (p4d_none(*p4d)) {
114                 void *p = early_alloc(PAGE_SIZE, nid, true);
115
116                 p4d_populate(&init_mm, p4d, p);
117         }
118
119         pud = pud_offset(p4d, addr);
120         do {
121                 next = pud_addr_end(addr, end);
122                 if (!pud_large(*pud))
123                         kasan_populate_pud(pud, addr, next, nid);
124         } while (pud++, addr = next, addr != end);
125 }
126
127 static void __init kasan_populate_pgd(pgd_t *pgd, unsigned long addr,
128                                       unsigned long end, int nid)
129 {
130         void *p;
131         p4d_t *p4d;
132         unsigned long next;
133
134         if (pgd_none(*pgd)) {
135                 p = early_alloc(PAGE_SIZE, nid, true);
136                 pgd_populate(&init_mm, pgd, p);
137         }
138
139         p4d = p4d_offset(pgd, addr);
140         do {
141                 next = p4d_addr_end(addr, end);
142                 kasan_populate_p4d(p4d, addr, next, nid);
143         } while (p4d++, addr = next, addr != end);
144 }
145
146 static void __init kasan_populate_shadow(unsigned long addr, unsigned long end,
147                                          int nid)
148 {
149         pgd_t *pgd;
150         unsigned long next;
151
152         addr = addr & PAGE_MASK;
153         end = round_up(end, PAGE_SIZE);
154         pgd = pgd_offset_k(addr);
155         do {
156                 next = pgd_addr_end(addr, end);
157                 kasan_populate_pgd(pgd, addr, next, nid);
158         } while (pgd++, addr = next, addr != end);
159 }
160
161 static void __init map_range(struct range *range)
162 {
163         unsigned long start;
164         unsigned long end;
165
166         start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start));
167         end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end));
168
169         kasan_populate_shadow(start, end, early_pfn_to_nid(range->start));
170 }
171
172 static void __init clear_pgds(unsigned long start,
173                         unsigned long end)
174 {
175         pgd_t *pgd;
176         /* See comment in kasan_init() */
177         unsigned long pgd_end = end & PGDIR_MASK;
178
179         for (; start < pgd_end; start += PGDIR_SIZE) {
180                 pgd = pgd_offset_k(start);
181                 /*
182                  * With folded p4d, pgd_clear() is nop, use p4d_clear()
183                  * instead.
184                  */
185                 if (pgtable_l5_enabled)
186                         pgd_clear(pgd);
187                 else
188                         p4d_clear(p4d_offset(pgd, start));
189         }
190
191         pgd = pgd_offset_k(start);
192         for (; start < end; start += P4D_SIZE)
193                 p4d_clear(p4d_offset(pgd, start));
194 }
195
196 static inline p4d_t *early_p4d_offset(pgd_t *pgd, unsigned long addr)
197 {
198         unsigned long p4d;
199
200         if (!pgtable_l5_enabled)
201                 return (p4d_t *)pgd;
202
203         p4d = __pa_nodebug(pgd_val(*pgd)) & PTE_PFN_MASK;
204         p4d += __START_KERNEL_map - phys_base;
205         return (p4d_t *)p4d + p4d_index(addr);
206 }
207
208 static void __init kasan_early_p4d_populate(pgd_t *pgd,
209                 unsigned long addr,
210                 unsigned long end)
211 {
212         pgd_t pgd_entry;
213         p4d_t *p4d, p4d_entry;
214         unsigned long next;
215
216         if (pgd_none(*pgd)) {
217                 pgd_entry = __pgd(_KERNPG_TABLE | __pa_nodebug(kasan_zero_p4d));
218                 set_pgd(pgd, pgd_entry);
219         }
220
221         p4d = early_p4d_offset(pgd, addr);
222         do {
223                 next = p4d_addr_end(addr, end);
224
225                 if (!p4d_none(*p4d))
226                         continue;
227
228                 p4d_entry = __p4d(_KERNPG_TABLE | __pa_nodebug(kasan_zero_pud));
229                 set_p4d(p4d, p4d_entry);
230         } while (p4d++, addr = next, addr != end && p4d_none(*p4d));
231 }
232
233 static void __init kasan_map_early_shadow(pgd_t *pgd)
234 {
235         /* See comment in kasan_init() */
236         unsigned long addr = KASAN_SHADOW_START & PGDIR_MASK;
237         unsigned long end = KASAN_SHADOW_END;
238         unsigned long next;
239
240         pgd += pgd_index(addr);
241         do {
242                 next = pgd_addr_end(addr, end);
243                 kasan_early_p4d_populate(pgd, addr, next);
244         } while (pgd++, addr = next, addr != end);
245 }
246
247 #ifdef CONFIG_KASAN_INLINE
248 static int kasan_die_handler(struct notifier_block *self,
249                              unsigned long val,
250                              void *data)
251 {
252         if (val == DIE_GPF) {
253                 pr_emerg("CONFIG_KASAN_INLINE enabled\n");
254                 pr_emerg("GPF could be caused by NULL-ptr deref or user memory access\n");
255         }
256         return NOTIFY_OK;
257 }
258
259 static struct notifier_block kasan_die_notifier = {
260         .notifier_call = kasan_die_handler,
261 };
262 #endif
263
264 void __init kasan_early_init(void)
265 {
266         int i;
267         pteval_t pte_val = __pa_nodebug(kasan_zero_page) | __PAGE_KERNEL | _PAGE_ENC;
268         pmdval_t pmd_val = __pa_nodebug(kasan_zero_pte) | _KERNPG_TABLE;
269         pudval_t pud_val = __pa_nodebug(kasan_zero_pmd) | _KERNPG_TABLE;
270         p4dval_t p4d_val = __pa_nodebug(kasan_zero_pud) | _KERNPG_TABLE;
271
272         /* Mask out unsupported __PAGE_KERNEL bits: */
273         pte_val &= __default_kernel_pte_mask;
274         pmd_val &= __default_kernel_pte_mask;
275         pud_val &= __default_kernel_pte_mask;
276         p4d_val &= __default_kernel_pte_mask;
277
278         for (i = 0; i < PTRS_PER_PTE; i++)
279                 kasan_zero_pte[i] = __pte(pte_val);
280
281         for (i = 0; i < PTRS_PER_PMD; i++)
282                 kasan_zero_pmd[i] = __pmd(pmd_val);
283
284         for (i = 0; i < PTRS_PER_PUD; i++)
285                 kasan_zero_pud[i] = __pud(pud_val);
286
287         for (i = 0; pgtable_l5_enabled && i < PTRS_PER_P4D; i++)
288                 kasan_zero_p4d[i] = __p4d(p4d_val);
289
290         kasan_map_early_shadow(early_top_pgt);
291         kasan_map_early_shadow(init_top_pgt);
292 }
293
294 void __init kasan_init(void)
295 {
296         int i;
297         void *shadow_cpu_entry_begin, *shadow_cpu_entry_end;
298
299 #ifdef CONFIG_KASAN_INLINE
300         register_die_notifier(&kasan_die_notifier);
301 #endif
302
303         memcpy(early_top_pgt, init_top_pgt, sizeof(early_top_pgt));
304
305         /*
306          * We use the same shadow offset for 4- and 5-level paging to
307          * facilitate boot-time switching between paging modes.
308          * As result in 5-level paging mode KASAN_SHADOW_START and
309          * KASAN_SHADOW_END are not aligned to PGD boundary.
310          *
311          * KASAN_SHADOW_START doesn't share PGD with anything else.
312          * We claim whole PGD entry to make things easier.
313          *
314          * KASAN_SHADOW_END lands in the last PGD entry and it collides with
315          * bunch of things like kernel code, modules, EFI mapping, etc.
316          * We need to take extra steps to not overwrite them.
317          */
318         if (pgtable_l5_enabled) {
319                 void *ptr;
320
321                 ptr = (void *)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_END));
322                 memcpy(tmp_p4d_table, (void *)ptr, sizeof(tmp_p4d_table));
323                 set_pgd(&early_top_pgt[pgd_index(KASAN_SHADOW_END)],
324                                 __pgd(__pa(tmp_p4d_table) | _KERNPG_TABLE));
325         }
326
327         load_cr3(early_top_pgt);
328         __flush_tlb_all();
329
330         clear_pgds(KASAN_SHADOW_START & PGDIR_MASK, KASAN_SHADOW_END);
331
332         kasan_populate_zero_shadow((void *)(KASAN_SHADOW_START & PGDIR_MASK),
333                         kasan_mem_to_shadow((void *)PAGE_OFFSET));
334
335         for (i = 0; i < E820_MAX_ENTRIES; i++) {
336                 if (pfn_mapped[i].end == 0)
337                         break;
338
339                 map_range(&pfn_mapped[i]);
340         }
341
342         shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE;
343         shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin);
344         shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin,
345                                                 PAGE_SIZE);
346
347         shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE +
348                                         CPU_ENTRY_AREA_MAP_SIZE);
349         shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end);
350         shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end,
351                                         PAGE_SIZE);
352
353         kasan_populate_zero_shadow(
354                 kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
355                 shadow_cpu_entry_begin);
356
357         kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin,
358                               (unsigned long)shadow_cpu_entry_end, 0);
359
360         kasan_populate_zero_shadow(shadow_cpu_entry_end,
361                                 kasan_mem_to_shadow((void *)__START_KERNEL_map));
362
363         kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext),
364                               (unsigned long)kasan_mem_to_shadow(_end),
365                               early_pfn_to_nid(__pa(_stext)));
366
367         kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
368                                 (void *)KASAN_SHADOW_END);
369
370         load_cr3(init_top_pgt);
371         __flush_tlb_all();
372
373         /*
374          * kasan_zero_page has been used as early shadow memory, thus it may
375          * contain some garbage. Now we can clear and write protect it, since
376          * after the TLB flush no one should write to it.
377          */
378         memset(kasan_zero_page, 0, PAGE_SIZE);
379         for (i = 0; i < PTRS_PER_PTE; i++) {
380                 pte_t pte;
381                 pgprot_t prot;
382
383                 prot = __pgprot(__PAGE_KERNEL_RO | _PAGE_ENC);
384                 pgprot_val(prot) &= __default_kernel_pte_mask;
385
386                 pte = __pte(__pa(kasan_zero_page) | pgprot_val(prot));
387                 set_pte(&kasan_zero_pte[i], pte);
388         }
389         /* Flush TLBs again to be sure that write protection applied. */
390         __flush_tlb_all();
391
392         init_task.kasan_depth = 0;
393         pr_info("KernelAddressSanitizer initialized\n");
394 }