x86/speculation/l1tf: Suggest what to do on systems with too much RAM
[muen/linux.git] / arch / x86 / kernel / cpu / bugs.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright (C) 1994  Linus Torvalds
4  *
5  *  Cyrix stuff, June 1998 by:
6  *      - Rafael R. Reilova (moved everything from head.S),
7  *        <rreilova@ececs.uc.edu>
8  *      - Channing Corn (tests & fixes),
9  *      - Andrew D. Balsa (code cleanup).
10  */
11 #include <linux/init.h>
12 #include <linux/utsname.h>
13 #include <linux/cpu.h>
14 #include <linux/module.h>
15 #include <linux/nospec.h>
16 #include <linux/prctl.h>
17
18 #include <asm/spec-ctrl.h>
19 #include <asm/cmdline.h>
20 #include <asm/bugs.h>
21 #include <asm/processor.h>
22 #include <asm/processor-flags.h>
23 #include <asm/fpu/internal.h>
24 #include <asm/msr.h>
25 #include <asm/vmx.h>
26 #include <asm/paravirt.h>
27 #include <asm/alternative.h>
28 #include <asm/pgtable.h>
29 #include <asm/set_memory.h>
30 #include <asm/intel-family.h>
31 #include <asm/e820/api.h>
32 #include <asm/hypervisor.h>
33
34 static void __init spectre_v2_select_mitigation(void);
35 static void __init ssb_select_mitigation(void);
36 static void __init l1tf_select_mitigation(void);
37
38 /*
39  * Our boot-time value of the SPEC_CTRL MSR. We read it once so that any
40  * writes to SPEC_CTRL contain whatever reserved bits have been set.
41  */
42 u64 __ro_after_init x86_spec_ctrl_base;
43 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
44
45 /*
46  * The vendor and possibly platform specific bits which can be modified in
47  * x86_spec_ctrl_base.
48  */
49 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
50
51 /*
52  * AMD specific MSR info for Speculative Store Bypass control.
53  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
54  */
55 u64 __ro_after_init x86_amd_ls_cfg_base;
56 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
57
58 void __init check_bugs(void)
59 {
60         identify_boot_cpu();
61
62         /*
63          * identify_boot_cpu() initialized SMT support information, let the
64          * core code know.
65          */
66         cpu_smt_check_topology_early();
67
68         if (!IS_ENABLED(CONFIG_SMP)) {
69                 pr_info("CPU: ");
70                 print_cpu_info(&boot_cpu_data);
71         }
72
73         /*
74          * Read the SPEC_CTRL MSR to account for reserved bits which may
75          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
76          * init code as it is not enumerated and depends on the family.
77          */
78         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
79                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
80
81         /* Allow STIBP in MSR_SPEC_CTRL if supported */
82         if (boot_cpu_has(X86_FEATURE_STIBP))
83                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
84
85         /* Select the proper spectre mitigation before patching alternatives */
86         spectre_v2_select_mitigation();
87
88         /*
89          * Select proper mitigation for any exposure to the Speculative Store
90          * Bypass vulnerability.
91          */
92         ssb_select_mitigation();
93
94         l1tf_select_mitigation();
95
96 #ifdef CONFIG_X86_32
97         /*
98          * Check whether we are able to run this kernel safely on SMP.
99          *
100          * - i386 is no longer supported.
101          * - In order to run on anything without a TSC, we need to be
102          *   compiled for a i486.
103          */
104         if (boot_cpu_data.x86 < 4)
105                 panic("Kernel requires i486+ for 'invlpg' and other features");
106
107         init_utsname()->machine[1] =
108                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
109         alternative_instructions();
110
111         fpu__init_check_bugs();
112 #else /* CONFIG_X86_64 */
113         alternative_instructions();
114
115         /*
116          * Make sure the first 2MB area is not mapped by huge pages
117          * There are typically fixed size MTRRs in there and overlapping
118          * MTRRs into large pages causes slow downs.
119          *
120          * Right now we don't do that with gbpages because there seems
121          * very little benefit for that case.
122          */
123         if (!direct_gbpages)
124                 set_memory_4k((unsigned long)__va(0), 1);
125 #endif
126 }
127
128 /* The kernel command line selection */
129 enum spectre_v2_mitigation_cmd {
130         SPECTRE_V2_CMD_NONE,
131         SPECTRE_V2_CMD_AUTO,
132         SPECTRE_V2_CMD_FORCE,
133         SPECTRE_V2_CMD_RETPOLINE,
134         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
135         SPECTRE_V2_CMD_RETPOLINE_AMD,
136 };
137
138 static const char *spectre_v2_strings[] = {
139         [SPECTRE_V2_NONE]                       = "Vulnerable",
140         [SPECTRE_V2_RETPOLINE_MINIMAL]          = "Vulnerable: Minimal generic ASM retpoline",
141         [SPECTRE_V2_RETPOLINE_MINIMAL_AMD]      = "Vulnerable: Minimal AMD ASM retpoline",
142         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
143         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
144         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
145 };
146
147 #undef pr_fmt
148 #define pr_fmt(fmt)     "Spectre V2 : " fmt
149
150 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
151         SPECTRE_V2_NONE;
152
153 void
154 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
155 {
156         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
157         struct thread_info *ti = current_thread_info();
158
159         /* Is MSR_SPEC_CTRL implemented ? */
160         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
161                 /*
162                  * Restrict guest_spec_ctrl to supported values. Clear the
163                  * modifiable bits in the host base value and or the
164                  * modifiable bits from the guest value.
165                  */
166                 guestval = hostval & ~x86_spec_ctrl_mask;
167                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
168
169                 /* SSBD controlled in MSR_SPEC_CTRL */
170                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
171                     static_cpu_has(X86_FEATURE_AMD_SSBD))
172                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
173
174                 if (hostval != guestval) {
175                         msrval = setguest ? guestval : hostval;
176                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
177                 }
178         }
179
180         /*
181          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
182          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
183          */
184         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
185             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
186                 return;
187
188         /*
189          * If the host has SSBD mitigation enabled, force it in the host's
190          * virtual MSR value. If its not permanently enabled, evaluate
191          * current's TIF_SSBD thread flag.
192          */
193         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
194                 hostval = SPEC_CTRL_SSBD;
195         else
196                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
197
198         /* Sanitize the guest value */
199         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
200
201         if (hostval != guestval) {
202                 unsigned long tif;
203
204                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
205                                  ssbd_spec_ctrl_to_tif(hostval);
206
207                 speculative_store_bypass_update(tif);
208         }
209 }
210 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
211
212 static void x86_amd_ssb_disable(void)
213 {
214         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
215
216         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
217                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
218         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
219                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
220 }
221
222 #ifdef RETPOLINE
223 static bool spectre_v2_bad_module;
224
225 bool retpoline_module_ok(bool has_retpoline)
226 {
227         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
228                 return true;
229
230         pr_err("System may be vulnerable to spectre v2\n");
231         spectre_v2_bad_module = true;
232         return false;
233 }
234
235 static inline const char *spectre_v2_module_string(void)
236 {
237         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
238 }
239 #else
240 static inline const char *spectre_v2_module_string(void) { return ""; }
241 #endif
242
243 static void __init spec2_print_if_insecure(const char *reason)
244 {
245         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
246                 pr_info("%s selected on command line.\n", reason);
247 }
248
249 static void __init spec2_print_if_secure(const char *reason)
250 {
251         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
252                 pr_info("%s selected on command line.\n", reason);
253 }
254
255 static inline bool retp_compiler(void)
256 {
257         return __is_defined(RETPOLINE);
258 }
259
260 static inline bool match_option(const char *arg, int arglen, const char *opt)
261 {
262         int len = strlen(opt);
263
264         return len == arglen && !strncmp(arg, opt, len);
265 }
266
267 static const struct {
268         const char *option;
269         enum spectre_v2_mitigation_cmd cmd;
270         bool secure;
271 } mitigation_options[] = {
272         { "off",               SPECTRE_V2_CMD_NONE,              false },
273         { "on",                SPECTRE_V2_CMD_FORCE,             true },
274         { "retpoline",         SPECTRE_V2_CMD_RETPOLINE,         false },
275         { "retpoline,amd",     SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
276         { "retpoline,generic", SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
277         { "auto",              SPECTRE_V2_CMD_AUTO,              false },
278 };
279
280 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
281 {
282         char arg[20];
283         int ret, i;
284         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
285
286         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2"))
287                 return SPECTRE_V2_CMD_NONE;
288         else {
289                 ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
290                 if (ret < 0)
291                         return SPECTRE_V2_CMD_AUTO;
292
293                 for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
294                         if (!match_option(arg, ret, mitigation_options[i].option))
295                                 continue;
296                         cmd = mitigation_options[i].cmd;
297                         break;
298                 }
299
300                 if (i >= ARRAY_SIZE(mitigation_options)) {
301                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
302                         return SPECTRE_V2_CMD_AUTO;
303                 }
304         }
305
306         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
307              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
308              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
309             !IS_ENABLED(CONFIG_RETPOLINE)) {
310                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
311                 return SPECTRE_V2_CMD_AUTO;
312         }
313
314         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
315             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
316                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
317                 return SPECTRE_V2_CMD_AUTO;
318         }
319
320         if (mitigation_options[i].secure)
321                 spec2_print_if_secure(mitigation_options[i].option);
322         else
323                 spec2_print_if_insecure(mitigation_options[i].option);
324
325         return cmd;
326 }
327
328 static void __init spectre_v2_select_mitigation(void)
329 {
330         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
331         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
332
333         /*
334          * If the CPU is not affected and the command line mode is NONE or AUTO
335          * then nothing to do.
336          */
337         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
338             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
339                 return;
340
341         switch (cmd) {
342         case SPECTRE_V2_CMD_NONE:
343                 return;
344
345         case SPECTRE_V2_CMD_FORCE:
346         case SPECTRE_V2_CMD_AUTO:
347                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
348                         mode = SPECTRE_V2_IBRS_ENHANCED;
349                         /* Force it so VMEXIT will restore correctly */
350                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
351                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
352                         goto specv2_set_mode;
353                 }
354                 if (IS_ENABLED(CONFIG_RETPOLINE))
355                         goto retpoline_auto;
356                 break;
357         case SPECTRE_V2_CMD_RETPOLINE_AMD:
358                 if (IS_ENABLED(CONFIG_RETPOLINE))
359                         goto retpoline_amd;
360                 break;
361         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
362                 if (IS_ENABLED(CONFIG_RETPOLINE))
363                         goto retpoline_generic;
364                 break;
365         case SPECTRE_V2_CMD_RETPOLINE:
366                 if (IS_ENABLED(CONFIG_RETPOLINE))
367                         goto retpoline_auto;
368                 break;
369         }
370         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
371         return;
372
373 retpoline_auto:
374         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
375         retpoline_amd:
376                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
377                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
378                         goto retpoline_generic;
379                 }
380                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_AMD :
381                                          SPECTRE_V2_RETPOLINE_MINIMAL_AMD;
382                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
383                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
384         } else {
385         retpoline_generic:
386                 mode = retp_compiler() ? SPECTRE_V2_RETPOLINE_GENERIC :
387                                          SPECTRE_V2_RETPOLINE_MINIMAL;
388                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
389         }
390
391 specv2_set_mode:
392         spectre_v2_enabled = mode;
393         pr_info("%s\n", spectre_v2_strings[mode]);
394
395         /*
396          * If spectre v2 protection has been enabled, unconditionally fill
397          * RSB during a context switch; this protects against two independent
398          * issues:
399          *
400          *      - RSB underflow (and switch to BTB) on Skylake+
401          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
402          */
403         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
404         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
405
406         /* Initialize Indirect Branch Prediction Barrier if supported */
407         if (boot_cpu_has(X86_FEATURE_IBPB)) {
408                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
409                 pr_info("Spectre v2 mitigation: Enabling Indirect Branch Prediction Barrier\n");
410         }
411
412         /*
413          * Retpoline means the kernel is safe because it has no indirect
414          * branches. Enhanced IBRS protects firmware too, so, enable restricted
415          * speculation around firmware calls only when Enhanced IBRS isn't
416          * supported.
417          *
418          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
419          * the user might select retpoline on the kernel command line and if
420          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
421          * enable IBRS around firmware calls.
422          */
423         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
424                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
425                 pr_info("Enabling Restricted Speculation for firmware calls\n");
426         }
427 }
428
429 #undef pr_fmt
430 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
431
432 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
433
434 /* The kernel command line selection */
435 enum ssb_mitigation_cmd {
436         SPEC_STORE_BYPASS_CMD_NONE,
437         SPEC_STORE_BYPASS_CMD_AUTO,
438         SPEC_STORE_BYPASS_CMD_ON,
439         SPEC_STORE_BYPASS_CMD_PRCTL,
440         SPEC_STORE_BYPASS_CMD_SECCOMP,
441 };
442
443 static const char *ssb_strings[] = {
444         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
445         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
446         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
447         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
448 };
449
450 static const struct {
451         const char *option;
452         enum ssb_mitigation_cmd cmd;
453 } ssb_mitigation_options[] = {
454         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
455         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
456         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
457         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
458         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
459 };
460
461 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
462 {
463         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
464         char arg[20];
465         int ret, i;
466
467         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable")) {
468                 return SPEC_STORE_BYPASS_CMD_NONE;
469         } else {
470                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
471                                           arg, sizeof(arg));
472                 if (ret < 0)
473                         return SPEC_STORE_BYPASS_CMD_AUTO;
474
475                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
476                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
477                                 continue;
478
479                         cmd = ssb_mitigation_options[i].cmd;
480                         break;
481                 }
482
483                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
484                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
485                         return SPEC_STORE_BYPASS_CMD_AUTO;
486                 }
487         }
488
489         return cmd;
490 }
491
492 static enum ssb_mitigation __init __ssb_select_mitigation(void)
493 {
494         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
495         enum ssb_mitigation_cmd cmd;
496
497         if (!boot_cpu_has(X86_FEATURE_SSBD))
498                 return mode;
499
500         cmd = ssb_parse_cmdline();
501         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
502             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
503              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
504                 return mode;
505
506         switch (cmd) {
507         case SPEC_STORE_BYPASS_CMD_AUTO:
508         case SPEC_STORE_BYPASS_CMD_SECCOMP:
509                 /*
510                  * Choose prctl+seccomp as the default mode if seccomp is
511                  * enabled.
512                  */
513                 if (IS_ENABLED(CONFIG_SECCOMP))
514                         mode = SPEC_STORE_BYPASS_SECCOMP;
515                 else
516                         mode = SPEC_STORE_BYPASS_PRCTL;
517                 break;
518         case SPEC_STORE_BYPASS_CMD_ON:
519                 mode = SPEC_STORE_BYPASS_DISABLE;
520                 break;
521         case SPEC_STORE_BYPASS_CMD_PRCTL:
522                 mode = SPEC_STORE_BYPASS_PRCTL;
523                 break;
524         case SPEC_STORE_BYPASS_CMD_NONE:
525                 break;
526         }
527
528         /*
529          * We have three CPU feature flags that are in play here:
530          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
531          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
532          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
533          */
534         if (mode == SPEC_STORE_BYPASS_DISABLE) {
535                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
536                 /*
537                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
538                  * use a completely different MSR and bit dependent on family.
539                  */
540                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
541                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
542                         x86_amd_ssb_disable();
543                 } else {
544                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
545                         x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
546                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
547                 }
548         }
549
550         return mode;
551 }
552
553 static void ssb_select_mitigation(void)
554 {
555         ssb_mode = __ssb_select_mitigation();
556
557         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
558                 pr_info("%s\n", ssb_strings[ssb_mode]);
559 }
560
561 #undef pr_fmt
562 #define pr_fmt(fmt)     "Speculation prctl: " fmt
563
564 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
565 {
566         bool update;
567
568         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
569             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
570                 return -ENXIO;
571
572         switch (ctrl) {
573         case PR_SPEC_ENABLE:
574                 /* If speculation is force disabled, enable is not allowed */
575                 if (task_spec_ssb_force_disable(task))
576                         return -EPERM;
577                 task_clear_spec_ssb_disable(task);
578                 update = test_and_clear_tsk_thread_flag(task, TIF_SSBD);
579                 break;
580         case PR_SPEC_DISABLE:
581                 task_set_spec_ssb_disable(task);
582                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
583                 break;
584         case PR_SPEC_FORCE_DISABLE:
585                 task_set_spec_ssb_disable(task);
586                 task_set_spec_ssb_force_disable(task);
587                 update = !test_and_set_tsk_thread_flag(task, TIF_SSBD);
588                 break;
589         default:
590                 return -ERANGE;
591         }
592
593         /*
594          * If being set on non-current task, delay setting the CPU
595          * mitigation until it is next scheduled.
596          */
597         if (task == current && update)
598                 speculative_store_bypass_update_current();
599
600         return 0;
601 }
602
603 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
604                              unsigned long ctrl)
605 {
606         switch (which) {
607         case PR_SPEC_STORE_BYPASS:
608                 return ssb_prctl_set(task, ctrl);
609         default:
610                 return -ENODEV;
611         }
612 }
613
614 #ifdef CONFIG_SECCOMP
615 void arch_seccomp_spec_mitigate(struct task_struct *task)
616 {
617         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
618                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
619 }
620 #endif
621
622 static int ssb_prctl_get(struct task_struct *task)
623 {
624         switch (ssb_mode) {
625         case SPEC_STORE_BYPASS_DISABLE:
626                 return PR_SPEC_DISABLE;
627         case SPEC_STORE_BYPASS_SECCOMP:
628         case SPEC_STORE_BYPASS_PRCTL:
629                 if (task_spec_ssb_force_disable(task))
630                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
631                 if (task_spec_ssb_disable(task))
632                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
633                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
634         default:
635                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
636                         return PR_SPEC_ENABLE;
637                 return PR_SPEC_NOT_AFFECTED;
638         }
639 }
640
641 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
642 {
643         switch (which) {
644         case PR_SPEC_STORE_BYPASS:
645                 return ssb_prctl_get(task);
646         default:
647                 return -ENODEV;
648         }
649 }
650
651 void x86_spec_ctrl_setup_ap(void)
652 {
653         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
654                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
655
656         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
657                 x86_amd_ssb_disable();
658 }
659
660 #undef pr_fmt
661 #define pr_fmt(fmt)     "L1TF: " fmt
662
663 /* Default mitigation for L1TF-affected CPUs */
664 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
665 #if IS_ENABLED(CONFIG_KVM_INTEL)
666 EXPORT_SYMBOL_GPL(l1tf_mitigation);
667 #endif
668 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
669 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
670
671 static void __init l1tf_select_mitigation(void)
672 {
673         u64 half_pa;
674
675         if (!boot_cpu_has_bug(X86_BUG_L1TF))
676                 return;
677
678         switch (l1tf_mitigation) {
679         case L1TF_MITIGATION_OFF:
680         case L1TF_MITIGATION_FLUSH_NOWARN:
681         case L1TF_MITIGATION_FLUSH:
682                 break;
683         case L1TF_MITIGATION_FLUSH_NOSMT:
684         case L1TF_MITIGATION_FULL:
685                 cpu_smt_disable(false);
686                 break;
687         case L1TF_MITIGATION_FULL_FORCE:
688                 cpu_smt_disable(true);
689                 break;
690         }
691
692 #if CONFIG_PGTABLE_LEVELS == 2
693         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
694         return;
695 #endif
696
697         /*
698          * This is extremely unlikely to happen because almost all
699          * systems have far more MAX_PA/2 than RAM can be fit into
700          * DIMM slots.
701          */
702         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
703         if (e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
704                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
705                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
706                                 half_pa);
707                 pr_info("However, doing so will make a part of your RAM unusable.\n");
708                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/l1tf.html might help you decide.\n");
709                 return;
710         }
711
712         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
713 }
714
715 static int __init l1tf_cmdline(char *str)
716 {
717         if (!boot_cpu_has_bug(X86_BUG_L1TF))
718                 return 0;
719
720         if (!str)
721                 return -EINVAL;
722
723         if (!strcmp(str, "off"))
724                 l1tf_mitigation = L1TF_MITIGATION_OFF;
725         else if (!strcmp(str, "flush,nowarn"))
726                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
727         else if (!strcmp(str, "flush"))
728                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
729         else if (!strcmp(str, "flush,nosmt"))
730                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
731         else if (!strcmp(str, "full"))
732                 l1tf_mitigation = L1TF_MITIGATION_FULL;
733         else if (!strcmp(str, "full,force"))
734                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
735
736         return 0;
737 }
738 early_param("l1tf", l1tf_cmdline);
739
740 #undef pr_fmt
741
742 #ifdef CONFIG_SYSFS
743
744 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
745
746 #if IS_ENABLED(CONFIG_KVM_INTEL)
747 static const char *l1tf_vmx_states[] = {
748         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
749         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
750         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
751         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
752         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
753         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
754 };
755
756 static ssize_t l1tf_show_state(char *buf)
757 {
758         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
759                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
760
761         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
762             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
763              cpu_smt_control == CPU_SMT_ENABLED))
764                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
765                                l1tf_vmx_states[l1tf_vmx_mitigation]);
766
767         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
768                        l1tf_vmx_states[l1tf_vmx_mitigation],
769                        cpu_smt_control == CPU_SMT_ENABLED ? "vulnerable" : "disabled");
770 }
771 #else
772 static ssize_t l1tf_show_state(char *buf)
773 {
774         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
775 }
776 #endif
777
778 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
779                                char *buf, unsigned int bug)
780 {
781         if (!boot_cpu_has_bug(bug))
782                 return sprintf(buf, "Not affected\n");
783
784         switch (bug) {
785         case X86_BUG_CPU_MELTDOWN:
786                 if (boot_cpu_has(X86_FEATURE_PTI))
787                         return sprintf(buf, "Mitigation: PTI\n");
788
789                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
790                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
791
792                 break;
793
794         case X86_BUG_SPECTRE_V1:
795                 return sprintf(buf, "Mitigation: __user pointer sanitization\n");
796
797         case X86_BUG_SPECTRE_V2:
798                 return sprintf(buf, "%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
799                                boot_cpu_has(X86_FEATURE_USE_IBPB) ? ", IBPB" : "",
800                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
801                                spectre_v2_module_string());
802
803         case X86_BUG_SPEC_STORE_BYPASS:
804                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
805
806         case X86_BUG_L1TF:
807                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
808                         return l1tf_show_state(buf);
809                 break;
810         default:
811                 break;
812         }
813
814         return sprintf(buf, "Vulnerable\n");
815 }
816
817 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
818 {
819         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
820 }
821
822 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
823 {
824         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
825 }
826
827 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
828 {
829         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
830 }
831
832 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
833 {
834         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
835 }
836
837 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
838 {
839         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
840 }
841 #endif