kvm: mmu: ITLB_MULTIHIT mitigation
[muen/linux.git] / arch / x86 / kernel / cpu / bugs.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  Copyright (C) 1994  Linus Torvalds
4  *
5  *  Cyrix stuff, June 1998 by:
6  *      - Rafael R. Reilova (moved everything from head.S),
7  *        <rreilova@ececs.uc.edu>
8  *      - Channing Corn (tests & fixes),
9  *      - Andrew D. Balsa (code cleanup).
10  */
11 #include <linux/init.h>
12 #include <linux/utsname.h>
13 #include <linux/cpu.h>
14 #include <linux/module.h>
15 #include <linux/nospec.h>
16 #include <linux/prctl.h>
17 #include <linux/sched/smt.h>
18
19 #include <asm/spec-ctrl.h>
20 #include <asm/cmdline.h>
21 #include <asm/bugs.h>
22 #include <asm/processor.h>
23 #include <asm/processor-flags.h>
24 #include <asm/fpu/internal.h>
25 #include <asm/msr.h>
26 #include <asm/vmx.h>
27 #include <asm/paravirt.h>
28 #include <asm/alternative.h>
29 #include <asm/pgtable.h>
30 #include <asm/set_memory.h>
31 #include <asm/intel-family.h>
32 #include <asm/e820/api.h>
33 #include <asm/hypervisor.h>
34
35 #include "cpu.h"
36
37 static void __init spectre_v1_select_mitigation(void);
38 static void __init spectre_v2_select_mitigation(void);
39 static void __init ssb_select_mitigation(void);
40 static void __init l1tf_select_mitigation(void);
41 static void __init mds_select_mitigation(void);
42 static void __init taa_select_mitigation(void);
43
44 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
45 u64 x86_spec_ctrl_base;
46 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
47 static DEFINE_MUTEX(spec_ctrl_mutex);
48
49 /*
50  * The vendor and possibly platform specific bits which can be modified in
51  * x86_spec_ctrl_base.
52  */
53 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
54
55 /*
56  * AMD specific MSR info for Speculative Store Bypass control.
57  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
58  */
59 u64 __ro_after_init x86_amd_ls_cfg_base;
60 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
61
62 /* Control conditional STIBP in switch_to() */
63 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
64 /* Control conditional IBPB in switch_mm() */
65 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
66 /* Control unconditional IBPB in switch_mm() */
67 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
68
69 /* Control MDS CPU buffer clear before returning to user space */
70 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
71 EXPORT_SYMBOL_GPL(mds_user_clear);
72 /* Control MDS CPU buffer clear before idling (halt, mwait) */
73 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
74 EXPORT_SYMBOL_GPL(mds_idle_clear);
75
76 void __init check_bugs(void)
77 {
78         identify_boot_cpu();
79
80         /*
81          * identify_boot_cpu() initialized SMT support information, let the
82          * core code know.
83          */
84         cpu_smt_check_topology();
85
86         if (!IS_ENABLED(CONFIG_SMP)) {
87                 pr_info("CPU: ");
88                 print_cpu_info(&boot_cpu_data);
89         }
90
91         /*
92          * Read the SPEC_CTRL MSR to account for reserved bits which may
93          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
94          * init code as it is not enumerated and depends on the family.
95          */
96         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
97                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
98
99         /* Allow STIBP in MSR_SPEC_CTRL if supported */
100         if (boot_cpu_has(X86_FEATURE_STIBP))
101                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
102
103         /* Select the proper CPU mitigations before patching alternatives: */
104         spectre_v1_select_mitigation();
105         spectre_v2_select_mitigation();
106         ssb_select_mitigation();
107         l1tf_select_mitigation();
108         mds_select_mitigation();
109         taa_select_mitigation();
110
111         arch_smt_update();
112
113 #ifdef CONFIG_X86_32
114         /*
115          * Check whether we are able to run this kernel safely on SMP.
116          *
117          * - i386 is no longer supported.
118          * - In order to run on anything without a TSC, we need to be
119          *   compiled for a i486.
120          */
121         if (boot_cpu_data.x86 < 4)
122                 panic("Kernel requires i486+ for 'invlpg' and other features");
123
124         init_utsname()->machine[1] =
125                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
126         alternative_instructions();
127
128         fpu__init_check_bugs();
129 #else /* CONFIG_X86_64 */
130         alternative_instructions();
131
132         /*
133          * Make sure the first 2MB area is not mapped by huge pages
134          * There are typically fixed size MTRRs in there and overlapping
135          * MTRRs into large pages causes slow downs.
136          *
137          * Right now we don't do that with gbpages because there seems
138          * very little benefit for that case.
139          */
140         if (!direct_gbpages)
141                 set_memory_4k((unsigned long)__va(0), 1);
142 #endif
143 }
144
145 void
146 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
147 {
148         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
149         struct thread_info *ti = current_thread_info();
150
151         /* Is MSR_SPEC_CTRL implemented ? */
152         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
153                 /*
154                  * Restrict guest_spec_ctrl to supported values. Clear the
155                  * modifiable bits in the host base value and or the
156                  * modifiable bits from the guest value.
157                  */
158                 guestval = hostval & ~x86_spec_ctrl_mask;
159                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
160
161                 /* SSBD controlled in MSR_SPEC_CTRL */
162                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
163                     static_cpu_has(X86_FEATURE_AMD_SSBD))
164                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
165
166                 /* Conditional STIBP enabled? */
167                 if (static_branch_unlikely(&switch_to_cond_stibp))
168                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
169
170                 if (hostval != guestval) {
171                         msrval = setguest ? guestval : hostval;
172                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
173                 }
174         }
175
176         /*
177          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
178          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
179          */
180         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
181             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
182                 return;
183
184         /*
185          * If the host has SSBD mitigation enabled, force it in the host's
186          * virtual MSR value. If its not permanently enabled, evaluate
187          * current's TIF_SSBD thread flag.
188          */
189         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
190                 hostval = SPEC_CTRL_SSBD;
191         else
192                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
193
194         /* Sanitize the guest value */
195         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
196
197         if (hostval != guestval) {
198                 unsigned long tif;
199
200                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
201                                  ssbd_spec_ctrl_to_tif(hostval);
202
203                 speculation_ctrl_update(tif);
204         }
205 }
206 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
207
208 static void x86_amd_ssb_disable(void)
209 {
210         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
211
212         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
213                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
214         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
215                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
216 }
217
218 #undef pr_fmt
219 #define pr_fmt(fmt)     "MDS: " fmt
220
221 /* Default mitigation for MDS-affected CPUs */
222 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
223 static bool mds_nosmt __ro_after_init = false;
224
225 static const char * const mds_strings[] = {
226         [MDS_MITIGATION_OFF]    = "Vulnerable",
227         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
228         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
229 };
230
231 static void __init mds_select_mitigation(void)
232 {
233         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
234                 mds_mitigation = MDS_MITIGATION_OFF;
235                 return;
236         }
237
238         if (mds_mitigation == MDS_MITIGATION_FULL) {
239                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
240                         mds_mitigation = MDS_MITIGATION_VMWERV;
241
242                 static_branch_enable(&mds_user_clear);
243
244                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
245                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
246                         cpu_smt_disable(false);
247         }
248
249         pr_info("%s\n", mds_strings[mds_mitigation]);
250 }
251
252 static int __init mds_cmdline(char *str)
253 {
254         if (!boot_cpu_has_bug(X86_BUG_MDS))
255                 return 0;
256
257         if (!str)
258                 return -EINVAL;
259
260         if (!strcmp(str, "off"))
261                 mds_mitigation = MDS_MITIGATION_OFF;
262         else if (!strcmp(str, "full"))
263                 mds_mitigation = MDS_MITIGATION_FULL;
264         else if (!strcmp(str, "full,nosmt")) {
265                 mds_mitigation = MDS_MITIGATION_FULL;
266                 mds_nosmt = true;
267         }
268
269         return 0;
270 }
271 early_param("mds", mds_cmdline);
272
273 #undef pr_fmt
274 #define pr_fmt(fmt)     "TAA: " fmt
275
276 /* Default mitigation for TAA-affected CPUs */
277 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
278 static bool taa_nosmt __ro_after_init;
279
280 static const char * const taa_strings[] = {
281         [TAA_MITIGATION_OFF]            = "Vulnerable",
282         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
283         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
284         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
285 };
286
287 static void __init taa_select_mitigation(void)
288 {
289         u64 ia32_cap;
290
291         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
292                 taa_mitigation = TAA_MITIGATION_OFF;
293                 return;
294         }
295
296         /* TSX previously disabled by tsx=off */
297         if (!boot_cpu_has(X86_FEATURE_RTM)) {
298                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
299                 goto out;
300         }
301
302         if (cpu_mitigations_off()) {
303                 taa_mitigation = TAA_MITIGATION_OFF;
304                 return;
305         }
306
307         /* TAA mitigation is turned off on the cmdline (tsx_async_abort=off) */
308         if (taa_mitigation == TAA_MITIGATION_OFF)
309                 goto out;
310
311         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
312                 taa_mitigation = TAA_MITIGATION_VERW;
313         else
314                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
315
316         /*
317          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
318          * A microcode update fixes this behavior to clear CPU buffers. It also
319          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
320          * ARCH_CAP_TSX_CTRL_MSR bit.
321          *
322          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
323          * update is required.
324          */
325         ia32_cap = x86_read_arch_cap_msr();
326         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
327             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
328                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
329
330         /*
331          * TSX is enabled, select alternate mitigation for TAA which is
332          * the same as MDS. Enable MDS static branch to clear CPU buffers.
333          *
334          * For guests that can't determine whether the correct microcode is
335          * present on host, enable the mitigation for UCODE_NEEDED as well.
336          */
337         static_branch_enable(&mds_user_clear);
338
339         if (taa_nosmt || cpu_mitigations_auto_nosmt())
340                 cpu_smt_disable(false);
341
342 out:
343         pr_info("%s\n", taa_strings[taa_mitigation]);
344 }
345
346 static int __init tsx_async_abort_parse_cmdline(char *str)
347 {
348         if (!boot_cpu_has_bug(X86_BUG_TAA))
349                 return 0;
350
351         if (!str)
352                 return -EINVAL;
353
354         if (!strcmp(str, "off")) {
355                 taa_mitigation = TAA_MITIGATION_OFF;
356         } else if (!strcmp(str, "full")) {
357                 taa_mitigation = TAA_MITIGATION_VERW;
358         } else if (!strcmp(str, "full,nosmt")) {
359                 taa_mitigation = TAA_MITIGATION_VERW;
360                 taa_nosmt = true;
361         }
362
363         return 0;
364 }
365 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
366
367 #undef pr_fmt
368 #define pr_fmt(fmt)     "Spectre V1 : " fmt
369
370 enum spectre_v1_mitigation {
371         SPECTRE_V1_MITIGATION_NONE,
372         SPECTRE_V1_MITIGATION_AUTO,
373 };
374
375 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
376         SPECTRE_V1_MITIGATION_AUTO;
377
378 static const char * const spectre_v1_strings[] = {
379         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
380         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
381 };
382
383 /*
384  * Does SMAP provide full mitigation against speculative kernel access to
385  * userspace?
386  */
387 static bool smap_works_speculatively(void)
388 {
389         if (!boot_cpu_has(X86_FEATURE_SMAP))
390                 return false;
391
392         /*
393          * On CPUs which are vulnerable to Meltdown, SMAP does not
394          * prevent speculative access to user data in the L1 cache.
395          * Consider SMAP to be non-functional as a mitigation on these
396          * CPUs.
397          */
398         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
399                 return false;
400
401         return true;
402 }
403
404 static void __init spectre_v1_select_mitigation(void)
405 {
406         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
407                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
408                 return;
409         }
410
411         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
412                 /*
413                  * With Spectre v1, a user can speculatively control either
414                  * path of a conditional swapgs with a user-controlled GS
415                  * value.  The mitigation is to add lfences to both code paths.
416                  *
417                  * If FSGSBASE is enabled, the user can put a kernel address in
418                  * GS, in which case SMAP provides no protection.
419                  *
420                  * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the
421                  *         FSGSBASE enablement patches have been merged. ]
422                  *
423                  * If FSGSBASE is disabled, the user can only put a user space
424                  * address in GS.  That makes an attack harder, but still
425                  * possible if there's no SMAP protection.
426                  */
427                 if (!smap_works_speculatively()) {
428                         /*
429                          * Mitigation can be provided from SWAPGS itself or
430                          * PTI as the CR3 write in the Meltdown mitigation
431                          * is serializing.
432                          *
433                          * If neither is there, mitigate with an LFENCE to
434                          * stop speculation through swapgs.
435                          */
436                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
437                             !boot_cpu_has(X86_FEATURE_PTI))
438                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
439
440                         /*
441                          * Enable lfences in the kernel entry (non-swapgs)
442                          * paths, to prevent user entry from speculatively
443                          * skipping swapgs.
444                          */
445                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
446                 }
447         }
448
449         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
450 }
451
452 static int __init nospectre_v1_cmdline(char *str)
453 {
454         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
455         return 0;
456 }
457 early_param("nospectre_v1", nospectre_v1_cmdline);
458
459 #undef pr_fmt
460 #define pr_fmt(fmt)     "Spectre V2 : " fmt
461
462 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
463         SPECTRE_V2_NONE;
464
465 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
466         SPECTRE_V2_USER_NONE;
467
468 #ifdef CONFIG_RETPOLINE
469 static bool spectre_v2_bad_module;
470
471 bool retpoline_module_ok(bool has_retpoline)
472 {
473         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
474                 return true;
475
476         pr_err("System may be vulnerable to spectre v2\n");
477         spectre_v2_bad_module = true;
478         return false;
479 }
480
481 static inline const char *spectre_v2_module_string(void)
482 {
483         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
484 }
485 #else
486 static inline const char *spectre_v2_module_string(void) { return ""; }
487 #endif
488
489 static inline bool match_option(const char *arg, int arglen, const char *opt)
490 {
491         int len = strlen(opt);
492
493         return len == arglen && !strncmp(arg, opt, len);
494 }
495
496 /* The kernel command line selection for spectre v2 */
497 enum spectre_v2_mitigation_cmd {
498         SPECTRE_V2_CMD_NONE,
499         SPECTRE_V2_CMD_AUTO,
500         SPECTRE_V2_CMD_FORCE,
501         SPECTRE_V2_CMD_RETPOLINE,
502         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
503         SPECTRE_V2_CMD_RETPOLINE_AMD,
504 };
505
506 enum spectre_v2_user_cmd {
507         SPECTRE_V2_USER_CMD_NONE,
508         SPECTRE_V2_USER_CMD_AUTO,
509         SPECTRE_V2_USER_CMD_FORCE,
510         SPECTRE_V2_USER_CMD_PRCTL,
511         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
512         SPECTRE_V2_USER_CMD_SECCOMP,
513         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
514 };
515
516 static const char * const spectre_v2_user_strings[] = {
517         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
518         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
519         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
520         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
521         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
522 };
523
524 static const struct {
525         const char                      *option;
526         enum spectre_v2_user_cmd        cmd;
527         bool                            secure;
528 } v2_user_options[] __initconst = {
529         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
530         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
531         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
532         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
533         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
534         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
535         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
536 };
537
538 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
539 {
540         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
541                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
542 }
543
544 static enum spectre_v2_user_cmd __init
545 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
546 {
547         char arg[20];
548         int ret, i;
549
550         switch (v2_cmd) {
551         case SPECTRE_V2_CMD_NONE:
552                 return SPECTRE_V2_USER_CMD_NONE;
553         case SPECTRE_V2_CMD_FORCE:
554                 return SPECTRE_V2_USER_CMD_FORCE;
555         default:
556                 break;
557         }
558
559         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
560                                   arg, sizeof(arg));
561         if (ret < 0)
562                 return SPECTRE_V2_USER_CMD_AUTO;
563
564         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
565                 if (match_option(arg, ret, v2_user_options[i].option)) {
566                         spec_v2_user_print_cond(v2_user_options[i].option,
567                                                 v2_user_options[i].secure);
568                         return v2_user_options[i].cmd;
569                 }
570         }
571
572         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
573         return SPECTRE_V2_USER_CMD_AUTO;
574 }
575
576 static void __init
577 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
578 {
579         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
580         bool smt_possible = IS_ENABLED(CONFIG_SMP);
581         enum spectre_v2_user_cmd cmd;
582
583         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
584                 return;
585
586         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
587             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
588                 smt_possible = false;
589
590         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
591         switch (cmd) {
592         case SPECTRE_V2_USER_CMD_NONE:
593                 goto set_mode;
594         case SPECTRE_V2_USER_CMD_FORCE:
595                 mode = SPECTRE_V2_USER_STRICT;
596                 break;
597         case SPECTRE_V2_USER_CMD_PRCTL:
598         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
599                 mode = SPECTRE_V2_USER_PRCTL;
600                 break;
601         case SPECTRE_V2_USER_CMD_AUTO:
602         case SPECTRE_V2_USER_CMD_SECCOMP:
603         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
604                 if (IS_ENABLED(CONFIG_SECCOMP))
605                         mode = SPECTRE_V2_USER_SECCOMP;
606                 else
607                         mode = SPECTRE_V2_USER_PRCTL;
608                 break;
609         }
610
611         /*
612          * At this point, an STIBP mode other than "off" has been set.
613          * If STIBP support is not being forced, check if STIBP always-on
614          * is preferred.
615          */
616         if (mode != SPECTRE_V2_USER_STRICT &&
617             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
618                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
619
620         /* Initialize Indirect Branch Prediction Barrier */
621         if (boot_cpu_has(X86_FEATURE_IBPB)) {
622                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
623
624                 switch (cmd) {
625                 case SPECTRE_V2_USER_CMD_FORCE:
626                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
627                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
628                         static_branch_enable(&switch_mm_always_ibpb);
629                         break;
630                 case SPECTRE_V2_USER_CMD_PRCTL:
631                 case SPECTRE_V2_USER_CMD_AUTO:
632                 case SPECTRE_V2_USER_CMD_SECCOMP:
633                         static_branch_enable(&switch_mm_cond_ibpb);
634                         break;
635                 default:
636                         break;
637                 }
638
639                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
640                         static_key_enabled(&switch_mm_always_ibpb) ?
641                         "always-on" : "conditional");
642         }
643
644         /* If enhanced IBRS is enabled no STIBP required */
645         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
646                 return;
647
648         /*
649          * If SMT is not possible or STIBP is not available clear the STIBP
650          * mode.
651          */
652         if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
653                 mode = SPECTRE_V2_USER_NONE;
654 set_mode:
655         spectre_v2_user = mode;
656         /* Only print the STIBP mode when SMT possible */
657         if (smt_possible)
658                 pr_info("%s\n", spectre_v2_user_strings[mode]);
659 }
660
661 static const char * const spectre_v2_strings[] = {
662         [SPECTRE_V2_NONE]                       = "Vulnerable",
663         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
664         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
665         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
666 };
667
668 static const struct {
669         const char *option;
670         enum spectre_v2_mitigation_cmd cmd;
671         bool secure;
672 } mitigation_options[] __initconst = {
673         { "off",                SPECTRE_V2_CMD_NONE,              false },
674         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
675         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
676         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
677         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
678         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
679 };
680
681 static void __init spec_v2_print_cond(const char *reason, bool secure)
682 {
683         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
684                 pr_info("%s selected on command line.\n", reason);
685 }
686
687 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
688 {
689         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
690         char arg[20];
691         int ret, i;
692
693         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
694             cpu_mitigations_off())
695                 return SPECTRE_V2_CMD_NONE;
696
697         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
698         if (ret < 0)
699                 return SPECTRE_V2_CMD_AUTO;
700
701         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
702                 if (!match_option(arg, ret, mitigation_options[i].option))
703                         continue;
704                 cmd = mitigation_options[i].cmd;
705                 break;
706         }
707
708         if (i >= ARRAY_SIZE(mitigation_options)) {
709                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
710                 return SPECTRE_V2_CMD_AUTO;
711         }
712
713         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
714              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
715              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
716             !IS_ENABLED(CONFIG_RETPOLINE)) {
717                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
718                 return SPECTRE_V2_CMD_AUTO;
719         }
720
721         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
722             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
723             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
724                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
725                 return SPECTRE_V2_CMD_AUTO;
726         }
727
728         spec_v2_print_cond(mitigation_options[i].option,
729                            mitigation_options[i].secure);
730         return cmd;
731 }
732
733 static void __init spectre_v2_select_mitigation(void)
734 {
735         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
736         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
737
738         /*
739          * If the CPU is not affected and the command line mode is NONE or AUTO
740          * then nothing to do.
741          */
742         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
743             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
744                 return;
745
746         switch (cmd) {
747         case SPECTRE_V2_CMD_NONE:
748                 return;
749
750         case SPECTRE_V2_CMD_FORCE:
751         case SPECTRE_V2_CMD_AUTO:
752                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
753                         mode = SPECTRE_V2_IBRS_ENHANCED;
754                         /* Force it so VMEXIT will restore correctly */
755                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
756                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
757                         goto specv2_set_mode;
758                 }
759                 if (IS_ENABLED(CONFIG_RETPOLINE))
760                         goto retpoline_auto;
761                 break;
762         case SPECTRE_V2_CMD_RETPOLINE_AMD:
763                 if (IS_ENABLED(CONFIG_RETPOLINE))
764                         goto retpoline_amd;
765                 break;
766         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
767                 if (IS_ENABLED(CONFIG_RETPOLINE))
768                         goto retpoline_generic;
769                 break;
770         case SPECTRE_V2_CMD_RETPOLINE:
771                 if (IS_ENABLED(CONFIG_RETPOLINE))
772                         goto retpoline_auto;
773                 break;
774         }
775         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
776         return;
777
778 retpoline_auto:
779         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
780             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
781         retpoline_amd:
782                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
783                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
784                         goto retpoline_generic;
785                 }
786                 mode = SPECTRE_V2_RETPOLINE_AMD;
787                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
788                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
789         } else {
790         retpoline_generic:
791                 mode = SPECTRE_V2_RETPOLINE_GENERIC;
792                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
793         }
794
795 specv2_set_mode:
796         spectre_v2_enabled = mode;
797         pr_info("%s\n", spectre_v2_strings[mode]);
798
799         /*
800          * If spectre v2 protection has been enabled, unconditionally fill
801          * RSB during a context switch; this protects against two independent
802          * issues:
803          *
804          *      - RSB underflow (and switch to BTB) on Skylake+
805          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
806          */
807         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
808         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
809
810         /*
811          * Retpoline means the kernel is safe because it has no indirect
812          * branches. Enhanced IBRS protects firmware too, so, enable restricted
813          * speculation around firmware calls only when Enhanced IBRS isn't
814          * supported.
815          *
816          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
817          * the user might select retpoline on the kernel command line and if
818          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
819          * enable IBRS around firmware calls.
820          */
821         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
822                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
823                 pr_info("Enabling Restricted Speculation for firmware calls\n");
824         }
825
826         /* Set up IBPB and STIBP depending on the general spectre V2 command */
827         spectre_v2_user_select_mitigation(cmd);
828 }
829
830 static void update_stibp_msr(void * __unused)
831 {
832         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
833 }
834
835 /* Update x86_spec_ctrl_base in case SMT state changed. */
836 static void update_stibp_strict(void)
837 {
838         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
839
840         if (sched_smt_active())
841                 mask |= SPEC_CTRL_STIBP;
842
843         if (mask == x86_spec_ctrl_base)
844                 return;
845
846         pr_info("Update user space SMT mitigation: STIBP %s\n",
847                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
848         x86_spec_ctrl_base = mask;
849         on_each_cpu(update_stibp_msr, NULL, 1);
850 }
851
852 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
853 static void update_indir_branch_cond(void)
854 {
855         if (sched_smt_active())
856                 static_branch_enable(&switch_to_cond_stibp);
857         else
858                 static_branch_disable(&switch_to_cond_stibp);
859 }
860
861 #undef pr_fmt
862 #define pr_fmt(fmt) fmt
863
864 /* Update the static key controlling the MDS CPU buffer clear in idle */
865 static void update_mds_branch_idle(void)
866 {
867         /*
868          * Enable the idle clearing if SMT is active on CPUs which are
869          * affected only by MSBDS and not any other MDS variant.
870          *
871          * The other variants cannot be mitigated when SMT is enabled, so
872          * clearing the buffers on idle just to prevent the Store Buffer
873          * repartitioning leak would be a window dressing exercise.
874          */
875         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
876                 return;
877
878         if (sched_smt_active())
879                 static_branch_enable(&mds_idle_clear);
880         else
881                 static_branch_disable(&mds_idle_clear);
882 }
883
884 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
885 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
886
887 void cpu_bugs_smt_update(void)
888 {
889         /* Enhanced IBRS implies STIBP. No update required. */
890         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
891                 return;
892
893         mutex_lock(&spec_ctrl_mutex);
894
895         switch (spectre_v2_user) {
896         case SPECTRE_V2_USER_NONE:
897                 break;
898         case SPECTRE_V2_USER_STRICT:
899         case SPECTRE_V2_USER_STRICT_PREFERRED:
900                 update_stibp_strict();
901                 break;
902         case SPECTRE_V2_USER_PRCTL:
903         case SPECTRE_V2_USER_SECCOMP:
904                 update_indir_branch_cond();
905                 break;
906         }
907
908         switch (mds_mitigation) {
909         case MDS_MITIGATION_FULL:
910         case MDS_MITIGATION_VMWERV:
911                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
912                         pr_warn_once(MDS_MSG_SMT);
913                 update_mds_branch_idle();
914                 break;
915         case MDS_MITIGATION_OFF:
916                 break;
917         }
918
919         switch (taa_mitigation) {
920         case TAA_MITIGATION_VERW:
921         case TAA_MITIGATION_UCODE_NEEDED:
922                 if (sched_smt_active())
923                         pr_warn_once(TAA_MSG_SMT);
924                 break;
925         case TAA_MITIGATION_TSX_DISABLED:
926         case TAA_MITIGATION_OFF:
927                 break;
928         }
929
930         mutex_unlock(&spec_ctrl_mutex);
931 }
932
933 #undef pr_fmt
934 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
935
936 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
937
938 /* The kernel command line selection */
939 enum ssb_mitigation_cmd {
940         SPEC_STORE_BYPASS_CMD_NONE,
941         SPEC_STORE_BYPASS_CMD_AUTO,
942         SPEC_STORE_BYPASS_CMD_ON,
943         SPEC_STORE_BYPASS_CMD_PRCTL,
944         SPEC_STORE_BYPASS_CMD_SECCOMP,
945 };
946
947 static const char * const ssb_strings[] = {
948         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
949         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
950         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
951         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
952 };
953
954 static const struct {
955         const char *option;
956         enum ssb_mitigation_cmd cmd;
957 } ssb_mitigation_options[]  __initconst = {
958         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
959         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
960         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
961         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
962         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
963 };
964
965 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
966 {
967         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
968         char arg[20];
969         int ret, i;
970
971         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
972             cpu_mitigations_off()) {
973                 return SPEC_STORE_BYPASS_CMD_NONE;
974         } else {
975                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
976                                           arg, sizeof(arg));
977                 if (ret < 0)
978                         return SPEC_STORE_BYPASS_CMD_AUTO;
979
980                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
981                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
982                                 continue;
983
984                         cmd = ssb_mitigation_options[i].cmd;
985                         break;
986                 }
987
988                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
989                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
990                         return SPEC_STORE_BYPASS_CMD_AUTO;
991                 }
992         }
993
994         return cmd;
995 }
996
997 static enum ssb_mitigation __init __ssb_select_mitigation(void)
998 {
999         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1000         enum ssb_mitigation_cmd cmd;
1001
1002         if (!boot_cpu_has(X86_FEATURE_SSBD))
1003                 return mode;
1004
1005         cmd = ssb_parse_cmdline();
1006         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1007             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1008              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1009                 return mode;
1010
1011         switch (cmd) {
1012         case SPEC_STORE_BYPASS_CMD_AUTO:
1013         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1014                 /*
1015                  * Choose prctl+seccomp as the default mode if seccomp is
1016                  * enabled.
1017                  */
1018                 if (IS_ENABLED(CONFIG_SECCOMP))
1019                         mode = SPEC_STORE_BYPASS_SECCOMP;
1020                 else
1021                         mode = SPEC_STORE_BYPASS_PRCTL;
1022                 break;
1023         case SPEC_STORE_BYPASS_CMD_ON:
1024                 mode = SPEC_STORE_BYPASS_DISABLE;
1025                 break;
1026         case SPEC_STORE_BYPASS_CMD_PRCTL:
1027                 mode = SPEC_STORE_BYPASS_PRCTL;
1028                 break;
1029         case SPEC_STORE_BYPASS_CMD_NONE:
1030                 break;
1031         }
1032
1033         /*
1034          * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1035          * bit in the mask to allow guests to use the mitigation even in the
1036          * case where the host does not enable it.
1037          */
1038         if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1039             static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1040                 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1041         }
1042
1043         /*
1044          * We have three CPU feature flags that are in play here:
1045          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1046          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1047          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1048          */
1049         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1050                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1051                 /*
1052                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1053                  * use a completely different MSR and bit dependent on family.
1054                  */
1055                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1056                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1057                         x86_amd_ssb_disable();
1058                 } else {
1059                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1060                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1061                 }
1062         }
1063
1064         return mode;
1065 }
1066
1067 static void ssb_select_mitigation(void)
1068 {
1069         ssb_mode = __ssb_select_mitigation();
1070
1071         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1072                 pr_info("%s\n", ssb_strings[ssb_mode]);
1073 }
1074
1075 #undef pr_fmt
1076 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1077
1078 static void task_update_spec_tif(struct task_struct *tsk)
1079 {
1080         /* Force the update of the real TIF bits */
1081         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1082
1083         /*
1084          * Immediately update the speculation control MSRs for the current
1085          * task, but for a non-current task delay setting the CPU
1086          * mitigation until it is scheduled next.
1087          *
1088          * This can only happen for SECCOMP mitigation. For PRCTL it's
1089          * always the current task.
1090          */
1091         if (tsk == current)
1092                 speculation_ctrl_update_current();
1093 }
1094
1095 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1096 {
1097         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1098             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1099                 return -ENXIO;
1100
1101         switch (ctrl) {
1102         case PR_SPEC_ENABLE:
1103                 /* If speculation is force disabled, enable is not allowed */
1104                 if (task_spec_ssb_force_disable(task))
1105                         return -EPERM;
1106                 task_clear_spec_ssb_disable(task);
1107                 task_clear_spec_ssb_noexec(task);
1108                 task_update_spec_tif(task);
1109                 break;
1110         case PR_SPEC_DISABLE:
1111                 task_set_spec_ssb_disable(task);
1112                 task_clear_spec_ssb_noexec(task);
1113                 task_update_spec_tif(task);
1114                 break;
1115         case PR_SPEC_FORCE_DISABLE:
1116                 task_set_spec_ssb_disable(task);
1117                 task_set_spec_ssb_force_disable(task);
1118                 task_clear_spec_ssb_noexec(task);
1119                 task_update_spec_tif(task);
1120                 break;
1121         case PR_SPEC_DISABLE_NOEXEC:
1122                 if (task_spec_ssb_force_disable(task))
1123                         return -EPERM;
1124                 task_set_spec_ssb_disable(task);
1125                 task_set_spec_ssb_noexec(task);
1126                 task_update_spec_tif(task);
1127                 break;
1128         default:
1129                 return -ERANGE;
1130         }
1131         return 0;
1132 }
1133
1134 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1135 {
1136         switch (ctrl) {
1137         case PR_SPEC_ENABLE:
1138                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1139                         return 0;
1140                 /*
1141                  * Indirect branch speculation is always disabled in strict
1142                  * mode.
1143                  */
1144                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1145                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1146                         return -EPERM;
1147                 task_clear_spec_ib_disable(task);
1148                 task_update_spec_tif(task);
1149                 break;
1150         case PR_SPEC_DISABLE:
1151         case PR_SPEC_FORCE_DISABLE:
1152                 /*
1153                  * Indirect branch speculation is always allowed when
1154                  * mitigation is force disabled.
1155                  */
1156                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1157                         return -EPERM;
1158                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1159                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1160                         return 0;
1161                 task_set_spec_ib_disable(task);
1162                 if (ctrl == PR_SPEC_FORCE_DISABLE)
1163                         task_set_spec_ib_force_disable(task);
1164                 task_update_spec_tif(task);
1165                 break;
1166         default:
1167                 return -ERANGE;
1168         }
1169         return 0;
1170 }
1171
1172 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1173                              unsigned long ctrl)
1174 {
1175         switch (which) {
1176         case PR_SPEC_STORE_BYPASS:
1177                 return ssb_prctl_set(task, ctrl);
1178         case PR_SPEC_INDIRECT_BRANCH:
1179                 return ib_prctl_set(task, ctrl);
1180         default:
1181                 return -ENODEV;
1182         }
1183 }
1184
1185 #ifdef CONFIG_SECCOMP
1186 void arch_seccomp_spec_mitigate(struct task_struct *task)
1187 {
1188         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1189                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1190         if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
1191                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1192 }
1193 #endif
1194
1195 static int ssb_prctl_get(struct task_struct *task)
1196 {
1197         switch (ssb_mode) {
1198         case SPEC_STORE_BYPASS_DISABLE:
1199                 return PR_SPEC_DISABLE;
1200         case SPEC_STORE_BYPASS_SECCOMP:
1201         case SPEC_STORE_BYPASS_PRCTL:
1202                 if (task_spec_ssb_force_disable(task))
1203                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1204                 if (task_spec_ssb_noexec(task))
1205                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
1206                 if (task_spec_ssb_disable(task))
1207                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1208                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1209         default:
1210                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1211                         return PR_SPEC_ENABLE;
1212                 return PR_SPEC_NOT_AFFECTED;
1213         }
1214 }
1215
1216 static int ib_prctl_get(struct task_struct *task)
1217 {
1218         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1219                 return PR_SPEC_NOT_AFFECTED;
1220
1221         switch (spectre_v2_user) {
1222         case SPECTRE_V2_USER_NONE:
1223                 return PR_SPEC_ENABLE;
1224         case SPECTRE_V2_USER_PRCTL:
1225         case SPECTRE_V2_USER_SECCOMP:
1226                 if (task_spec_ib_force_disable(task))
1227                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1228                 if (task_spec_ib_disable(task))
1229                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1230                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1231         case SPECTRE_V2_USER_STRICT:
1232         case SPECTRE_V2_USER_STRICT_PREFERRED:
1233                 return PR_SPEC_DISABLE;
1234         default:
1235                 return PR_SPEC_NOT_AFFECTED;
1236         }
1237 }
1238
1239 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1240 {
1241         switch (which) {
1242         case PR_SPEC_STORE_BYPASS:
1243                 return ssb_prctl_get(task);
1244         case PR_SPEC_INDIRECT_BRANCH:
1245                 return ib_prctl_get(task);
1246         default:
1247                 return -ENODEV;
1248         }
1249 }
1250
1251 void x86_spec_ctrl_setup_ap(void)
1252 {
1253         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1254                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1255
1256         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1257                 x86_amd_ssb_disable();
1258 }
1259
1260 bool itlb_multihit_kvm_mitigation;
1261 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
1262
1263 #undef pr_fmt
1264 #define pr_fmt(fmt)     "L1TF: " fmt
1265
1266 /* Default mitigation for L1TF-affected CPUs */
1267 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1268 #if IS_ENABLED(CONFIG_KVM_INTEL)
1269 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1270 #endif
1271 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1272 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1273
1274 /*
1275  * These CPUs all support 44bits physical address space internally in the
1276  * cache but CPUID can report a smaller number of physical address bits.
1277  *
1278  * The L1TF mitigation uses the top most address bit for the inversion of
1279  * non present PTEs. When the installed memory reaches into the top most
1280  * address bit due to memory holes, which has been observed on machines
1281  * which report 36bits physical address bits and have 32G RAM installed,
1282  * then the mitigation range check in l1tf_select_mitigation() triggers.
1283  * This is a false positive because the mitigation is still possible due to
1284  * the fact that the cache uses 44bit internally. Use the cache bits
1285  * instead of the reported physical bits and adjust them on the affected
1286  * machines to 44bit if the reported bits are less than 44.
1287  */
1288 static void override_cache_bits(struct cpuinfo_x86 *c)
1289 {
1290         if (c->x86 != 6)
1291                 return;
1292
1293         switch (c->x86_model) {
1294         case INTEL_FAM6_NEHALEM:
1295         case INTEL_FAM6_WESTMERE:
1296         case INTEL_FAM6_SANDYBRIDGE:
1297         case INTEL_FAM6_IVYBRIDGE:
1298         case INTEL_FAM6_HASWELL:
1299         case INTEL_FAM6_HASWELL_L:
1300         case INTEL_FAM6_HASWELL_G:
1301         case INTEL_FAM6_BROADWELL:
1302         case INTEL_FAM6_BROADWELL_G:
1303         case INTEL_FAM6_SKYLAKE_L:
1304         case INTEL_FAM6_SKYLAKE:
1305         case INTEL_FAM6_KABYLAKE_L:
1306         case INTEL_FAM6_KABYLAKE:
1307                 if (c->x86_cache_bits < 44)
1308                         c->x86_cache_bits = 44;
1309                 break;
1310         }
1311 }
1312
1313 static void __init l1tf_select_mitigation(void)
1314 {
1315         u64 half_pa;
1316
1317         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1318                 return;
1319
1320         if (cpu_mitigations_off())
1321                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1322         else if (cpu_mitigations_auto_nosmt())
1323                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1324
1325         override_cache_bits(&boot_cpu_data);
1326
1327         switch (l1tf_mitigation) {
1328         case L1TF_MITIGATION_OFF:
1329         case L1TF_MITIGATION_FLUSH_NOWARN:
1330         case L1TF_MITIGATION_FLUSH:
1331                 break;
1332         case L1TF_MITIGATION_FLUSH_NOSMT:
1333         case L1TF_MITIGATION_FULL:
1334                 cpu_smt_disable(false);
1335                 break;
1336         case L1TF_MITIGATION_FULL_FORCE:
1337                 cpu_smt_disable(true);
1338                 break;
1339         }
1340
1341 #if CONFIG_PGTABLE_LEVELS == 2
1342         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1343         return;
1344 #endif
1345
1346         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1347         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1348                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
1349                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1350                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1351                                 half_pa);
1352                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1353                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1354                 return;
1355         }
1356
1357         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1358 }
1359
1360 static int __init l1tf_cmdline(char *str)
1361 {
1362         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1363                 return 0;
1364
1365         if (!str)
1366                 return -EINVAL;
1367
1368         if (!strcmp(str, "off"))
1369                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1370         else if (!strcmp(str, "flush,nowarn"))
1371                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1372         else if (!strcmp(str, "flush"))
1373                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1374         else if (!strcmp(str, "flush,nosmt"))
1375                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1376         else if (!strcmp(str, "full"))
1377                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1378         else if (!strcmp(str, "full,force"))
1379                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1380
1381         return 0;
1382 }
1383 early_param("l1tf", l1tf_cmdline);
1384
1385 #undef pr_fmt
1386 #define pr_fmt(fmt) fmt
1387
1388 #ifdef CONFIG_SYSFS
1389
1390 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1391
1392 #if IS_ENABLED(CONFIG_KVM_INTEL)
1393 static const char * const l1tf_vmx_states[] = {
1394         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1395         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1396         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1397         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1398         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1399         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1400 };
1401
1402 static ssize_t l1tf_show_state(char *buf)
1403 {
1404         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1405                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1406
1407         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1408             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1409              sched_smt_active())) {
1410                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1411                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1412         }
1413
1414         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1415                        l1tf_vmx_states[l1tf_vmx_mitigation],
1416                        sched_smt_active() ? "vulnerable" : "disabled");
1417 }
1418
1419 static ssize_t itlb_multihit_show_state(char *buf)
1420 {
1421         if (itlb_multihit_kvm_mitigation)
1422                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
1423         else
1424                 return sprintf(buf, "KVM: Vulnerable\n");
1425 }
1426 #else
1427 static ssize_t l1tf_show_state(char *buf)
1428 {
1429         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1430 }
1431
1432 static ssize_t itlb_multihit_show_state(char *buf)
1433 {
1434         return sprintf(buf, "Processor vulnerable\n");
1435 }
1436 #endif
1437
1438 static ssize_t mds_show_state(char *buf)
1439 {
1440         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1441                 return sprintf(buf, "%s; SMT Host state unknown\n",
1442                                mds_strings[mds_mitigation]);
1443         }
1444
1445         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1446                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1447                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
1448                                 sched_smt_active() ? "mitigated" : "disabled"));
1449         }
1450
1451         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1452                        sched_smt_active() ? "vulnerable" : "disabled");
1453 }
1454
1455 static ssize_t tsx_async_abort_show_state(char *buf)
1456 {
1457         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
1458             (taa_mitigation == TAA_MITIGATION_OFF))
1459                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
1460
1461         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1462                 return sprintf(buf, "%s; SMT Host state unknown\n",
1463                                taa_strings[taa_mitigation]);
1464         }
1465
1466         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
1467                        sched_smt_active() ? "vulnerable" : "disabled");
1468 }
1469
1470 static char *stibp_state(void)
1471 {
1472         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1473                 return "";
1474
1475         switch (spectre_v2_user) {
1476         case SPECTRE_V2_USER_NONE:
1477                 return ", STIBP: disabled";
1478         case SPECTRE_V2_USER_STRICT:
1479                 return ", STIBP: forced";
1480         case SPECTRE_V2_USER_STRICT_PREFERRED:
1481                 return ", STIBP: always-on";
1482         case SPECTRE_V2_USER_PRCTL:
1483         case SPECTRE_V2_USER_SECCOMP:
1484                 if (static_key_enabled(&switch_to_cond_stibp))
1485                         return ", STIBP: conditional";
1486         }
1487         return "";
1488 }
1489
1490 static char *ibpb_state(void)
1491 {
1492         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1493                 if (static_key_enabled(&switch_mm_always_ibpb))
1494                         return ", IBPB: always-on";
1495                 if (static_key_enabled(&switch_mm_cond_ibpb))
1496                         return ", IBPB: conditional";
1497                 return ", IBPB: disabled";
1498         }
1499         return "";
1500 }
1501
1502 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1503                                char *buf, unsigned int bug)
1504 {
1505         if (!boot_cpu_has_bug(bug))
1506                 return sprintf(buf, "Not affected\n");
1507
1508         switch (bug) {
1509         case X86_BUG_CPU_MELTDOWN:
1510                 if (boot_cpu_has(X86_FEATURE_PTI))
1511                         return sprintf(buf, "Mitigation: PTI\n");
1512
1513                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
1514                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
1515
1516                 break;
1517
1518         case X86_BUG_SPECTRE_V1:
1519                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1520
1521         case X86_BUG_SPECTRE_V2:
1522                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1523                                ibpb_state(),
1524                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1525                                stibp_state(),
1526                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1527                                spectre_v2_module_string());
1528
1529         case X86_BUG_SPEC_STORE_BYPASS:
1530                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1531
1532         case X86_BUG_L1TF:
1533                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1534                         return l1tf_show_state(buf);
1535                 break;
1536
1537         case X86_BUG_MDS:
1538                 return mds_show_state(buf);
1539
1540         case X86_BUG_TAA:
1541                 return tsx_async_abort_show_state(buf);
1542
1543         case X86_BUG_ITLB_MULTIHIT:
1544                 return itlb_multihit_show_state(buf);
1545
1546         default:
1547                 break;
1548         }
1549
1550         return sprintf(buf, "Vulnerable\n");
1551 }
1552
1553 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1554 {
1555         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1556 }
1557
1558 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1559 {
1560         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1561 }
1562
1563 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1564 {
1565         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1566 }
1567
1568 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1569 {
1570         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1571 }
1572
1573 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1574 {
1575         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1576 }
1577
1578 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1579 {
1580         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1581 }
1582
1583 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
1584 {
1585         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
1586 }
1587
1588 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
1589 {
1590         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
1591 }
1592 #endif