7f9b699969c7f62187b2f0d8f774e54a7efd13ac
[muen/linux.git] / arch / arm64 / kernel / cpu_errata.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Contains CPU specific errata definitions
4  *
5  * Copyright (C) 2014 ARM Ltd.
6  */
7
8 #include <linux/arm-smccc.h>
9 #include <linux/psci.h>
10 #include <linux/types.h>
11 #include <linux/cpu.h>
12 #include <asm/cpu.h>
13 #include <asm/cputype.h>
14 #include <asm/cpufeature.h>
15 #include <asm/smp_plat.h>
16
17 static bool __maybe_unused
18 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
19 {
20         const struct arm64_midr_revidr *fix;
21         u32 midr = read_cpuid_id(), revidr;
22
23         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
24         if (!is_midr_in_range(midr, &entry->midr_range))
25                 return false;
26
27         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
28         revidr = read_cpuid(REVIDR_EL1);
29         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
30                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
31                         return false;
32
33         return true;
34 }
35
36 static bool __maybe_unused
37 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
38                             int scope)
39 {
40         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
41         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
42 }
43
44 static bool __maybe_unused
45 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
46 {
47         u32 model;
48
49         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
50
51         model = read_cpuid_id();
52         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
53                  MIDR_ARCHITECTURE_MASK;
54
55         return model == entry->midr_range.model;
56 }
57
58 static bool
59 has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
60                           int scope)
61 {
62         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
63         u64 sys = arm64_ftr_reg_ctrel0.sys_val & mask;
64         u64 ctr_raw, ctr_real;
65
66         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
67
68         /*
69          * We want to make sure that all the CPUs in the system expose
70          * a consistent CTR_EL0 to make sure that applications behaves
71          * correctly with migration.
72          *
73          * If a CPU has CTR_EL0.IDC but does not advertise it via CTR_EL0 :
74          *
75          * 1) It is safe if the system doesn't support IDC, as CPU anyway
76          *    reports IDC = 0, consistent with the rest.
77          *
78          * 2) If the system has IDC, it is still safe as we trap CTR_EL0
79          *    access on this CPU via the ARM64_HAS_CACHE_IDC capability.
80          *
81          * So, we need to make sure either the raw CTR_EL0 or the effective
82          * CTR_EL0 matches the system's copy to allow a secondary CPU to boot.
83          */
84         ctr_raw = read_cpuid_cachetype() & mask;
85         ctr_real = read_cpuid_effective_cachetype() & mask;
86
87         return (ctr_real != sys) && (ctr_raw != sys);
88 }
89
90 static void
91 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
92 {
93         u64 mask = arm64_ftr_reg_ctrel0.strict_mask;
94
95         /* Trap CTR_EL0 access on this CPU, only if it has a mismatch */
96         if ((read_cpuid_cachetype() & mask) !=
97             (arm64_ftr_reg_ctrel0.sys_val & mask))
98                 sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
99 }
100
101 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
102
103 #include <asm/mmu_context.h>
104 #include <asm/cacheflush.h>
105
106 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
107
108 #ifdef CONFIG_KVM_INDIRECT_VECTORS
109 extern char __smccc_workaround_1_smc_start[];
110 extern char __smccc_workaround_1_smc_end[];
111
112 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
113                                 const char *hyp_vecs_end)
114 {
115         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
116         int i;
117
118         for (i = 0; i < SZ_2K; i += 0x80)
119                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
120
121         __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
122 }
123
124 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
125                                     const char *hyp_vecs_start,
126                                     const char *hyp_vecs_end)
127 {
128         static DEFINE_RAW_SPINLOCK(bp_lock);
129         int cpu, slot = -1;
130
131         /*
132          * detect_harden_bp_fw() passes NULL for the hyp_vecs start/end if
133          * we're a guest. Skip the hyp-vectors work.
134          */
135         if (!hyp_vecs_start) {
136                 __this_cpu_write(bp_hardening_data.fn, fn);
137                 return;
138         }
139
140         raw_spin_lock(&bp_lock);
141         for_each_possible_cpu(cpu) {
142                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
143                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
144                         break;
145                 }
146         }
147
148         if (slot == -1) {
149                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
150                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
151                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
152         }
153
154         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
155         __this_cpu_write(bp_hardening_data.fn, fn);
156         raw_spin_unlock(&bp_lock);
157 }
158 #else
159 #define __smccc_workaround_1_smc_start          NULL
160 #define __smccc_workaround_1_smc_end            NULL
161
162 static void install_bp_hardening_cb(bp_hardening_cb_t fn,
163                                       const char *hyp_vecs_start,
164                                       const char *hyp_vecs_end)
165 {
166         __this_cpu_write(bp_hardening_data.fn, fn);
167 }
168 #endif  /* CONFIG_KVM_INDIRECT_VECTORS */
169
170 #include <uapi/linux/psci.h>
171 #include <linux/arm-smccc.h>
172 #include <linux/psci.h>
173
174 static void call_smc_arch_workaround_1(void)
175 {
176         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
177 }
178
179 static void call_hvc_arch_workaround_1(void)
180 {
181         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
182 }
183
184 static void qcom_link_stack_sanitization(void)
185 {
186         u64 tmp;
187
188         asm volatile("mov       %0, x30         \n"
189                      ".rept     16              \n"
190                      "bl        . + 4           \n"
191                      ".endr                     \n"
192                      "mov       x30, %0         \n"
193                      : "=&r" (tmp));
194 }
195
196 static bool __nospectre_v2;
197 static int __init parse_nospectre_v2(char *str)
198 {
199         __nospectre_v2 = true;
200         return 0;
201 }
202 early_param("nospectre_v2", parse_nospectre_v2);
203
204 /*
205  * -1: No workaround
206  *  0: No workaround required
207  *  1: Workaround installed
208  */
209 static int detect_harden_bp_fw(void)
210 {
211         bp_hardening_cb_t cb;
212         void *smccc_start, *smccc_end;
213         struct arm_smccc_res res;
214         u32 midr = read_cpuid_id();
215
216         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
217                 return -1;
218
219         switch (psci_ops.conduit) {
220         case PSCI_CONDUIT_HVC:
221                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
222                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
223                 switch ((int)res.a0) {
224                 case 1:
225                         /* Firmware says we're just fine */
226                         return 0;
227                 case 0:
228                         cb = call_hvc_arch_workaround_1;
229                         /* This is a guest, no need to patch KVM vectors */
230                         smccc_start = NULL;
231                         smccc_end = NULL;
232                         break;
233                 default:
234                         return -1;
235                 }
236                 break;
237
238         case PSCI_CONDUIT_SMC:
239                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
240                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
241                 switch ((int)res.a0) {
242                 case 1:
243                         /* Firmware says we're just fine */
244                         return 0;
245                 case 0:
246                         cb = call_smc_arch_workaround_1;
247                         smccc_start = __smccc_workaround_1_smc_start;
248                         smccc_end = __smccc_workaround_1_smc_end;
249                         break;
250                 default:
251                         return -1;
252                 }
253                 break;
254
255         default:
256                 return -1;
257         }
258
259         if (((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR) ||
260             ((midr & MIDR_CPU_MODEL_MASK) == MIDR_QCOM_FALKOR_V1))
261                 cb = qcom_link_stack_sanitization;
262
263         if (IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR))
264                 install_bp_hardening_cb(cb, smccc_start, smccc_end);
265
266         return 1;
267 }
268
269 DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);
270
271 int ssbd_state __read_mostly = ARM64_SSBD_KERNEL;
272 static bool __ssb_safe = true;
273
274 static const struct ssbd_options {
275         const char      *str;
276         int             state;
277 } ssbd_options[] = {
278         { "force-on",   ARM64_SSBD_FORCE_ENABLE, },
279         { "force-off",  ARM64_SSBD_FORCE_DISABLE, },
280         { "kernel",     ARM64_SSBD_KERNEL, },
281 };
282
283 static int __init ssbd_cfg(char *buf)
284 {
285         int i;
286
287         if (!buf || !buf[0])
288                 return -EINVAL;
289
290         for (i = 0; i < ARRAY_SIZE(ssbd_options); i++) {
291                 int len = strlen(ssbd_options[i].str);
292
293                 if (strncmp(buf, ssbd_options[i].str, len))
294                         continue;
295
296                 ssbd_state = ssbd_options[i].state;
297                 return 0;
298         }
299
300         return -EINVAL;
301 }
302 early_param("ssbd", ssbd_cfg);
303
304 void __init arm64_update_smccc_conduit(struct alt_instr *alt,
305                                        __le32 *origptr, __le32 *updptr,
306                                        int nr_inst)
307 {
308         u32 insn;
309
310         BUG_ON(nr_inst != 1);
311
312         switch (psci_ops.conduit) {
313         case PSCI_CONDUIT_HVC:
314                 insn = aarch64_insn_get_hvc_value();
315                 break;
316         case PSCI_CONDUIT_SMC:
317                 insn = aarch64_insn_get_smc_value();
318                 break;
319         default:
320                 return;
321         }
322
323         *updptr = cpu_to_le32(insn);
324 }
325
326 void __init arm64_enable_wa2_handling(struct alt_instr *alt,
327                                       __le32 *origptr, __le32 *updptr,
328                                       int nr_inst)
329 {
330         BUG_ON(nr_inst != 1);
331         /*
332          * Only allow mitigation on EL1 entry/exit and guest
333          * ARCH_WORKAROUND_2 handling if the SSBD state allows it to
334          * be flipped.
335          */
336         if (arm64_get_ssbd_state() == ARM64_SSBD_KERNEL)
337                 *updptr = cpu_to_le32(aarch64_insn_gen_nop());
338 }
339
340 void arm64_set_ssbd_mitigation(bool state)
341 {
342         if (!IS_ENABLED(CONFIG_ARM64_SSBD)) {
343                 pr_info_once("SSBD disabled by kernel configuration\n");
344                 return;
345         }
346
347         if (this_cpu_has_cap(ARM64_SSBS)) {
348                 if (state)
349                         asm volatile(SET_PSTATE_SSBS(0));
350                 else
351                         asm volatile(SET_PSTATE_SSBS(1));
352                 return;
353         }
354
355         switch (psci_ops.conduit) {
356         case PSCI_CONDUIT_HVC:
357                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
358                 break;
359
360         case PSCI_CONDUIT_SMC:
361                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_2, state, NULL);
362                 break;
363
364         default:
365                 WARN_ON_ONCE(1);
366                 break;
367         }
368 }
369
370 static bool has_ssbd_mitigation(const struct arm64_cpu_capabilities *entry,
371                                     int scope)
372 {
373         struct arm_smccc_res res;
374         bool required = true;
375         s32 val;
376         bool this_cpu_safe = false;
377
378         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
379
380         if (cpu_mitigations_off())
381                 ssbd_state = ARM64_SSBD_FORCE_DISABLE;
382
383         /* delay setting __ssb_safe until we get a firmware response */
384         if (is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list))
385                 this_cpu_safe = true;
386
387         if (this_cpu_has_cap(ARM64_SSBS)) {
388                 if (!this_cpu_safe)
389                         __ssb_safe = false;
390                 required = false;
391                 goto out_printmsg;
392         }
393
394         if (psci_ops.smccc_version == SMCCC_VERSION_1_0) {
395                 ssbd_state = ARM64_SSBD_UNKNOWN;
396                 if (!this_cpu_safe)
397                         __ssb_safe = false;
398                 return false;
399         }
400
401         switch (psci_ops.conduit) {
402         case PSCI_CONDUIT_HVC:
403                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
404                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
405                 break;
406
407         case PSCI_CONDUIT_SMC:
408                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
409                                   ARM_SMCCC_ARCH_WORKAROUND_2, &res);
410                 break;
411
412         default:
413                 ssbd_state = ARM64_SSBD_UNKNOWN;
414                 if (!this_cpu_safe)
415                         __ssb_safe = false;
416                 return false;
417         }
418
419         val = (s32)res.a0;
420
421         switch (val) {
422         case SMCCC_RET_NOT_SUPPORTED:
423                 ssbd_state = ARM64_SSBD_UNKNOWN;
424                 if (!this_cpu_safe)
425                         __ssb_safe = false;
426                 return false;
427
428         /* machines with mixed mitigation requirements must not return this */
429         case SMCCC_RET_NOT_REQUIRED:
430                 pr_info_once("%s mitigation not required\n", entry->desc);
431                 ssbd_state = ARM64_SSBD_MITIGATED;
432                 return false;
433
434         case SMCCC_RET_SUCCESS:
435                 __ssb_safe = false;
436                 required = true;
437                 break;
438
439         case 1: /* Mitigation not required on this CPU */
440                 required = false;
441                 break;
442
443         default:
444                 WARN_ON(1);
445                 if (!this_cpu_safe)
446                         __ssb_safe = false;
447                 return false;
448         }
449
450         switch (ssbd_state) {
451         case ARM64_SSBD_FORCE_DISABLE:
452                 arm64_set_ssbd_mitigation(false);
453                 required = false;
454                 break;
455
456         case ARM64_SSBD_KERNEL:
457                 if (required) {
458                         __this_cpu_write(arm64_ssbd_callback_required, 1);
459                         arm64_set_ssbd_mitigation(true);
460                 }
461                 break;
462
463         case ARM64_SSBD_FORCE_ENABLE:
464                 arm64_set_ssbd_mitigation(true);
465                 required = true;
466                 break;
467
468         default:
469                 WARN_ON(1);
470                 break;
471         }
472
473 out_printmsg:
474         switch (ssbd_state) {
475         case ARM64_SSBD_FORCE_DISABLE:
476                 pr_info_once("%s disabled from command-line\n", entry->desc);
477                 break;
478
479         case ARM64_SSBD_FORCE_ENABLE:
480                 pr_info_once("%s forced from command-line\n", entry->desc);
481                 break;
482         }
483
484         return required;
485 }
486
487 /* known invulnerable cores */
488 static const struct midr_range arm64_ssb_cpus[] = {
489         MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
490         MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
491         MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
492         {},
493 };
494
495 #ifdef CONFIG_ARM64_ERRATUM_1463225
496 DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
497
498 static bool
499 has_cortex_a76_erratum_1463225(const struct arm64_cpu_capabilities *entry,
500                                int scope)
501 {
502         u32 midr = read_cpuid_id();
503         /* Cortex-A76 r0p0 - r3p1 */
504         struct midr_range range = MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1);
505
506         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
507         return is_midr_in_range(midr, &range) && is_kernel_in_hyp_mode();
508 }
509 #endif
510
511 static void __maybe_unused
512 cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
513 {
514         sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
515 }
516
517 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
518         .matches = is_affected_midr_range,                      \
519         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
520
521 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
522         .matches = is_affected_midr_range,                              \
523         .midr_range = MIDR_ALL_VERSIONS(model)
524
525 #define MIDR_FIXED(rev, revidr_mask) \
526         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
527
528 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
529         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
530         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
531
532 #define CAP_MIDR_RANGE_LIST(list)                               \
533         .matches = is_affected_midr_range_list,                 \
534         .midr_range_list = list
535
536 /* Errata affecting a range of revisions of  given model variant */
537 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
538         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
539
540 /* Errata affecting a single variant/revision of a model */
541 #define ERRATA_MIDR_REV(model, var, rev)        \
542         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
543
544 /* Errata affecting all variants/revisions of a given a model */
545 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
546         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
547         CAP_MIDR_ALL_VERSIONS(model)
548
549 /* Errata affecting a list of midr ranges, with same work around */
550 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
551         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
552         CAP_MIDR_RANGE_LIST(midr_list)
553
554 /* Track overall mitigation state. We are only mitigated if all cores are ok */
555 static bool __hardenbp_enab = true;
556 static bool __spectrev2_safe = true;
557
558 int get_spectre_v2_workaround_state(void)
559 {
560         if (__spectrev2_safe)
561                 return ARM64_BP_HARDEN_NOT_REQUIRED;
562
563         if (!__hardenbp_enab)
564                 return ARM64_BP_HARDEN_UNKNOWN;
565
566         return ARM64_BP_HARDEN_WA_NEEDED;
567 }
568
569 /*
570  * List of CPUs that do not need any Spectre-v2 mitigation at all.
571  */
572 static const struct midr_range spectre_v2_safe_list[] = {
573         MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
574         MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
575         MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
576         { /* sentinel */ }
577 };
578
579 /*
580  * Track overall bp hardening for all heterogeneous cores in the machine.
581  * We are only considered "safe" if all booted cores are known safe.
582  */
583 static bool __maybe_unused
584 check_branch_predictor(const struct arm64_cpu_capabilities *entry, int scope)
585 {
586         int need_wa;
587
588         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
589
590         /* If the CPU has CSV2 set, we're safe */
591         if (cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64PFR0_EL1),
592                                                  ID_AA64PFR0_CSV2_SHIFT))
593                 return false;
594
595         /* Alternatively, we have a list of unaffected CPUs */
596         if (is_midr_in_range_list(read_cpuid_id(), spectre_v2_safe_list))
597                 return false;
598
599         /* Fallback to firmware detection */
600         need_wa = detect_harden_bp_fw();
601         if (!need_wa)
602                 return false;
603
604         __spectrev2_safe = false;
605
606         if (!IS_ENABLED(CONFIG_HARDEN_BRANCH_PREDICTOR)) {
607                 pr_warn_once("spectrev2 mitigation disabled by kernel configuration\n");
608                 __hardenbp_enab = false;
609                 return false;
610         }
611
612         /* forced off */
613         if (__nospectre_v2 || cpu_mitigations_off()) {
614                 pr_info_once("spectrev2 mitigation disabled by command line option\n");
615                 __hardenbp_enab = false;
616                 return false;
617         }
618
619         if (need_wa < 0) {
620                 pr_warn_once("ARM_SMCCC_ARCH_WORKAROUND_1 missing from firmware\n");
621                 __hardenbp_enab = false;
622         }
623
624         return (need_wa > 0);
625 }
626
627 static const __maybe_unused struct midr_range tx2_family_cpus[] = {
628         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
629         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
630         {},
631 };
632
633 static bool __maybe_unused
634 needs_tx2_tvm_workaround(const struct arm64_cpu_capabilities *entry,
635                          int scope)
636 {
637         int i;
638
639         if (!is_affected_midr_range_list(entry, scope) ||
640             !is_hyp_mode_available())
641                 return false;
642
643         for_each_possible_cpu(i) {
644                 if (MPIDR_AFFINITY_LEVEL(cpu_logical_map(i), 0) != 0)
645                         return true;
646         }
647
648         return false;
649 }
650
651 #ifdef CONFIG_HARDEN_EL2_VECTORS
652
653 static const struct midr_range arm64_harden_el2_vectors[] = {
654         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
655         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
656         {},
657 };
658
659 #endif
660
661 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
662
663 static const struct midr_range arm64_repeat_tlbi_cpus[] = {
664 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
665         MIDR_RANGE(MIDR_QCOM_FALKOR_V1, 0, 0, 0, 0),
666 #endif
667 #ifdef CONFIG_ARM64_ERRATUM_1286807
668         MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 0),
669 #endif
670         {},
671 };
672
673 #endif
674
675 #ifdef CONFIG_CAVIUM_ERRATUM_27456
676 const struct midr_range cavium_erratum_27456_cpus[] = {
677         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
678         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
679         /* Cavium ThunderX, T81 pass 1.0 */
680         MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
681         {},
682 };
683 #endif
684
685 #ifdef CONFIG_CAVIUM_ERRATUM_30115
686 static const struct midr_range cavium_erratum_30115_cpus[] = {
687         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
688         MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 2),
689         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
690         MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
691         /* Cavium ThunderX, T83 pass 1.0 */
692         MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
693         {},
694 };
695 #endif
696
697 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
698 static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = {
699         {
700                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
701         },
702         {
703                 .midr_range.model = MIDR_QCOM_KRYO,
704                 .matches = is_kryo_midr,
705         },
706         {},
707 };
708 #endif
709
710 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
711 static const struct midr_range workaround_clean_cache[] = {
712 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
713         defined(CONFIG_ARM64_ERRATUM_827319) || \
714         defined(CONFIG_ARM64_ERRATUM_824069)
715         /* Cortex-A53 r0p[012]: ARM errata 826319, 827319, 824069 */
716         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
717 #endif
718 #ifdef  CONFIG_ARM64_ERRATUM_819472
719         /* Cortex-A53 r0p[01] : ARM errata 819472 */
720         MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
721 #endif
722         {},
723 };
724 #endif
725
726 #ifdef CONFIG_ARM64_ERRATUM_1418040
727 /*
728  * - 1188873 affects r0p0 to r2p0
729  * - 1418040 affects r0p0 to r3p1
730  */
731 static const struct midr_range erratum_1418040_list[] = {
732         /* Cortex-A76 r0p0 to r3p1 */
733         MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 3, 1),
734         /* Neoverse-N1 r0p0 to r3p1 */
735         MIDR_RANGE(MIDR_NEOVERSE_N1, 0, 0, 3, 1),
736         {},
737 };
738 #endif
739
740 const struct arm64_cpu_capabilities arm64_errata[] = {
741 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
742         {
743                 .desc = "ARM errata 826319, 827319, 824069, 819472",
744                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
745                 ERRATA_MIDR_RANGE_LIST(workaround_clean_cache),
746                 .cpu_enable = cpu_enable_cache_maint_trap,
747         },
748 #endif
749 #ifdef CONFIG_ARM64_ERRATUM_832075
750         {
751         /* Cortex-A57 r0p0 - r1p2 */
752                 .desc = "ARM erratum 832075",
753                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
754                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
755                                   0, 0,
756                                   1, 2),
757         },
758 #endif
759 #ifdef CONFIG_ARM64_ERRATUM_834220
760         {
761         /* Cortex-A57 r0p0 - r1p2 */
762                 .desc = "ARM erratum 834220",
763                 .capability = ARM64_WORKAROUND_834220,
764                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
765                                   0, 0,
766                                   1, 2),
767         },
768 #endif
769 #ifdef CONFIG_ARM64_ERRATUM_843419
770         {
771         /* Cortex-A53 r0p[01234] */
772                 .desc = "ARM erratum 843419",
773                 .capability = ARM64_WORKAROUND_843419,
774                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
775                 MIDR_FIXED(0x4, BIT(8)),
776         },
777 #endif
778 #ifdef CONFIG_ARM64_ERRATUM_845719
779         {
780         /* Cortex-A53 r0p[01234] */
781                 .desc = "ARM erratum 845719",
782                 .capability = ARM64_WORKAROUND_845719,
783                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
784         },
785 #endif
786 #ifdef CONFIG_CAVIUM_ERRATUM_23154
787         {
788         /* Cavium ThunderX, pass 1.x */
789                 .desc = "Cavium erratum 23154",
790                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
791                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
792         },
793 #endif
794 #ifdef CONFIG_CAVIUM_ERRATUM_27456
795         {
796                 .desc = "Cavium erratum 27456",
797                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
798                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_27456_cpus),
799         },
800 #endif
801 #ifdef CONFIG_CAVIUM_ERRATUM_30115
802         {
803                 .desc = "Cavium erratum 30115",
804                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
805                 ERRATA_MIDR_RANGE_LIST(cavium_erratum_30115_cpus),
806         },
807 #endif
808         {
809                 .desc = "Mismatched cache type (CTR_EL0)",
810                 .capability = ARM64_MISMATCHED_CACHE_TYPE,
811                 .matches = has_mismatched_cache_type,
812                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
813                 .cpu_enable = cpu_enable_trap_ctr_access,
814         },
815 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
816         {
817                 .desc = "Qualcomm Technologies Falkor/Kryo erratum 1003",
818                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
819                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
820                 .matches = cpucap_multi_entry_cap_matches,
821                 .match_list = qcom_erratum_1003_list,
822         },
823 #endif
824 #ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
825         {
826                 .desc = "Qualcomm erratum 1009, ARM erratum 1286807",
827                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
828                 ERRATA_MIDR_RANGE_LIST(arm64_repeat_tlbi_cpus),
829         },
830 #endif
831 #ifdef CONFIG_ARM64_ERRATUM_858921
832         {
833         /* Cortex-A73 all versions */
834                 .desc = "ARM erratum 858921",
835                 .capability = ARM64_WORKAROUND_858921,
836                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
837         },
838 #endif
839         {
840                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
841                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
842                 .matches = check_branch_predictor,
843         },
844 #ifdef CONFIG_HARDEN_EL2_VECTORS
845         {
846                 .desc = "EL2 vector hardening",
847                 .capability = ARM64_HARDEN_EL2_VECTORS,
848                 ERRATA_MIDR_RANGE_LIST(arm64_harden_el2_vectors),
849         },
850 #endif
851         {
852                 .desc = "Speculative Store Bypass Disable",
853                 .capability = ARM64_SSBD,
854                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
855                 .matches = has_ssbd_mitigation,
856                 .midr_range_list = arm64_ssb_cpus,
857         },
858 #ifdef CONFIG_ARM64_ERRATUM_1418040
859         {
860                 .desc = "ARM erratum 1418040",
861                 .capability = ARM64_WORKAROUND_1418040,
862                 ERRATA_MIDR_RANGE_LIST(erratum_1418040_list),
863         },
864 #endif
865 #ifdef CONFIG_ARM64_ERRATUM_1165522
866         {
867                 /* Cortex-A76 r0p0 to r2p0 */
868                 .desc = "ARM erratum 1165522",
869                 .capability = ARM64_WORKAROUND_1165522,
870                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
871         },
872 #endif
873 #ifdef CONFIG_ARM64_ERRATUM_1463225
874         {
875                 .desc = "ARM erratum 1463225",
876                 .capability = ARM64_WORKAROUND_1463225,
877                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
878                 .matches = has_cortex_a76_erratum_1463225,
879         },
880 #endif
881 #ifdef CONFIG_CAVIUM_TX2_ERRATUM_219
882         {
883                 .desc = "Cavium ThunderX2 erratum 219 (KVM guest sysreg trapping)",
884                 .capability = ARM64_WORKAROUND_CAVIUM_TX2_219_TVM,
885                 ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
886                 .matches = needs_tx2_tvm_workaround,
887         },
888         {
889                 .desc = "Cavium ThunderX2 erratum 219 (PRFM removal)",
890                 .capability = ARM64_WORKAROUND_CAVIUM_TX2_219_PRFM,
891                 ERRATA_MIDR_RANGE_LIST(tx2_family_cpus),
892         },
893 #endif
894         {
895         }
896 };
897
898 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
899                             char *buf)
900 {
901         return sprintf(buf, "Mitigation: __user pointer sanitization\n");
902 }
903
904 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
905                 char *buf)
906 {
907         switch (get_spectre_v2_workaround_state()) {
908         case ARM64_BP_HARDEN_NOT_REQUIRED:
909                 return sprintf(buf, "Not affected\n");
910         case ARM64_BP_HARDEN_WA_NEEDED:
911                 return sprintf(buf, "Mitigation: Branch predictor hardening\n");
912         case ARM64_BP_HARDEN_UNKNOWN:
913         default:
914                 return sprintf(buf, "Vulnerable\n");
915         }
916 }
917
918 ssize_t cpu_show_spec_store_bypass(struct device *dev,
919                 struct device_attribute *attr, char *buf)
920 {
921         if (__ssb_safe)
922                 return sprintf(buf, "Not affected\n");
923
924         switch (ssbd_state) {
925         case ARM64_SSBD_KERNEL:
926         case ARM64_SSBD_FORCE_ENABLE:
927                 if (IS_ENABLED(CONFIG_ARM64_SSBD))
928                         return sprintf(buf,
929                             "Mitigation: Speculative Store Bypass disabled via prctl\n");
930         }
931
932         return sprintf(buf, "Vulnerable\n");
933 }