Merge tag 'for-linus' of git://git.kernel.org/pub/scm/virt/kvm/kvm
[muen/linux.git] / arch / arm64 / kernel / cpu_errata.c
1 /*
2  * Contains CPU specific errata definitions
3  *
4  * Copyright (C) 2014 ARM Ltd.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #include <linux/types.h>
20 #include <asm/cpu.h>
21 #include <asm/cputype.h>
22 #include <asm/cpufeature.h>
23
24 static bool __maybe_unused
25 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
26 {
27         const struct arm64_midr_revidr *fix;
28         u32 midr = read_cpuid_id(), revidr;
29
30         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
31         if (!is_midr_in_range(midr, &entry->midr_range))
32                 return false;
33
34         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
35         revidr = read_cpuid(REVIDR_EL1);
36         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
37                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
38                         return false;
39
40         return true;
41 }
42
43 static bool __maybe_unused
44 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
45                             int scope)
46 {
47         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
48         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
49 }
50
51 static bool __maybe_unused
52 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
53 {
54         u32 model;
55
56         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
57
58         model = read_cpuid_id();
59         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
60                  MIDR_ARCHITECTURE_MASK;
61
62         return model == entry->midr_range.model;
63 }
64
65 static bool
66 has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
67                                 int scope)
68 {
69         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
70         return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
71                 (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
72 }
73
74 static void
75 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
76 {
77         /* Clear SCTLR_EL1.UCT */
78         config_sctlr_el1(SCTLR_EL1_UCT, 0);
79 }
80
81 atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
82
83 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
84 #include <asm/mmu_context.h>
85 #include <asm/cacheflush.h>
86
87 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
88
89 #ifdef CONFIG_KVM
90 extern char __qcom_hyp_sanitize_link_stack_start[];
91 extern char __qcom_hyp_sanitize_link_stack_end[];
92 extern char __smccc_workaround_1_smc_start[];
93 extern char __smccc_workaround_1_smc_end[];
94 extern char __smccc_workaround_1_hvc_start[];
95 extern char __smccc_workaround_1_hvc_end[];
96
97 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
98                                 const char *hyp_vecs_end)
99 {
100         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
101         int i;
102
103         for (i = 0; i < SZ_2K; i += 0x80)
104                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
105
106         flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
107 }
108
109 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
110                                       const char *hyp_vecs_start,
111                                       const char *hyp_vecs_end)
112 {
113         static DEFINE_SPINLOCK(bp_lock);
114         int cpu, slot = -1;
115
116         spin_lock(&bp_lock);
117         for_each_possible_cpu(cpu) {
118                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
119                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
120                         break;
121                 }
122         }
123
124         if (slot == -1) {
125                 slot = atomic_inc_return(&arm64_el2_vector_last_slot);
126                 BUG_ON(slot >= BP_HARDEN_EL2_SLOTS);
127                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
128         }
129
130         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
131         __this_cpu_write(bp_hardening_data.fn, fn);
132         spin_unlock(&bp_lock);
133 }
134 #else
135 #define __qcom_hyp_sanitize_link_stack_start    NULL
136 #define __qcom_hyp_sanitize_link_stack_end      NULL
137 #define __smccc_workaround_1_smc_start          NULL
138 #define __smccc_workaround_1_smc_end            NULL
139 #define __smccc_workaround_1_hvc_start          NULL
140 #define __smccc_workaround_1_hvc_end            NULL
141
142 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
143                                       const char *hyp_vecs_start,
144                                       const char *hyp_vecs_end)
145 {
146         __this_cpu_write(bp_hardening_data.fn, fn);
147 }
148 #endif  /* CONFIG_KVM */
149
150 static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
151                                      bp_hardening_cb_t fn,
152                                      const char *hyp_vecs_start,
153                                      const char *hyp_vecs_end)
154 {
155         u64 pfr0;
156
157         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
158                 return;
159
160         pfr0 = read_cpuid(ID_AA64PFR0_EL1);
161         if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
162                 return;
163
164         __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
165 }
166
167 #include <uapi/linux/psci.h>
168 #include <linux/arm-smccc.h>
169 #include <linux/psci.h>
170
171 static void call_smc_arch_workaround_1(void)
172 {
173         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
174 }
175
176 static void call_hvc_arch_workaround_1(void)
177 {
178         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
179 }
180
181 static void
182 enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
183 {
184         bp_hardening_cb_t cb;
185         void *smccc_start, *smccc_end;
186         struct arm_smccc_res res;
187
188         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
189                 return;
190
191         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
192                 return;
193
194         switch (psci_ops.conduit) {
195         case PSCI_CONDUIT_HVC:
196                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
197                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
198                 if ((int)res.a0 < 0)
199                         return;
200                 cb = call_hvc_arch_workaround_1;
201                 smccc_start = __smccc_workaround_1_hvc_start;
202                 smccc_end = __smccc_workaround_1_hvc_end;
203                 break;
204
205         case PSCI_CONDUIT_SMC:
206                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
207                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
208                 if ((int)res.a0 < 0)
209                         return;
210                 cb = call_smc_arch_workaround_1;
211                 smccc_start = __smccc_workaround_1_smc_start;
212                 smccc_end = __smccc_workaround_1_smc_end;
213                 break;
214
215         default:
216                 return;
217         }
218
219         install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
220
221         return;
222 }
223
224 static void qcom_link_stack_sanitization(void)
225 {
226         u64 tmp;
227
228         asm volatile("mov       %0, x30         \n"
229                      ".rept     16              \n"
230                      "bl        . + 4           \n"
231                      ".endr                     \n"
232                      "mov       x30, %0         \n"
233                      : "=&r" (tmp));
234 }
235
236 static void
237 qcom_enable_link_stack_sanitization(const struct arm64_cpu_capabilities *entry)
238 {
239         install_bp_hardening_cb(entry, qcom_link_stack_sanitization,
240                                 __qcom_hyp_sanitize_link_stack_start,
241                                 __qcom_hyp_sanitize_link_stack_end);
242 }
243 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
244
245 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
246         .matches = is_affected_midr_range,                      \
247         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
248
249 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
250         .matches = is_affected_midr_range,                              \
251         .midr_range = MIDR_ALL_VERSIONS(model)
252
253 #define MIDR_FIXED(rev, revidr_mask) \
254         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
255
256 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
257         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
258         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
259
260 #define CAP_MIDR_RANGE_LIST(list)                               \
261         .matches = is_affected_midr_range_list,                 \
262         .midr_range_list = list
263
264 /* Errata affecting a range of revisions of  given model variant */
265 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
266         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
267
268 /* Errata affecting a single variant/revision of a model */
269 #define ERRATA_MIDR_REV(model, var, rev)        \
270         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
271
272 /* Errata affecting all variants/revisions of a given a model */
273 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
274         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
275         CAP_MIDR_ALL_VERSIONS(model)
276
277 /* Errata affecting a list of midr ranges, with same work around */
278 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
279         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
280         CAP_MIDR_RANGE_LIST(midr_list)
281
282 /*
283  * Generic helper for handling capabilties with multiple (match,enable) pairs
284  * of call backs, sharing the same capability bit.
285  * Iterate over each entry to see if at least one matches.
286  */
287 static bool __maybe_unused
288 multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
289 {
290         const struct arm64_cpu_capabilities *caps;
291
292         for (caps = entry->match_list; caps->matches; caps++)
293                 if (caps->matches(caps, scope))
294                         return true;
295
296         return false;
297 }
298
299 /*
300  * Take appropriate action for all matching entries in the shared capability
301  * entry.
302  */
303 static void __maybe_unused
304 multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
305 {
306         const struct arm64_cpu_capabilities *caps;
307
308         for (caps = entry->match_list; caps->matches; caps++)
309                 if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
310                     caps->cpu_enable)
311                         caps->cpu_enable(caps);
312 }
313
314 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
315
316 /*
317  * List of CPUs where we need to issue a psci call to
318  * harden the branch predictor.
319  */
320 static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
321         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
322         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
323         MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
324         MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
325         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
326         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
327         {},
328 };
329
330 static const struct midr_range qcom_bp_harden_cpus[] = {
331         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
332         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
333         {},
334 };
335
336 static const struct arm64_cpu_capabilities arm64_bp_harden_list[] = {
337         {
338                 CAP_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
339                 .cpu_enable = enable_smccc_arch_workaround_1,
340         },
341         {
342                 CAP_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
343                 .cpu_enable = qcom_enable_link_stack_sanitization,
344         },
345         {},
346 };
347
348 #endif
349
350 #ifndef ERRATA_MIDR_ALL_VERSIONS
351 #define ERRATA_MIDR_ALL_VERSIONS(x)     MIDR_ALL_VERSIONS(x)
352 #endif
353
354 const struct arm64_cpu_capabilities arm64_errata[] = {
355 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
356         defined(CONFIG_ARM64_ERRATUM_827319) || \
357         defined(CONFIG_ARM64_ERRATUM_824069)
358         {
359         /* Cortex-A53 r0p[012] */
360                 .desc = "ARM errata 826319, 827319, 824069",
361                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
362                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
363                 .cpu_enable = cpu_enable_cache_maint_trap,
364         },
365 #endif
366 #ifdef CONFIG_ARM64_ERRATUM_819472
367         {
368         /* Cortex-A53 r0p[01] */
369                 .desc = "ARM errata 819472",
370                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
371                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
372                 .cpu_enable = cpu_enable_cache_maint_trap,
373         },
374 #endif
375 #ifdef CONFIG_ARM64_ERRATUM_832075
376         {
377         /* Cortex-A57 r0p0 - r1p2 */
378                 .desc = "ARM erratum 832075",
379                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
380                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
381                                   0, 0,
382                                   1, 2),
383         },
384 #endif
385 #ifdef CONFIG_ARM64_ERRATUM_834220
386         {
387         /* Cortex-A57 r0p0 - r1p2 */
388                 .desc = "ARM erratum 834220",
389                 .capability = ARM64_WORKAROUND_834220,
390                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
391                                   0, 0,
392                                   1, 2),
393         },
394 #endif
395 #ifdef CONFIG_ARM64_ERRATUM_843419
396         {
397         /* Cortex-A53 r0p[01234] */
398                 .desc = "ARM erratum 843419",
399                 .capability = ARM64_WORKAROUND_843419,
400                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
401                 MIDR_FIXED(0x4, BIT(8)),
402         },
403 #endif
404 #ifdef CONFIG_ARM64_ERRATUM_845719
405         {
406         /* Cortex-A53 r0p[01234] */
407                 .desc = "ARM erratum 845719",
408                 .capability = ARM64_WORKAROUND_845719,
409                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
410         },
411 #endif
412 #ifdef CONFIG_CAVIUM_ERRATUM_23154
413         {
414         /* Cavium ThunderX, pass 1.x */
415                 .desc = "Cavium erratum 23154",
416                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
417                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
418         },
419 #endif
420 #ifdef CONFIG_CAVIUM_ERRATUM_27456
421         {
422         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
423                 .desc = "Cavium erratum 27456",
424                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
425                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
426                                   0, 0,
427                                   1, 1),
428         },
429         {
430         /* Cavium ThunderX, T81 pass 1.0 */
431                 .desc = "Cavium erratum 27456",
432                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
433                 ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
434         },
435 #endif
436 #ifdef CONFIG_CAVIUM_ERRATUM_30115
437         {
438         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
439                 .desc = "Cavium erratum 30115",
440                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
441                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
442                                       0, 0,
443                                       1, 2),
444         },
445         {
446         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
447                 .desc = "Cavium erratum 30115",
448                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
449                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
450         },
451         {
452         /* Cavium ThunderX, T83 pass 1.0 */
453                 .desc = "Cavium erratum 30115",
454                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
455                 ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
456         },
457 #endif
458         {
459                 .desc = "Mismatched cache line size",
460                 .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
461                 .matches = has_mismatched_cache_line_size,
462                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
463                 .cpu_enable = cpu_enable_trap_ctr_access,
464         },
465 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
466         {
467                 .desc = "Qualcomm Technologies Falkor erratum 1003",
468                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
469                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
470         },
471         {
472                 .desc = "Qualcomm Technologies Kryo erratum 1003",
473                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
474                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
475                 .midr_range.model = MIDR_QCOM_KRYO,
476                 .matches = is_kryo_midr,
477         },
478 #endif
479 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
480         {
481                 .desc = "Qualcomm Technologies Falkor erratum 1009",
482                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
483                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
484         },
485 #endif
486 #ifdef CONFIG_ARM64_ERRATUM_858921
487         {
488         /* Cortex-A73 all versions */
489                 .desc = "ARM erratum 858921",
490                 .capability = ARM64_WORKAROUND_858921,
491                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
492         },
493 #endif
494 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
495         {
496                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
497                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
498                 .matches = multi_entry_cap_matches,
499                 .cpu_enable = multi_entry_cap_cpu_enable,
500                 .match_list = arm64_bp_harden_list,
501         },
502         {
503                 .capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
504                 ERRATA_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
505         },
506 #endif
507 #ifdef CONFIG_HARDEN_EL2_VECTORS
508         {
509                 .desc = "Cortex-A57 EL2 vector hardening",
510                 .capability = ARM64_HARDEN_EL2_VECTORS,
511                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
512         },
513         {
514                 .desc = "Cortex-A72 EL2 vector hardening",
515                 .capability = ARM64_HARDEN_EL2_VECTORS,
516                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
517         },
518 #endif
519         {
520         }
521 };