arm64: uaccess: Fix omissions from usercopy whitelist
[muen/linux.git] / arch / arm64 / kernel / cpu_errata.c
1 /*
2  * Contains CPU specific errata definitions
3  *
4  * Copyright (C) 2014 ARM Ltd.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #include <linux/types.h>
20 #include <asm/cpu.h>
21 #include <asm/cputype.h>
22 #include <asm/cpufeature.h>
23
24 static bool __maybe_unused
25 is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope)
26 {
27         const struct arm64_midr_revidr *fix;
28         u32 midr = read_cpuid_id(), revidr;
29
30         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
31         if (!is_midr_in_range(midr, &entry->midr_range))
32                 return false;
33
34         midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
35         revidr = read_cpuid(REVIDR_EL1);
36         for (fix = entry->fixed_revs; fix && fix->revidr_mask; fix++)
37                 if (midr == fix->midr_rv && (revidr & fix->revidr_mask))
38                         return false;
39
40         return true;
41 }
42
43 static bool __maybe_unused
44 is_affected_midr_range_list(const struct arm64_cpu_capabilities *entry,
45                             int scope)
46 {
47         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
48         return is_midr_in_range_list(read_cpuid_id(), entry->midr_range_list);
49 }
50
51 static bool __maybe_unused
52 is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope)
53 {
54         u32 model;
55
56         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
57
58         model = read_cpuid_id();
59         model &= MIDR_IMPLEMENTOR_MASK | (0xf00 << MIDR_PARTNUM_SHIFT) |
60                  MIDR_ARCHITECTURE_MASK;
61
62         return model == entry->midr_range.model;
63 }
64
65 static bool
66 has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
67                                 int scope)
68 {
69         WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
70         return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
71                 (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
72 }
73
74 static void
75 cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
76 {
77         /* Clear SCTLR_EL1.UCT */
78         config_sctlr_el1(SCTLR_EL1_UCT, 0);
79 }
80
81 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
82 #include <asm/mmu_context.h>
83 #include <asm/cacheflush.h>
84
85 DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
86
87 #ifdef CONFIG_KVM
88 extern char __qcom_hyp_sanitize_link_stack_start[];
89 extern char __qcom_hyp_sanitize_link_stack_end[];
90 extern char __smccc_workaround_1_smc_start[];
91 extern char __smccc_workaround_1_smc_end[];
92 extern char __smccc_workaround_1_hvc_start[];
93 extern char __smccc_workaround_1_hvc_end[];
94
95 static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
96                                 const char *hyp_vecs_end)
97 {
98         void *dst = lm_alias(__bp_harden_hyp_vecs_start + slot * SZ_2K);
99         int i;
100
101         for (i = 0; i < SZ_2K; i += 0x80)
102                 memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
103
104         flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
105 }
106
107 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
108                                       const char *hyp_vecs_start,
109                                       const char *hyp_vecs_end)
110 {
111         static int last_slot = -1;
112         static DEFINE_SPINLOCK(bp_lock);
113         int cpu, slot = -1;
114
115         spin_lock(&bp_lock);
116         for_each_possible_cpu(cpu) {
117                 if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
118                         slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
119                         break;
120                 }
121         }
122
123         if (slot == -1) {
124                 last_slot++;
125                 BUG_ON(((__bp_harden_hyp_vecs_end - __bp_harden_hyp_vecs_start)
126                         / SZ_2K) <= last_slot);
127                 slot = last_slot;
128                 __copy_hyp_vect_bpi(slot, hyp_vecs_start, hyp_vecs_end);
129         }
130
131         __this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
132         __this_cpu_write(bp_hardening_data.fn, fn);
133         spin_unlock(&bp_lock);
134 }
135 #else
136 #define __qcom_hyp_sanitize_link_stack_start    NULL
137 #define __qcom_hyp_sanitize_link_stack_end      NULL
138 #define __smccc_workaround_1_smc_start          NULL
139 #define __smccc_workaround_1_smc_end            NULL
140 #define __smccc_workaround_1_hvc_start          NULL
141 #define __smccc_workaround_1_hvc_end            NULL
142
143 static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
144                                       const char *hyp_vecs_start,
145                                       const char *hyp_vecs_end)
146 {
147         __this_cpu_write(bp_hardening_data.fn, fn);
148 }
149 #endif  /* CONFIG_KVM */
150
151 static void  install_bp_hardening_cb(const struct arm64_cpu_capabilities *entry,
152                                      bp_hardening_cb_t fn,
153                                      const char *hyp_vecs_start,
154                                      const char *hyp_vecs_end)
155 {
156         u64 pfr0;
157
158         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
159                 return;
160
161         pfr0 = read_cpuid(ID_AA64PFR0_EL1);
162         if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
163                 return;
164
165         __install_bp_hardening_cb(fn, hyp_vecs_start, hyp_vecs_end);
166 }
167
168 #include <uapi/linux/psci.h>
169 #include <linux/arm-smccc.h>
170 #include <linux/psci.h>
171
172 static void call_smc_arch_workaround_1(void)
173 {
174         arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
175 }
176
177 static void call_hvc_arch_workaround_1(void)
178 {
179         arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
180 }
181
182 static void
183 enable_smccc_arch_workaround_1(const struct arm64_cpu_capabilities *entry)
184 {
185         bp_hardening_cb_t cb;
186         void *smccc_start, *smccc_end;
187         struct arm_smccc_res res;
188
189         if (!entry->matches(entry, SCOPE_LOCAL_CPU))
190                 return;
191
192         if (psci_ops.smccc_version == SMCCC_VERSION_1_0)
193                 return;
194
195         switch (psci_ops.conduit) {
196         case PSCI_CONDUIT_HVC:
197                 arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
198                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
199                 if (res.a0)
200                         return;
201                 cb = call_hvc_arch_workaround_1;
202                 smccc_start = __smccc_workaround_1_hvc_start;
203                 smccc_end = __smccc_workaround_1_hvc_end;
204                 break;
205
206         case PSCI_CONDUIT_SMC:
207                 arm_smccc_1_1_smc(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
208                                   ARM_SMCCC_ARCH_WORKAROUND_1, &res);
209                 if (res.a0)
210                         return;
211                 cb = call_smc_arch_workaround_1;
212                 smccc_start = __smccc_workaround_1_smc_start;
213                 smccc_end = __smccc_workaround_1_smc_end;
214                 break;
215
216         default:
217                 return;
218         }
219
220         install_bp_hardening_cb(entry, cb, smccc_start, smccc_end);
221
222         return;
223 }
224
225 static void qcom_link_stack_sanitization(void)
226 {
227         u64 tmp;
228
229         asm volatile("mov       %0, x30         \n"
230                      ".rept     16              \n"
231                      "bl        . + 4           \n"
232                      ".endr                     \n"
233                      "mov       x30, %0         \n"
234                      : "=&r" (tmp));
235 }
236
237 static void
238 qcom_enable_link_stack_sanitization(const struct arm64_cpu_capabilities *entry)
239 {
240         install_bp_hardening_cb(entry, qcom_link_stack_sanitization,
241                                 __qcom_hyp_sanitize_link_stack_start,
242                                 __qcom_hyp_sanitize_link_stack_end);
243 }
244 #endif  /* CONFIG_HARDEN_BRANCH_PREDICTOR */
245
246 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)       \
247         .matches = is_affected_midr_range,                      \
248         .midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
249
250 #define CAP_MIDR_ALL_VERSIONS(model)                                    \
251         .matches = is_affected_midr_range,                              \
252         .midr_range = MIDR_ALL_VERSIONS(model)
253
254 #define MIDR_FIXED(rev, revidr_mask) \
255         .fixed_revs = (struct arm64_midr_revidr[]){{ (rev), (revidr_mask) }, {}}
256
257 #define ERRATA_MIDR_RANGE(model, v_min, r_min, v_max, r_max)            \
258         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                         \
259         CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)
260
261 #define CAP_MIDR_RANGE_LIST(list)                               \
262         .matches = is_affected_midr_range_list,                 \
263         .midr_range_list = list
264
265 /* Errata affecting a range of revisions of  given model variant */
266 #define ERRATA_MIDR_REV_RANGE(m, var, r_min, r_max)      \
267         ERRATA_MIDR_RANGE(m, var, r_min, var, r_max)
268
269 /* Errata affecting a single variant/revision of a model */
270 #define ERRATA_MIDR_REV(model, var, rev)        \
271         ERRATA_MIDR_RANGE(model, var, rev, var, rev)
272
273 /* Errata affecting all variants/revisions of a given a model */
274 #define ERRATA_MIDR_ALL_VERSIONS(model)                         \
275         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
276         CAP_MIDR_ALL_VERSIONS(model)
277
278 /* Errata affecting a list of midr ranges, with same work around */
279 #define ERRATA_MIDR_RANGE_LIST(midr_list)                       \
280         .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,                 \
281         CAP_MIDR_RANGE_LIST(midr_list)
282
283 /*
284  * Generic helper for handling capabilties with multiple (match,enable) pairs
285  * of call backs, sharing the same capability bit.
286  * Iterate over each entry to see if at least one matches.
287  */
288 static bool __maybe_unused
289 multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
290 {
291         const struct arm64_cpu_capabilities *caps;
292
293         for (caps = entry->match_list; caps->matches; caps++)
294                 if (caps->matches(caps, scope))
295                         return true;
296
297         return false;
298 }
299
300 /*
301  * Take appropriate action for all matching entries in the shared capability
302  * entry.
303  */
304 static void __maybe_unused
305 multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
306 {
307         const struct arm64_cpu_capabilities *caps;
308
309         for (caps = entry->match_list; caps->matches; caps++)
310                 if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
311                     caps->cpu_enable)
312                         caps->cpu_enable(caps);
313 }
314
315 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
316
317 /*
318  * List of CPUs where we need to issue a psci call to
319  * harden the branch predictor.
320  */
321 static const struct midr_range arm64_bp_harden_smccc_cpus[] = {
322         MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
323         MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
324         MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
325         MIDR_ALL_VERSIONS(MIDR_CORTEX_A75),
326         MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
327         MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
328         {},
329 };
330
331 static const struct midr_range qcom_bp_harden_cpus[] = {
332         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR_V1),
333         MIDR_ALL_VERSIONS(MIDR_QCOM_FALKOR),
334         {},
335 };
336
337 static const struct arm64_cpu_capabilities arm64_bp_harden_list[] = {
338         {
339                 CAP_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
340                 .cpu_enable = enable_smccc_arch_workaround_1,
341         },
342         {
343                 CAP_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
344                 .cpu_enable = qcom_enable_link_stack_sanitization,
345         },
346         {},
347 };
348
349 #endif
350
351 const struct arm64_cpu_capabilities arm64_errata[] = {
352 #if     defined(CONFIG_ARM64_ERRATUM_826319) || \
353         defined(CONFIG_ARM64_ERRATUM_827319) || \
354         defined(CONFIG_ARM64_ERRATUM_824069)
355         {
356         /* Cortex-A53 r0p[012] */
357                 .desc = "ARM errata 826319, 827319, 824069",
358                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
359                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
360                 .cpu_enable = cpu_enable_cache_maint_trap,
361         },
362 #endif
363 #ifdef CONFIG_ARM64_ERRATUM_819472
364         {
365         /* Cortex-A53 r0p[01] */
366                 .desc = "ARM errata 819472",
367                 .capability = ARM64_WORKAROUND_CLEAN_CACHE,
368                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
369                 .cpu_enable = cpu_enable_cache_maint_trap,
370         },
371 #endif
372 #ifdef CONFIG_ARM64_ERRATUM_832075
373         {
374         /* Cortex-A57 r0p0 - r1p2 */
375                 .desc = "ARM erratum 832075",
376                 .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE,
377                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
378                                   0, 0,
379                                   1, 2),
380         },
381 #endif
382 #ifdef CONFIG_ARM64_ERRATUM_834220
383         {
384         /* Cortex-A57 r0p0 - r1p2 */
385                 .desc = "ARM erratum 834220",
386                 .capability = ARM64_WORKAROUND_834220,
387                 ERRATA_MIDR_RANGE(MIDR_CORTEX_A57,
388                                   0, 0,
389                                   1, 2),
390         },
391 #endif
392 #ifdef CONFIG_ARM64_ERRATUM_843419
393         {
394         /* Cortex-A53 r0p[01234] */
395                 .desc = "ARM erratum 843419",
396                 .capability = ARM64_WORKAROUND_843419,
397                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
398                 MIDR_FIXED(0x4, BIT(8)),
399         },
400 #endif
401 #ifdef CONFIG_ARM64_ERRATUM_845719
402         {
403         /* Cortex-A53 r0p[01234] */
404                 .desc = "ARM erratum 845719",
405                 .capability = ARM64_WORKAROUND_845719,
406                 ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 4),
407         },
408 #endif
409 #ifdef CONFIG_CAVIUM_ERRATUM_23154
410         {
411         /* Cavium ThunderX, pass 1.x */
412                 .desc = "Cavium erratum 23154",
413                 .capability = ARM64_WORKAROUND_CAVIUM_23154,
414                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX, 0, 0, 1),
415         },
416 #endif
417 #ifdef CONFIG_CAVIUM_ERRATUM_27456
418         {
419         /* Cavium ThunderX, T88 pass 1.x - 2.1 */
420                 .desc = "Cavium erratum 27456",
421                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
422                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
423                                   0, 0,
424                                   1, 1),
425         },
426         {
427         /* Cavium ThunderX, T81 pass 1.0 */
428                 .desc = "Cavium erratum 27456",
429                 .capability = ARM64_WORKAROUND_CAVIUM_27456,
430                 ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
431         },
432 #endif
433 #ifdef CONFIG_CAVIUM_ERRATUM_30115
434         {
435         /* Cavium ThunderX, T88 pass 1.x - 2.2 */
436                 .desc = "Cavium erratum 30115",
437                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
438                 ERRATA_MIDR_RANGE(MIDR_THUNDERX,
439                                       0, 0,
440                                       1, 2),
441         },
442         {
443         /* Cavium ThunderX, T81 pass 1.0 - 1.2 */
444                 .desc = "Cavium erratum 30115",
445                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
446                 ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
447         },
448         {
449         /* Cavium ThunderX, T83 pass 1.0 */
450                 .desc = "Cavium erratum 30115",
451                 .capability = ARM64_WORKAROUND_CAVIUM_30115,
452                 ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
453         },
454 #endif
455         {
456                 .desc = "Mismatched cache line size",
457                 .capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
458                 .matches = has_mismatched_cache_line_size,
459                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
460                 .cpu_enable = cpu_enable_trap_ctr_access,
461         },
462 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
463         {
464                 .desc = "Qualcomm Technologies Falkor erratum 1003",
465                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
466                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
467         },
468         {
469                 .desc = "Qualcomm Technologies Kryo erratum 1003",
470                 .capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
471                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
472                 .midr_range.model = MIDR_QCOM_KRYO,
473                 .matches = is_kryo_midr,
474         },
475 #endif
476 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009
477         {
478                 .desc = "Qualcomm Technologies Falkor erratum 1009",
479                 .capability = ARM64_WORKAROUND_REPEAT_TLBI,
480                 ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
481         },
482 #endif
483 #ifdef CONFIG_ARM64_ERRATUM_858921
484         {
485         /* Cortex-A73 all versions */
486                 .desc = "ARM erratum 858921",
487                 .capability = ARM64_WORKAROUND_858921,
488                 ERRATA_MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
489         },
490 #endif
491 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
492         {
493                 .capability = ARM64_HARDEN_BRANCH_PREDICTOR,
494                 .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
495                 .matches = multi_entry_cap_matches,
496                 .cpu_enable = multi_entry_cap_cpu_enable,
497                 .match_list = arm64_bp_harden_list,
498         },
499         {
500                 .capability = ARM64_HARDEN_BP_POST_GUEST_EXIT,
501                 ERRATA_MIDR_RANGE_LIST(qcom_bp_harden_cpus),
502         },
503 #endif
504         {
505         }
506 };