1954dd5552a2e2fbeaf21937ad4c6d98c6ba0aff
[muen/linux.git] / arch / x86 / include / asm / uaccess.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_UACCESS_H
3 #define _ASM_X86_UACCESS_H
4 /*
5  * User space memory access functions
6  */
7 #include <linux/compiler.h>
8 #include <linux/kasan-checks.h>
9 #include <linux/string.h>
10 #include <asm/asm.h>
11 #include <asm/page.h>
12 #include <asm/smap.h>
13 #include <asm/extable.h>
14
15 /*
16  * The fs value determines whether argument validity checking should be
17  * performed or not.  If get_fs() == USER_DS, checking is performed, with
18  * get_fs() == KERNEL_DS, checking is bypassed.
19  *
20  * For historical reasons, these macros are grossly misnamed.
21  */
22
23 #define MAKE_MM_SEG(s)  ((mm_segment_t) { (s) })
24
25 #define KERNEL_DS       MAKE_MM_SEG(-1UL)
26 #define USER_DS         MAKE_MM_SEG(TASK_SIZE_MAX)
27
28 #define get_fs()        (current->thread.addr_limit)
29 static inline void set_fs(mm_segment_t fs)
30 {
31         current->thread.addr_limit = fs;
32         /* On user-mode return, check fs is correct */
33         set_thread_flag(TIF_FSCHECK);
34 }
35
36 #define segment_eq(a, b)        ((a).seg == (b).seg)
37 #define user_addr_max() (current->thread.addr_limit.seg)
38
39 /*
40  * Test whether a block of memory is a valid user space address.
41  * Returns 0 if the range is valid, nonzero otherwise.
42  */
43 static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
44 {
45         /*
46          * If we have used "sizeof()" for the size,
47          * we know it won't overflow the limit (but
48          * it might overflow the 'addr', so it's
49          * important to subtract the size from the
50          * limit, not add it to the address).
51          */
52         if (__builtin_constant_p(size))
53                 return unlikely(addr > limit - size);
54
55         /* Arbitrary sizes? Be careful about overflow */
56         addr += size;
57         if (unlikely(addr < size))
58                 return true;
59         return unlikely(addr > limit);
60 }
61
62 #define __range_not_ok(addr, size, limit)                               \
63 ({                                                                      \
64         __chk_user_ptr(addr);                                           \
65         __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
66 })
67
68 #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
69 # define WARN_ON_IN_IRQ()       WARN_ON_ONCE(!in_task())
70 #else
71 # define WARN_ON_IN_IRQ()
72 #endif
73
74 /**
75  * access_ok - Checks if a user space pointer is valid
76  * @addr: User space pointer to start of block to check
77  * @size: Size of block to check
78  *
79  * Context: User context only. This function may sleep if pagefaults are
80  *          enabled.
81  *
82  * Checks if a pointer to a block of memory in user space is valid.
83  *
84  * Note that, depending on architecture, this function probably just
85  * checks that the pointer is in the user space range - after calling
86  * this function, memory access functions may still return -EFAULT.
87  *
88  * Return: true (nonzero) if the memory block may be valid, false (zero)
89  * if it is definitely invalid.
90  */
91 #define access_ok(addr, size)                                   \
92 ({                                                                      \
93         WARN_ON_IN_IRQ();                                               \
94         likely(!__range_not_ok(addr, size, user_addr_max()));           \
95 })
96
97 /*
98  * These are the main single-value transfer routines.  They automatically
99  * use the right size if we just have the right pointer type.
100  *
101  * This gets kind of ugly. We want to return _two_ values in "get_user()"
102  * and yet we don't want to do any pointers, because that is too much
103  * of a performance impact. Thus we have a few rather ugly macros here,
104  * and hide all the ugliness from the user.
105  *
106  * The "__xxx" versions of the user access functions are versions that
107  * do not verify the address space, that must have been done previously
108  * with a separate "access_ok()" call (this is used when we do multiple
109  * accesses to the same area of user memory).
110  */
111
112 extern int __get_user_1(void);
113 extern int __get_user_2(void);
114 extern int __get_user_4(void);
115 extern int __get_user_8(void);
116 extern int __get_user_bad(void);
117
118 #define __uaccess_begin() stac()
119 #define __uaccess_end()   clac()
120 #define __uaccess_begin_nospec()        \
121 ({                                      \
122         stac();                         \
123         barrier_nospec();               \
124 })
125
126 /*
127  * This is a type: either unsigned long, if the argument fits into
128  * that type, or otherwise unsigned long long.
129  */
130 #define __inttype(x) \
131 __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
132
133 /**
134  * get_user - Get a simple variable from user space.
135  * @x:   Variable to store result.
136  * @ptr: Source address, in user space.
137  *
138  * Context: User context only. This function may sleep if pagefaults are
139  *          enabled.
140  *
141  * This macro copies a single simple variable from user space to kernel
142  * space.  It supports simple types like char and int, but not larger
143  * data types like structures or arrays.
144  *
145  * @ptr must have pointer-to-simple-variable type, and the result of
146  * dereferencing @ptr must be assignable to @x without a cast.
147  *
148  * Return: zero on success, or -EFAULT on error.
149  * On error, the variable @x is set to zero.
150  */
151 /*
152  * Careful: we have to cast the result to the type of the pointer
153  * for sign reasons.
154  *
155  * The use of _ASM_DX as the register specifier is a bit of a
156  * simplification, as gcc only cares about it as the starting point
157  * and not size: for a 64-bit value it will use %ecx:%edx on 32 bits
158  * (%ecx being the next register in gcc's x86 register sequence), and
159  * %rdx on 64 bits.
160  *
161  * Clang/LLVM cares about the size of the register, but still wants
162  * the base register for something that ends up being a pair.
163  */
164 #define get_user(x, ptr)                                                \
165 ({                                                                      \
166         int __ret_gu;                                                   \
167         register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX);            \
168         __chk_user_ptr(ptr);                                            \
169         might_fault();                                                  \
170         asm volatile("call __get_user_%P4"                              \
171                      : "=a" (__ret_gu), "=r" (__val_gu),                \
172                         ASM_CALL_CONSTRAINT                             \
173                      : "0" (ptr), "i" (sizeof(*(ptr))));                \
174         (x) = (__force __typeof__(*(ptr))) __val_gu;                    \
175         __builtin_expect(__ret_gu, 0);                                  \
176 })
177
178 #define __put_user_x(size, x, ptr, __ret_pu)                    \
179         asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
180                      : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
181
182
183
184 #ifdef CONFIG_X86_32
185 #define __put_user_goto_u64(x, addr, label)                     \
186         asm_volatile_goto("\n"                                  \
187                      "1:        movl %%eax,0(%1)\n"             \
188                      "2:        movl %%edx,4(%1)\n"             \
189                      _ASM_EXTABLE_UA(1b, %l2)                   \
190                      _ASM_EXTABLE_UA(2b, %l2)                   \
191                      : : "A" (x), "r" (addr)                    \
192                      : : label)
193
194 #define __put_user_asm_ex_u64(x, addr)                                  \
195         asm volatile("\n"                                               \
196                      "1:        movl %%eax,0(%1)\n"                     \
197                      "2:        movl %%edx,4(%1)\n"                     \
198                      "3:"                                               \
199                      _ASM_EXTABLE_EX(1b, 2b)                            \
200                      _ASM_EXTABLE_EX(2b, 3b)                            \
201                      : : "A" (x), "r" (addr))
202
203 #define __put_user_x8(x, ptr, __ret_pu)                         \
204         asm volatile("call __put_user_8" : "=a" (__ret_pu)      \
205                      : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
206 #else
207 #define __put_user_goto_u64(x, ptr, label) \
208         __put_user_goto(x, ptr, "q", "", "er", label)
209 #define __put_user_asm_ex_u64(x, addr)  \
210         __put_user_asm_ex(x, addr, "q", "", "er")
211 #define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
212 #endif
213
214 extern void __put_user_bad(void);
215
216 /*
217  * Strange magic calling convention: pointer in %ecx,
218  * value in %eax(:%edx), return value in %eax. clobbers %rbx
219  */
220 extern void __put_user_1(void);
221 extern void __put_user_2(void);
222 extern void __put_user_4(void);
223 extern void __put_user_8(void);
224
225 /**
226  * put_user - Write a simple value into user space.
227  * @x:   Value to copy to user space.
228  * @ptr: Destination address, in user space.
229  *
230  * Context: User context only. This function may sleep if pagefaults are
231  *          enabled.
232  *
233  * This macro copies a single simple value from kernel space to user
234  * space.  It supports simple types like char and int, but not larger
235  * data types like structures or arrays.
236  *
237  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
238  * to the result of dereferencing @ptr.
239  *
240  * Return: zero on success, or -EFAULT on error.
241  */
242 #define put_user(x, ptr)                                        \
243 ({                                                              \
244         int __ret_pu;                                           \
245         __typeof__(*(ptr)) __pu_val;                            \
246         __chk_user_ptr(ptr);                                    \
247         might_fault();                                          \
248         __pu_val = x;                                           \
249         switch (sizeof(*(ptr))) {                               \
250         case 1:                                                 \
251                 __put_user_x(1, __pu_val, ptr, __ret_pu);       \
252                 break;                                          \
253         case 2:                                                 \
254                 __put_user_x(2, __pu_val, ptr, __ret_pu);       \
255                 break;                                          \
256         case 4:                                                 \
257                 __put_user_x(4, __pu_val, ptr, __ret_pu);       \
258                 break;                                          \
259         case 8:                                                 \
260                 __put_user_x8(__pu_val, ptr, __ret_pu);         \
261                 break;                                          \
262         default:                                                \
263                 __put_user_x(X, __pu_val, ptr, __ret_pu);       \
264                 break;                                          \
265         }                                                       \
266         __builtin_expect(__ret_pu, 0);                          \
267 })
268
269 #define __put_user_size(x, ptr, size, label)                            \
270 do {                                                                    \
271         __chk_user_ptr(ptr);                                            \
272         switch (size) {                                                 \
273         case 1:                                                         \
274                 __put_user_goto(x, ptr, "b", "b", "iq", label); \
275                 break;                                                  \
276         case 2:                                                         \
277                 __put_user_goto(x, ptr, "w", "w", "ir", label);         \
278                 break;                                                  \
279         case 4:                                                         \
280                 __put_user_goto(x, ptr, "l", "k", "ir", label);         \
281                 break;                                                  \
282         case 8:                                                         \
283                 __put_user_goto_u64(x, ptr, label);                     \
284                 break;                                                  \
285         default:                                                        \
286                 __put_user_bad();                                       \
287         }                                                               \
288 } while (0)
289
290 /*
291  * This doesn't do __uaccess_begin/end - the exception handling
292  * around it must do that.
293  */
294 #define __put_user_size_ex(x, ptr, size)                                \
295 do {                                                                    \
296         __chk_user_ptr(ptr);                                            \
297         switch (size) {                                                 \
298         case 1:                                                         \
299                 __put_user_asm_ex(x, ptr, "b", "b", "iq");              \
300                 break;                                                  \
301         case 2:                                                         \
302                 __put_user_asm_ex(x, ptr, "w", "w", "ir");              \
303                 break;                                                  \
304         case 4:                                                         \
305                 __put_user_asm_ex(x, ptr, "l", "k", "ir");              \
306                 break;                                                  \
307         case 8:                                                         \
308                 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr);      \
309                 break;                                                  \
310         default:                                                        \
311                 __put_user_bad();                                       \
312         }                                                               \
313 } while (0)
314
315 #ifdef CONFIG_X86_32
316 #define __get_user_asm_u64(x, ptr, retval, errret)                      \
317 ({                                                                      \
318         __typeof__(ptr) __ptr = (ptr);                                  \
319         asm volatile("\n"                                       \
320                      "1:        movl %2,%%eax\n"                        \
321                      "2:        movl %3,%%edx\n"                        \
322                      "3:\n"                             \
323                      ".section .fixup,\"ax\"\n"                         \
324                      "4:        mov %4,%0\n"                            \
325                      "  xorl %%eax,%%eax\n"                             \
326                      "  xorl %%edx,%%edx\n"                             \
327                      "  jmp 3b\n"                                       \
328                      ".previous\n"                                      \
329                      _ASM_EXTABLE_UA(1b, 4b)                            \
330                      _ASM_EXTABLE_UA(2b, 4b)                            \
331                      : "=r" (retval), "=&A"(x)                          \
332                      : "m" (__m(__ptr)), "m" __m(((u32 __user *)(__ptr)) + 1),  \
333                        "i" (errret), "0" (retval));                     \
334 })
335
336 #define __get_user_asm_ex_u64(x, ptr)                   (x) = __get_user_bad()
337 #else
338 #define __get_user_asm_u64(x, ptr, retval, errret) \
339          __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
340 #define __get_user_asm_ex_u64(x, ptr) \
341          __get_user_asm_ex(x, ptr, "q", "", "=r")
342 #endif
343
344 #define __get_user_size(x, ptr, size, retval, errret)                   \
345 do {                                                                    \
346         retval = 0;                                                     \
347         __chk_user_ptr(ptr);                                            \
348         switch (size) {                                                 \
349         case 1:                                                         \
350                 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
351                 break;                                                  \
352         case 2:                                                         \
353                 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
354                 break;                                                  \
355         case 4:                                                         \
356                 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
357                 break;                                                  \
358         case 8:                                                         \
359                 __get_user_asm_u64(x, ptr, retval, errret);             \
360                 break;                                                  \
361         default:                                                        \
362                 (x) = __get_user_bad();                                 \
363         }                                                               \
364 } while (0)
365
366 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)       \
367         asm volatile("\n"                                               \
368                      "1:        mov"itype" %2,%"rtype"1\n"              \
369                      "2:\n"                                             \
370                      ".section .fixup,\"ax\"\n"                         \
371                      "3:        mov %3,%0\n"                            \
372                      "  xor"itype" %"rtype"1,%"rtype"1\n"               \
373                      "  jmp 2b\n"                                       \
374                      ".previous\n"                                      \
375                      _ASM_EXTABLE_UA(1b, 3b)                            \
376                      : "=r" (err), ltype(x)                             \
377                      : "m" (__m(addr)), "i" (errret), "0" (err))
378
379 #define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret)        \
380         asm volatile("\n"                                               \
381                      "1:        mov"itype" %2,%"rtype"1\n"              \
382                      "2:\n"                                             \
383                      ".section .fixup,\"ax\"\n"                         \
384                      "3:        mov %3,%0\n"                            \
385                      "  jmp 2b\n"                                       \
386                      ".previous\n"                                      \
387                      _ASM_EXTABLE_UA(1b, 3b)                            \
388                      : "=r" (err), ltype(x)                             \
389                      : "m" (__m(addr)), "i" (errret), "0" (err))
390
391 /*
392  * This doesn't do __uaccess_begin/end - the exception handling
393  * around it must do that.
394  */
395 #define __get_user_size_ex(x, ptr, size)                                \
396 do {                                                                    \
397         __chk_user_ptr(ptr);                                            \
398         switch (size) {                                                 \
399         case 1:                                                         \
400                 __get_user_asm_ex(x, ptr, "b", "b", "=q");              \
401                 break;                                                  \
402         case 2:                                                         \
403                 __get_user_asm_ex(x, ptr, "w", "w", "=r");              \
404                 break;                                                  \
405         case 4:                                                         \
406                 __get_user_asm_ex(x, ptr, "l", "k", "=r");              \
407                 break;                                                  \
408         case 8:                                                         \
409                 __get_user_asm_ex_u64(x, ptr);                          \
410                 break;                                                  \
411         default:                                                        \
412                 (x) = __get_user_bad();                                 \
413         }                                                               \
414 } while (0)
415
416 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)                 \
417         asm volatile("1:        mov"itype" %1,%"rtype"0\n"              \
418                      "2:\n"                                             \
419                      ".section .fixup,\"ax\"\n"                         \
420                      "3:xor"itype" %"rtype"0,%"rtype"0\n"               \
421                      "  jmp 2b\n"                                       \
422                      ".previous\n"                                      \
423                      _ASM_EXTABLE_EX(1b, 3b)                            \
424                      : ltype(x) : "m" (__m(addr)))
425
426 #define __put_user_nocheck(x, ptr, size)                        \
427 ({                                                              \
428         __label__ __pu_label;                                   \
429         int __pu_err = -EFAULT;                                 \
430         __typeof__(*(ptr)) __pu_val;                            \
431         __pu_val = x;                                           \
432         __uaccess_begin();                                      \
433         __put_user_size(__pu_val, (ptr), (size), __pu_label);   \
434         __pu_err = 0;                                           \
435 __pu_label:                                                     \
436         __uaccess_end();                                        \
437         __builtin_expect(__pu_err, 0);                          \
438 })
439
440 #define __get_user_nocheck(x, ptr, size)                                \
441 ({                                                                      \
442         int __gu_err;                                                   \
443         __inttype(*(ptr)) __gu_val;                                     \
444         __uaccess_begin_nospec();                                       \
445         __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT);    \
446         __uaccess_end();                                                \
447         (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
448         __builtin_expect(__gu_err, 0);                                  \
449 })
450
451 /* FIXME: this hack is definitely wrong -AK */
452 struct __large_struct { unsigned long buf[100]; };
453 #define __m(x) (*(struct __large_struct __user *)(x))
454
455 /*
456  * Tell gcc we read from memory instead of writing: this is because
457  * we do not write to any memory gcc knows about, so there are no
458  * aliasing issues.
459  */
460 #define __put_user_goto(x, addr, itype, rtype, ltype, label)    \
461         asm_volatile_goto("\n"                                          \
462                 "1:     mov"itype" %"rtype"0,%1\n"                      \
463                 _ASM_EXTABLE_UA(1b, %l2)                                        \
464                 : : ltype(x), "m" (__m(addr))                           \
465                 : : label)
466
467 #define __put_user_failed(x, addr, itype, rtype, ltype, errret)         \
468         ({      __label__ __puflab;                                     \
469                 int __pufret = errret;                                  \
470                 __put_user_goto(x,addr,itype,rtype,ltype,__puflab);     \
471                 __pufret = 0;                                           \
472         __puflab: __pufret; })
473
474 #define __put_user_asm(x, addr, retval, itype, rtype, ltype, errret)    do {    \
475         retval = __put_user_failed(x, addr, itype, rtype, ltype, errret);       \
476 } while (0)
477
478 #define __put_user_asm_ex(x, addr, itype, rtype, ltype)                 \
479         asm volatile("1:        mov"itype" %"rtype"0,%1\n"              \
480                      "2:\n"                                             \
481                      _ASM_EXTABLE_EX(1b, 2b)                            \
482                      : : ltype(x), "m" (__m(addr)))
483
484 /*
485  * uaccess_try and catch
486  */
487 #define uaccess_try     do {                                            \
488         current->thread.uaccess_err = 0;                                \
489         __uaccess_begin();                                              \
490         barrier();
491
492 #define uaccess_try_nospec do {                                         \
493         current->thread.uaccess_err = 0;                                \
494         __uaccess_begin_nospec();                                       \
495
496 #define uaccess_catch(err)                                              \
497         __uaccess_end();                                                \
498         (err) |= (current->thread.uaccess_err ? -EFAULT : 0);           \
499 } while (0)
500
501 /**
502  * __get_user - Get a simple variable from user space, with less checking.
503  * @x:   Variable to store result.
504  * @ptr: Source address, in user space.
505  *
506  * Context: User context only. This function may sleep if pagefaults are
507  *          enabled.
508  *
509  * This macro copies a single simple variable from user space to kernel
510  * space.  It supports simple types like char and int, but not larger
511  * data types like structures or arrays.
512  *
513  * @ptr must have pointer-to-simple-variable type, and the result of
514  * dereferencing @ptr must be assignable to @x without a cast.
515  *
516  * Caller must check the pointer with access_ok() before calling this
517  * function.
518  *
519  * Return: zero on success, or -EFAULT on error.
520  * On error, the variable @x is set to zero.
521  */
522
523 #define __get_user(x, ptr)                                              \
524         __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
525
526 /**
527  * __put_user - Write a simple value into user space, with less checking.
528  * @x:   Value to copy to user space.
529  * @ptr: Destination address, in user space.
530  *
531  * Context: User context only. This function may sleep if pagefaults are
532  *          enabled.
533  *
534  * This macro copies a single simple value from kernel space to user
535  * space.  It supports simple types like char and int, but not larger
536  * data types like structures or arrays.
537  *
538  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
539  * to the result of dereferencing @ptr.
540  *
541  * Caller must check the pointer with access_ok() before calling this
542  * function.
543  *
544  * Return: zero on success, or -EFAULT on error.
545  */
546
547 #define __put_user(x, ptr)                                              \
548         __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
549
550 /*
551  * {get|put}_user_try and catch
552  *
553  * get_user_try {
554  *      get_user_ex(...);
555  * } get_user_catch(err)
556  */
557 #define get_user_try            uaccess_try_nospec
558 #define get_user_catch(err)     uaccess_catch(err)
559
560 #define get_user_ex(x, ptr)     do {                                    \
561         unsigned long __gue_val;                                        \
562         __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr))));       \
563         (x) = (__force __typeof__(*(ptr)))__gue_val;                    \
564 } while (0)
565
566 #define put_user_try            uaccess_try
567 #define put_user_catch(err)     uaccess_catch(err)
568
569 #define put_user_ex(x, ptr)                                             \
570         __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
571
572 extern unsigned long
573 copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
574 extern __must_check long
575 strncpy_from_user(char *dst, const char __user *src, long count);
576
577 extern __must_check long strnlen_user(const char __user *str, long n);
578
579 unsigned long __must_check clear_user(void __user *mem, unsigned long len);
580 unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
581
582 extern void __cmpxchg_wrong_size(void)
583         __compiletime_error("Bad argument size for cmpxchg");
584
585 #define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size)       \
586 ({                                                                      \
587         int __ret = 0;                                                  \
588         __typeof__(ptr) __uval = (uval);                                \
589         __typeof__(*(ptr)) __old = (old);                               \
590         __typeof__(*(ptr)) __new = (new);                               \
591         __uaccess_begin_nospec();                                       \
592         switch (size) {                                                 \
593         case 1:                                                         \
594         {                                                               \
595                 asm volatile("\n"                                       \
596                         "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n"          \
597                         "2:\n"                                          \
598                         "\t.section .fixup, \"ax\"\n"                   \
599                         "3:\tmov     %3, %0\n"                          \
600                         "\tjmp     2b\n"                                \
601                         "\t.previous\n"                                 \
602                         _ASM_EXTABLE_UA(1b, 3b)                         \
603                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
604                         : "i" (-EFAULT), "q" (__new), "1" (__old)       \
605                         : "memory"                                      \
606                 );                                                      \
607                 break;                                                  \
608         }                                                               \
609         case 2:                                                         \
610         {                                                               \
611                 asm volatile("\n"                                       \
612                         "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n"          \
613                         "2:\n"                                          \
614                         "\t.section .fixup, \"ax\"\n"                   \
615                         "3:\tmov     %3, %0\n"                          \
616                         "\tjmp     2b\n"                                \
617                         "\t.previous\n"                                 \
618                         _ASM_EXTABLE_UA(1b, 3b)                         \
619                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
620                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
621                         : "memory"                                      \
622                 );                                                      \
623                 break;                                                  \
624         }                                                               \
625         case 4:                                                         \
626         {                                                               \
627                 asm volatile("\n"                                       \
628                         "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"          \
629                         "2:\n"                                          \
630                         "\t.section .fixup, \"ax\"\n"                   \
631                         "3:\tmov     %3, %0\n"                          \
632                         "\tjmp     2b\n"                                \
633                         "\t.previous\n"                                 \
634                         _ASM_EXTABLE_UA(1b, 3b)                         \
635                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
636                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
637                         : "memory"                                      \
638                 );                                                      \
639                 break;                                                  \
640         }                                                               \
641         case 8:                                                         \
642         {                                                               \
643                 if (!IS_ENABLED(CONFIG_X86_64))                         \
644                         __cmpxchg_wrong_size();                         \
645                                                                         \
646                 asm volatile("\n"                                       \
647                         "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n"          \
648                         "2:\n"                                          \
649                         "\t.section .fixup, \"ax\"\n"                   \
650                         "3:\tmov     %3, %0\n"                          \
651                         "\tjmp     2b\n"                                \
652                         "\t.previous\n"                                 \
653                         _ASM_EXTABLE_UA(1b, 3b)                         \
654                         : "+r" (__ret), "=a" (__old), "+m" (*(ptr))     \
655                         : "i" (-EFAULT), "r" (__new), "1" (__old)       \
656                         : "memory"                                      \
657                 );                                                      \
658                 break;                                                  \
659         }                                                               \
660         default:                                                        \
661                 __cmpxchg_wrong_size();                                 \
662         }                                                               \
663         __uaccess_end();                                                \
664         *__uval = __old;                                                \
665         __ret;                                                          \
666 })
667
668 #define user_atomic_cmpxchg_inatomic(uval, ptr, old, new)               \
669 ({                                                                      \
670         access_ok((ptr), sizeof(*(ptr))) ?              \
671                 __user_atomic_cmpxchg_inatomic((uval), (ptr),           \
672                                 (old), (new), sizeof(*(ptr))) :         \
673                 -EFAULT;                                                \
674 })
675
676 /*
677  * movsl can be slow when source and dest are not both 8-byte aligned
678  */
679 #ifdef CONFIG_X86_INTEL_USERCOPY
680 extern struct movsl_mask {
681         int mask;
682 } ____cacheline_aligned_in_smp movsl_mask;
683 #endif
684
685 #define ARCH_HAS_NOCACHE_UACCESS 1
686
687 #ifdef CONFIG_X86_32
688 # include <asm/uaccess_32.h>
689 #else
690 # include <asm/uaccess_64.h>
691 #endif
692
693 /*
694  * We rely on the nested NMI work to allow atomic faults from the NMI path; the
695  * nested NMI paths are careful to preserve CR2.
696  *
697  * Caller must use pagefault_enable/disable, or run in interrupt context,
698  * and also do a uaccess_ok() check
699  */
700 #define __copy_from_user_nmi __copy_from_user_inatomic
701
702 /*
703  * The "unsafe" user accesses aren't really "unsafe", but the naming
704  * is a big fat warning: you have to not only do the access_ok()
705  * checking before using them, but you have to surround them with the
706  * user_access_begin/end() pair.
707  */
708 static __must_check inline bool user_access_begin(const void __user *ptr, size_t len)
709 {
710         if (unlikely(!access_ok(ptr,len)))
711                 return 0;
712         __uaccess_begin_nospec();
713         return 1;
714 }
715 #define user_access_begin(a,b)  user_access_begin(a,b)
716 #define user_access_end()       __uaccess_end()
717
718 #define unsafe_put_user(x, ptr, label)  \
719         __put_user_size((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
720
721 #define unsafe_get_user(x, ptr, err_label)                                      \
722 do {                                                                            \
723         int __gu_err;                                                           \
724         __inttype(*(ptr)) __gu_val;                                             \
725         __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT);    \
726         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
727         if (unlikely(__gu_err)) goto err_label;                                 \
728 } while (0)
729
730 #endif /* _ASM_X86_UACCESS_H */
731