4e853d1582c71ab318f70fec1022e67c6b3091c1
[muen/linux.git] / arch / sparc / vdso / vclock_gettime.c
1 /*
2  * Copyright 2006 Andi Kleen, SUSE Labs.
3  * Subject to the GNU Public License, v.2
4  *
5  * Fast user context implementation of clock_gettime, gettimeofday, and time.
6  *
7  * The code should have no internal unresolved relocations.
8  * Check with readelf after changing.
9  * Also alternative() doesn't work.
10  */
11 /*
12  * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
13  */
14
15 /* Disable profiling for userspace code: */
16 #ifndef DISABLE_BRANCH_PROFILING
17 #define DISABLE_BRANCH_PROFILING
18 #endif
19
20 #include <linux/kernel.h>
21 #include <linux/time.h>
22 #include <linux/string.h>
23 #include <asm/io.h>
24 #include <asm/unistd.h>
25 #include <asm/timex.h>
26 #include <asm/clocksource.h>
27 #include <asm/vvar.h>
28
29 #ifdef  CONFIG_SPARC64
30 #define SYSCALL_STRING                                                  \
31         "ta     0x6d;"                                                  \
32         "bcs,a  1f;"                                                    \
33         " sub   %%g0, %%o0, %%o0;"                                      \
34         "1:"
35 #else
36 #define SYSCALL_STRING                                                  \
37         "ta     0x10;"                                                  \
38         "bcs,a  1f;"                                                    \
39         " sub   %%g0, %%o0, %%o0;"                                      \
40         "1:"
41 #endif
42
43 #define SYSCALL_CLOBBERS                                                \
44         "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",                 \
45         "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",           \
46         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",         \
47         "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",         \
48         "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",         \
49         "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",         \
50         "cc", "memory"
51
52 /*
53  * Compute the vvar page's address in the process address space, and return it
54  * as a pointer to the vvar_data.
55  */
56 notrace static __always_inline struct vvar_data *get_vvar_data(void)
57 {
58         unsigned long ret;
59
60         /*
61          * vdso data page is the first vDSO page so grab the PC
62          * and move up a page to get to the data page.
63          */
64         __asm__("rd %%pc, %0" : "=r" (ret));
65         ret &= ~(8192 - 1);
66         ret -= 8192;
67
68         return (struct vvar_data *) ret;
69 }
70
71 notrace static long vdso_fallback_gettime(long clock, struct timespec *ts)
72 {
73         register long num __asm__("g1") = __NR_clock_gettime;
74         register long o0 __asm__("o0") = clock;
75         register long o1 __asm__("o1") = (long) ts;
76
77         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
78                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
79         return o0;
80 }
81
82 notrace static long vdso_fallback_gettimeofday(struct timeval *tv, struct timezone *tz)
83 {
84         register long num __asm__("g1") = __NR_gettimeofday;
85         register long o0 __asm__("o0") = (long) tv;
86         register long o1 __asm__("o1") = (long) tz;
87
88         __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
89                              "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
90         return o0;
91 }
92
93 #ifdef  CONFIG_SPARC64
94 notrace static __always_inline u64 vread_tick(void)
95 {
96         u64     ret;
97
98         __asm__ __volatile__("1:\n\t"
99                              "rd                %%tick, %0\n\t"
100                              ".pushsection      .tick_patch, \"a\"\n\t"
101                              ".word             1b - ., 1f - .\n\t"
102                              ".popsection\n\t"
103                              ".pushsection      .tick_patch_replacement, \"ax\"\n\t"
104                              "1:\n\t"
105                              "rd                %%asr24, %0\n\t"
106                              ".popsection\n"
107                              : "=r" (ret));
108         return ret;
109 }
110 #else
111 notrace static __always_inline u64 vread_tick(void)
112 {
113         register unsigned long long ret asm("o4");
114
115         __asm__ __volatile__("1:\n\t"
116                              "rd                %%tick, %L0\n\t"
117                              "srlx              %L0, 32, %H0\n\t"
118                              ".pushsection      .tick_patch, \"a\"\n\t"
119                              ".word             1b - ., 1f - .\n\t"
120                              ".popsection\n\t"
121                              ".pushsection      .tick_patch_replacement, \"ax\"\n\t"
122                              "1:\n\t"
123                              "rd                %%asr24, %L0\n\t"
124                              ".popsection\n"
125                              : "=r" (ret));
126         return ret;
127 }
128 #endif
129
130 notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
131 {
132         u64 v;
133         u64 cycles;
134
135         cycles = vread_tick();
136         v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
137         return v * vvar->clock.mult;
138 }
139
140 notrace static __always_inline int do_realtime(struct vvar_data *vvar,
141                                                struct timespec *ts)
142 {
143         unsigned long seq;
144         u64 ns;
145
146         ts->tv_nsec = 0;
147         do {
148                 seq = vvar_read_begin(vvar);
149                 ts->tv_sec = vvar->wall_time_sec;
150                 ns = vvar->wall_time_snsec;
151                 ns += vgetsns(vvar);
152                 ns >>= vvar->clock.shift;
153         } while (unlikely(vvar_read_retry(vvar, seq)));
154
155         timespec_add_ns(ts, ns);
156
157         return 0;
158 }
159
160 notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
161                                                 struct timespec *ts)
162 {
163         unsigned long seq;
164         u64 ns;
165
166         ts->tv_nsec = 0;
167         do {
168                 seq = vvar_read_begin(vvar);
169                 ts->tv_sec = vvar->monotonic_time_sec;
170                 ns = vvar->monotonic_time_snsec;
171                 ns += vgetsns(vvar);
172                 ns >>= vvar->clock.shift;
173         } while (unlikely(vvar_read_retry(vvar, seq)));
174
175         timespec_add_ns(ts, ns);
176
177         return 0;
178 }
179
180 notrace static int do_realtime_coarse(struct vvar_data *vvar,
181                                       struct timespec *ts)
182 {
183         unsigned long seq;
184
185         do {
186                 seq = vvar_read_begin(vvar);
187                 ts->tv_sec = vvar->wall_time_coarse_sec;
188                 ts->tv_nsec = vvar->wall_time_coarse_nsec;
189         } while (unlikely(vvar_read_retry(vvar, seq)));
190         return 0;
191 }
192
193 notrace static int do_monotonic_coarse(struct vvar_data *vvar,
194                                        struct timespec *ts)
195 {
196         unsigned long seq;
197
198         do {
199                 seq = vvar_read_begin(vvar);
200                 ts->tv_sec = vvar->monotonic_time_coarse_sec;
201                 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
202         } while (unlikely(vvar_read_retry(vvar, seq)));
203
204         return 0;
205 }
206
207 notrace int
208 __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
209 {
210         struct vvar_data *vvd = get_vvar_data();
211
212         switch (clock) {
213         case CLOCK_REALTIME:
214                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
215                         break;
216                 return do_realtime(vvd, ts);
217         case CLOCK_MONOTONIC:
218                 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
219                         break;
220                 return do_monotonic(vvd, ts);
221         case CLOCK_REALTIME_COARSE:
222                 return do_realtime_coarse(vvd, ts);
223         case CLOCK_MONOTONIC_COARSE:
224                 return do_monotonic_coarse(vvd, ts);
225         }
226         /*
227          * Unknown clock ID ? Fall back to the syscall.
228          */
229         return vdso_fallback_gettime(clock, ts);
230 }
231 int
232 clock_gettime(clockid_t, struct timespec *)
233         __attribute__((weak, alias("__vdso_clock_gettime")));
234
235 notrace int
236 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
237 {
238         struct vvar_data *vvd = get_vvar_data();
239
240         if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
241                 if (likely(tv != NULL)) {
242                         union tstv_t {
243                                 struct timespec ts;
244                                 struct timeval tv;
245                         } *tstv = (union tstv_t *) tv;
246                         do_realtime(vvd, &tstv->ts);
247                         /*
248                          * Assign before dividing to ensure that the division is
249                          * done in the type of tv_usec, not tv_nsec.
250                          *
251                          * There cannot be > 1 billion usec in a second:
252                          * do_realtime() has already distributed such overflow
253                          * into tv_sec.  So we can assign it to an int safely.
254                          */
255                         tstv->tv.tv_usec = tstv->ts.tv_nsec;
256                         tstv->tv.tv_usec /= 1000;
257                 }
258                 if (unlikely(tz != NULL)) {
259                         /* Avoid memcpy. Some old compilers fail to inline it */
260                         tz->tz_minuteswest = vvd->tz_minuteswest;
261                         tz->tz_dsttime = vvd->tz_dsttime;
262                 }
263                 return 0;
264         }
265         return vdso_fallback_gettimeofday(tv, tz);
266 }
267 int
268 gettimeofday(struct timeval *, struct timezone *)
269         __attribute__((weak, alias("__vdso_gettimeofday")));