Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64...
[muen/linux.git] / arch / arm64 / kernel / probes / kprobes.c
1 /*
2  * arch/arm64/kernel/probes/kprobes.c
3  *
4  * Kprobes support for ARM64
5  *
6  * Copyright (C) 2013 Linaro Limited.
7  * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  */
19 #include <linux/kasan.h>
20 #include <linux/kernel.h>
21 #include <linux/kprobes.h>
22 #include <linux/extable.h>
23 #include <linux/slab.h>
24 #include <linux/stop_machine.h>
25 #include <linux/sched/debug.h>
26 #include <linux/set_memory.h>
27 #include <linux/stringify.h>
28 #include <linux/vmalloc.h>
29 #include <asm/traps.h>
30 #include <asm/ptrace.h>
31 #include <asm/cacheflush.h>
32 #include <asm/debug-monitors.h>
33 #include <asm/system_misc.h>
34 #include <asm/insn.h>
35 #include <linux/uaccess.h>
36 #include <asm/irq.h>
37 #include <asm/sections.h>
38
39 #include "decode-insn.h"
40
41 DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
42 DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
43
44 static void __kprobes
45 post_kprobe_handler(struct kprobe_ctlblk *, struct pt_regs *);
46
47 static int __kprobes patch_text(kprobe_opcode_t *addr, u32 opcode)
48 {
49         void *addrs[1];
50         u32 insns[1];
51
52         addrs[0] = addr;
53         insns[0] = opcode;
54
55         return aarch64_insn_patch_text(addrs, insns, 1);
56 }
57
58 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
59 {
60         /* prepare insn slot */
61         patch_text(p->ainsn.api.insn, p->opcode);
62
63         flush_icache_range((uintptr_t) (p->ainsn.api.insn),
64                            (uintptr_t) (p->ainsn.api.insn) +
65                            MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
66
67         /*
68          * Needs restoring of return address after stepping xol.
69          */
70         p->ainsn.api.restore = (unsigned long) p->addr +
71           sizeof(kprobe_opcode_t);
72 }
73
74 static void __kprobes arch_prepare_simulate(struct kprobe *p)
75 {
76         /* This instructions is not executed xol. No need to adjust the PC */
77         p->ainsn.api.restore = 0;
78 }
79
80 static void __kprobes arch_simulate_insn(struct kprobe *p, struct pt_regs *regs)
81 {
82         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
83
84         if (p->ainsn.api.handler)
85                 p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs);
86
87         /* single step simulated, now go for post processing */
88         post_kprobe_handler(kcb, regs);
89 }
90
91 int __kprobes arch_prepare_kprobe(struct kprobe *p)
92 {
93         unsigned long probe_addr = (unsigned long)p->addr;
94         extern char __start_rodata[];
95         extern char __end_rodata[];
96
97         if (probe_addr & 0x3)
98                 return -EINVAL;
99
100         /* copy instruction */
101         p->opcode = le32_to_cpu(*p->addr);
102
103         if (in_exception_text(probe_addr))
104                 return -EINVAL;
105         if (probe_addr >= (unsigned long) __start_rodata &&
106             probe_addr <= (unsigned long) __end_rodata)
107                 return -EINVAL;
108
109         /* decode instruction */
110         switch (arm_kprobe_decode_insn(p->addr, &p->ainsn)) {
111         case INSN_REJECTED:     /* insn not supported */
112                 return -EINVAL;
113
114         case INSN_GOOD_NO_SLOT: /* insn need simulation */
115                 p->ainsn.api.insn = NULL;
116                 break;
117
118         case INSN_GOOD: /* instruction uses slot */
119                 p->ainsn.api.insn = get_insn_slot();
120                 if (!p->ainsn.api.insn)
121                         return -ENOMEM;
122                 break;
123         }
124
125         /* prepare the instruction */
126         if (p->ainsn.api.insn)
127                 arch_prepare_ss_slot(p);
128         else
129                 arch_prepare_simulate(p);
130
131         return 0;
132 }
133
134 void *alloc_insn_page(void)
135 {
136         void *page;
137
138         page = vmalloc_exec(PAGE_SIZE);
139         if (page)
140                 set_memory_ro((unsigned long)page, 1);
141
142         return page;
143 }
144
145 /* arm kprobe: install breakpoint in text */
146 void __kprobes arch_arm_kprobe(struct kprobe *p)
147 {
148         patch_text(p->addr, BRK64_OPCODE_KPROBES);
149 }
150
151 /* disarm kprobe: remove breakpoint from text */
152 void __kprobes arch_disarm_kprobe(struct kprobe *p)
153 {
154         patch_text(p->addr, p->opcode);
155 }
156
157 void __kprobes arch_remove_kprobe(struct kprobe *p)
158 {
159         if (p->ainsn.api.insn) {
160                 free_insn_slot(p->ainsn.api.insn, 0);
161                 p->ainsn.api.insn = NULL;
162         }
163 }
164
165 static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
166 {
167         kcb->prev_kprobe.kp = kprobe_running();
168         kcb->prev_kprobe.status = kcb->kprobe_status;
169 }
170
171 static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
172 {
173         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
174         kcb->kprobe_status = kcb->prev_kprobe.status;
175 }
176
177 static void __kprobes set_current_kprobe(struct kprobe *p)
178 {
179         __this_cpu_write(current_kprobe, p);
180 }
181
182 /*
183  * When PSTATE.D is set (masked), then software step exceptions can not be
184  * generated.
185  * SPSR's D bit shows the value of PSTATE.D immediately before the
186  * exception was taken. PSTATE.D is set while entering into any exception
187  * mode, however software clears it for any normal (none-debug-exception)
188  * mode in the exception entry. Therefore, when we are entering into kprobe
189  * breakpoint handler from any normal mode then SPSR.D bit is already
190  * cleared, however it is set when we are entering from any debug exception
191  * mode.
192  * Since we always need to generate single step exception after a kprobe
193  * breakpoint exception therefore we need to clear it unconditionally, when
194  * we become sure that the current breakpoint exception is for kprobe.
195  */
196 static void __kprobes
197 spsr_set_debug_flag(struct pt_regs *regs, int mask)
198 {
199         unsigned long spsr = regs->pstate;
200
201         if (mask)
202                 spsr |= PSR_D_BIT;
203         else
204                 spsr &= ~PSR_D_BIT;
205
206         regs->pstate = spsr;
207 }
208
209 /*
210  * Interrupts need to be disabled before single-step mode is set, and not
211  * reenabled until after single-step mode ends.
212  * Without disabling interrupt on local CPU, there is a chance of
213  * interrupt occurrence in the period of exception return and  start of
214  * out-of-line single-step, that result in wrongly single stepping
215  * into the interrupt handler.
216  */
217 static void __kprobes kprobes_save_local_irqflag(struct kprobe_ctlblk *kcb,
218                                                 struct pt_regs *regs)
219 {
220         kcb->saved_irqflag = regs->pstate;
221         regs->pstate |= PSR_I_BIT;
222 }
223
224 static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb,
225                                                 struct pt_regs *regs)
226 {
227         if (kcb->saved_irqflag & PSR_I_BIT)
228                 regs->pstate |= PSR_I_BIT;
229         else
230                 regs->pstate &= ~PSR_I_BIT;
231 }
232
233 static void __kprobes
234 set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
235 {
236         kcb->ss_ctx.ss_pending = true;
237         kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
238 }
239
240 static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
241 {
242         kcb->ss_ctx.ss_pending = false;
243         kcb->ss_ctx.match_addr = 0;
244 }
245
246 static void __kprobes setup_singlestep(struct kprobe *p,
247                                        struct pt_regs *regs,
248                                        struct kprobe_ctlblk *kcb, int reenter)
249 {
250         unsigned long slot;
251
252         if (reenter) {
253                 save_previous_kprobe(kcb);
254                 set_current_kprobe(p);
255                 kcb->kprobe_status = KPROBE_REENTER;
256         } else {
257                 kcb->kprobe_status = KPROBE_HIT_SS;
258         }
259
260
261         if (p->ainsn.api.insn) {
262                 /* prepare for single stepping */
263                 slot = (unsigned long)p->ainsn.api.insn;
264
265                 set_ss_context(kcb, slot);      /* mark pending ss */
266
267                 spsr_set_debug_flag(regs, 0);
268
269                 /* IRQs and single stepping do not mix well. */
270                 kprobes_save_local_irqflag(kcb, regs);
271                 kernel_enable_single_step(regs);
272                 instruction_pointer_set(regs, slot);
273         } else {
274                 /* insn simulation */
275                 arch_simulate_insn(p, regs);
276         }
277 }
278
279 static int __kprobes reenter_kprobe(struct kprobe *p,
280                                     struct pt_regs *regs,
281                                     struct kprobe_ctlblk *kcb)
282 {
283         switch (kcb->kprobe_status) {
284         case KPROBE_HIT_SSDONE:
285         case KPROBE_HIT_ACTIVE:
286                 kprobes_inc_nmissed_count(p);
287                 setup_singlestep(p, regs, kcb, 1);
288                 break;
289         case KPROBE_HIT_SS:
290         case KPROBE_REENTER:
291                 pr_warn("Unrecoverable kprobe detected.\n");
292                 dump_kprobe(p);
293                 BUG();
294                 break;
295         default:
296                 WARN_ON(1);
297                 return 0;
298         }
299
300         return 1;
301 }
302
303 static void __kprobes
304 post_kprobe_handler(struct kprobe_ctlblk *kcb, struct pt_regs *regs)
305 {
306         struct kprobe *cur = kprobe_running();
307
308         if (!cur)
309                 return;
310
311         /* return addr restore if non-branching insn */
312         if (cur->ainsn.api.restore != 0)
313                 instruction_pointer_set(regs, cur->ainsn.api.restore);
314
315         /* restore back original saved kprobe variables and continue */
316         if (kcb->kprobe_status == KPROBE_REENTER) {
317                 restore_previous_kprobe(kcb);
318                 return;
319         }
320         /* call post handler */
321         kcb->kprobe_status = KPROBE_HIT_SSDONE;
322         if (cur->post_handler)  {
323                 /* post_handler can hit breakpoint and single step
324                  * again, so we enable D-flag for recursive exception.
325                  */
326                 cur->post_handler(cur, regs, 0);
327         }
328
329         reset_current_kprobe();
330 }
331
332 int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr)
333 {
334         struct kprobe *cur = kprobe_running();
335         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
336
337         switch (kcb->kprobe_status) {
338         case KPROBE_HIT_SS:
339         case KPROBE_REENTER:
340                 /*
341                  * We are here because the instruction being single
342                  * stepped caused a page fault. We reset the current
343                  * kprobe and the ip points back to the probe address
344                  * and allow the page fault handler to continue as a
345                  * normal page fault.
346                  */
347                 instruction_pointer_set(regs, (unsigned long) cur->addr);
348                 if (!instruction_pointer(regs))
349                         BUG();
350
351                 kernel_disable_single_step();
352
353                 if (kcb->kprobe_status == KPROBE_REENTER)
354                         restore_previous_kprobe(kcb);
355                 else
356                         reset_current_kprobe();
357
358                 break;
359         case KPROBE_HIT_ACTIVE:
360         case KPROBE_HIT_SSDONE:
361                 /*
362                  * We increment the nmissed count for accounting,
363                  * we can also use npre/npostfault count for accounting
364                  * these specific fault cases.
365                  */
366                 kprobes_inc_nmissed_count(cur);
367
368                 /*
369                  * We come here because instructions in the pre/post
370                  * handler caused the page_fault, this could happen
371                  * if handler tries to access user space by
372                  * copy_from_user(), get_user() etc. Let the
373                  * user-specified handler try to fix it first.
374                  */
375                 if (cur->fault_handler && cur->fault_handler(cur, regs, fsr))
376                         return 1;
377
378                 /*
379                  * In case the user-specified fault handler returned
380                  * zero, try to fix up.
381                  */
382                 if (fixup_exception(regs))
383                         return 1;
384         }
385         return 0;
386 }
387
388 static void __kprobes kprobe_handler(struct pt_regs *regs)
389 {
390         struct kprobe *p, *cur_kprobe;
391         struct kprobe_ctlblk *kcb;
392         unsigned long addr = instruction_pointer(regs);
393
394         kcb = get_kprobe_ctlblk();
395         cur_kprobe = kprobe_running();
396
397         p = get_kprobe((kprobe_opcode_t *) addr);
398
399         if (p) {
400                 if (cur_kprobe) {
401                         if (reenter_kprobe(p, regs, kcb))
402                                 return;
403                 } else {
404                         /* Probe hit */
405                         set_current_kprobe(p);
406                         kcb->kprobe_status = KPROBE_HIT_ACTIVE;
407
408                         /*
409                          * If we have no pre-handler or it returned 0, we
410                          * continue with normal processing.  If we have a
411                          * pre-handler and it returned non-zero, it will
412                          * modify the execution path and no need to single
413                          * stepping. Let's just reset current kprobe and exit.
414                          *
415                          * pre_handler can hit a breakpoint and can step thru
416                          * before return, keep PSTATE D-flag enabled until
417                          * pre_handler return back.
418                          */
419                         if (!p->pre_handler || !p->pre_handler(p, regs)) {
420                                 setup_singlestep(p, regs, kcb, 0);
421                         } else
422                                 reset_current_kprobe();
423                 }
424         }
425         /*
426          * The breakpoint instruction was removed right
427          * after we hit it.  Another cpu has removed
428          * either a probepoint or a debugger breakpoint
429          * at this address.  In either case, no further
430          * handling of this interrupt is appropriate.
431          * Return back to original instruction, and continue.
432          */
433 }
434
435 static int __kprobes
436 kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
437 {
438         if ((kcb->ss_ctx.ss_pending)
439             && (kcb->ss_ctx.match_addr == addr)) {
440                 clear_ss_context(kcb);  /* clear pending ss */
441                 return DBG_HOOK_HANDLED;
442         }
443         /* not ours, kprobes should ignore it */
444         return DBG_HOOK_ERROR;
445 }
446
447 int __kprobes
448 kprobe_single_step_handler(struct pt_regs *regs, unsigned int esr)
449 {
450         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
451         int retval;
452
453         /* return error if this is not our step */
454         retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
455
456         if (retval == DBG_HOOK_HANDLED) {
457                 kprobes_restore_local_irqflag(kcb, regs);
458                 kernel_disable_single_step();
459
460                 post_kprobe_handler(kcb, regs);
461         }
462
463         return retval;
464 }
465
466 int __kprobes
467 kprobe_breakpoint_handler(struct pt_regs *regs, unsigned int esr)
468 {
469         kprobe_handler(regs);
470         return DBG_HOOK_HANDLED;
471 }
472
473 bool arch_within_kprobe_blacklist(unsigned long addr)
474 {
475         if ((addr >= (unsigned long)__kprobes_text_start &&
476             addr < (unsigned long)__kprobes_text_end) ||
477             (addr >= (unsigned long)__entry_text_start &&
478             addr < (unsigned long)__entry_text_end) ||
479             (addr >= (unsigned long)__idmap_text_start &&
480             addr < (unsigned long)__idmap_text_end) ||
481             !!search_exception_tables(addr))
482                 return true;
483
484         if (!is_kernel_in_hyp_mode()) {
485                 if ((addr >= (unsigned long)__hyp_text_start &&
486                     addr < (unsigned long)__hyp_text_end) ||
487                     (addr >= (unsigned long)__hyp_idmap_text_start &&
488                     addr < (unsigned long)__hyp_idmap_text_end))
489                         return true;
490         }
491
492         return false;
493 }
494
495 void __kprobes __used *trampoline_probe_handler(struct pt_regs *regs)
496 {
497         struct kretprobe_instance *ri = NULL;
498         struct hlist_head *head, empty_rp;
499         struct hlist_node *tmp;
500         unsigned long flags, orig_ret_address = 0;
501         unsigned long trampoline_address =
502                 (unsigned long)&kretprobe_trampoline;
503         kprobe_opcode_t *correct_ret_addr = NULL;
504
505         INIT_HLIST_HEAD(&empty_rp);
506         kretprobe_hash_lock(current, &head, &flags);
507
508         /*
509          * It is possible to have multiple instances associated with a given
510          * task either because multiple functions in the call path have
511          * return probes installed on them, and/or more than one
512          * return probe was registered for a target function.
513          *
514          * We can handle this because:
515          *     - instances are always pushed into the head of the list
516          *     - when multiple return probes are registered for the same
517          *       function, the (chronologically) first instance's ret_addr
518          *       will be the real return address, and all the rest will
519          *       point to kretprobe_trampoline.
520          */
521         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
522                 if (ri->task != current)
523                         /* another task is sharing our hash bucket */
524                         continue;
525
526                 orig_ret_address = (unsigned long)ri->ret_addr;
527
528                 if (orig_ret_address != trampoline_address)
529                         /*
530                          * This is the real return address. Any other
531                          * instances associated with this task are for
532                          * other calls deeper on the call stack
533                          */
534                         break;
535         }
536
537         kretprobe_assert(ri, orig_ret_address, trampoline_address);
538
539         correct_ret_addr = ri->ret_addr;
540         hlist_for_each_entry_safe(ri, tmp, head, hlist) {
541                 if (ri->task != current)
542                         /* another task is sharing our hash bucket */
543                         continue;
544
545                 orig_ret_address = (unsigned long)ri->ret_addr;
546                 if (ri->rp && ri->rp->handler) {
547                         __this_cpu_write(current_kprobe, &ri->rp->kp);
548                         get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
549                         ri->ret_addr = correct_ret_addr;
550                         ri->rp->handler(ri, regs);
551                         __this_cpu_write(current_kprobe, NULL);
552                 }
553
554                 recycle_rp_inst(ri, &empty_rp);
555
556                 if (orig_ret_address != trampoline_address)
557                         /*
558                          * This is the real return address. Any other
559                          * instances associated with this task are for
560                          * other calls deeper on the call stack
561                          */
562                         break;
563         }
564
565         kretprobe_hash_unlock(current, &flags);
566
567         hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
568                 hlist_del(&ri->hlist);
569                 kfree(ri);
570         }
571         return (void *)orig_ret_address;
572 }
573
574 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
575                                       struct pt_regs *regs)
576 {
577         ri->ret_addr = (kprobe_opcode_t *)regs->regs[30];
578
579         /* replace return addr (x30) with trampoline */
580         regs->regs[30] = (long)&kretprobe_trampoline;
581 }
582
583 int __kprobes arch_trampoline_kprobe(struct kprobe *p)
584 {
585         return 0;
586 }
587
588 int __init arch_init_kprobes(void)
589 {
590         return 0;
591 }