x86/xen: split off smp_hvm.c
[muen/linux.git] / arch / x86 / xen / smp.c
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 #include <linux/irq_work.h>
20 #include <linux/tick.h>
21 #include <linux/nmi.h>
22
23 #include <asm/paravirt.h>
24 #include <asm/desc.h>
25 #include <asm/pgtable.h>
26 #include <asm/cpu.h>
27
28 #include <xen/interface/xen.h>
29 #include <xen/interface/vcpu.h>
30 #include <xen/interface/xenpmu.h>
31
32 #include <asm/xen/interface.h>
33 #include <asm/xen/hypercall.h>
34
35 #include <xen/xen.h>
36 #include <xen/page.h>
37 #include <xen/events.h>
38
39 #include <xen/hvc-console.h>
40 #include "xen-ops.h"
41 #include "mmu.h"
42 #include "smp.h"
43 #include "pmu.h"
44
45 cpumask_var_t xen_cpu_initialized_map;
46
47 struct xen_common_irq {
48         int irq;
49         char *name;
50 };
51 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
53 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
54 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
55 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
56 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
57
58 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
59 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
60 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
61
62 /*
63  * Reschedule call back.
64  */
65 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
66 {
67         inc_irq_stat(irq_resched_count);
68         scheduler_ipi();
69
70         return IRQ_HANDLED;
71 }
72
73 static void cpu_bringup(void)
74 {
75         int cpu;
76
77         cpu_init();
78         touch_softlockup_watchdog();
79         preempt_disable();
80
81         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
82         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
83                 xen_enable_sysenter();
84                 xen_enable_syscall();
85         }
86         cpu = smp_processor_id();
87         smp_store_cpu_info(cpu);
88         cpu_data(cpu).x86_max_cores = 1;
89         set_cpu_sibling_map(cpu);
90
91         xen_setup_cpu_clockevents();
92
93         notify_cpu_starting(cpu);
94
95         set_cpu_online(cpu, true);
96
97         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
98
99         /* We can take interrupts now: we're officially "up". */
100         local_irq_enable();
101 }
102
103 asmlinkage __visible void cpu_bringup_and_idle(void)
104 {
105         cpu_bringup();
106         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
107 }
108
109 void xen_smp_intr_free(unsigned int cpu)
110 {
111         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
112                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
113                 per_cpu(xen_resched_irq, cpu).irq = -1;
114                 kfree(per_cpu(xen_resched_irq, cpu).name);
115                 per_cpu(xen_resched_irq, cpu).name = NULL;
116         }
117         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
118                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
119                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
120                 kfree(per_cpu(xen_callfunc_irq, cpu).name);
121                 per_cpu(xen_callfunc_irq, cpu).name = NULL;
122         }
123         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
124                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
125                 per_cpu(xen_debug_irq, cpu).irq = -1;
126                 kfree(per_cpu(xen_debug_irq, cpu).name);
127                 per_cpu(xen_debug_irq, cpu).name = NULL;
128         }
129         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
130                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
131                                        NULL);
132                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
133                 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
134                 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
135         }
136 }
137
138 void xen_smp_intr_free_pv(unsigned int cpu)
139 {
140         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
141                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
142                 per_cpu(xen_irq_work, cpu).irq = -1;
143                 kfree(per_cpu(xen_irq_work, cpu).name);
144                 per_cpu(xen_irq_work, cpu).name = NULL;
145         }
146
147         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
148                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
149                 per_cpu(xen_pmu_irq, cpu).irq = -1;
150                 kfree(per_cpu(xen_pmu_irq, cpu).name);
151                 per_cpu(xen_pmu_irq, cpu).name = NULL;
152         }
153 }
154
155 int xen_smp_intr_init(unsigned int cpu)
156 {
157         int rc;
158         char *resched_name, *callfunc_name, *debug_name;
159
160         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
161         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
162                                     cpu,
163                                     xen_reschedule_interrupt,
164                                     IRQF_PERCPU|IRQF_NOBALANCING,
165                                     resched_name,
166                                     NULL);
167         if (rc < 0)
168                 goto fail;
169         per_cpu(xen_resched_irq, cpu).irq = rc;
170         per_cpu(xen_resched_irq, cpu).name = resched_name;
171
172         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
173         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
174                                     cpu,
175                                     xen_call_function_interrupt,
176                                     IRQF_PERCPU|IRQF_NOBALANCING,
177                                     callfunc_name,
178                                     NULL);
179         if (rc < 0)
180                 goto fail;
181         per_cpu(xen_callfunc_irq, cpu).irq = rc;
182         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
183
184         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
185         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
186                                      IRQF_PERCPU | IRQF_NOBALANCING,
187                                      debug_name, NULL);
188         if (rc < 0)
189                 goto fail;
190         per_cpu(xen_debug_irq, cpu).irq = rc;
191         per_cpu(xen_debug_irq, cpu).name = debug_name;
192
193         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
194         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
195                                     cpu,
196                                     xen_call_function_single_interrupt,
197                                     IRQF_PERCPU|IRQF_NOBALANCING,
198                                     callfunc_name,
199                                     NULL);
200         if (rc < 0)
201                 goto fail;
202         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
203         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
204
205         return 0;
206
207  fail:
208         xen_smp_intr_free(cpu);
209         return rc;
210 }
211
212 int xen_smp_intr_init_pv(unsigned int cpu)
213 {
214         int rc;
215         char *callfunc_name, *pmu_name;
216
217         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
218         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
219                                     cpu,
220                                     xen_irq_work_interrupt,
221                                     IRQF_PERCPU|IRQF_NOBALANCING,
222                                     callfunc_name,
223                                     NULL);
224         if (rc < 0)
225                 goto fail;
226         per_cpu(xen_irq_work, cpu).irq = rc;
227         per_cpu(xen_irq_work, cpu).name = callfunc_name;
228
229         if (is_xen_pmu(cpu)) {
230                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
231                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
232                                              xen_pmu_irq_handler,
233                                              IRQF_PERCPU|IRQF_NOBALANCING,
234                                              pmu_name, NULL);
235                 if (rc < 0)
236                         goto fail;
237                 per_cpu(xen_pmu_irq, cpu).irq = rc;
238                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
239         }
240
241         return 0;
242
243  fail:
244         xen_smp_intr_free_pv(cpu);
245         return rc;
246 }
247
248 static void __init xen_fill_possible_map(void)
249 {
250         int i, rc;
251
252         if (xen_initial_domain())
253                 return;
254
255         for (i = 0; i < nr_cpu_ids; i++) {
256                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
257                 if (rc >= 0) {
258                         num_processors++;
259                         set_cpu_possible(i, true);
260                 }
261         }
262 }
263
264 static void __init xen_filter_cpu_maps(void)
265 {
266         int i, rc;
267         unsigned int subtract = 0;
268
269         if (!xen_initial_domain())
270                 return;
271
272         num_processors = 0;
273         disabled_cpus = 0;
274         for (i = 0; i < nr_cpu_ids; i++) {
275                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
276                 if (rc >= 0) {
277                         num_processors++;
278                         set_cpu_possible(i, true);
279                 } else {
280                         set_cpu_possible(i, false);
281                         set_cpu_present(i, false);
282                         subtract++;
283                 }
284         }
285 #ifdef CONFIG_HOTPLUG_CPU
286         /* This is akin to using 'nr_cpus' on the Linux command line.
287          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
288          * have up to X, while nr_cpu_ids is greater than X. This
289          * normally is not a problem, except when CPU hotplugging
290          * is involved and then there might be more than X CPUs
291          * in the guest - which will not work as there is no
292          * hypercall to expand the max number of VCPUs an already
293          * running guest has. So cap it up to X. */
294         if (subtract)
295                 nr_cpu_ids = nr_cpu_ids - subtract;
296 #endif
297
298 }
299
300 static void __init xen_pv_smp_prepare_boot_cpu(void)
301 {
302         BUG_ON(smp_processor_id() != 0);
303         native_smp_prepare_boot_cpu();
304
305         if (!xen_feature(XENFEAT_writable_page_tables))
306                 /* We've switched to the "real" per-cpu gdt, so make
307                  * sure the old memory can be recycled. */
308                 make_lowmem_page_readwrite(xen_initial_gdt);
309
310 #ifdef CONFIG_X86_32
311         /*
312          * Xen starts us with XEN_FLAT_RING1_DS, but linux code
313          * expects __USER_DS
314          */
315         loadsegment(ds, __USER_DS);
316         loadsegment(es, __USER_DS);
317 #endif
318
319         xen_filter_cpu_maps();
320         xen_setup_vcpu_info_placement();
321
322         /*
323          * The alternative logic (which patches the unlock/lock) runs before
324          * the smp bootup up code is activated. Hence we need to set this up
325          * the core kernel is being patched. Otherwise we will have only
326          * modules patched but not core code.
327          */
328         xen_init_spinlocks();
329 }
330
331 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
332 {
333         unsigned cpu;
334         unsigned int i;
335
336         if (skip_ioapic_setup) {
337                 char *m = (max_cpus == 0) ?
338                         "The nosmp parameter is incompatible with Xen; " \
339                         "use Xen dom0_max_vcpus=1 parameter" :
340                         "The noapic parameter is incompatible with Xen";
341
342                 xen_raw_printk(m);
343                 panic(m);
344         }
345         xen_init_lock_cpu(0);
346
347         smp_store_boot_cpu_info();
348         cpu_data(0).x86_max_cores = 1;
349
350         for_each_possible_cpu(i) {
351                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
352                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
353                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
354         }
355         set_cpu_sibling_map(0);
356
357         xen_pmu_init(0);
358
359         if (xen_smp_intr_init(0))
360                 BUG();
361
362         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
363                 panic("could not allocate xen_cpu_initialized_map\n");
364
365         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
366
367         /* Restrict the possible_map according to max_cpus. */
368         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
369                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
370                         continue;
371                 set_cpu_possible(cpu, false);
372         }
373
374         for_each_possible_cpu(cpu)
375                 set_cpu_present(cpu, true);
376 }
377
378 static int
379 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
380 {
381         struct vcpu_guest_context *ctxt;
382         struct desc_struct *gdt;
383         unsigned long gdt_mfn;
384
385         /* used to tell cpu_init() that it can proceed with initialization */
386         cpumask_set_cpu(cpu, cpu_callout_mask);
387         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
388                 return 0;
389
390         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
391         if (ctxt == NULL)
392                 return -ENOMEM;
393
394         gdt = get_cpu_gdt_rw(cpu);
395
396 #ifdef CONFIG_X86_32
397         ctxt->user_regs.fs = __KERNEL_PERCPU;
398         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
399 #endif
400         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
401
402         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
403         ctxt->flags = VGCF_IN_KERNEL;
404         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
405         ctxt->user_regs.ds = __USER_DS;
406         ctxt->user_regs.es = __USER_DS;
407         ctxt->user_regs.ss = __KERNEL_DS;
408
409         xen_copy_trap_info(ctxt->trap_ctxt);
410
411         ctxt->ldt_ents = 0;
412
413         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
414
415         gdt_mfn = arbitrary_virt_to_mfn(gdt);
416         make_lowmem_page_readonly(gdt);
417         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
418
419         ctxt->gdt_frames[0] = gdt_mfn;
420         ctxt->gdt_ents      = GDT_ENTRIES;
421
422         ctxt->kernel_ss = __KERNEL_DS;
423         ctxt->kernel_sp = idle->thread.sp0;
424
425 #ifdef CONFIG_X86_32
426         ctxt->event_callback_cs     = __KERNEL_CS;
427         ctxt->failsafe_callback_cs  = __KERNEL_CS;
428 #else
429         ctxt->gs_base_kernel = per_cpu_offset(cpu);
430 #endif
431         ctxt->event_callback_eip    =
432                 (unsigned long)xen_hypervisor_callback;
433         ctxt->failsafe_callback_eip =
434                 (unsigned long)xen_failsafe_callback;
435         ctxt->user_regs.cs = __KERNEL_CS;
436         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
437
438         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
439         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
440         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
441                 BUG();
442
443         kfree(ctxt);
444         return 0;
445 }
446
447 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
448 {
449         int rc;
450
451         common_cpu_up(cpu, idle);
452
453         xen_setup_runstate_info(cpu);
454
455         /*
456          * PV VCPUs are always successfully taken down (see 'while' loop
457          * in xen_cpu_die()), so -EBUSY is an error.
458          */
459         rc = cpu_check_up_prepare(cpu);
460         if (rc)
461                 return rc;
462
463         /* make sure interrupts start blocked */
464         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
465
466         rc = cpu_initialize_context(cpu, idle);
467         if (rc)
468                 return rc;
469
470         xen_pmu_init(cpu);
471
472         rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
473         BUG_ON(rc);
474
475         while (cpu_report_state(cpu) != CPU_ONLINE)
476                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
477
478         return 0;
479 }
480
481 static void xen_smp_cpus_done(unsigned int max_cpus)
482 {
483 }
484
485 #ifdef CONFIG_HOTPLUG_CPU
486 static int xen_cpu_disable(void)
487 {
488         unsigned int cpu = smp_processor_id();
489         if (cpu == 0)
490                 return -EBUSY;
491
492         cpu_disable_common();
493
494         load_cr3(swapper_pg_dir);
495         return 0;
496 }
497
498 static void xen_pv_cpu_die(unsigned int cpu)
499 {
500         while (HYPERVISOR_vcpu_op(VCPUOP_is_up,
501                                   xen_vcpu_nr(cpu), NULL)) {
502                 __set_current_state(TASK_UNINTERRUPTIBLE);
503                 schedule_timeout(HZ/10);
504         }
505
506         if (common_cpu_die(cpu) == 0) {
507                 xen_smp_intr_free(cpu);
508                 xen_uninit_lock_cpu(cpu);
509                 xen_teardown_timer(cpu);
510                 xen_pmu_finish(cpu);
511         }
512 }
513
514 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
515 {
516         play_dead_common();
517         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
518         cpu_bringup();
519         /*
520          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
521          * clears certain data that the cpu_idle loop (which called us
522          * and that we return from) expects. The only way to get that
523          * data back is to call:
524          */
525         tick_nohz_idle_enter();
526
527         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
528 }
529
530 #else /* !CONFIG_HOTPLUG_CPU */
531 static int xen_cpu_disable(void)
532 {
533         return -ENOSYS;
534 }
535
536 static void xen_pv_cpu_die(unsigned int cpu)
537 {
538         BUG();
539 }
540
541 static void xen_play_dead(void)
542 {
543         BUG();
544 }
545
546 #endif
547 static void stop_self(void *v)
548 {
549         int cpu = smp_processor_id();
550
551         /* make sure we're not pinning something down */
552         load_cr3(swapper_pg_dir);
553         /* should set up a minimal gdt */
554
555         set_cpu_online(cpu, false);
556
557         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
558         BUG();
559 }
560
561 static void xen_stop_other_cpus(int wait)
562 {
563         smp_call_function(stop_self, NULL, wait);
564 }
565
566 void xen_smp_send_reschedule(int cpu)
567 {
568         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
569 }
570
571 static void __xen_send_IPI_mask(const struct cpumask *mask,
572                               int vector)
573 {
574         unsigned cpu;
575
576         for_each_cpu_and(cpu, mask, cpu_online_mask)
577                 xen_send_IPI_one(cpu, vector);
578 }
579
580 void xen_smp_send_call_function_ipi(const struct cpumask *mask)
581 {
582         int cpu;
583
584         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
585
586         /* Make sure other vcpus get a chance to run if they need to. */
587         for_each_cpu(cpu, mask) {
588                 if (xen_vcpu_stolen(cpu)) {
589                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
590                         break;
591                 }
592         }
593 }
594
595 void xen_smp_send_call_function_single_ipi(int cpu)
596 {
597         __xen_send_IPI_mask(cpumask_of(cpu),
598                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
599 }
600
601 static inline int xen_map_vector(int vector)
602 {
603         int xen_vector;
604
605         switch (vector) {
606         case RESCHEDULE_VECTOR:
607                 xen_vector = XEN_RESCHEDULE_VECTOR;
608                 break;
609         case CALL_FUNCTION_VECTOR:
610                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
611                 break;
612         case CALL_FUNCTION_SINGLE_VECTOR:
613                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
614                 break;
615         case IRQ_WORK_VECTOR:
616                 xen_vector = XEN_IRQ_WORK_VECTOR;
617                 break;
618 #ifdef CONFIG_X86_64
619         case NMI_VECTOR:
620         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
621                 xen_vector = XEN_NMI_VECTOR;
622                 break;
623 #endif
624         default:
625                 xen_vector = -1;
626                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
627                         vector);
628         }
629
630         return xen_vector;
631 }
632
633 void xen_send_IPI_mask(const struct cpumask *mask,
634                               int vector)
635 {
636         int xen_vector = xen_map_vector(vector);
637
638         if (xen_vector >= 0)
639                 __xen_send_IPI_mask(mask, xen_vector);
640 }
641
642 void xen_send_IPI_all(int vector)
643 {
644         int xen_vector = xen_map_vector(vector);
645
646         if (xen_vector >= 0)
647                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
648 }
649
650 void xen_send_IPI_self(int vector)
651 {
652         int xen_vector = xen_map_vector(vector);
653
654         if (xen_vector >= 0)
655                 xen_send_IPI_one(smp_processor_id(), xen_vector);
656 }
657
658 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
659                                 int vector)
660 {
661         unsigned cpu;
662         unsigned int this_cpu = smp_processor_id();
663         int xen_vector = xen_map_vector(vector);
664
665         if (!(num_online_cpus() > 1) || (xen_vector < 0))
666                 return;
667
668         for_each_cpu_and(cpu, mask, cpu_online_mask) {
669                 if (this_cpu == cpu)
670                         continue;
671
672                 xen_send_IPI_one(cpu, xen_vector);
673         }
674 }
675
676 void xen_send_IPI_allbutself(int vector)
677 {
678         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
679 }
680
681 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
682 {
683         irq_enter();
684         generic_smp_call_function_interrupt();
685         inc_irq_stat(irq_call_count);
686         irq_exit();
687
688         return IRQ_HANDLED;
689 }
690
691 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
692 {
693         irq_enter();
694         generic_smp_call_function_single_interrupt();
695         inc_irq_stat(irq_call_count);
696         irq_exit();
697
698         return IRQ_HANDLED;
699 }
700
701 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
702 {
703         irq_enter();
704         irq_work_run();
705         inc_irq_stat(apic_irq_work_irqs);
706         irq_exit();
707
708         return IRQ_HANDLED;
709 }
710
711 static const struct smp_ops xen_smp_ops __initconst = {
712         .smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
713         .smp_prepare_cpus = xen_smp_prepare_cpus,
714         .smp_cpus_done = xen_smp_cpus_done,
715
716         .cpu_up = xen_cpu_up,
717         .cpu_die = xen_pv_cpu_die,
718         .cpu_disable = xen_cpu_disable,
719         .play_dead = xen_play_dead,
720
721         .stop_other_cpus = xen_stop_other_cpus,
722         .smp_send_reschedule = xen_smp_send_reschedule,
723
724         .send_call_func_ipi = xen_smp_send_call_function_ipi,
725         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
726 };
727
728 void __init xen_smp_init(void)
729 {
730         smp_ops = xen_smp_ops;
731         xen_fill_possible_map();
732 }