00c623bef72f86afa7ed49e5b44588a8891cd09e
[muen/linux.git] / arch / x86 / xen / smp.c
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 #include <linux/irq_work.h>
20 #include <linux/tick.h>
21 #include <linux/nmi.h>
22
23 #include <asm/paravirt.h>
24 #include <asm/desc.h>
25 #include <asm/pgtable.h>
26 #include <asm/cpu.h>
27
28 #include <xen/interface/xen.h>
29 #include <xen/interface/vcpu.h>
30 #include <xen/interface/xenpmu.h>
31
32 #include <asm/xen/interface.h>
33 #include <asm/xen/hypercall.h>
34
35 #include <xen/xen.h>
36 #include <xen/page.h>
37 #include <xen/events.h>
38
39 #include <xen/hvc-console.h>
40 #include "xen-ops.h"
41 #include "mmu.h"
42 #include "smp.h"
43 #include "pmu.h"
44
45 cpumask_var_t xen_cpu_initialized_map;
46
47 struct xen_common_irq {
48         int irq;
49         char *name;
50 };
51 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
53 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
54 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
55 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
56 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
57
58 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
59 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
60 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
61
62 /*
63  * Reschedule call back.
64  */
65 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
66 {
67         inc_irq_stat(irq_resched_count);
68         scheduler_ipi();
69
70         return IRQ_HANDLED;
71 }
72
73 static void cpu_bringup(void)
74 {
75         int cpu;
76
77         cpu_init();
78         touch_softlockup_watchdog();
79         preempt_disable();
80
81         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
82         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
83                 xen_enable_sysenter();
84                 xen_enable_syscall();
85         }
86         cpu = smp_processor_id();
87         smp_store_cpu_info(cpu);
88         cpu_data(cpu).x86_max_cores = 1;
89         set_cpu_sibling_map(cpu);
90
91         xen_setup_cpu_clockevents();
92
93         notify_cpu_starting(cpu);
94
95         set_cpu_online(cpu, true);
96
97         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
98
99         /* We can take interrupts now: we're officially "up". */
100         local_irq_enable();
101 }
102
103 asmlinkage __visible void cpu_bringup_and_idle(void)
104 {
105         cpu_bringup();
106         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
107 }
108
109 void xen_smp_intr_free(unsigned int cpu)
110 {
111         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
112                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
113                 per_cpu(xen_resched_irq, cpu).irq = -1;
114                 kfree(per_cpu(xen_resched_irq, cpu).name);
115                 per_cpu(xen_resched_irq, cpu).name = NULL;
116         }
117         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
118                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
119                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
120                 kfree(per_cpu(xen_callfunc_irq, cpu).name);
121                 per_cpu(xen_callfunc_irq, cpu).name = NULL;
122         }
123         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
124                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
125                 per_cpu(xen_debug_irq, cpu).irq = -1;
126                 kfree(per_cpu(xen_debug_irq, cpu).name);
127                 per_cpu(xen_debug_irq, cpu).name = NULL;
128         }
129         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
130                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
131                                        NULL);
132                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
133                 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
134                 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
135         }
136 }
137
138 void xen_smp_intr_free_pv(unsigned int cpu)
139 {
140         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
141                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
142                 per_cpu(xen_irq_work, cpu).irq = -1;
143                 kfree(per_cpu(xen_irq_work, cpu).name);
144                 per_cpu(xen_irq_work, cpu).name = NULL;
145         }
146
147         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
148                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
149                 per_cpu(xen_pmu_irq, cpu).irq = -1;
150                 kfree(per_cpu(xen_pmu_irq, cpu).name);
151                 per_cpu(xen_pmu_irq, cpu).name = NULL;
152         }
153 }
154
155 int xen_smp_intr_init(unsigned int cpu)
156 {
157         int rc;
158         char *resched_name, *callfunc_name, *debug_name;
159
160         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
161         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
162                                     cpu,
163                                     xen_reschedule_interrupt,
164                                     IRQF_PERCPU|IRQF_NOBALANCING,
165                                     resched_name,
166                                     NULL);
167         if (rc < 0)
168                 goto fail;
169         per_cpu(xen_resched_irq, cpu).irq = rc;
170         per_cpu(xen_resched_irq, cpu).name = resched_name;
171
172         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
173         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
174                                     cpu,
175                                     xen_call_function_interrupt,
176                                     IRQF_PERCPU|IRQF_NOBALANCING,
177                                     callfunc_name,
178                                     NULL);
179         if (rc < 0)
180                 goto fail;
181         per_cpu(xen_callfunc_irq, cpu).irq = rc;
182         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
183
184         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
185         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
186                                      IRQF_PERCPU | IRQF_NOBALANCING,
187                                      debug_name, NULL);
188         if (rc < 0)
189                 goto fail;
190         per_cpu(xen_debug_irq, cpu).irq = rc;
191         per_cpu(xen_debug_irq, cpu).name = debug_name;
192
193         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
194         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
195                                     cpu,
196                                     xen_call_function_single_interrupt,
197                                     IRQF_PERCPU|IRQF_NOBALANCING,
198                                     callfunc_name,
199                                     NULL);
200         if (rc < 0)
201                 goto fail;
202         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
203         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
204
205         return 0;
206
207  fail:
208         xen_smp_intr_free(cpu);
209         return rc;
210 }
211
212 int xen_smp_intr_init_pv(unsigned int cpu)
213 {
214         int rc;
215         char *callfunc_name, *pmu_name;
216
217         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
218         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
219                                     cpu,
220                                     xen_irq_work_interrupt,
221                                     IRQF_PERCPU|IRQF_NOBALANCING,
222                                     callfunc_name,
223                                     NULL);
224         if (rc < 0)
225                 goto fail;
226         per_cpu(xen_irq_work, cpu).irq = rc;
227         per_cpu(xen_irq_work, cpu).name = callfunc_name;
228
229         if (is_xen_pmu(cpu)) {
230                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
231                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
232                                              xen_pmu_irq_handler,
233                                              IRQF_PERCPU|IRQF_NOBALANCING,
234                                              pmu_name, NULL);
235                 if (rc < 0)
236                         goto fail;
237                 per_cpu(xen_pmu_irq, cpu).irq = rc;
238                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
239         }
240
241         return 0;
242
243  fail:
244         xen_smp_intr_free_pv(cpu);
245         return rc;
246 }
247
248 static void __init xen_fill_possible_map(void)
249 {
250         int i, rc;
251
252         if (xen_initial_domain())
253                 return;
254
255         for (i = 0; i < nr_cpu_ids; i++) {
256                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
257                 if (rc >= 0) {
258                         num_processors++;
259                         set_cpu_possible(i, true);
260                 }
261         }
262 }
263
264 static void __init xen_filter_cpu_maps(void)
265 {
266         int i, rc;
267         unsigned int subtract = 0;
268
269         if (!xen_initial_domain())
270                 return;
271
272         num_processors = 0;
273         disabled_cpus = 0;
274         for (i = 0; i < nr_cpu_ids; i++) {
275                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
276                 if (rc >= 0) {
277                         num_processors++;
278                         set_cpu_possible(i, true);
279                 } else {
280                         set_cpu_possible(i, false);
281                         set_cpu_present(i, false);
282                         subtract++;
283                 }
284         }
285 #ifdef CONFIG_HOTPLUG_CPU
286         /* This is akin to using 'nr_cpus' on the Linux command line.
287          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
288          * have up to X, while nr_cpu_ids is greater than X. This
289          * normally is not a problem, except when CPU hotplugging
290          * is involved and then there might be more than X CPUs
291          * in the guest - which will not work as there is no
292          * hypercall to expand the max number of VCPUs an already
293          * running guest has. So cap it up to X. */
294         if (subtract)
295                 nr_cpu_ids = nr_cpu_ids - subtract;
296 #endif
297
298 }
299
300 static void __init xen_pv_smp_prepare_boot_cpu(void)
301 {
302         BUG_ON(smp_processor_id() != 0);
303         native_smp_prepare_boot_cpu();
304
305         if (!xen_feature(XENFEAT_writable_page_tables))
306                 /* We've switched to the "real" per-cpu gdt, so make
307                  * sure the old memory can be recycled. */
308                 make_lowmem_page_readwrite(xen_initial_gdt);
309
310 #ifdef CONFIG_X86_32
311         /*
312          * Xen starts us with XEN_FLAT_RING1_DS, but linux code
313          * expects __USER_DS
314          */
315         loadsegment(ds, __USER_DS);
316         loadsegment(es, __USER_DS);
317 #endif
318
319         xen_filter_cpu_maps();
320         xen_setup_vcpu_info_placement();
321
322         /*
323          * The alternative logic (which patches the unlock/lock) runs before
324          * the smp bootup up code is activated. Hence we need to set this up
325          * the core kernel is being patched. Otherwise we will have only
326          * modules patched but not core code.
327          */
328         xen_init_spinlocks();
329 }
330
331 static void __init xen_hvm_smp_prepare_boot_cpu(void)
332 {
333         BUG_ON(smp_processor_id() != 0);
334         native_smp_prepare_boot_cpu();
335
336         /*
337          * Setup vcpu_info for boot CPU.
338          */
339         xen_vcpu_setup(0);
340
341         /*
342          * The alternative logic (which patches the unlock/lock) runs before
343          * the smp bootup up code is activated. Hence we need to set this up
344          * the core kernel is being patched. Otherwise we will have only
345          * modules patched but not core code.
346          */
347         xen_init_spinlocks();
348 }
349
350 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
351 {
352         unsigned cpu;
353         unsigned int i;
354
355         if (skip_ioapic_setup) {
356                 char *m = (max_cpus == 0) ?
357                         "The nosmp parameter is incompatible with Xen; " \
358                         "use Xen dom0_max_vcpus=1 parameter" :
359                         "The noapic parameter is incompatible with Xen";
360
361                 xen_raw_printk(m);
362                 panic(m);
363         }
364         xen_init_lock_cpu(0);
365
366         smp_store_boot_cpu_info();
367         cpu_data(0).x86_max_cores = 1;
368
369         for_each_possible_cpu(i) {
370                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
371                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
372                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
373         }
374         set_cpu_sibling_map(0);
375
376         xen_pmu_init(0);
377
378         if (xen_smp_intr_init(0))
379                 BUG();
380
381         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
382                 panic("could not allocate xen_cpu_initialized_map\n");
383
384         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
385
386         /* Restrict the possible_map according to max_cpus. */
387         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
388                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
389                         continue;
390                 set_cpu_possible(cpu, false);
391         }
392
393         for_each_possible_cpu(cpu)
394                 set_cpu_present(cpu, true);
395 }
396
397 static int
398 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
399 {
400         struct vcpu_guest_context *ctxt;
401         struct desc_struct *gdt;
402         unsigned long gdt_mfn;
403
404         /* used to tell cpu_init() that it can proceed with initialization */
405         cpumask_set_cpu(cpu, cpu_callout_mask);
406         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
407                 return 0;
408
409         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
410         if (ctxt == NULL)
411                 return -ENOMEM;
412
413         gdt = get_cpu_gdt_rw(cpu);
414
415 #ifdef CONFIG_X86_32
416         ctxt->user_regs.fs = __KERNEL_PERCPU;
417         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
418 #endif
419         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
420
421         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
422         ctxt->flags = VGCF_IN_KERNEL;
423         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
424         ctxt->user_regs.ds = __USER_DS;
425         ctxt->user_regs.es = __USER_DS;
426         ctxt->user_regs.ss = __KERNEL_DS;
427
428         xen_copy_trap_info(ctxt->trap_ctxt);
429
430         ctxt->ldt_ents = 0;
431
432         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
433
434         gdt_mfn = arbitrary_virt_to_mfn(gdt);
435         make_lowmem_page_readonly(gdt);
436         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
437
438         ctxt->gdt_frames[0] = gdt_mfn;
439         ctxt->gdt_ents      = GDT_ENTRIES;
440
441         ctxt->kernel_ss = __KERNEL_DS;
442         ctxt->kernel_sp = idle->thread.sp0;
443
444 #ifdef CONFIG_X86_32
445         ctxt->event_callback_cs     = __KERNEL_CS;
446         ctxt->failsafe_callback_cs  = __KERNEL_CS;
447 #else
448         ctxt->gs_base_kernel = per_cpu_offset(cpu);
449 #endif
450         ctxt->event_callback_eip    =
451                 (unsigned long)xen_hypervisor_callback;
452         ctxt->failsafe_callback_eip =
453                 (unsigned long)xen_failsafe_callback;
454         ctxt->user_regs.cs = __KERNEL_CS;
455         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
456
457         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
458         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
459         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
460                 BUG();
461
462         kfree(ctxt);
463         return 0;
464 }
465
466 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
467 {
468         int rc;
469
470         common_cpu_up(cpu, idle);
471
472         xen_setup_runstate_info(cpu);
473
474         /*
475          * PV VCPUs are always successfully taken down (see 'while' loop
476          * in xen_cpu_die()), so -EBUSY is an error.
477          */
478         rc = cpu_check_up_prepare(cpu);
479         if (rc)
480                 return rc;
481
482         /* make sure interrupts start blocked */
483         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
484
485         rc = cpu_initialize_context(cpu, idle);
486         if (rc)
487                 return rc;
488
489         xen_pmu_init(cpu);
490
491         rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
492         BUG_ON(rc);
493
494         while (cpu_report_state(cpu) != CPU_ONLINE)
495                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
496
497         return 0;
498 }
499
500 static void xen_smp_cpus_done(unsigned int max_cpus)
501 {
502 }
503
504 #ifdef CONFIG_HOTPLUG_CPU
505 static int xen_cpu_disable(void)
506 {
507         unsigned int cpu = smp_processor_id();
508         if (cpu == 0)
509                 return -EBUSY;
510
511         cpu_disable_common();
512
513         load_cr3(swapper_pg_dir);
514         return 0;
515 }
516
517 static void xen_cpu_die(unsigned int cpu)
518 {
519         while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up,
520                                                      xen_vcpu_nr(cpu), NULL)) {
521                 __set_current_state(TASK_UNINTERRUPTIBLE);
522                 schedule_timeout(HZ/10);
523         }
524
525         if (common_cpu_die(cpu) == 0) {
526                 xen_smp_intr_free(cpu);
527                 xen_uninit_lock_cpu(cpu);
528                 xen_teardown_timer(cpu);
529                 xen_pmu_finish(cpu);
530         }
531 }
532
533 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
534 {
535         play_dead_common();
536         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
537         cpu_bringup();
538         /*
539          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
540          * clears certain data that the cpu_idle loop (which called us
541          * and that we return from) expects. The only way to get that
542          * data back is to call:
543          */
544         tick_nohz_idle_enter();
545
546         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
547 }
548
549 #else /* !CONFIG_HOTPLUG_CPU */
550 static int xen_cpu_disable(void)
551 {
552         return -ENOSYS;
553 }
554
555 static void xen_cpu_die(unsigned int cpu)
556 {
557         BUG();
558 }
559
560 static void xen_play_dead(void)
561 {
562         BUG();
563 }
564
565 #endif
566 static void stop_self(void *v)
567 {
568         int cpu = smp_processor_id();
569
570         /* make sure we're not pinning something down */
571         load_cr3(swapper_pg_dir);
572         /* should set up a minimal gdt */
573
574         set_cpu_online(cpu, false);
575
576         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
577         BUG();
578 }
579
580 static void xen_stop_other_cpus(int wait)
581 {
582         smp_call_function(stop_self, NULL, wait);
583 }
584
585 static void xen_smp_send_reschedule(int cpu)
586 {
587         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
588 }
589
590 static void __xen_send_IPI_mask(const struct cpumask *mask,
591                               int vector)
592 {
593         unsigned cpu;
594
595         for_each_cpu_and(cpu, mask, cpu_online_mask)
596                 xen_send_IPI_one(cpu, vector);
597 }
598
599 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
600 {
601         int cpu;
602
603         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
604
605         /* Make sure other vcpus get a chance to run if they need to. */
606         for_each_cpu(cpu, mask) {
607                 if (xen_vcpu_stolen(cpu)) {
608                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
609                         break;
610                 }
611         }
612 }
613
614 static void xen_smp_send_call_function_single_ipi(int cpu)
615 {
616         __xen_send_IPI_mask(cpumask_of(cpu),
617                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
618 }
619
620 static inline int xen_map_vector(int vector)
621 {
622         int xen_vector;
623
624         switch (vector) {
625         case RESCHEDULE_VECTOR:
626                 xen_vector = XEN_RESCHEDULE_VECTOR;
627                 break;
628         case CALL_FUNCTION_VECTOR:
629                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
630                 break;
631         case CALL_FUNCTION_SINGLE_VECTOR:
632                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
633                 break;
634         case IRQ_WORK_VECTOR:
635                 xen_vector = XEN_IRQ_WORK_VECTOR;
636                 break;
637 #ifdef CONFIG_X86_64
638         case NMI_VECTOR:
639         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
640                 xen_vector = XEN_NMI_VECTOR;
641                 break;
642 #endif
643         default:
644                 xen_vector = -1;
645                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
646                         vector);
647         }
648
649         return xen_vector;
650 }
651
652 void xen_send_IPI_mask(const struct cpumask *mask,
653                               int vector)
654 {
655         int xen_vector = xen_map_vector(vector);
656
657         if (xen_vector >= 0)
658                 __xen_send_IPI_mask(mask, xen_vector);
659 }
660
661 void xen_send_IPI_all(int vector)
662 {
663         int xen_vector = xen_map_vector(vector);
664
665         if (xen_vector >= 0)
666                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
667 }
668
669 void xen_send_IPI_self(int vector)
670 {
671         int xen_vector = xen_map_vector(vector);
672
673         if (xen_vector >= 0)
674                 xen_send_IPI_one(smp_processor_id(), xen_vector);
675 }
676
677 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
678                                 int vector)
679 {
680         unsigned cpu;
681         unsigned int this_cpu = smp_processor_id();
682         int xen_vector = xen_map_vector(vector);
683
684         if (!(num_online_cpus() > 1) || (xen_vector < 0))
685                 return;
686
687         for_each_cpu_and(cpu, mask, cpu_online_mask) {
688                 if (this_cpu == cpu)
689                         continue;
690
691                 xen_send_IPI_one(cpu, xen_vector);
692         }
693 }
694
695 void xen_send_IPI_allbutself(int vector)
696 {
697         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
698 }
699
700 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
701 {
702         irq_enter();
703         generic_smp_call_function_interrupt();
704         inc_irq_stat(irq_call_count);
705         irq_exit();
706
707         return IRQ_HANDLED;
708 }
709
710 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
711 {
712         irq_enter();
713         generic_smp_call_function_single_interrupt();
714         inc_irq_stat(irq_call_count);
715         irq_exit();
716
717         return IRQ_HANDLED;
718 }
719
720 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
721 {
722         irq_enter();
723         irq_work_run();
724         inc_irq_stat(apic_irq_work_irqs);
725         irq_exit();
726
727         return IRQ_HANDLED;
728 }
729
730 static const struct smp_ops xen_smp_ops __initconst = {
731         .smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
732         .smp_prepare_cpus = xen_smp_prepare_cpus,
733         .smp_cpus_done = xen_smp_cpus_done,
734
735         .cpu_up = xen_cpu_up,
736         .cpu_die = xen_cpu_die,
737         .cpu_disable = xen_cpu_disable,
738         .play_dead = xen_play_dead,
739
740         .stop_other_cpus = xen_stop_other_cpus,
741         .smp_send_reschedule = xen_smp_send_reschedule,
742
743         .send_call_func_ipi = xen_smp_send_call_function_ipi,
744         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
745 };
746
747 void __init xen_smp_init(void)
748 {
749         smp_ops = xen_smp_ops;
750         xen_fill_possible_map();
751 }
752
753 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
754 {
755         native_smp_prepare_cpus(max_cpus);
756         WARN_ON(xen_smp_intr_init(0));
757
758         xen_init_lock_cpu(0);
759 }
760
761 void __init xen_hvm_smp_init(void)
762 {
763         smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
764         smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
765         smp_ops.cpu_die = xen_cpu_die;
766         smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
767         smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
768         smp_ops.smp_prepare_boot_cpu = xen_hvm_smp_prepare_boot_cpu;
769 }