x86/xen: split xen_cpu_die()
[muen/linux.git] / arch / x86 / xen / smp.c
1 /*
2  * Xen SMP support
3  *
4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
5  * very straightforward.  Bringing a CPU up is simply a matter of
6  * loading its initial context and setting it running.
7  *
8  * IPIs are handled through the Xen event mechanism.
9  *
10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
11  * useful topology information for the kernel to make use of.  As a
12  * result, all CPUs are treated as if they're single-core and
13  * single-threaded.
14  */
15 #include <linux/sched.h>
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/smp.h>
19 #include <linux/irq_work.h>
20 #include <linux/tick.h>
21 #include <linux/nmi.h>
22
23 #include <asm/paravirt.h>
24 #include <asm/desc.h>
25 #include <asm/pgtable.h>
26 #include <asm/cpu.h>
27
28 #include <xen/interface/xen.h>
29 #include <xen/interface/vcpu.h>
30 #include <xen/interface/xenpmu.h>
31
32 #include <asm/xen/interface.h>
33 #include <asm/xen/hypercall.h>
34
35 #include <xen/xen.h>
36 #include <xen/page.h>
37 #include <xen/events.h>
38
39 #include <xen/hvc-console.h>
40 #include "xen-ops.h"
41 #include "mmu.h"
42 #include "smp.h"
43 #include "pmu.h"
44
45 cpumask_var_t xen_cpu_initialized_map;
46
47 struct xen_common_irq {
48         int irq;
49         char *name;
50 };
51 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
52 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
53 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
54 static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
55 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
56 static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
57
58 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
59 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
60 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
61
62 /*
63  * Reschedule call back.
64  */
65 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
66 {
67         inc_irq_stat(irq_resched_count);
68         scheduler_ipi();
69
70         return IRQ_HANDLED;
71 }
72
73 static void cpu_bringup(void)
74 {
75         int cpu;
76
77         cpu_init();
78         touch_softlockup_watchdog();
79         preempt_disable();
80
81         /* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
82         if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
83                 xen_enable_sysenter();
84                 xen_enable_syscall();
85         }
86         cpu = smp_processor_id();
87         smp_store_cpu_info(cpu);
88         cpu_data(cpu).x86_max_cores = 1;
89         set_cpu_sibling_map(cpu);
90
91         xen_setup_cpu_clockevents();
92
93         notify_cpu_starting(cpu);
94
95         set_cpu_online(cpu, true);
96
97         cpu_set_state_online(cpu);  /* Implies full memory barrier. */
98
99         /* We can take interrupts now: we're officially "up". */
100         local_irq_enable();
101 }
102
103 asmlinkage __visible void cpu_bringup_and_idle(void)
104 {
105         cpu_bringup();
106         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
107 }
108
109 void xen_smp_intr_free(unsigned int cpu)
110 {
111         if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
112                 unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
113                 per_cpu(xen_resched_irq, cpu).irq = -1;
114                 kfree(per_cpu(xen_resched_irq, cpu).name);
115                 per_cpu(xen_resched_irq, cpu).name = NULL;
116         }
117         if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
118                 unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
119                 per_cpu(xen_callfunc_irq, cpu).irq = -1;
120                 kfree(per_cpu(xen_callfunc_irq, cpu).name);
121                 per_cpu(xen_callfunc_irq, cpu).name = NULL;
122         }
123         if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
124                 unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
125                 per_cpu(xen_debug_irq, cpu).irq = -1;
126                 kfree(per_cpu(xen_debug_irq, cpu).name);
127                 per_cpu(xen_debug_irq, cpu).name = NULL;
128         }
129         if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
130                 unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
131                                        NULL);
132                 per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
133                 kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
134                 per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
135         }
136 }
137
138 void xen_smp_intr_free_pv(unsigned int cpu)
139 {
140         if (per_cpu(xen_irq_work, cpu).irq >= 0) {
141                 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
142                 per_cpu(xen_irq_work, cpu).irq = -1;
143                 kfree(per_cpu(xen_irq_work, cpu).name);
144                 per_cpu(xen_irq_work, cpu).name = NULL;
145         }
146
147         if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
148                 unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
149                 per_cpu(xen_pmu_irq, cpu).irq = -1;
150                 kfree(per_cpu(xen_pmu_irq, cpu).name);
151                 per_cpu(xen_pmu_irq, cpu).name = NULL;
152         }
153 }
154
155 int xen_smp_intr_init(unsigned int cpu)
156 {
157         int rc;
158         char *resched_name, *callfunc_name, *debug_name;
159
160         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
161         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
162                                     cpu,
163                                     xen_reschedule_interrupt,
164                                     IRQF_PERCPU|IRQF_NOBALANCING,
165                                     resched_name,
166                                     NULL);
167         if (rc < 0)
168                 goto fail;
169         per_cpu(xen_resched_irq, cpu).irq = rc;
170         per_cpu(xen_resched_irq, cpu).name = resched_name;
171
172         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
173         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
174                                     cpu,
175                                     xen_call_function_interrupt,
176                                     IRQF_PERCPU|IRQF_NOBALANCING,
177                                     callfunc_name,
178                                     NULL);
179         if (rc < 0)
180                 goto fail;
181         per_cpu(xen_callfunc_irq, cpu).irq = rc;
182         per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
183
184         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
185         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
186                                      IRQF_PERCPU | IRQF_NOBALANCING,
187                                      debug_name, NULL);
188         if (rc < 0)
189                 goto fail;
190         per_cpu(xen_debug_irq, cpu).irq = rc;
191         per_cpu(xen_debug_irq, cpu).name = debug_name;
192
193         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
194         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
195                                     cpu,
196                                     xen_call_function_single_interrupt,
197                                     IRQF_PERCPU|IRQF_NOBALANCING,
198                                     callfunc_name,
199                                     NULL);
200         if (rc < 0)
201                 goto fail;
202         per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
203         per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
204
205         return 0;
206
207  fail:
208         xen_smp_intr_free(cpu);
209         return rc;
210 }
211
212 int xen_smp_intr_init_pv(unsigned int cpu)
213 {
214         int rc;
215         char *callfunc_name, *pmu_name;
216
217         callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
218         rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
219                                     cpu,
220                                     xen_irq_work_interrupt,
221                                     IRQF_PERCPU|IRQF_NOBALANCING,
222                                     callfunc_name,
223                                     NULL);
224         if (rc < 0)
225                 goto fail;
226         per_cpu(xen_irq_work, cpu).irq = rc;
227         per_cpu(xen_irq_work, cpu).name = callfunc_name;
228
229         if (is_xen_pmu(cpu)) {
230                 pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
231                 rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
232                                              xen_pmu_irq_handler,
233                                              IRQF_PERCPU|IRQF_NOBALANCING,
234                                              pmu_name, NULL);
235                 if (rc < 0)
236                         goto fail;
237                 per_cpu(xen_pmu_irq, cpu).irq = rc;
238                 per_cpu(xen_pmu_irq, cpu).name = pmu_name;
239         }
240
241         return 0;
242
243  fail:
244         xen_smp_intr_free_pv(cpu);
245         return rc;
246 }
247
248 static void __init xen_fill_possible_map(void)
249 {
250         int i, rc;
251
252         if (xen_initial_domain())
253                 return;
254
255         for (i = 0; i < nr_cpu_ids; i++) {
256                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
257                 if (rc >= 0) {
258                         num_processors++;
259                         set_cpu_possible(i, true);
260                 }
261         }
262 }
263
264 static void __init xen_filter_cpu_maps(void)
265 {
266         int i, rc;
267         unsigned int subtract = 0;
268
269         if (!xen_initial_domain())
270                 return;
271
272         num_processors = 0;
273         disabled_cpus = 0;
274         for (i = 0; i < nr_cpu_ids; i++) {
275                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
276                 if (rc >= 0) {
277                         num_processors++;
278                         set_cpu_possible(i, true);
279                 } else {
280                         set_cpu_possible(i, false);
281                         set_cpu_present(i, false);
282                         subtract++;
283                 }
284         }
285 #ifdef CONFIG_HOTPLUG_CPU
286         /* This is akin to using 'nr_cpus' on the Linux command line.
287          * Which is OK as when we use 'dom0_max_vcpus=X' we can only
288          * have up to X, while nr_cpu_ids is greater than X. This
289          * normally is not a problem, except when CPU hotplugging
290          * is involved and then there might be more than X CPUs
291          * in the guest - which will not work as there is no
292          * hypercall to expand the max number of VCPUs an already
293          * running guest has. So cap it up to X. */
294         if (subtract)
295                 nr_cpu_ids = nr_cpu_ids - subtract;
296 #endif
297
298 }
299
300 static void __init xen_pv_smp_prepare_boot_cpu(void)
301 {
302         BUG_ON(smp_processor_id() != 0);
303         native_smp_prepare_boot_cpu();
304
305         if (!xen_feature(XENFEAT_writable_page_tables))
306                 /* We've switched to the "real" per-cpu gdt, so make
307                  * sure the old memory can be recycled. */
308                 make_lowmem_page_readwrite(xen_initial_gdt);
309
310 #ifdef CONFIG_X86_32
311         /*
312          * Xen starts us with XEN_FLAT_RING1_DS, but linux code
313          * expects __USER_DS
314          */
315         loadsegment(ds, __USER_DS);
316         loadsegment(es, __USER_DS);
317 #endif
318
319         xen_filter_cpu_maps();
320         xen_setup_vcpu_info_placement();
321
322         /*
323          * The alternative logic (which patches the unlock/lock) runs before
324          * the smp bootup up code is activated. Hence we need to set this up
325          * the core kernel is being patched. Otherwise we will have only
326          * modules patched but not core code.
327          */
328         xen_init_spinlocks();
329 }
330
331 static void __init xen_hvm_smp_prepare_boot_cpu(void)
332 {
333         BUG_ON(smp_processor_id() != 0);
334         native_smp_prepare_boot_cpu();
335
336         /*
337          * Setup vcpu_info for boot CPU.
338          */
339         xen_vcpu_setup(0);
340
341         /*
342          * The alternative logic (which patches the unlock/lock) runs before
343          * the smp bootup up code is activated. Hence we need to set this up
344          * the core kernel is being patched. Otherwise we will have only
345          * modules patched but not core code.
346          */
347         xen_init_spinlocks();
348 }
349
350 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
351 {
352         unsigned cpu;
353         unsigned int i;
354
355         if (skip_ioapic_setup) {
356                 char *m = (max_cpus == 0) ?
357                         "The nosmp parameter is incompatible with Xen; " \
358                         "use Xen dom0_max_vcpus=1 parameter" :
359                         "The noapic parameter is incompatible with Xen";
360
361                 xen_raw_printk(m);
362                 panic(m);
363         }
364         xen_init_lock_cpu(0);
365
366         smp_store_boot_cpu_info();
367         cpu_data(0).x86_max_cores = 1;
368
369         for_each_possible_cpu(i) {
370                 zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
371                 zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
372                 zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
373         }
374         set_cpu_sibling_map(0);
375
376         xen_pmu_init(0);
377
378         if (xen_smp_intr_init(0))
379                 BUG();
380
381         if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
382                 panic("could not allocate xen_cpu_initialized_map\n");
383
384         cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
385
386         /* Restrict the possible_map according to max_cpus. */
387         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
388                 for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
389                         continue;
390                 set_cpu_possible(cpu, false);
391         }
392
393         for_each_possible_cpu(cpu)
394                 set_cpu_present(cpu, true);
395 }
396
397 static int
398 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
399 {
400         struct vcpu_guest_context *ctxt;
401         struct desc_struct *gdt;
402         unsigned long gdt_mfn;
403
404         /* used to tell cpu_init() that it can proceed with initialization */
405         cpumask_set_cpu(cpu, cpu_callout_mask);
406         if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
407                 return 0;
408
409         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
410         if (ctxt == NULL)
411                 return -ENOMEM;
412
413         gdt = get_cpu_gdt_rw(cpu);
414
415 #ifdef CONFIG_X86_32
416         ctxt->user_regs.fs = __KERNEL_PERCPU;
417         ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
418 #endif
419         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
420
421         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
422         ctxt->flags = VGCF_IN_KERNEL;
423         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
424         ctxt->user_regs.ds = __USER_DS;
425         ctxt->user_regs.es = __USER_DS;
426         ctxt->user_regs.ss = __KERNEL_DS;
427
428         xen_copy_trap_info(ctxt->trap_ctxt);
429
430         ctxt->ldt_ents = 0;
431
432         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
433
434         gdt_mfn = arbitrary_virt_to_mfn(gdt);
435         make_lowmem_page_readonly(gdt);
436         make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
437
438         ctxt->gdt_frames[0] = gdt_mfn;
439         ctxt->gdt_ents      = GDT_ENTRIES;
440
441         ctxt->kernel_ss = __KERNEL_DS;
442         ctxt->kernel_sp = idle->thread.sp0;
443
444 #ifdef CONFIG_X86_32
445         ctxt->event_callback_cs     = __KERNEL_CS;
446         ctxt->failsafe_callback_cs  = __KERNEL_CS;
447 #else
448         ctxt->gs_base_kernel = per_cpu_offset(cpu);
449 #endif
450         ctxt->event_callback_eip    =
451                 (unsigned long)xen_hypervisor_callback;
452         ctxt->failsafe_callback_eip =
453                 (unsigned long)xen_failsafe_callback;
454         ctxt->user_regs.cs = __KERNEL_CS;
455         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
456
457         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
458         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
459         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
460                 BUG();
461
462         kfree(ctxt);
463         return 0;
464 }
465
466 static int xen_cpu_up(unsigned int cpu, struct task_struct *idle)
467 {
468         int rc;
469
470         common_cpu_up(cpu, idle);
471
472         xen_setup_runstate_info(cpu);
473
474         /*
475          * PV VCPUs are always successfully taken down (see 'while' loop
476          * in xen_cpu_die()), so -EBUSY is an error.
477          */
478         rc = cpu_check_up_prepare(cpu);
479         if (rc)
480                 return rc;
481
482         /* make sure interrupts start blocked */
483         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
484
485         rc = cpu_initialize_context(cpu, idle);
486         if (rc)
487                 return rc;
488
489         xen_pmu_init(cpu);
490
491         rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
492         BUG_ON(rc);
493
494         while (cpu_report_state(cpu) != CPU_ONLINE)
495                 HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
496
497         return 0;
498 }
499
500 static void xen_smp_cpus_done(unsigned int max_cpus)
501 {
502 }
503
504 #ifdef CONFIG_HOTPLUG_CPU
505 static int xen_cpu_disable(void)
506 {
507         unsigned int cpu = smp_processor_id();
508         if (cpu == 0)
509                 return -EBUSY;
510
511         cpu_disable_common();
512
513         load_cr3(swapper_pg_dir);
514         return 0;
515 }
516
517 static void xen_pv_cpu_die(unsigned int cpu)
518 {
519         while (HYPERVISOR_vcpu_op(VCPUOP_is_up,
520                                   xen_vcpu_nr(cpu), NULL)) {
521                 __set_current_state(TASK_UNINTERRUPTIBLE);
522                 schedule_timeout(HZ/10);
523         }
524
525         if (common_cpu_die(cpu) == 0) {
526                 xen_smp_intr_free(cpu);
527                 xen_uninit_lock_cpu(cpu);
528                 xen_teardown_timer(cpu);
529                 xen_pmu_finish(cpu);
530         }
531 }
532
533 static void xen_hvm_cpu_die(unsigned int cpu)
534 {
535         if (common_cpu_die(cpu) == 0) {
536                 xen_smp_intr_free(cpu);
537                 xen_uninit_lock_cpu(cpu);
538                 xen_teardown_timer(cpu);
539         }
540 }
541
542 static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
543 {
544         play_dead_common();
545         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
546         cpu_bringup();
547         /*
548          * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
549          * clears certain data that the cpu_idle loop (which called us
550          * and that we return from) expects. The only way to get that
551          * data back is to call:
552          */
553         tick_nohz_idle_enter();
554
555         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
556 }
557
558 #else /* !CONFIG_HOTPLUG_CPU */
559 static int xen_cpu_disable(void)
560 {
561         return -ENOSYS;
562 }
563
564 static void xen_pv_cpu_die(unsigned int cpu)
565 {
566         BUG();
567 }
568
569 static void xen_hvm_cpu_die(unsigned int cpu)
570 {
571         BUG();
572 }
573
574 static void xen_play_dead(void)
575 {
576         BUG();
577 }
578
579 #endif
580 static void stop_self(void *v)
581 {
582         int cpu = smp_processor_id();
583
584         /* make sure we're not pinning something down */
585         load_cr3(swapper_pg_dir);
586         /* should set up a minimal gdt */
587
588         set_cpu_online(cpu, false);
589
590         HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
591         BUG();
592 }
593
594 static void xen_stop_other_cpus(int wait)
595 {
596         smp_call_function(stop_self, NULL, wait);
597 }
598
599 static void xen_smp_send_reschedule(int cpu)
600 {
601         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
602 }
603
604 static void __xen_send_IPI_mask(const struct cpumask *mask,
605                               int vector)
606 {
607         unsigned cpu;
608
609         for_each_cpu_and(cpu, mask, cpu_online_mask)
610                 xen_send_IPI_one(cpu, vector);
611 }
612
613 static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
614 {
615         int cpu;
616
617         __xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
618
619         /* Make sure other vcpus get a chance to run if they need to. */
620         for_each_cpu(cpu, mask) {
621                 if (xen_vcpu_stolen(cpu)) {
622                         HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
623                         break;
624                 }
625         }
626 }
627
628 static void xen_smp_send_call_function_single_ipi(int cpu)
629 {
630         __xen_send_IPI_mask(cpumask_of(cpu),
631                           XEN_CALL_FUNCTION_SINGLE_VECTOR);
632 }
633
634 static inline int xen_map_vector(int vector)
635 {
636         int xen_vector;
637
638         switch (vector) {
639         case RESCHEDULE_VECTOR:
640                 xen_vector = XEN_RESCHEDULE_VECTOR;
641                 break;
642         case CALL_FUNCTION_VECTOR:
643                 xen_vector = XEN_CALL_FUNCTION_VECTOR;
644                 break;
645         case CALL_FUNCTION_SINGLE_VECTOR:
646                 xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
647                 break;
648         case IRQ_WORK_VECTOR:
649                 xen_vector = XEN_IRQ_WORK_VECTOR;
650                 break;
651 #ifdef CONFIG_X86_64
652         case NMI_VECTOR:
653         case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
654                 xen_vector = XEN_NMI_VECTOR;
655                 break;
656 #endif
657         default:
658                 xen_vector = -1;
659                 printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
660                         vector);
661         }
662
663         return xen_vector;
664 }
665
666 void xen_send_IPI_mask(const struct cpumask *mask,
667                               int vector)
668 {
669         int xen_vector = xen_map_vector(vector);
670
671         if (xen_vector >= 0)
672                 __xen_send_IPI_mask(mask, xen_vector);
673 }
674
675 void xen_send_IPI_all(int vector)
676 {
677         int xen_vector = xen_map_vector(vector);
678
679         if (xen_vector >= 0)
680                 __xen_send_IPI_mask(cpu_online_mask, xen_vector);
681 }
682
683 void xen_send_IPI_self(int vector)
684 {
685         int xen_vector = xen_map_vector(vector);
686
687         if (xen_vector >= 0)
688                 xen_send_IPI_one(smp_processor_id(), xen_vector);
689 }
690
691 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
692                                 int vector)
693 {
694         unsigned cpu;
695         unsigned int this_cpu = smp_processor_id();
696         int xen_vector = xen_map_vector(vector);
697
698         if (!(num_online_cpus() > 1) || (xen_vector < 0))
699                 return;
700
701         for_each_cpu_and(cpu, mask, cpu_online_mask) {
702                 if (this_cpu == cpu)
703                         continue;
704
705                 xen_send_IPI_one(cpu, xen_vector);
706         }
707 }
708
709 void xen_send_IPI_allbutself(int vector)
710 {
711         xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
712 }
713
714 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
715 {
716         irq_enter();
717         generic_smp_call_function_interrupt();
718         inc_irq_stat(irq_call_count);
719         irq_exit();
720
721         return IRQ_HANDLED;
722 }
723
724 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
725 {
726         irq_enter();
727         generic_smp_call_function_single_interrupt();
728         inc_irq_stat(irq_call_count);
729         irq_exit();
730
731         return IRQ_HANDLED;
732 }
733
734 static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
735 {
736         irq_enter();
737         irq_work_run();
738         inc_irq_stat(apic_irq_work_irqs);
739         irq_exit();
740
741         return IRQ_HANDLED;
742 }
743
744 static const struct smp_ops xen_smp_ops __initconst = {
745         .smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
746         .smp_prepare_cpus = xen_smp_prepare_cpus,
747         .smp_cpus_done = xen_smp_cpus_done,
748
749         .cpu_up = xen_cpu_up,
750         .cpu_die = xen_pv_cpu_die,
751         .cpu_disable = xen_cpu_disable,
752         .play_dead = xen_play_dead,
753
754         .stop_other_cpus = xen_stop_other_cpus,
755         .smp_send_reschedule = xen_smp_send_reschedule,
756
757         .send_call_func_ipi = xen_smp_send_call_function_ipi,
758         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
759 };
760
761 void __init xen_smp_init(void)
762 {
763         smp_ops = xen_smp_ops;
764         xen_fill_possible_map();
765 }
766
767 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
768 {
769         native_smp_prepare_cpus(max_cpus);
770         WARN_ON(xen_smp_intr_init(0));
771
772         xen_init_lock_cpu(0);
773 }
774
775 void __init xen_hvm_smp_init(void)
776 {
777         smp_ops.smp_prepare_cpus = xen_hvm_smp_prepare_cpus;
778         smp_ops.smp_send_reschedule = xen_smp_send_reschedule;
779         smp_ops.cpu_die = xen_hvm_cpu_die;
780         smp_ops.send_call_func_ipi = xen_smp_send_call_function_ipi;
781         smp_ops.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi;
782         smp_ops.smp_prepare_boot_cpu = xen_hvm_smp_prepare_boot_cpu;
783 }