[Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? diff -r 8c806c66a597 xen/arch/ia64/linux-xen/setup.c --- a/xen/arch/ia64/linux-xen/setup.c Tue Apr 14 14:04:58 2009 +0900 +++ b/xen/arch/ia64/linux-xen/setup.c Tue Apr 21 12:03:44 2009 +0900 @@ -67,6 +67,7 @@ #include xen/kexec.h #endif +int enable_dom0_con_tasklet=0; #if defined(CONFIG_SMP) (IA64_CPU_SIZE PAGE_SIZE) # error struct cpuinfo_ia64 too big! #endif @@ -592,6 +593,9 @@ late_setup_arch (char **cmdline_p) #endif cpu_init(); /* initialize the bootstrap CPU */ + + enable_dom0_con_tasklet=1; + printk(AAA: enable_dom0_con_tasklet\n); #ifdef CONFIG_ACPI_BOOT acpi_boot_init(); diff -r 8c806c66a597 xen/drivers/char/console.c --- a/xen/drivers/char/console.cTue Apr 14 14:04:58 2009 +0900 +++ b/xen/drivers/char/console.cTue Apr 21 12:03:44 2009 +0900 @@ -117,6 +117,8 @@ custom_param(loglvl, parse_loglvl); custom_param(loglvl, parse_loglvl); custom_param(guest_loglvl, parse_guest_loglvl); +extern int enable_dom0_con_tasklet; + static atomic_t print_everything = ATOMIC_INIT(0); #define ___parse_loglvl(s, ps, lvlstr, lvlnum) \ @@ -354,7 +356,10 @@ static long guest_console_write(XEN_GUES { for ( kptr = kbuf; *kptr != '\0'; kptr++ ) putchar_console_ring(*kptr); -tasklet_schedule(notify_dom0_con_ring_tasklet); + if (enable_dom0_con_tasklet ) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } spin_unlock_irq(console_lock); @@ -435,7 +440,10 @@ static void __putstr(const char *str) { while ( (c = *str++) != '\0' ) putchar_console_ring(c); -tasklet_schedule(notify_dom0_con_ring_tasklet); +if (enable_dom0_con_tasklet ) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } } Best Regards, Akio Takebe ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
On Tue, Apr 21, 2009 at 10:27:02AM +0900, Akio Takebe wrote: Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? Unfortunately, it happened to boot on my tiger4 so that I pushed out the change set. I Understood the issue. Looking into the boot sequence, it seems to somewhat difficult to move down init_console() after cpu_init() and remove all the printk() before cpu_init(). Hmm, it needs some consideration. BTW, is there similar issue on ia64 linux case before? thanks, diff -r 8c806c66a597 xen/arch/ia64/linux-xen/setup.c --- a/xen/arch/ia64/linux-xen/setup.c Tue Apr 14 14:04:58 2009 +0900 +++ b/xen/arch/ia64/linux-xen/setup.c Tue Apr 21 12:03:44 2009 +0900 @@ -67,6 +67,7 @@ #include xen/kexec.h #endif +int enable_dom0_con_tasklet=0; #if defined(CONFIG_SMP) (IA64_CPU_SIZE PAGE_SIZE) # error struct cpuinfo_ia64 too big! #endif @@ -592,6 +593,9 @@ late_setup_arch (char **cmdline_p) #endif cpu_init(); /* initialize the bootstrap CPU */ + + enable_dom0_con_tasklet=1; + printk(AAA: enable_dom0_con_tasklet\n); #ifdef CONFIG_ACPI_BOOT acpi_boot_init(); diff -r 8c806c66a597 xen/drivers/char/console.c --- a/xen/drivers/char/console.c Tue Apr 14 14:04:58 2009 +0900 +++ b/xen/drivers/char/console.c Tue Apr 21 12:03:44 2009 +0900 @@ -117,6 +117,8 @@ custom_param(loglvl, parse_loglvl); custom_param(loglvl, parse_loglvl); custom_param(guest_loglvl, parse_guest_loglvl); +extern int enable_dom0_con_tasklet; + static atomic_t print_everything = ATOMIC_INIT(0); #define ___parse_loglvl(s, ps, lvlstr, lvlnum) \ @@ -354,7 +356,10 @@ static long guest_console_write(XEN_GUES { for ( kptr = kbuf; *kptr != '\0'; kptr++ ) putchar_console_ring(*kptr); -tasklet_schedule(notify_dom0_con_ring_tasklet); + if (enable_dom0_con_tasklet ) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } spin_unlock_irq(console_lock); @@ -435,7 +440,10 @@ static void __putstr(const char *str) { while ( (c = *str++) != '\0' ) putchar_console_ring(c); -tasklet_schedule(notify_dom0_con_ring_tasklet); +if (enable_dom0_con_tasklet ) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } } Best Regards, Akio Takebe ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel -- yamahata ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
Hi, Thank you for comments. Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 10:27:02AM +0900, Akio Takebe wrote: Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? Unfortunately, it happened to boot on my tiger4 so that I pushed out the change set. I Understood the issue. Looking into the boot sequence, it seems to somewhat difficult to move down init_console() after cpu_init() and remove all the printk() before cpu_init(). Hmm, it needs some consideration. I also move some function after cpu_init(). But it failed to boot. So we may need such a early_printk(). I think we should not call tasklet_schedule() before softirq_init() at least on x86. So how about the attached patch? BTW, is there similar issue on ia64 linux case before? No, I have not met. Best Regards, Akio Takebe --- diff -r 8c806c66a597 xen/arch/ia64/xen/xensetup.c --- a/xen/arch/ia64/xen/xensetup.c Tue Apr 14 14:04:58 2009 +0900 +++ b/xen/arch/ia64/xen/xensetup.c Tue Apr 21 13:42:26 2009 +0900 @@ -564,9 +564,9 @@ skip_move: end_boot_allocator(); +late_setup_arch(cmdline); + softirq_init(); - -late_setup_arch(cmdline); scheduler_init(); idle_vcpu[0] = (struct vcpu*) ia64_r13; diff -r 8c806c66a597 xen/common/softirq.c --- a/xen/common/softirq.c Tue Apr 14 14:04:58 2009 +0900 +++ b/xen/common/softirq.c Tue Apr 21 13:42:26 2009 +0900 @@ -20,7 +20,7 @@ irq_cpustat_t irq_stat[NR_CPUS]; irq_cpustat_t irq_stat[NR_CPUS]; #endif -static softirq_handler softirq_handlers[NR_SOFTIRQS]; +softirq_handler softirq_handlers[NR_SOFTIRQS]; asmlinkage void do_softirq(void) { diff -r 8c806c66a597 xen/drivers/char/console.c --- a/xen/drivers/char/console.cTue Apr 14 14:04:58 2009 +0900 +++ b/xen/drivers/char/console.cTue Apr 21 13:42:26 2009 +0900 @@ -117,6 +117,8 @@ custom_param(loglvl, parse_loglvl); custom_param(loglvl, parse_loglvl); custom_param(guest_loglvl, parse_guest_loglvl); +extern softirq_handler softirq_handlers[NR_SOFTIRQS]; + static atomic_t print_everything = ATOMIC_INIT(0); #define ___parse_loglvl(s, ps, lvlstr, lvlnum) \ @@ -354,7 +356,10 @@ static long guest_console_write(XEN_GUES { for ( kptr = kbuf; *kptr != '\0'; kptr++ ) putchar_console_ring(*kptr); -tasklet_schedule(notify_dom0_con_ring_tasklet); + if (softirq_handlers[TASKLET_SOFTIRQ]) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } spin_unlock_irq(console_lock); @@ -435,7 +440,10 @@ static void __putstr(const char *str) { while ( (c = *str++) != '\0' ) putchar_console_ring(c); -tasklet_schedule(notify_dom0_con_ring_tasklet); +if (softirq_handlers[TASKLET_SOFTIRQ]) +tasklet_schedule(notify_dom0_con_ring_tasklet); + else +send_guest_global_virq(dom0, VIRQ_CON_RING); } } ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
On Tue, Apr 21, 2009 at 11:19:06AM +0900, Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 10:27:02AM +0900, Akio Takebe wrote: Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? Unfortunately, it happened to boot on my tiger4 so that I pushed out the change set. I Understood the issue. Looking into the boot sequence, it seems to somewhat difficult to move down init_console() after cpu_init() and remove all the printk() before cpu_init(). Hmm, it needs some consideration. BTW, is there similar issue on ia64 linux case before? Yes, there was. commit 10617bbe84628eb18ab5f723d3ba35005adde143 and commit c459ce8b5a7d933a3bcf6915ab17ac1e036e2ac4 The mails were Jul 14 Christian Kande Initialization order problem and Aug 07 Luck, Tony [RFC] Fix early access to per-cpu variables thread. -- yamahata ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
On Tue, Apr 21, 2009 at 11:49:55AM +0900, Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 11:19:06AM +0900, Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 10:27:02AM +0900, Akio Takebe wrote: Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? Unfortunately, it happened to boot on my tiger4 so that I pushed out the change set. I Understood the issue. Looking into the boot sequence, it seems to somewhat difficult to move down init_console() after cpu_init() and remove all the printk() before cpu_init(). Hmm, it needs some consideration. BTW, is there similar issue on ia64 linux case before? Yes, there was. commit 10617bbe84628eb18ab5f723d3ba35005adde143 and commit c459ce8b5a7d933a3bcf6915ab17ac1e036e2ac4 The mails were Jul 14 Christian Kande Initialization order problem and Aug 07 Luck, Tony [RFC] Fix early access to per-cpu variables thread. I created the patch following ia64 linux way. Could you try it? This is not intrusive to the common code, and prevent future similar breakage. [IA64] fix early access to per cpu area. The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. There was a similar issueson Linux/ia64. The following change sets resolved it. 10617bbe84628eb18ab5f723d3ba35005adde143 c459ce8b5a7d933a3bcf6915ab17ac1e036e2ac4 This patch fixes the issue following the linux/ia64 solution. Allocate per cpu area for cpu0 in .data section and initialize it early. reported-by: Akio Takebe takebe_a...@jp.fujitsu.com Signed-off-by: Isaku Yamahata yamah...@valinux.co.jp diff --git a/xen/arch/ia64/linux-xen/head.S b/xen/arch/ia64/linux-xen/head.S --- a/xen/arch/ia64/linux-xen/head.S +++ b/xen/arch/ia64/linux-xen/head.S @@ -382,6 +382,35 @@ 1: // now we are in virtual mode mov ar.rsc=0// place RSE in enforced lazy mode ;; loadrs // clear the dirty partition +#ifdef XEN +(isAP) br.few 2f + movl r19=__phys_per_cpu_start + mov r18=PERCPU_PAGE_SIZE +#ifndef CONFIG_SMP + add r19=r19,r18 + ;; +#else + movl r20=__cpu0_per_cpu + ;; + shr.u r18=r18,3 +1: + ld8 r21=[r19],8 ;; + st8[r20]=r21,8 + adds r18=-1,r18 + ;; + cmp4.lt p7,p6=0,r18 +(p7) br.cond.dptk.few 1b + ;; +#endif + movl r18=__per_cpu_offset + movl r19=__cpu0_per_cpu + movl r20=__per_cpu_start + ;; + sub r20=r19,r20 + ;; + st8 [r18]=r20 +2: +#endif ;; mov ar.bspstore=r2 // establish the new RSE stack ;; diff --git a/xen/arch/ia64/linux-xen/mm_contig.c b/xen/arch/ia64/linux-xen/mm_contig.c --- a/xen/arch/ia64/linux-xen/mm_contig.c +++ b/xen/arch/ia64/linux-xen/mm_contig.c @@ -183,7 +183,7 @@ void *percpu_area __initdata = NULL; void* __init per_cpu_allocate(void *xen_heap_start, unsigned long end_in_pa) { - int order = get_order(NR_CPUS * PERCPU_PAGE_SIZE); + int order = get_order((NR_CPUS - 1) * PERCPU_PAGE_SIZE); unsigned long size = 1UL (order + PAGE_SHIFT); unsigned long start = ALIGN_UP((unsigned long)xen_heap_start, PERCPU_PAGE_SIZE); @@ -226,19 +226,31 @@ per_cpu_init (void) */ if (smp_processor_id() == 0) { #ifdef XEN + void *cpu0_data = __cpu0_per_cpu; + + __per_cpu_offset[0] = (char *)cpu0_data - __per_cpu_start; + per_cpu(local_per_cpu_offset, 0) = __per_cpu_offset[0]; + cpu_data = get_per_cpu_area(); if (cpu_data == NULL) panic(can't allocate per cpu area.\n); + + for (cpu = 1; cpu NR_CPUS; cpu++) { + memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; + cpu_data += PERCPU_PAGE_SIZE; + per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; + } #else cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS, PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS)); -#endif for (cpu = 0; cpu
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
Hi, Isaku Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 11:49:55AM +0900, Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 11:19:06AM +0900, Isaku Yamahata wrote: On Tue, Apr 21, 2009 at 10:27:02AM +0900, Akio Takebe wrote: Hi, The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. I make the following patch for this investigation. It can boot xen-ia64 on the ia64 boxes. I'm not sure why Tiger4 can boot the latest xen-ia64. I didn't find a good-looking solution, what do you think about it? Unfortunately, it happened to boot on my tiger4 so that I pushed out the change set. I Understood the issue. Looking into the boot sequence, it seems to somewhat difficult to move down init_console() after cpu_init() and remove all the printk() before cpu_init(). Hmm, it needs some consideration. BTW, is there similar issue on ia64 linux case before? Yes, there was. commit 10617bbe84628eb18ab5f723d3ba35005adde143 and commit c459ce8b5a7d933a3bcf6915ab17ac1e036e2ac4 The mails were Jul 14 Christian Kande Initialization order problem and Aug 07 Luck, Tony [RFC] Fix early access to per-cpu variables thread. I created the patch following ia64 linux way. Could you try it? This is not intrusive to the common code, and prevent future similar breakage. I tried it, and I could boot the latest xen-ia64 on the ia64 box. I agree with you. Please commit this. Best Regards, Akio Takebe [IA64] fix early access to per cpu area. The following changeset broke booting xen-ia64 on some kinds of ia64 boxes. http://xenbits.xensource.com/ext/ia64/xen-unstable.hg/rev/3fd8f9b34941 The tasklet_schedule call raise_softirq(). Because raise_softirq() use per_cpu, if we access per_cpu before cpu_init() the behavior would be unexpected. There was a similar issueson Linux/ia64. The following change sets resolved it. 10617bbe84628eb18ab5f723d3ba35005adde143 c459ce8b5a7d933a3bcf6915ab17ac1e036e2ac4 This patch fixes the issue following the linux/ia64 solution. Allocate per cpu area for cpu0 in .data section and initialize it early. reported-by: Akio Takebe takebe_a...@jp.fujitsu.com Signed-off-by: Isaku Yamahata yamah...@valinux.co.jp diff --git a/xen/arch/ia64/linux-xen/head.S b/xen/arch/ia64/linux-xen/head.S --- a/xen/arch/ia64/linux-xen/head.S +++ b/xen/arch/ia64/linux-xen/head.S @@ -382,6 +382,35 @@ 1: // now we are in virtual mode mov ar.rsc=0// place RSE in enforced lazy mode ;; loadrs // clear the dirty partition +#ifdef XEN +(isAP) br.few 2f + movl r19=__phys_per_cpu_start + mov r18=PERCPU_PAGE_SIZE +#ifndef CONFIG_SMP + add r19=r19,r18 + ;; +#else + movl r20=__cpu0_per_cpu + ;; + shr.u r18=r18,3 +1: + ld8 r21=[r19],8 ;; + st8[r20]=r21,8 + adds r18=-1,r18 + ;; + cmp4.lt p7,p6=0,r18 +(p7) br.cond.dptk.few 1b + ;; +#endif + movl r18=__per_cpu_offset + movl r19=__cpu0_per_cpu + movl r20=__per_cpu_start + ;; + sub r20=r19,r20 + ;; + st8 [r18]=r20 +2: +#endif ;; mov ar.bspstore=r2 // establish the new RSE stack ;; diff --git a/xen/arch/ia64/linux-xen/mm_contig.c b/xen/arch/ia64/linux-xen/mm_contig.c --- a/xen/arch/ia64/linux-xen/mm_contig.c +++ b/xen/arch/ia64/linux-xen/mm_contig.c @@ -183,7 +183,7 @@ void *percpu_area __initdata = NULL; void* __init per_cpu_allocate(void *xen_heap_start, unsigned long end_in_pa) { - int order = get_order(NR_CPUS * PERCPU_PAGE_SIZE); + int order = get_order((NR_CPUS - 1) * PERCPU_PAGE_SIZE); unsigned long size = 1UL (order + PAGE_SHIFT); unsigned long start = ALIGN_UP((unsigned long)xen_heap_start, PERCPU_PAGE_SIZE); @@ -226,19 +226,31 @@ per_cpu_init (void) */ if (smp_processor_id() == 0) { #ifdef XEN + void *cpu0_data = __cpu0_per_cpu; + + __per_cpu_offset[0] = (char *)cpu0_data - __per_cpu_start; + per_cpu(local_per_cpu_offset, 0) = __per_cpu_offset[0]; + cpu_data = get_per_cpu_area(); if (cpu_data == NULL) panic(can't allocate per cpu area.\n); + + for (cpu = 1; cpu NR_CPUS; cpu++) { + memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); + __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; + cpu_data += PERCPU_PAGE_SIZE; + per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; + } #else cpu_data = __alloc_bootmem(PERCPU_PAGE_SIZE * NR_CPUS,
Re: [Xen-ia64-devel] [RFC][Patch] fix boot xen-ia64 on some ia64 platforms
On Tue, Apr 21, 2009 at 11:43:59AM +0900, Akio Takebe wrote: I think we should not call tasklet_schedule() before softirq_init() at least on x86. So how about the attached patch? I don't get why. Could you please elaborate? -- yamahata ___ Xen-ia64-devel mailing list Xen-ia64-devel@lists.xensource.com http://lists.xensource.com/xen-ia64-devel