Re: [next-20170609] Oops while running CPU off-on (cpuset.c/cpuset_can_attach)

[Abdul Haleem]

On Wed, 2017-07-05 at 11:28 -0400, Tejun Heo wrote:
> Hello, Abdul.
> 
> Thanks for the debug info.  Can you please see whether the following
> patch fixes the issue?  

It is my pleasure and yes the patch fixes the problem.

> If the problem is too difficult to reproduce

The problem was reproducible all the time. 

With the patch fix, I tried multiple times and long runs of cpu off-on
cycles but no Oops is seen.

Thank you for spending your valuable time on fixing this issue.

Reported-and-tested-by : Abdul Haleem 

> to confirm the fix by seeing whether it no longer triggers, please let
> me know.  We can instead apply a patch which triggers WARN on the
> failing condition to confirm the diagnosis.
> 
> Thanks.
> 
> diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
> index 793565c05742..8b4c3c2f2509 100644
> --- a/kernel/cgroup/cgroup-internal.h
> +++ b/kernel/cgroup/cgroup-internal.h
> @@ -33,6 +33,9 @@ struct cgroup_taskset {
>   struct list_headsrc_csets;
>   struct list_headdst_csets;
> 
> + /* the number of tasks in the set */
> + int nr_tasks;
> +
>   /* the subsys currently being processed */
>   int ssid;
> 
> diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
> index dbfd7028b1c6..e3c4152741a3 100644
> --- a/kernel/cgroup/cgroup.c
> +++ b/kernel/cgroup/cgroup.c
> @@ -1954,6 +1954,8 @@ static void cgroup_migrate_add_task(struct task_struct 
> *task,
>   if (!cset->mg_src_cgrp)
>   return;
> 
> + mgctx->tset.nr_tasks++;
> +
>   list_move_tail(>cg_list, >mg_tasks);
>   if (list_empty(>mg_node))
>   list_add_tail(>mg_node,
> @@ -2047,16 +2049,18 @@ static int cgroup_migrate_execute(struct cgroup_mgctx 
> *mgctx)
>   return 0;
> 
>   /* check that we can legitimately attach to the cgroup */
> - do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> - if (ss->can_attach) {
> - tset->ssid = ssid;
> - ret = ss->can_attach(tset);
> - if (ret) {
> - failed_ssid = ssid;
> - goto out_cancel_attach;
> + if (tset->nr_tasks) {
> + do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> + if (ss->can_attach) {
> + tset->ssid = ssid;
> + ret = ss->can_attach(tset);
> + if (ret) {
> + failed_ssid = ssid;
> + goto out_cancel_attach;
> + }
>   }
> - }
> - } while_each_subsys_mask();
> + } while_each_subsys_mask();
> + }
> 
>   /*
>* Now that we're guaranteed success, proceed to move all tasks to
> @@ -2085,25 +2089,29 @@ static int cgroup_migrate_execute(struct cgroup_mgctx 
> *mgctx)
>*/
>   tset->csets = >dst_csets;
> 
> - do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> - if (ss->attach) {
> - tset->ssid = ssid;
> - ss->attach(tset);
> - }
> - } while_each_subsys_mask();
> + if (tset->nr_tasks) {
> + do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> + if (ss->attach) {
> + tset->ssid = ssid;
> + ss->attach(tset);
> + }
> + } while_each_subsys_mask();
> + }
> 
>   ret = 0;
>   goto out_release_tset;
> 
>  out_cancel_attach:
> - do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> - if (ssid == failed_ssid)
> - break;
> - if (ss->cancel_attach) {
> - tset->ssid = ssid;
> - ss->cancel_attach(tset);
> - }
> - } while_each_subsys_mask();
> + if (tset->nr_tasks) {
> + do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
> + if (ssid == failed_ssid)
> + break;
> + if (ss->cancel_attach) {
> + tset->ssid = ssid;
> + ss->cancel_attach(tset);
> + }
> + } while_each_subsys_mask();
> + }
>  out_release_tset:
>   spin_lock_irq(_set_lock);
>   list_splice_init(>dst_csets, >src_csets);
> 


-- 
Regard's

Abdul Haleem
IBM Linux Technology Centre

Re: [next-20170609] Oops while running CPU off-on (cpuset.c/cpuset_can_attach)

[Tejun Heo]

Hello, Abdul.

Thanks for the debug info.  Can you please see whether the following
patch fixes the issue?  If the problem is too difficult to reproduce
to confirm the fix by seeing whether it no longer triggers, please let
me know.  We can instead apply a patch which triggers WARN on the
failing condition to confirm the diagnosis.

Thanks.

diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index 793565c05742..8b4c3c2f2509 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -33,6 +33,9 @@ struct cgroup_taskset {
struct list_headsrc_csets;
struct list_headdst_csets;
 
+   /* the number of tasks in the set */
+   int nr_tasks;
+
/* the subsys currently being processed */
int ssid;
 
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index dbfd7028b1c6..e3c4152741a3 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -1954,6 +1954,8 @@ static void cgroup_migrate_add_task(struct task_struct 
*task,
if (!cset->mg_src_cgrp)
return;
 
+   mgctx->tset.nr_tasks++;
+
list_move_tail(>cg_list, >mg_tasks);
if (list_empty(>mg_node))
list_add_tail(>mg_node,
@@ -2047,16 +2049,18 @@ static int cgroup_migrate_execute(struct cgroup_mgctx 
*mgctx)
return 0;
 
/* check that we can legitimately attach to the cgroup */
-   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
-   if (ss->can_attach) {
-   tset->ssid = ssid;
-   ret = ss->can_attach(tset);
-   if (ret) {
-   failed_ssid = ssid;
-   goto out_cancel_attach;
+   if (tset->nr_tasks) {
+   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
+   if (ss->can_attach) {
+   tset->ssid = ssid;
+   ret = ss->can_attach(tset);
+   if (ret) {
+   failed_ssid = ssid;
+   goto out_cancel_attach;
+   }
}
-   }
-   } while_each_subsys_mask();
+   } while_each_subsys_mask();
+   }
 
/*
 * Now that we're guaranteed success, proceed to move all tasks to
@@ -2085,25 +2089,29 @@ static int cgroup_migrate_execute(struct cgroup_mgctx 
*mgctx)
 */
tset->csets = >dst_csets;
 
-   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
-   if (ss->attach) {
-   tset->ssid = ssid;
-   ss->attach(tset);
-   }
-   } while_each_subsys_mask();
+   if (tset->nr_tasks) {
+   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
+   if (ss->attach) {
+   tset->ssid = ssid;
+   ss->attach(tset);
+   }
+   } while_each_subsys_mask();
+   }
 
ret = 0;
goto out_release_tset;
 
 out_cancel_attach:
-   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
-   if (ssid == failed_ssid)
-   break;
-   if (ss->cancel_attach) {
-   tset->ssid = ssid;
-   ss->cancel_attach(tset);
-   }
-   } while_each_subsys_mask();
+   if (tset->nr_tasks) {
+   do_each_subsys_mask(ss, ssid, mgctx->ss_mask) {
+   if (ssid == failed_ssid)
+   break;
+   if (ss->cancel_attach) {
+   tset->ssid = ssid;
+   ss->cancel_attach(tset);
+   }
+   } while_each_subsys_mask();
+   }
 out_release_tset:
spin_lock_irq(_set_lock);
list_splice_init(>dst_csets, >src_csets);

Re: [next-20170609] Oops while running CPU off-on (cpuset.c/cpuset_can_attach)

[Abdul Haleem]

On Tue, 2017-06-27 at 11:36 -0400, Tejun Heo wrote:
> Hello, Abdul.
> 
> Sorry about the long delay.
> 
> On Mon, Jun 12, 2017 at 04:53:42PM +0530, Abdul Haleem wrote:
> > linux-next kernel crashed while running CPU offline and online.
> > 
> > Machine: Power 8 LPAR
> > Kernel : 4.12.0-rc4-next-20170609
> > gcc : version 5.2.1
> > config: attached
> > testcase: CPU off/on
> > 
> > for i in $(seq 100);do 
> > for j in $(seq 0 15);do 
> > echo 0 >  /sys/devices/system/cpu/cpu$j/online
> > sleep 5
> > echo 1 > /sys/devices/system/cpu/cpu$j/online
> > done
> > done
> > 
> ...
> > NIP [c01d6868] cpuset_can_attach+0x58/0x1b0
> 
> Can you please map this to the source line?

Hi Tejun,

Was able to recreate on latest next kernel, from the new trace.

Unable to handle kernel paging request for data at address 0x09e0
Faulting instruction address: 0xc01dd688

which is:
c01dd688  e0 09 23 e9 ld  r9,2528(r3)

r9 = c00775cd7950, 2528() = 0x09e0


Oops: Kernel access of bad area, sig: 11 [#1]
SMP NR_CPUS=2048 
NUMA 
pSeries
Modules linked in: xt_addrtype xt_conntrack ipt_MASQUERADE
nf_nat_masquerade_ipv4 iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4
nf_nat_ipv4 iptable_filter ip_tables x_tables nf_nat nf_conntrack bridge
stp llc dm_thin_pool dm_persistent_data dm_bio_prison dm_bufio libcrc32c
vmx_crypto rtc_generic pseries_rng autofs4
CPU: 15 PID: 120 Comm: kworker/15:1 Tainted: GW
4.12.0-rc7-next-20170630-autotest #1
Workqueue: events cpuset_hotplug_workfn
task: c00775c5e300 task.stack: c00775cd4000
NIP: c01dd688 LR: c01dd678 CTR: c01dd630
REGS: c00775cd7730 TRAP: 0300   Tainted: GW
(4.12.0-rc7-next-20170630-autotest)
MSR: 80010280b033 
  CR: 44a3  XER: 2000
CFAR: c0008718 DAR: 09e0 DSISR: 4000 SOFTE: 1
GPR00: c01dd678 c00775cd79b0 c154a400
 
GPR04: c00775cd79d0  c00775cd7ad0
c000fb1de480
GPR08: c000fb1dda98 c00775cd7950 c174a400
  
GPR12: c01dd630 ce789600 c0128dc8
c00776bb0080 
GPR16:   cbcc91c0
cbcc91e0 
GPR20: cbcc90c0  
 
GPR24:  c00775cd7a30 c1428020
c1745cdc 
GPR28: c1427130 c00775cd7ac0 cb41b588
 
NIP [c01dd688] cpuset_can_attach+0x58/0x1b0

> gdb -batch vmlinux -ex 'list *(0xc01dd688)'

0xc01dd688 is in cpuset_can_attach (./include/linux/compiler.h:250).
245 })
246 
247 static __always_inline
248 void __read_once_size(const volatile void *p, void *res, int size)
249 {
250 __READ_ONCE_SIZE;
251 }
252 
253 #ifdef CONFIG_KASAN
254 /*

Does this helps, please let me know if you need more debugging. Thanks

-- 
Regard's

Abdul Haleem
IBM Linux Technology Centre

Re: [next-20170609] Oops while running CPU off-on (cpuset.c/cpuset_can_attach)

[Tejun Heo]

Hello, Abdul.

Sorry about the long delay.

On Mon, Jun 12, 2017 at 04:53:42PM +0530, Abdul Haleem wrote:
> linux-next kernel crashed while running CPU offline and online.
> 
> Machine: Power 8 LPAR
> Kernel : 4.12.0-rc4-next-20170609
> gcc : version 5.2.1
> config: attached
> testcase: CPU off/on
> 
> for i in $(seq 100);do 
> for j in $(seq 0 15);do 
> echo 0 >  /sys/devices/system/cpu/cpu$j/online
> sleep 5
> echo 1 > /sys/devices/system/cpu/cpu$j/online
> done
> done
> 
...
> NIP [c01d6868] cpuset_can_attach+0x58/0x1b0

Can you please map this to the source line?

> LR [c01d6858] cpuset_can_attach+0x48/0x1b0
> Call Trace:
> [cc72b9a0] [c01d6858] cpuset_can_attach+0x48/0x1b0
> (unreliable)
> [cc72ba00] [c01cbe80] cgroup_migrate_execute+0xb0/0x450
> [cc72ba80] [c01d3754] cgroup_transfer_tasks+0x1c4/0x360
> [cc72bba0] [c01d923c] cpuset_hotplug_workfn+0x86c/0xa20
> [cc72bca0] [c011aa44] process_one_work+0x1e4/0x580
> [cc72bd30] [c011ae78] worker_thread+0x98/0x5c0
> [cc72bdc0] [c0124058] kthread+0x168/0x1b0
> [cc72be30] [c000b2e8] ret_from_kernel_thread+0x5c/0x74
> Instruction dump:
> f821ffa1 7c7d1b78 6000 6000 38810020 7fa3eb78 3f42ffed 4bff4c25 
> 6000 3b5a0448 3d420020 eb610020  7f43d378 e929
> f92af200 

Thanks.

-- 
tejun

Re: [next-20170609] Oops while running CPU off-on (cpuset.c/cpuset_can_attach)

[Stephen Rothwell]

Hi all,

On Tue, 13 Jun 2017 09:56:41 -0400 Tejun Heo  wrote:
>
> (forwarding to Li w/ full body)
> 
> Li, can you please take a look at this?
> 
> Thanks.
> 
> On Mon, Jun 12, 2017 at 04:53:42PM +0530, Abdul Haleem wrote:
> > Hi,
> > 
> > linux-next kernel crashed while running CPU offline and online.
> > 
> > Machine: Power 8 LPAR
> > Kernel : 4.12.0-rc4-next-20170609
> > gcc : version 5.2.1
> > config: attached
> > testcase: CPU off/on
> > 
> > for i in $(seq 100);do 
> > for j in $(seq 0 15);do 
> > echo 0 >  /sys/devices/system/cpu/cpu$j/online
> > sleep 5
> > echo 1 > /sys/devices/system/cpu/cpu$j/online
> > done
> > done
> > 
> > kernel trace:
> > --
> > Unable to handle kernel paging request for data at address 0x0960
> > Faulting instruction address: 0xc01d6868
> > Oops: Kernel access of bad area, sig: 11 [#1]
> > SMP NR_CPUS=2048
> > NUMA
> > pSeries
> > Modules linked in: dlci mpls_router af_key 8021q garp mrp nfc af_alg
> > caif_socket caif pn_pep phonet fcrypt pcbc rxrpc hidp hid cmtp
> > kernelcapi bnep rfcomm bluetooth ecdh_generic can_bcm can_raw can pptp
> > gre l2tp_ppp l2tp_netlink l2tp_core ip6_udp_tunnel udp_tunnel pppoe
> > pppox irda xfrm_user xfrm_algo nfnetlink scsi_transport_iscsi dn_rtmsg
> > llc2 dccp_ipv6 atm appletalk ipx p8023 p8022 psnap sctp dccp_ipv4 dccp
> > xt_addrtype xt_conntrack ipt_MASQUERADE nf_nat_masquerade_ipv4
> > iptable_nat nf_conntrack_ipv4 nf_defrag_ipv4 nf_nat_ipv4 iptable_filter
> > ip_tables x_tables nf_nat nf_conntrack bridge stp llc dm_thin_pool
> > dm_persistent_data dm_bio_prison dm_bufio libcrc32c rtc_generic
> > vmx_crypto pseries_rng autofs4
> > CPU: 14 PID: 16947 Comm: kworker/14:0 Tainted: GW
> > 4.12.0-rc4-next-20170609 #2
> > Workqueue: events cpuset_hotplug_workfn
> > task: cca60580 task.stack: cc728000
> > NIP: c01d6868 LR: c01d6858 CTR: c01d6810
> > REGS: cc72b720 TRAP: 0300   Tainted: GW
> > (4.12.0-rc4-next-20170609)
> > MSR: 80009033 
> >   CR: 44722422  XER: 2000  
> > CFAR: c0008710 DAR: 0960 DSISR: 4000 SOFTE: 1 
> > GPR00: c01d6858 cc72b9a0 c1536e00
> >  
> > GPR04: cc72b9c0  cc72bad0
> > c00766367678 
> > GPR08: c00766366d10 cc72b958 c1736e00
> >  
> > GPR12: c01d6810 ce749300 c0123ef8
> > c00775af4180 
> > GPR16:   c0075480e9c0
> > c0075480e9e0 
> > GPR20: c0075480e8c0 0001 
> > cc72ba20 
> > GPR24: cc72baa0 cc72bac0 c1407248
> > cc72ba20 
> > GPR28: c141fc80 cc72bac0 cc6bc790
> >  
> > NIP [c01d6868] cpuset_can_attach+0x58/0x1b0
> > LR [c01d6858] cpuset_can_attach+0x48/0x1b0
> > Call Trace:
> > [cc72b9a0] [c01d6858] cpuset_can_attach+0x48/0x1b0
> > (unreliable)
> > [cc72ba00] [c01cbe80] cgroup_migrate_execute+0xb0/0x450
> > [cc72ba80] [c01d3754] cgroup_transfer_tasks+0x1c4/0x360
> > [cc72bba0] [c01d923c] cpuset_hotplug_workfn+0x86c/0xa20
> > [cc72bca0] [c011aa44] process_one_work+0x1e4/0x580
> > [cc72bd30] [c011ae78] worker_thread+0x98/0x5c0
> > [cc72bdc0] [c0124058] kthread+0x168/0x1b0
> > [cc72be30] [c000b2e8] ret_from_kernel_thread+0x5c/0x74
> > Instruction dump:
> > f821ffa1 7c7d1b78 6000 6000 38810020 7fa3eb78 3f42ffed 4bff4c25 
> > 6000 3b5a0448 3d420020 eb610020  7f43d378 e929
> > f92af200 
> > ---[ end trace dcaaf98fb36d9e64 ]---

Has there been any progress on this?
-- 
Cheers,
Stephen Rothwell