Public bug reported:
1 create two-arm LB with 1client and 1 backend server on a tenant
2 repeat step1 to create 350 tenants
3 after step 2 , the LB network unstable
log and cpu infor
[root@nsj4 ~]# ip netns exec qlbaas-ade3e627-5fda-4781-855e-25a0f5217d17 ping
20.1.1.100
PING 20.1.1.100 (20.1.1.100) 56(84) bytes of data.
>From 20.1.1.101 icmp_seq=1 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=2 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=3 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=4 Destination Host Unreachable<<<<<<<<<<<<<<<<
64 bytes from 20.1.1.100: icmp_seq=6 ttl=64 time=0.100 ms
64 bytes from 20.1.1.100: icmp_seq=7 ttl=64 time=0.067 ms
64 bytes from 20.1.1.100: icmp_seq=8 ttl=64 time=0.081 ms
>From 20.1.1.101 icmp_seq=5 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=9 ttl=64 time=0.054 ms
64 bytes from 20.1.1.100: icmp_seq=10 ttl=64 time=0.050 ms
64 bytes from 20.1.1.100: icmp_seq=11 ttl=64 time=0.087 ms
64 bytes from 20.1.1.100: icmp_seq=12 ttl=64 time=0.065 ms
64 bytes from 20.1.1.100: icmp_seq=13 ttl=64 time=0.079 ms
64 bytes from 20.1.1.100: icmp_seq=14 ttl=64 time=0.073 ms
64 bytes from 20.1.1.100: icmp_seq=15 ttl=64 time=0.062 ms
64 bytes from 20.1.1.100: icmp_seq=16 ttl=64 time=0.070 ms
64 bytes from 20.1.1.100: icmp_seq=17 ttl=64 time=0.058 ms
64 bytes from 20.1.1.100: icmp_seq=18 ttl=64 time=0.146 ms
64 bytes from 20.1.1.100: icmp_seq=19 ttl=64 time=0.060 ms
64 bytes from 20.1.1.100: icmp_seq=20 ttl=64 time=0.097 ms
64 bytes from 20.1.1.100: icmp_seq=21 ttl=64 time=0.114 ms
64 bytes from 20.1.1.100: icmp_seq=22 ttl=64 time=0.091 ms
64 bytes from 20.1.1.100: icmp_seq=23 ttl=64 time=0.122 ms
64 bytes from 20.1.1.100: icmp_seq=24 ttl=64 time=0.074 ms
64 bytes from 20.1.1.100: icmp_seq=25 ttl=64 time=0.091 ms
>From 20.1.1.101 icmp_seq=26 Destination Host Unreachable<<<<<<<<<<<<<<<<<<
>From 20.1.1.101 icmp_seq=27 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=28 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=29 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=30 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=31 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=32 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=33 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=34 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=35 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=36 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=37 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=38 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=39 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=40 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=41 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=42 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=43 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=44 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=45 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=46 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=47 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=48 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=49 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=50 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=51 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=52 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=53 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=54 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=55 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=56 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=57 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=58 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=59 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=60 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=64 ttl=64 time=0.117 ms
>From 20.1.1.101 icmp_seq=61 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=62 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=63 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=65 ttl=64 time=0.097 ms
64 bytes from 20.1.1.100: icmp_seq=66 ttl=64 time=0.060 ms
64 bytes from 20.1.1.100: icmp_seq=67 ttl=64 time=0.083 ms
64 bytes from 20.1.1.100: icmp_seq=68 ttl=64 time=0.059 ms
64 bytes from 20.1.1.100: icmp_seq=69 ttl=64 time=0.173 ms
64 bytes from 20.1.1.100: icmp_seq=70 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=71 ttl=64 time=0.067 ms
64 bytes from 20.1.1.100: icmp_seq=72 ttl=64 time=0.075 ms
64 bytes from 20.1.1.100: icmp_seq=73 ttl=64 time=0.051 ms
64 bytes from 20.1.1.100: icmp_seq=74 ttl=64 time=0.078 ms
64 bytes from 20.1.1.100: icmp_seq=75 ttl=64 time=0.106 ms
64 bytes from 20.1.1.100: icmp_seq=76 ttl=64 time=0.100 ms
64 bytes from 20.1.1.100: icmp_seq=77 ttl=64 time=0.205 ms
64 bytes from 20.1.1.100: icmp_seq=78 ttl=64 time=0.116 ms
64 bytes from 20.1.1.100: icmp_seq=79 ttl=64 time=0.110 ms
64 bytes from 20.1.1.100: icmp_seq=80 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=81 ttl=64 time=0.068 ms
64 bytes from 20.1.1.100: icmp_seq=82 ttl=64 time=0.091 ms
64 bytes from 20.1.1.100: icmp_seq=83 ttl=64 time=0.092 ms
64 bytes from 20.1.1.100: icmp_seq=84 ttl=64 time=0.082 ms
64 bytes from 20.1.1.100: icmp_seq=85 ttl=64 time=0.056 ms
64 bytes from 20.1.1.100: icmp_seq=86 ttl=64 time=0.063 ms
64 bytes from 20.1.1.100: icmp_seq=87 ttl=64 time=0.118 ms
64 bytes from 20.1.1.100: icmp_seq=88 ttl=64 time=0.104 ms
64 bytes from 20.1.1.100: icmp_seq=89 ttl=64 time=0.074 ms
64 bytes from 20.1.1.100: icmp_seq=90 ttl=64 time=0.143 ms
64 bytes from 20.1.1.100: icmp_seq=91 ttl=64 time=0.049 ms
64 bytes from 20.1.1.100: icmp_seq=92 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=93 ttl=64 time=0.076 ms
>From 20.1.1.101 icmp_seq=94 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=95 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=96 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=97 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=98 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=99 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=100 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=101 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=102 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=103 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=104 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=105 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=106 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=107 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=108 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=109 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=110 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=111 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=112 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=113 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=114 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=115 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=116 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=117 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=118 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=119 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=123 ttl=64 time=0.079 ms
>From 20.1.1.101 icmp_seq=120 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=121 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=122 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=124 ttl=64 time=0.066 ms
64 bytes from 20.1.1.100: icmp_seq=125 ttl=64 time=0.050 ms
64 bytes from 20.1.1.100: icmp_seq=126 ttl=64 time=0.039 ms
64 bytes from 20.1.1.100: icmp_seq=127 ttl=64 time=0.088 ms
64 bytes from 20.1.1.100: icmp_seq=128 ttl=64 time=0.082 ms
64 bytes from 20.1.1.100: icmp_seq=129 ttl=64 time=0.079 ms
64 bytes from 20.1.1.100: icmp_seq=130 ttl=64 time=0.104 ms
64 bytes from 20.1.1.100: icmp_seq=131 ttl=64 time=0.069 ms
64 bytes from 20.1.1.100: icmp_seq=132 ttl=64 time=0.101 ms
64 bytes from 20.1.1.100: icmp_seq=133 ttl=64 time=0.088 ms
>From 20.1.1.101 icmp_seq=134 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=135 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=136 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=137 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=138 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=139 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=140 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=141 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=142 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=143 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=144 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=145 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=146 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=147 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=148 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=149 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=150 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=151 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=152 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=153 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=154 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=155 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=156 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=157 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=158 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=159 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=160 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=161 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=162 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=163 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=164 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=165 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=166 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=167 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=168 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=169 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=170 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=171 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=172 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=173 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=174 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=175 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=176 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=177 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=178 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=179 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=180 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=181 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=182 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=183 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=184 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=185 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=186 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=187 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=188 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=189 Destination Host Unreachable
^C
--- 20.1.1.100 ping statistics ---
189 packets transmitted, 61 received, +128 errors, 67% packet loss, time
188402ms
rtt min/avg/max/mdev = 0.039/0.085/0.205/0.030 ms, pipe 4
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]# sar -P ALL 60 1
Linux 3.10.0-229.11.1.el7.x86_64 (nsj4) 08/28/2015 _x86_64_
(80 CPU)
03:42:48 AM CPU %user %nice %system %iowait %steal %idle
03:43:48 AM all 3.09 0.00 75.23 0.00 0.00 21.68
03:43:48 AM 0 2.24 0.00 84.93 0.00 0.00 12.83
03:43:48 AM 1 2.48 0.00 81.40 0.00 0.00 16.12
03:43:48 AM 2 2.36 0.00 77.36 0.00 0.00 20.28
03:43:48 AM 3 2.43 0.00 79.85 0.00 0.00 17.72
03:43:48 AM 4 2.26 0.00 76.10 0.00 0.00 21.64
03:43:48 AM 5 2.60 0.00 76.77 0.00 0.00 20.62
03:43:48 AM 6 2.77 0.00 78.80 0.00 0.00 18.43
03:43:48 AM 7 2.48 0.00 75.81 0.00 0.00 21.71
03:43:48 AM 8 2.49 0.00 78.44 0.00 0.00 19.07
03:43:48 AM 9 3.25 0.00 76.81 0.00 0.00 19.94
03:43:48 AM 10 2.80 0.00 79.90 0.00 0.00 17.30
03:43:48 AM 11 2.41 0.00 77.32 0.00 0.00 20.28
03:43:48 AM 12 2.17 0.00 75.10 0.00 0.00 22.73
03:43:48 AM 13 2.92 0.00 77.01 0.00 0.00 20.06
03:43:48 AM 14 2.41 0.00 76.10 0.00 0.00 21.48
03:43:48 AM 15 2.08 0.00 73.94 0.00 0.00 23.98
03:43:48 AM 16 2.29 0.00 74.58 0.00 0.00 23.14
03:43:48 AM 17 2.72 0.00 76.94 0.00 0.00 20.34
03:43:48 AM 18 2.38 0.00 75.70 0.00 0.00 21.92
03:43:48 AM 19 2.37 0.00 76.11 0.00 0.00 21.53
03:43:48 AM 20 1.86 0.00 78.34 0.00 0.00 19.79
03:43:48 AM 21 1.76 0.00 77.52 0.00 0.00 20.72
03:43:48 AM 22 1.76 0.00 77.80 0.00 0.00 20.44
03:43:48 AM 23 1.84 0.00 75.49 0.00 0.00 22.67
03:43:48 AM 24 1.59 0.00 74.62 0.00 0.00 23.79
03:43:48 AM 25 1.88 0.00 74.26 0.00 0.00 23.86
03:43:48 AM 26 1.79 0.00 73.82 0.00 0.00 24.38
03:43:48 AM 27 1.58 0.00 76.51 0.00 0.00 21.92
03:43:48 AM 28 1.79 0.00 77.89 0.00 0.00 20.32
03:43:48 AM 29 1.89 0.00 78.22 0.00 0.00 19.88
03:43:48 AM 30 3.04 0.00 76.31 0.00 0.00 20.65
03:43:48 AM 31 3.35 0.00 77.76 0.00 0.00 18.89
03:43:48 AM 32 3.07 0.00 74.56 0.00 0.00 22.37
03:43:48 AM 33 3.15 0.00 74.31 0.00 0.00 22.53
03:43:48 AM 34 2.90 0.00 74.33 0.00 0.00 22.77
03:43:48 AM 35 3.09 0.00 75.38 0.00 0.00 21.53
03:43:48 AM 36 2.90 0.00 75.25 0.02 0.00 21.84
03:43:48 AM 37 2.92 0.00 74.80 0.00 0.00 22.28
03:43:48 AM 38 3.24 0.00 74.99 0.00 0.00 21.77
03:43:48 AM 39 3.11 0.00 75.46 0.00 0.00 21.43
03:43:48 AM 40 3.30 0.00 72.86 0.00 0.00 23.83
03:43:48 AM 41 3.25 0.00 71.73 0.00 0.00 25.01
03:43:48 AM 42 3.51 0.00 73.49 0.00 0.00 23.00
03:43:48 AM 43 3.25 0.00 73.79 0.00 0.00 22.96
03:43:48 AM 44 2.97 0.00 73.59 0.00 0.00 23.44
03:43:48 AM 45 3.14 0.00 74.78 0.00 0.00 22.08
03:43:48 AM 46 3.58 0.00 68.07 0.00 0.00 28.36
03:43:48 AM 47 3.46 0.00 71.68 0.00 0.00 24.86
03:43:48 AM 48 3.28 0.00 73.14 0.00 0.00 23.58
03:43:48 AM 49 3.71 0.00 69.21 0.00 0.00 27.08
03:43:48 AM 50 3.12 0.00 71.34 0.00 0.00 25.54
03:43:48 AM 51 3.01 0.00 76.15 0.00 0.00 20.84
03:43:48 AM 52 27.12 0.00 59.04 0.00 0.00 13.84
03:43:48 AM 53 3.18 0.00 76.32 0.00 0.00 20.50
03:43:48 AM 54 3.04 0.00 72.01 0.00 0.00 24.95
03:43:48 AM 55 2.95 0.00 74.96 0.00 0.00 22.09
03:43:48 AM 56 3.11 0.00 76.32 0.00 0.00 20.56
03:43:48 AM 57 2.92 0.00 72.79 0.00 0.00 24.29
03:43:48 AM 58 3.13 0.00 75.64 0.00 0.00 21.23
03:43:48 AM 59 3.08 0.00 74.22 0.00 0.00 22.69
03:43:48 AM 60 2.27 0.00 76.90 0.00 0.00 20.82
03:43:48 AM 61 2.26 0.00 73.16 0.00 0.00 24.58
03:43:48 AM 62 2.41 0.00 80.87 0.00 0.00 16.72
03:43:48 AM 63 2.41 0.00 76.70 0.00 0.00 20.89
03:43:48 AM 64 2.76 0.00 77.53 0.00 0.00 19.71
03:43:48 AM 65 2.34 0.00 77.57 0.00 0.00 20.08
03:43:48 AM 66 2.41 0.00 77.48 0.00 0.00 20.11
03:43:48 AM 67 2.32 0.00 78.66 0.00 0.00 19.02
03:43:48 AM 68 2.22 0.00 72.75 0.00 0.00 25.03
03:43:48 AM 69 2.21 0.00 72.78 0.00 0.00 25.01
03:43:48 AM 70 3.43 0.00 71.67 0.00 0.00 24.91
03:43:48 AM 71 3.70 0.00 75.48 0.00 0.00 20.82
03:43:48 AM 72 3.92 0.00 74.17 0.00 0.00 21.91
03:43:48 AM 73 4.00 0.00 74.42 0.00 0.00 21.58
03:43:48 AM 74 3.85 0.00 75.61 0.00 0.00 20.54
03:43:48 AM 75 3.80 0.00 76.05 0.00 0.00 20.14
03:43:48 AM 76 3.35 0.00 75.91 0.00 0.00 20.74
03:43:48 AM 77 3.41 0.00 74.09 0.00 0.00 22.49
03:43:48 AM 78 3.47 0.00 71.43 0.00 0.00 25.10
03:43:48 AM 79 3.57 0.00 68.43 0.00 0.00 27.99
Average: CPU %user %nice %system %iowait %steal %idle
Average: all 3.09 0.00 75.23 0.00 0.00 21.68
Average: 0 2.24 0.00 84.93 0.00 0.00 12.83
Average: 1 2.48 0.00 81.40 0.00 0.00 16.12
Average: 2 2.36 0.00 77.36 0.00 0.00 20.28
Average: 3 2.43 0.00 79.85 0.00 0.00 17.72
Average: 4 2.26 0.00 76.10 0.00 0.00 21.64
Average: 5 2.60 0.00 76.77 0.00 0.00 20.62
Average: 6 2.77 0.00 78.80 0.00 0.00 18.43
Average: 7 2.48 0.00 75.81 0.00 0.00 21.71
Average: 8 2.49 0.00 78.44 0.00 0.00 19.07
Average: 9 3.25 0.00 76.81 0.00 0.00 19.94
Average: 10 2.80 0.00 79.90 0.00 0.00 17.30
Average: 11 2.41 0.00 77.32 0.00 0.00 20.28
Average: 12 2.17 0.00 75.10 0.00 0.00 22.73
Average: 13 2.92 0.00 77.01 0.00 0.00 20.06
Average: 14 2.41 0.00 76.10 0.00 0.00 21.48
Average: 15 2.08 0.00 73.94 0.00 0.00 23.98
Average: 16 2.29 0.00 74.58 0.00 0.00 23.14
Average: 17 2.72 0.00 76.94 0.00 0.00 20.34
Average: 18 2.38 0.00 75.70 0.00 0.00 21.92
Average: 19 2.37 0.00 76.11 0.00 0.00 21.53
Average: 20 1.86 0.00 78.34 0.00 0.00 19.79
Average: 21 1.76 0.00 77.52 0.00 0.00 20.72
Average: 22 1.76 0.00 77.80 0.00 0.00 20.44
Average: 23 1.84 0.00 75.49 0.00 0.00 22.67
Average: 24 1.59 0.00 74.62 0.00 0.00 23.79
Average: 25 1.88 0.00 74.26 0.00 0.00 23.86
Average: 26 1.79 0.00 73.82 0.00 0.00 24.38
Average: 27 1.58 0.00 76.51 0.00 0.00 21.92
Average: 28 1.79 0.00 77.89 0.00 0.00 20.32
Average: 29 1.89 0.00 78.22 0.00 0.00 19.88
Average: 30 3.04 0.00 76.31 0.00 0.00 20.65
Average: 31 3.35 0.00 77.76 0.00 0.00 18.89
Average: 32 3.07 0.00 74.56 0.00 0.00 22.37
Average: 33 3.15 0.00 74.31 0.00 0.00 22.53
Average: 34 2.90 0.00 74.33 0.00 0.00 22.77
Average: 35 3.09 0.00 75.38 0.00 0.00 21.53
Average: 36 2.90 0.00 75.25 0.02 0.00 21.84
Average: 37 2.92 0.00 74.80 0.00 0.00 22.28
Average: 38 3.24 0.00 74.99 0.00 0.00 21.77
Average: 39 3.11 0.00 75.46 0.00 0.00 21.43
Average: 40 3.30 0.00 72.86 0.00 0.00 23.83
Average: 41 3.25 0.00 71.73 0.00 0.00 25.01
Average: 42 3.51 0.00 73.49 0.00 0.00 23.00
Average: 43 3.25 0.00 73.79 0.00 0.00 22.96
Average: 44 2.97 0.00 73.59 0.00 0.00 23.44
Average: 45 3.14 0.00 74.78 0.00 0.00 22.08
Average: 46 3.58 0.00 68.07 0.00 0.00 28.36
Average: 47 3.46 0.00 71.68 0.00 0.00 24.86
Average: 48 3.28 0.00 73.14 0.00 0.00 23.58
Average: 49 3.71 0.00 69.21 0.00 0.00 27.08
Average: 50 3.12 0.00 71.34 0.00 0.00 25.54
Average: 51 3.01 0.00 76.15 0.00 0.00 20.84
Average: 52 27.12 0.00 59.04 0.00 0.00 13.84
Average: 53 3.18 0.00 76.32 0.00 0.00 20.50
Average: 54 3.04 0.00 72.01 0.00 0.00 24.95
Average: 55 2.95 0.00 74.96 0.00 0.00 22.09
Average: 56 3.11 0.00 76.32 0.00 0.00 20.56
Average: 57 2.92 0.00 72.79 0.00 0.00 24.29
Average: 58 3.13 0.00 75.64 0.00 0.00 21.23
Average: 59 3.08 0.00 74.22 0.00 0.00 22.69
Average: 60 2.27 0.00 76.90 0.00 0.00 20.82
Average: 61 2.26 0.00 73.16 0.00 0.00 24.58
Average: 62 2.41 0.00 80.87 0.00 0.00 16.72
Average: 63 2.41 0.00 76.70 0.00 0.00 20.89
Average: 64 2.76 0.00 77.53 0.00 0.00 19.71
Average: 65 2.34 0.00 77.57 0.00 0.00 20.08
Average: 66 2.41 0.00 77.48 0.00 0.00 20.11
Average: 67 2.32 0.00 78.66 0.00 0.00 19.02
Average: 68 2.22 0.00 72.75 0.00 0.00 25.03
Average: 69 2.21 0.00 72.78 0.00 0.00 25.01
Average: 70 3.43 0.00 71.67 0.00 0.00 24.91
Average: 71 3.70 0.00 75.48 0.00 0.00 20.82
Average: 72 3.92 0.00 74.17 0.00 0.00 21.91
Average: 73 4.00 0.00 74.42 0.00 0.00 21.58
Average: 74 3.85 0.00 75.61 0.00 0.00 20.54
Average: 75 3.80 0.00 76.05 0.00 0.00 20.14
Average: 76 3.35 0.00 75.91 0.00 0.00 20.74
Average: 77 3.41 0.00 74.09 0.00 0.00 22.49
Average: 78 3.47 0.00 71.43 0.00 0.00 25.10
Average: 79 3.57 0.00 68.43 0.00 0.00 27.99
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]# ip netns exec qlbaas-ade3e627-5fda-4781-855e-25a0f5217d17 ping
20.1.1.100
PING 20.1.1.100 (20.1.1.100) 56(84) bytes of data.
>From 20.1.1.101 icmp_seq=1 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=2 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=3 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=4 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=5 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=6 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=7 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=8 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=9 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=10 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=11 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=12 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=13 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=14 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=15 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=16 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=17 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=18 Destination Host Unreachable
>From 20.1.1.101 icmp_seq=19 Destination Host Unreachable
^C
--- 20.1.1.100 ping statistics ---
20 packets transmitted, 0 received, +19 errors, 100% packet loss, time 19044ms
pipe 4
[root@nsj4 ~]# ^C
[root@nsj4 ~]# top
top - 03:44:38 up 10:20, 2 users, load average: 106.37, 107.24, 107.97
Tasks: 2897 total, 68 running, 2611 sleeping, 0 stopped, 218 zombie
%Cpu(s): 1.5 us, 21.7 sy, 0.0 ni, 76.8 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st
KiB Mem : 26396150+total, 24796382+free, 11314276 used, 4683396 buff/cache
KiB Swap: 1048572 total, 1048572 free, 0 used. 24887622+avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
7009 neutron 20 0 317952 72268 4444 R 92.9 0.0 148:13.18
neutron-openvsw
536 root rt 0 0 0 0 S 85.0 0.0 4:57.94
migration/73
1025 root 20 0 134048 61472 59008 R 70.7 0.0 80:33.70
systemd-journal
39205 root 20 0 118948 1000 776 R 50.7 0.0 0:00.71 killall
39216 root 20 0 118948 1000 776 R 45.7 0.0 0:00.64 killall
556 root rt 0 0 0 0 S 45.0 0.0 5:43.53
migration/77
38994 root 20 0 132928 4704 1220 R 45.0 0.0 0:01.46 top
183 root rt 0 0 0 0 S 42.1 0.0 5:29.76 migration/3
39219 root 20 0 118948 1004 776 R 42.1 0.0 0:00.59 killall
39236 root 20 0 118948 1000 776 R 37.9 0.0 0:00.53 killall
486 root rt 0 0 0 0 S 37.1 0.0 7:06.88
migration/63
471 root rt 0 0 0 0 S 35.7 0.0 6:16.86
migration/60
39280 root 20 0 118948 1000 776 R 32.9 0.0 0:00.46 killall
39277 root 20 0 118948 1000 776 R 30.0 0.0 0:00.42 killall
39304 root 20 0 118948 1000 776 R 30.0 0.0 0:00.42 killall
39286 root 20 0 118948 1000 776 R 29.3 0.0 0:00.41 killall
39325 root 20 0 118948 1020 776 R 29.3 0.0 0:00.41 killall
39303 root 20 0 118948 1004 776 R 27.9 0.0 0:00.39 killall
39260 root 20 0 118952 1032 776 R 27.1 0.0 0:00.38 killall
39262 root 20 0 118948 996 776 R 27.1 0.0 0:00.38 killall
39328 root 20 0 118948 996 776 R 26.4 0.0 0:00.37 killall
39278 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39290 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39349 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39353 root 20 0 118948 1000 776 R 25.7 0.0 0:00.36 killall
39293 root 20 0 118948 1004 776 R 25.0 0.0 0:00.35 killall
39300 root 20 0 118948 996 776 R 25.0 0.0 0:00.35 killall
39314 root 20 0 118948 1004 776 R 25.0 0.0 0:00.35 killall
39316 root 20 0 118948 1000 776 R 25.0 0.0 0:00.35 killall
39323 root 20 0 118948 1000 776 R 25.0 0.0 0:00.35 killall
39345 root 20 0 118948 1000 776 R 23.6 0.0 0:00.33 killall
39370 root 20 0 118948 1020 776 R 23.6 0.0 0:00.33 killall
39354 root 20 0 118948 1000 776 R 22.9 0.0 0:00.32 killall
39335 root 20 0 118948 1000 776 R 22.1 0.0 0:00.31 killall
39320 root 20 0 118948 1000 776 R 21.4 0.0 0:00.30 killall
39356 root 20 0 118952 1012 776 R 21.4 0.0 0:00.30 killall
39362 root 20 0 118948 1000 776 R 21.4 0.0 0:00.30 killall
39361 root 20 0 118948 1000 776 R 20.7 0.0 0:00.29 killall
39368 root 20 0 118948 1000 776 R 20.0 0.0 0:00.28 killall
39374 root 20 0 118952 1012 776 R 20.0 0.0 0:00.28 killall
39380 root 20 0 118948 1000 776 R 19.3 0.0 0:00.27 killall
39382 root 20 0 118952 1016 776 R 18.6 0.0 0:00.26 killall
39390 root 20 0 118948 996 776 R 17.1 0.0 0:00.24 killall
39396 root 20 0 118948 1000 776 R 17.1 0.0 0:00.24 killall
39398 root 20 0 118948 1000 776 R 16.4 0.0 0:00.23 killall
39404 root 20 0 118948 1000 776 R 16.4 0.0 0:00.23 killall
39407 root 20 0 118948 996 776 R 16.4 0.0 0:00.23 killall
39394 root 20 0 118948 996 776 R 15.7 0.0 0:00.22 killall
28742 root 20 0 0 0 0 S 13.6 0.0 4:07.63 kworker/8:0
67592 root 20 0 0 0 0 R 13.6 0.0 4:53.96
kworker/49:2
12851 root 20 0 0 0 0 S 10.7 0.0 14:01.48
kworker/16:0
25321 root 20 0 0 0 0 S 10.7 0.0 4:20.20
kworker/58:2
35314 root 20 0 0 0 0 S 10.7 0.0 8:05.81
kworker/69:1
67884 root 20 0 0 0 0 S 10.0 0.0 7:03.74
kworker/64:2
39418 root 20 0 118948 996 776 R 9.3 0.0 0:00.13 killall
39422 root 20 0 118948 992 776 R 9.3 0.0 0:00.13 killall
39426 root 20 0 118948 1000 776 R 9.3 0.0 0:00.13 killall
39428 root 20 0 118948 996 776 R 9.3 0.0 0:00.13 killall
451 root rt 0 0 0 0 S 8.6 0.0 5:35.46
migration/56
26106 root 20 0 0 0 0 S 8.6 0.0 9:16.24
kworker/65:2
[root@nsj4 ~]# free
total used free shared buff/cache available
** Affects: neutron
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1498502
Title:
LB_NFV[Alchemy]after create 301 LB pool , the lb network unstable
Status in neutron:
New
Bug description:
1 create two-arm LB with 1client and 1 backend server on a tenant
2 repeat step1 to create 350 tenants
3 after step 2 , the LB network unstable
log and cpu infor
[root@nsj4 ~]# ip netns exec qlbaas-ade3e627-5fda-4781-855e-25a0f5217d17 ping
20.1.1.100
PING 20.1.1.100 (20.1.1.100) 56(84) bytes of data.
From 20.1.1.101 icmp_seq=1 Destination Host Unreachable
From 20.1.1.101 icmp_seq=2 Destination Host Unreachable
From 20.1.1.101 icmp_seq=3 Destination Host Unreachable
From 20.1.1.101 icmp_seq=4 Destination Host Unreachable<<<<<<<<<<<<<<<<
64 bytes from 20.1.1.100: icmp_seq=6 ttl=64 time=0.100 ms
64 bytes from 20.1.1.100: icmp_seq=7 ttl=64 time=0.067 ms
64 bytes from 20.1.1.100: icmp_seq=8 ttl=64 time=0.081 ms
From 20.1.1.101 icmp_seq=5 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=9 ttl=64 time=0.054 ms
64 bytes from 20.1.1.100: icmp_seq=10 ttl=64 time=0.050 ms
64 bytes from 20.1.1.100: icmp_seq=11 ttl=64 time=0.087 ms
64 bytes from 20.1.1.100: icmp_seq=12 ttl=64 time=0.065 ms
64 bytes from 20.1.1.100: icmp_seq=13 ttl=64 time=0.079 ms
64 bytes from 20.1.1.100: icmp_seq=14 ttl=64 time=0.073 ms
64 bytes from 20.1.1.100: icmp_seq=15 ttl=64 time=0.062 ms
64 bytes from 20.1.1.100: icmp_seq=16 ttl=64 time=0.070 ms
64 bytes from 20.1.1.100: icmp_seq=17 ttl=64 time=0.058 ms
64 bytes from 20.1.1.100: icmp_seq=18 ttl=64 time=0.146 ms
64 bytes from 20.1.1.100: icmp_seq=19 ttl=64 time=0.060 ms
64 bytes from 20.1.1.100: icmp_seq=20 ttl=64 time=0.097 ms
64 bytes from 20.1.1.100: icmp_seq=21 ttl=64 time=0.114 ms
64 bytes from 20.1.1.100: icmp_seq=22 ttl=64 time=0.091 ms
64 bytes from 20.1.1.100: icmp_seq=23 ttl=64 time=0.122 ms
64 bytes from 20.1.1.100: icmp_seq=24 ttl=64 time=0.074 ms
64 bytes from 20.1.1.100: icmp_seq=25 ttl=64 time=0.091 ms
From 20.1.1.101 icmp_seq=26 Destination Host Unreachable<<<<<<<<<<<<<<<<<<
From 20.1.1.101 icmp_seq=27 Destination Host Unreachable
From 20.1.1.101 icmp_seq=28 Destination Host Unreachable
From 20.1.1.101 icmp_seq=29 Destination Host Unreachable
From 20.1.1.101 icmp_seq=30 Destination Host Unreachable
From 20.1.1.101 icmp_seq=31 Destination Host Unreachable
From 20.1.1.101 icmp_seq=32 Destination Host Unreachable
From 20.1.1.101 icmp_seq=33 Destination Host Unreachable
From 20.1.1.101 icmp_seq=34 Destination Host Unreachable
From 20.1.1.101 icmp_seq=35 Destination Host Unreachable
From 20.1.1.101 icmp_seq=36 Destination Host Unreachable
From 20.1.1.101 icmp_seq=37 Destination Host Unreachable
From 20.1.1.101 icmp_seq=38 Destination Host Unreachable
From 20.1.1.101 icmp_seq=39 Destination Host Unreachable
From 20.1.1.101 icmp_seq=40 Destination Host Unreachable
From 20.1.1.101 icmp_seq=41 Destination Host Unreachable
From 20.1.1.101 icmp_seq=42 Destination Host Unreachable
From 20.1.1.101 icmp_seq=43 Destination Host Unreachable
From 20.1.1.101 icmp_seq=44 Destination Host Unreachable
From 20.1.1.101 icmp_seq=45 Destination Host Unreachable
From 20.1.1.101 icmp_seq=46 Destination Host Unreachable
From 20.1.1.101 icmp_seq=47 Destination Host Unreachable
From 20.1.1.101 icmp_seq=48 Destination Host Unreachable
From 20.1.1.101 icmp_seq=49 Destination Host Unreachable
From 20.1.1.101 icmp_seq=50 Destination Host Unreachable
From 20.1.1.101 icmp_seq=51 Destination Host Unreachable
From 20.1.1.101 icmp_seq=52 Destination Host Unreachable
From 20.1.1.101 icmp_seq=53 Destination Host Unreachable
From 20.1.1.101 icmp_seq=54 Destination Host Unreachable
From 20.1.1.101 icmp_seq=55 Destination Host Unreachable
From 20.1.1.101 icmp_seq=56 Destination Host Unreachable
From 20.1.1.101 icmp_seq=57 Destination Host Unreachable
From 20.1.1.101 icmp_seq=58 Destination Host Unreachable
From 20.1.1.101 icmp_seq=59 Destination Host Unreachable
From 20.1.1.101 icmp_seq=60 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=64 ttl=64 time=0.117 ms
From 20.1.1.101 icmp_seq=61 Destination Host Unreachable
From 20.1.1.101 icmp_seq=62 Destination Host Unreachable
From 20.1.1.101 icmp_seq=63 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=65 ttl=64 time=0.097 ms
64 bytes from 20.1.1.100: icmp_seq=66 ttl=64 time=0.060 ms
64 bytes from 20.1.1.100: icmp_seq=67 ttl=64 time=0.083 ms
64 bytes from 20.1.1.100: icmp_seq=68 ttl=64 time=0.059 ms
64 bytes from 20.1.1.100: icmp_seq=69 ttl=64 time=0.173 ms
64 bytes from 20.1.1.100: icmp_seq=70 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=71 ttl=64 time=0.067 ms
64 bytes from 20.1.1.100: icmp_seq=72 ttl=64 time=0.075 ms
64 bytes from 20.1.1.100: icmp_seq=73 ttl=64 time=0.051 ms
64 bytes from 20.1.1.100: icmp_seq=74 ttl=64 time=0.078 ms
64 bytes from 20.1.1.100: icmp_seq=75 ttl=64 time=0.106 ms
64 bytes from 20.1.1.100: icmp_seq=76 ttl=64 time=0.100 ms
64 bytes from 20.1.1.100: icmp_seq=77 ttl=64 time=0.205 ms
64 bytes from 20.1.1.100: icmp_seq=78 ttl=64 time=0.116 ms
64 bytes from 20.1.1.100: icmp_seq=79 ttl=64 time=0.110 ms
64 bytes from 20.1.1.100: icmp_seq=80 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=81 ttl=64 time=0.068 ms
64 bytes from 20.1.1.100: icmp_seq=82 ttl=64 time=0.091 ms
64 bytes from 20.1.1.100: icmp_seq=83 ttl=64 time=0.092 ms
64 bytes from 20.1.1.100: icmp_seq=84 ttl=64 time=0.082 ms
64 bytes from 20.1.1.100: icmp_seq=85 ttl=64 time=0.056 ms
64 bytes from 20.1.1.100: icmp_seq=86 ttl=64 time=0.063 ms
64 bytes from 20.1.1.100: icmp_seq=87 ttl=64 time=0.118 ms
64 bytes from 20.1.1.100: icmp_seq=88 ttl=64 time=0.104 ms
64 bytes from 20.1.1.100: icmp_seq=89 ttl=64 time=0.074 ms
64 bytes from 20.1.1.100: icmp_seq=90 ttl=64 time=0.143 ms
64 bytes from 20.1.1.100: icmp_seq=91 ttl=64 time=0.049 ms
64 bytes from 20.1.1.100: icmp_seq=92 ttl=64 time=0.072 ms
64 bytes from 20.1.1.100: icmp_seq=93 ttl=64 time=0.076 ms
From 20.1.1.101 icmp_seq=94 Destination Host Unreachable
From 20.1.1.101 icmp_seq=95 Destination Host Unreachable
From 20.1.1.101 icmp_seq=96 Destination Host Unreachable
From 20.1.1.101 icmp_seq=97 Destination Host Unreachable
From 20.1.1.101 icmp_seq=98 Destination Host Unreachable
From 20.1.1.101 icmp_seq=99 Destination Host Unreachable
From 20.1.1.101 icmp_seq=100 Destination Host Unreachable
From 20.1.1.101 icmp_seq=101 Destination Host Unreachable
From 20.1.1.101 icmp_seq=102 Destination Host Unreachable
From 20.1.1.101 icmp_seq=103 Destination Host Unreachable
From 20.1.1.101 icmp_seq=104 Destination Host Unreachable
From 20.1.1.101 icmp_seq=105 Destination Host Unreachable
From 20.1.1.101 icmp_seq=106 Destination Host Unreachable
From 20.1.1.101 icmp_seq=107 Destination Host Unreachable
From 20.1.1.101 icmp_seq=108 Destination Host Unreachable
From 20.1.1.101 icmp_seq=109 Destination Host Unreachable
From 20.1.1.101 icmp_seq=110 Destination Host Unreachable
From 20.1.1.101 icmp_seq=111 Destination Host Unreachable
From 20.1.1.101 icmp_seq=112 Destination Host Unreachable
From 20.1.1.101 icmp_seq=113 Destination Host Unreachable
From 20.1.1.101 icmp_seq=114 Destination Host Unreachable
From 20.1.1.101 icmp_seq=115 Destination Host Unreachable
From 20.1.1.101 icmp_seq=116 Destination Host Unreachable
From 20.1.1.101 icmp_seq=117 Destination Host Unreachable
From 20.1.1.101 icmp_seq=118 Destination Host Unreachable
From 20.1.1.101 icmp_seq=119 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=123 ttl=64 time=0.079 ms
From 20.1.1.101 icmp_seq=120 Destination Host Unreachable
From 20.1.1.101 icmp_seq=121 Destination Host Unreachable
From 20.1.1.101 icmp_seq=122 Destination Host Unreachable
64 bytes from 20.1.1.100: icmp_seq=124 ttl=64 time=0.066 ms
64 bytes from 20.1.1.100: icmp_seq=125 ttl=64 time=0.050 ms
64 bytes from 20.1.1.100: icmp_seq=126 ttl=64 time=0.039 ms
64 bytes from 20.1.1.100: icmp_seq=127 ttl=64 time=0.088 ms
64 bytes from 20.1.1.100: icmp_seq=128 ttl=64 time=0.082 ms
64 bytes from 20.1.1.100: icmp_seq=129 ttl=64 time=0.079 ms
64 bytes from 20.1.1.100: icmp_seq=130 ttl=64 time=0.104 ms
64 bytes from 20.1.1.100: icmp_seq=131 ttl=64 time=0.069 ms
64 bytes from 20.1.1.100: icmp_seq=132 ttl=64 time=0.101 ms
64 bytes from 20.1.1.100: icmp_seq=133 ttl=64 time=0.088 ms
From 20.1.1.101 icmp_seq=134 Destination Host Unreachable
From 20.1.1.101 icmp_seq=135 Destination Host Unreachable
From 20.1.1.101 icmp_seq=136 Destination Host Unreachable
From 20.1.1.101 icmp_seq=137 Destination Host Unreachable
From 20.1.1.101 icmp_seq=138 Destination Host Unreachable
From 20.1.1.101 icmp_seq=139 Destination Host Unreachable
From 20.1.1.101 icmp_seq=140 Destination Host Unreachable
From 20.1.1.101 icmp_seq=141 Destination Host Unreachable
From 20.1.1.101 icmp_seq=142 Destination Host Unreachable
From 20.1.1.101 icmp_seq=143 Destination Host Unreachable
From 20.1.1.101 icmp_seq=144 Destination Host Unreachable
From 20.1.1.101 icmp_seq=145 Destination Host Unreachable
From 20.1.1.101 icmp_seq=146 Destination Host Unreachable
From 20.1.1.101 icmp_seq=147 Destination Host Unreachable
From 20.1.1.101 icmp_seq=148 Destination Host Unreachable
From 20.1.1.101 icmp_seq=149 Destination Host Unreachable
From 20.1.1.101 icmp_seq=150 Destination Host Unreachable
From 20.1.1.101 icmp_seq=151 Destination Host Unreachable
From 20.1.1.101 icmp_seq=152 Destination Host Unreachable
From 20.1.1.101 icmp_seq=153 Destination Host Unreachable
From 20.1.1.101 icmp_seq=154 Destination Host Unreachable
From 20.1.1.101 icmp_seq=155 Destination Host Unreachable
From 20.1.1.101 icmp_seq=156 Destination Host Unreachable
From 20.1.1.101 icmp_seq=157 Destination Host Unreachable
From 20.1.1.101 icmp_seq=158 Destination Host Unreachable
From 20.1.1.101 icmp_seq=159 Destination Host Unreachable
From 20.1.1.101 icmp_seq=160 Destination Host Unreachable
From 20.1.1.101 icmp_seq=161 Destination Host Unreachable
From 20.1.1.101 icmp_seq=162 Destination Host Unreachable
From 20.1.1.101 icmp_seq=163 Destination Host Unreachable
From 20.1.1.101 icmp_seq=164 Destination Host Unreachable
From 20.1.1.101 icmp_seq=165 Destination Host Unreachable
From 20.1.1.101 icmp_seq=166 Destination Host Unreachable
From 20.1.1.101 icmp_seq=167 Destination Host Unreachable
From 20.1.1.101 icmp_seq=168 Destination Host Unreachable
From 20.1.1.101 icmp_seq=169 Destination Host Unreachable
From 20.1.1.101 icmp_seq=170 Destination Host Unreachable
From 20.1.1.101 icmp_seq=171 Destination Host Unreachable
From 20.1.1.101 icmp_seq=172 Destination Host Unreachable
From 20.1.1.101 icmp_seq=173 Destination Host Unreachable
From 20.1.1.101 icmp_seq=174 Destination Host Unreachable
From 20.1.1.101 icmp_seq=175 Destination Host Unreachable
From 20.1.1.101 icmp_seq=176 Destination Host Unreachable
From 20.1.1.101 icmp_seq=177 Destination Host Unreachable
From 20.1.1.101 icmp_seq=178 Destination Host Unreachable
From 20.1.1.101 icmp_seq=179 Destination Host Unreachable
From 20.1.1.101 icmp_seq=180 Destination Host Unreachable
From 20.1.1.101 icmp_seq=181 Destination Host Unreachable
From 20.1.1.101 icmp_seq=182 Destination Host Unreachable
From 20.1.1.101 icmp_seq=183 Destination Host Unreachable
From 20.1.1.101 icmp_seq=184 Destination Host Unreachable
From 20.1.1.101 icmp_seq=185 Destination Host Unreachable
From 20.1.1.101 icmp_seq=186 Destination Host Unreachable
From 20.1.1.101 icmp_seq=187 Destination Host Unreachable
From 20.1.1.101 icmp_seq=188 Destination Host Unreachable
From 20.1.1.101 icmp_seq=189 Destination Host Unreachable
^C
--- 20.1.1.100 ping statistics ---
189 packets transmitted, 61 received, +128 errors, 67% packet loss, time
188402ms
rtt min/avg/max/mdev = 0.039/0.085/0.205/0.030 ms, pipe 4
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]# sar -P ALL 60 1
Linux 3.10.0-229.11.1.el7.x86_64 (nsj4) 08/28/2015 _x86_64_
(80 CPU)
03:42:48 AM CPU %user %nice %system %iowait %steal
%idle
03:43:48 AM all 3.09 0.00 75.23 0.00 0.00
21.68
03:43:48 AM 0 2.24 0.00 84.93 0.00 0.00
12.83
03:43:48 AM 1 2.48 0.00 81.40 0.00 0.00
16.12
03:43:48 AM 2 2.36 0.00 77.36 0.00 0.00
20.28
03:43:48 AM 3 2.43 0.00 79.85 0.00 0.00
17.72
03:43:48 AM 4 2.26 0.00 76.10 0.00 0.00
21.64
03:43:48 AM 5 2.60 0.00 76.77 0.00 0.00
20.62
03:43:48 AM 6 2.77 0.00 78.80 0.00 0.00
18.43
03:43:48 AM 7 2.48 0.00 75.81 0.00 0.00
21.71
03:43:48 AM 8 2.49 0.00 78.44 0.00 0.00
19.07
03:43:48 AM 9 3.25 0.00 76.81 0.00 0.00
19.94
03:43:48 AM 10 2.80 0.00 79.90 0.00 0.00
17.30
03:43:48 AM 11 2.41 0.00 77.32 0.00 0.00
20.28
03:43:48 AM 12 2.17 0.00 75.10 0.00 0.00
22.73
03:43:48 AM 13 2.92 0.00 77.01 0.00 0.00
20.06
03:43:48 AM 14 2.41 0.00 76.10 0.00 0.00
21.48
03:43:48 AM 15 2.08 0.00 73.94 0.00 0.00
23.98
03:43:48 AM 16 2.29 0.00 74.58 0.00 0.00
23.14
03:43:48 AM 17 2.72 0.00 76.94 0.00 0.00
20.34
03:43:48 AM 18 2.38 0.00 75.70 0.00 0.00
21.92
03:43:48 AM 19 2.37 0.00 76.11 0.00 0.00
21.53
03:43:48 AM 20 1.86 0.00 78.34 0.00 0.00
19.79
03:43:48 AM 21 1.76 0.00 77.52 0.00 0.00
20.72
03:43:48 AM 22 1.76 0.00 77.80 0.00 0.00
20.44
03:43:48 AM 23 1.84 0.00 75.49 0.00 0.00
22.67
03:43:48 AM 24 1.59 0.00 74.62 0.00 0.00
23.79
03:43:48 AM 25 1.88 0.00 74.26 0.00 0.00
23.86
03:43:48 AM 26 1.79 0.00 73.82 0.00 0.00
24.38
03:43:48 AM 27 1.58 0.00 76.51 0.00 0.00
21.92
03:43:48 AM 28 1.79 0.00 77.89 0.00 0.00
20.32
03:43:48 AM 29 1.89 0.00 78.22 0.00 0.00
19.88
03:43:48 AM 30 3.04 0.00 76.31 0.00 0.00
20.65
03:43:48 AM 31 3.35 0.00 77.76 0.00 0.00
18.89
03:43:48 AM 32 3.07 0.00 74.56 0.00 0.00
22.37
03:43:48 AM 33 3.15 0.00 74.31 0.00 0.00
22.53
03:43:48 AM 34 2.90 0.00 74.33 0.00 0.00
22.77
03:43:48 AM 35 3.09 0.00 75.38 0.00 0.00
21.53
03:43:48 AM 36 2.90 0.00 75.25 0.02 0.00
21.84
03:43:48 AM 37 2.92 0.00 74.80 0.00 0.00
22.28
03:43:48 AM 38 3.24 0.00 74.99 0.00 0.00
21.77
03:43:48 AM 39 3.11 0.00 75.46 0.00 0.00
21.43
03:43:48 AM 40 3.30 0.00 72.86 0.00 0.00
23.83
03:43:48 AM 41 3.25 0.00 71.73 0.00 0.00
25.01
03:43:48 AM 42 3.51 0.00 73.49 0.00 0.00
23.00
03:43:48 AM 43 3.25 0.00 73.79 0.00 0.00
22.96
03:43:48 AM 44 2.97 0.00 73.59 0.00 0.00
23.44
03:43:48 AM 45 3.14 0.00 74.78 0.00 0.00
22.08
03:43:48 AM 46 3.58 0.00 68.07 0.00 0.00
28.36
03:43:48 AM 47 3.46 0.00 71.68 0.00 0.00
24.86
03:43:48 AM 48 3.28 0.00 73.14 0.00 0.00
23.58
03:43:48 AM 49 3.71 0.00 69.21 0.00 0.00
27.08
03:43:48 AM 50 3.12 0.00 71.34 0.00 0.00
25.54
03:43:48 AM 51 3.01 0.00 76.15 0.00 0.00
20.84
03:43:48 AM 52 27.12 0.00 59.04 0.00 0.00
13.84
03:43:48 AM 53 3.18 0.00 76.32 0.00 0.00
20.50
03:43:48 AM 54 3.04 0.00 72.01 0.00 0.00
24.95
03:43:48 AM 55 2.95 0.00 74.96 0.00 0.00
22.09
03:43:48 AM 56 3.11 0.00 76.32 0.00 0.00
20.56
03:43:48 AM 57 2.92 0.00 72.79 0.00 0.00
24.29
03:43:48 AM 58 3.13 0.00 75.64 0.00 0.00
21.23
03:43:48 AM 59 3.08 0.00 74.22 0.00 0.00
22.69
03:43:48 AM 60 2.27 0.00 76.90 0.00 0.00
20.82
03:43:48 AM 61 2.26 0.00 73.16 0.00 0.00
24.58
03:43:48 AM 62 2.41 0.00 80.87 0.00 0.00
16.72
03:43:48 AM 63 2.41 0.00 76.70 0.00 0.00
20.89
03:43:48 AM 64 2.76 0.00 77.53 0.00 0.00
19.71
03:43:48 AM 65 2.34 0.00 77.57 0.00 0.00
20.08
03:43:48 AM 66 2.41 0.00 77.48 0.00 0.00
20.11
03:43:48 AM 67 2.32 0.00 78.66 0.00 0.00
19.02
03:43:48 AM 68 2.22 0.00 72.75 0.00 0.00
25.03
03:43:48 AM 69 2.21 0.00 72.78 0.00 0.00
25.01
03:43:48 AM 70 3.43 0.00 71.67 0.00 0.00
24.91
03:43:48 AM 71 3.70 0.00 75.48 0.00 0.00
20.82
03:43:48 AM 72 3.92 0.00 74.17 0.00 0.00
21.91
03:43:48 AM 73 4.00 0.00 74.42 0.00 0.00
21.58
03:43:48 AM 74 3.85 0.00 75.61 0.00 0.00
20.54
03:43:48 AM 75 3.80 0.00 76.05 0.00 0.00
20.14
03:43:48 AM 76 3.35 0.00 75.91 0.00 0.00
20.74
03:43:48 AM 77 3.41 0.00 74.09 0.00 0.00
22.49
03:43:48 AM 78 3.47 0.00 71.43 0.00 0.00
25.10
03:43:48 AM 79 3.57 0.00 68.43 0.00 0.00
27.99
Average: CPU %user %nice %system %iowait %steal
%idle
Average: all 3.09 0.00 75.23 0.00 0.00
21.68
Average: 0 2.24 0.00 84.93 0.00 0.00
12.83
Average: 1 2.48 0.00 81.40 0.00 0.00
16.12
Average: 2 2.36 0.00 77.36 0.00 0.00
20.28
Average: 3 2.43 0.00 79.85 0.00 0.00
17.72
Average: 4 2.26 0.00 76.10 0.00 0.00
21.64
Average: 5 2.60 0.00 76.77 0.00 0.00
20.62
Average: 6 2.77 0.00 78.80 0.00 0.00
18.43
Average: 7 2.48 0.00 75.81 0.00 0.00
21.71
Average: 8 2.49 0.00 78.44 0.00 0.00
19.07
Average: 9 3.25 0.00 76.81 0.00 0.00
19.94
Average: 10 2.80 0.00 79.90 0.00 0.00
17.30
Average: 11 2.41 0.00 77.32 0.00 0.00
20.28
Average: 12 2.17 0.00 75.10 0.00 0.00
22.73
Average: 13 2.92 0.00 77.01 0.00 0.00
20.06
Average: 14 2.41 0.00 76.10 0.00 0.00
21.48
Average: 15 2.08 0.00 73.94 0.00 0.00
23.98
Average: 16 2.29 0.00 74.58 0.00 0.00
23.14
Average: 17 2.72 0.00 76.94 0.00 0.00
20.34
Average: 18 2.38 0.00 75.70 0.00 0.00
21.92
Average: 19 2.37 0.00 76.11 0.00 0.00
21.53
Average: 20 1.86 0.00 78.34 0.00 0.00
19.79
Average: 21 1.76 0.00 77.52 0.00 0.00
20.72
Average: 22 1.76 0.00 77.80 0.00 0.00
20.44
Average: 23 1.84 0.00 75.49 0.00 0.00
22.67
Average: 24 1.59 0.00 74.62 0.00 0.00
23.79
Average: 25 1.88 0.00 74.26 0.00 0.00
23.86
Average: 26 1.79 0.00 73.82 0.00 0.00
24.38
Average: 27 1.58 0.00 76.51 0.00 0.00
21.92
Average: 28 1.79 0.00 77.89 0.00 0.00
20.32
Average: 29 1.89 0.00 78.22 0.00 0.00
19.88
Average: 30 3.04 0.00 76.31 0.00 0.00
20.65
Average: 31 3.35 0.00 77.76 0.00 0.00
18.89
Average: 32 3.07 0.00 74.56 0.00 0.00
22.37
Average: 33 3.15 0.00 74.31 0.00 0.00
22.53
Average: 34 2.90 0.00 74.33 0.00 0.00
22.77
Average: 35 3.09 0.00 75.38 0.00 0.00
21.53
Average: 36 2.90 0.00 75.25 0.02 0.00
21.84
Average: 37 2.92 0.00 74.80 0.00 0.00
22.28
Average: 38 3.24 0.00 74.99 0.00 0.00
21.77
Average: 39 3.11 0.00 75.46 0.00 0.00
21.43
Average: 40 3.30 0.00 72.86 0.00 0.00
23.83
Average: 41 3.25 0.00 71.73 0.00 0.00
25.01
Average: 42 3.51 0.00 73.49 0.00 0.00
23.00
Average: 43 3.25 0.00 73.79 0.00 0.00
22.96
Average: 44 2.97 0.00 73.59 0.00 0.00
23.44
Average: 45 3.14 0.00 74.78 0.00 0.00
22.08
Average: 46 3.58 0.00 68.07 0.00 0.00
28.36
Average: 47 3.46 0.00 71.68 0.00 0.00
24.86
Average: 48 3.28 0.00 73.14 0.00 0.00
23.58
Average: 49 3.71 0.00 69.21 0.00 0.00
27.08
Average: 50 3.12 0.00 71.34 0.00 0.00
25.54
Average: 51 3.01 0.00 76.15 0.00 0.00
20.84
Average: 52 27.12 0.00 59.04 0.00 0.00
13.84
Average: 53 3.18 0.00 76.32 0.00 0.00
20.50
Average: 54 3.04 0.00 72.01 0.00 0.00
24.95
Average: 55 2.95 0.00 74.96 0.00 0.00
22.09
Average: 56 3.11 0.00 76.32 0.00 0.00
20.56
Average: 57 2.92 0.00 72.79 0.00 0.00
24.29
Average: 58 3.13 0.00 75.64 0.00 0.00
21.23
Average: 59 3.08 0.00 74.22 0.00 0.00
22.69
Average: 60 2.27 0.00 76.90 0.00 0.00
20.82
Average: 61 2.26 0.00 73.16 0.00 0.00
24.58
Average: 62 2.41 0.00 80.87 0.00 0.00
16.72
Average: 63 2.41 0.00 76.70 0.00 0.00
20.89
Average: 64 2.76 0.00 77.53 0.00 0.00
19.71
Average: 65 2.34 0.00 77.57 0.00 0.00
20.08
Average: 66 2.41 0.00 77.48 0.00 0.00
20.11
Average: 67 2.32 0.00 78.66 0.00 0.00
19.02
Average: 68 2.22 0.00 72.75 0.00 0.00
25.03
Average: 69 2.21 0.00 72.78 0.00 0.00
25.01
Average: 70 3.43 0.00 71.67 0.00 0.00
24.91
Average: 71 3.70 0.00 75.48 0.00 0.00
20.82
Average: 72 3.92 0.00 74.17 0.00 0.00
21.91
Average: 73 4.00 0.00 74.42 0.00 0.00
21.58
Average: 74 3.85 0.00 75.61 0.00 0.00
20.54
Average: 75 3.80 0.00 76.05 0.00 0.00
20.14
Average: 76 3.35 0.00 75.91 0.00 0.00
20.74
Average: 77 3.41 0.00 74.09 0.00 0.00
22.49
Average: 78 3.47 0.00 71.43 0.00 0.00
25.10
Average: 79 3.57 0.00 68.43 0.00 0.00
27.99
[root@nsj4 ~]#
[root@nsj4 ~]#
[root@nsj4 ~]# ip netns exec qlbaas-ade3e627-5fda-4781-855e-25a0f5217d17 ping
20.1.1.100
PING 20.1.1.100 (20.1.1.100) 56(84) bytes of data.
From 20.1.1.101 icmp_seq=1 Destination Host Unreachable
From 20.1.1.101 icmp_seq=2 Destination Host Unreachable
From 20.1.1.101 icmp_seq=3 Destination Host Unreachable
From 20.1.1.101 icmp_seq=4 Destination Host Unreachable
From 20.1.1.101 icmp_seq=5 Destination Host Unreachable
From 20.1.1.101 icmp_seq=6 Destination Host Unreachable
From 20.1.1.101 icmp_seq=7 Destination Host Unreachable
From 20.1.1.101 icmp_seq=8 Destination Host Unreachable
From 20.1.1.101 icmp_seq=9 Destination Host Unreachable
From 20.1.1.101 icmp_seq=10 Destination Host Unreachable
From 20.1.1.101 icmp_seq=11 Destination Host Unreachable
From 20.1.1.101 icmp_seq=12 Destination Host Unreachable
From 20.1.1.101 icmp_seq=13 Destination Host Unreachable
From 20.1.1.101 icmp_seq=14 Destination Host Unreachable
From 20.1.1.101 icmp_seq=15 Destination Host Unreachable
From 20.1.1.101 icmp_seq=16 Destination Host Unreachable
From 20.1.1.101 icmp_seq=17 Destination Host Unreachable
From 20.1.1.101 icmp_seq=18 Destination Host Unreachable
From 20.1.1.101 icmp_seq=19 Destination Host Unreachable
^C
--- 20.1.1.100 ping statistics ---
20 packets transmitted, 0 received, +19 errors, 100% packet loss, time 19044ms
pipe 4
[root@nsj4 ~]# ^C
[root@nsj4 ~]# top
top - 03:44:38 up 10:20, 2 users, load average: 106.37, 107.24, 107.97
Tasks: 2897 total, 68 running, 2611 sleeping, 0 stopped, 218 zombie
%Cpu(s): 1.5 us, 21.7 sy, 0.0 ni, 76.8 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0
st
KiB Mem : 26396150+total, 24796382+free, 11314276 used, 4683396 buff/cache
KiB Swap: 1048572 total, 1048572 free, 0 used. 24887622+avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
7009 neutron 20 0 317952 72268 4444 R 92.9 0.0 148:13.18
neutron-openvsw
536 root rt 0 0 0 0 S 85.0 0.0 4:57.94
migration/73
1025 root 20 0 134048 61472 59008 R 70.7 0.0 80:33.70
systemd-journal
39205 root 20 0 118948 1000 776 R 50.7 0.0 0:00.71 killall
39216 root 20 0 118948 1000 776 R 45.7 0.0 0:00.64 killall
556 root rt 0 0 0 0 S 45.0 0.0 5:43.53
migration/77
38994 root 20 0 132928 4704 1220 R 45.0 0.0 0:01.46 top
183 root rt 0 0 0 0 S 42.1 0.0 5:29.76
migration/3
39219 root 20 0 118948 1004 776 R 42.1 0.0 0:00.59 killall
39236 root 20 0 118948 1000 776 R 37.9 0.0 0:00.53 killall
486 root rt 0 0 0 0 S 37.1 0.0 7:06.88
migration/63
471 root rt 0 0 0 0 S 35.7 0.0 6:16.86
migration/60
39280 root 20 0 118948 1000 776 R 32.9 0.0 0:00.46 killall
39277 root 20 0 118948 1000 776 R 30.0 0.0 0:00.42 killall
39304 root 20 0 118948 1000 776 R 30.0 0.0 0:00.42 killall
39286 root 20 0 118948 1000 776 R 29.3 0.0 0:00.41 killall
39325 root 20 0 118948 1020 776 R 29.3 0.0 0:00.41 killall
39303 root 20 0 118948 1004 776 R 27.9 0.0 0:00.39 killall
39260 root 20 0 118952 1032 776 R 27.1 0.0 0:00.38 killall
39262 root 20 0 118948 996 776 R 27.1 0.0 0:00.38 killall
39328 root 20 0 118948 996 776 R 26.4 0.0 0:00.37 killall
39278 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39290 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39349 root 20 0 118948 1004 776 R 25.7 0.0 0:00.36 killall
39353 root 20 0 118948 1000 776 R 25.7 0.0 0:00.36 killall
39293 root 20 0 118948 1004 776 R 25.0 0.0 0:00.35 killall
39300 root 20 0 118948 996 776 R 25.0 0.0 0:00.35 killall
39314 root 20 0 118948 1004 776 R 25.0 0.0 0:00.35 killall
39316 root 20 0 118948 1000 776 R 25.0 0.0 0:00.35 killall
39323 root 20 0 118948 1000 776 R 25.0 0.0 0:00.35 killall
39345 root 20 0 118948 1000 776 R 23.6 0.0 0:00.33 killall
39370 root 20 0 118948 1020 776 R 23.6 0.0 0:00.33 killall
39354 root 20 0 118948 1000 776 R 22.9 0.0 0:00.32 killall
39335 root 20 0 118948 1000 776 R 22.1 0.0 0:00.31 killall
39320 root 20 0 118948 1000 776 R 21.4 0.0 0:00.30 killall
39356 root 20 0 118952 1012 776 R 21.4 0.0 0:00.30 killall
39362 root 20 0 118948 1000 776 R 21.4 0.0 0:00.30 killall
39361 root 20 0 118948 1000 776 R 20.7 0.0 0:00.29 killall
39368 root 20 0 118948 1000 776 R 20.0 0.0 0:00.28 killall
39374 root 20 0 118952 1012 776 R 20.0 0.0 0:00.28 killall
39380 root 20 0 118948 1000 776 R 19.3 0.0 0:00.27 killall
39382 root 20 0 118952 1016 776 R 18.6 0.0 0:00.26 killall
39390 root 20 0 118948 996 776 R 17.1 0.0 0:00.24 killall
39396 root 20 0 118948 1000 776 R 17.1 0.0 0:00.24 killall
39398 root 20 0 118948 1000 776 R 16.4 0.0 0:00.23 killall
39404 root 20 0 118948 1000 776 R 16.4 0.0 0:00.23 killall
39407 root 20 0 118948 996 776 R 16.4 0.0 0:00.23 killall
39394 root 20 0 118948 996 776 R 15.7 0.0 0:00.22 killall
28742 root 20 0 0 0 0 S 13.6 0.0 4:07.63
kworker/8:0
67592 root 20 0 0 0 0 R 13.6 0.0 4:53.96
kworker/49:2
12851 root 20 0 0 0 0 S 10.7 0.0 14:01.48
kworker/16:0
25321 root 20 0 0 0 0 S 10.7 0.0 4:20.20
kworker/58:2
35314 root 20 0 0 0 0 S 10.7 0.0 8:05.81
kworker/69:1
67884 root 20 0 0 0 0 S 10.0 0.0 7:03.74
kworker/64:2
39418 root 20 0 118948 996 776 R 9.3 0.0 0:00.13 killall
39422 root 20 0 118948 992 776 R 9.3 0.0 0:00.13 killall
39426 root 20 0 118948 1000 776 R 9.3 0.0 0:00.13 killall
39428 root 20 0 118948 996 776 R 9.3 0.0 0:00.13 killall
451 root rt 0 0 0 0 S 8.6 0.0 5:35.46
migration/56
26106 root 20 0 0 0 0 S 8.6 0.0 9:16.24
kworker/65:2
[root@nsj4 ~]# free
total used free shared buff/cache
available
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1498502/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
