Hi All On Monday I reverted the SmartOS distro being used back to vanilla Joyent and upgraded to the latest version.
I was using the UQ EAIT builds found here: http://imgapi.uqcloud.net/builds So far the issue hasn’t reoccured but I’ll keep an eye on it. arekinath (maintainer of the builds) thought it might be related to the AMD patches that he introduced, but it appears to be resolved all the same. Will keep an eye on it though. Thanks, Dave On 31 Aug 2014, at 7:28 pm, David Finster via smartos-discuss <[email protected]<mailto:[email protected]>> wrote: Which VirtIO drivers you have? NIC: Red Hat VirtIO Ethernet Adapter Driver Provider: Red Hat Inc. Driver Date: 20/11/2013 Driver Version: 62.65.104.7400 Digital Signer: Red Hat. Inc Disk: Red Hat VirtIO SCSI Controller Driver Provider: Red Hat Inc. Driver Date: 20/11/2013 Driver Version: 62.65.104.7400 Digital Signer: Red Hat. Inc Latest ones from the Fedora/Redhat site. I don’t get a power management option for the NIC. (BTW, enable zfs compression pool wide if not already). Any particular reason? Currently turned off. On 31 Aug 2014, at 6:48 pm, Micky <[email protected]<mailto:[email protected]>> wrote: Those look fine. Now let's get back to Windows side. Which VirtIO drivers you have? (BTW, enable zfs compression pool wide if not already). On Sun, Aug 31, 2014 at 1:30 PM, David Finster <[email protected]<mailto:[email protected]>> wrote: (Forgot to reply all again…) It’s a curious issue for me too in that it’s only affecting some KVMs and not others and the bizarre behaviour being exhibited around the debugging process. VM XML: <!DOCTYPE zone PUBLIC "-//Sun Microsystems Inc//DTD Zones//EN" "file:///usr/share/lib/xml/dtd/zonecfg.dtd.1"> <!-- DO NOT EDIT THIS FILE. Use zonecfg(1M) instead. --> <zone name="122a2197-9889-42b3-9e00-71baface6f59" zonepath="/zones/122a2197-9889-42b3-9e00-71baface6f59" autoboot="false" brand="kvm" ip-type="exclusive" limitpriv="default,-file_link_any,-net_access,-proc_fork,-proc_info,-proc_session" debugid="40"> <attr name="vm-version" type="string" value="1"/> <attr name="create-timestamp" type="string" value="2014-06-06T13:37:15.492Z"/> <rctl name="zone.cpu-shares"> <rctl-value priv="privileged" limit="100" action="none"/> </rctl> <rctl name="zone.zfs-io-priority"> <rctl-value priv="privileged" limit="100" action="none"/> </rctl> <rctl name="zone.max-lwps"> <rctl-value priv="privileged" limit="2000" action="deny"/> </rctl> <attr name="billing-id" type="string" value="00000000-0000-0000-0000-000000000000"/> <attr name="owner-uuid" type="string" value="00000000-0000-0000-0000-000000000000"/> <attr name="hostname" type="string" value="HO-CLUS-08"/> <attr name="resolvers" type="string" value="10.4.2.1,10.4.2.7"/> <attr name="alias" type="string" value="aG8tY2x1cy0wOA=="/> <attr name="vcpus" type="string" value="2"/> <attr name="vnc-port" type="string" value="37422"/> <device match="/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk0"> <net-attr name="boot" value="true"/> <net-attr name="model" value="virtio"/> <net-attr name="media" value="disk"/> <net-attr name="size" value="102400"/> </device> <device match="/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk1"> <net-attr name="boot" value="false"/> <net-attr name="model" value="virtio"/> <net-attr name="media" value="disk"/> <net-attr name="size" value="307200"/> </device> <network physical="net0" mac-addr="42:7f:f8:fc:90:c8" global-nic="admin"> <net-attr name="gateway" value="10.4.2.215"/> <net-attr name="ip" value="10.4.3.253"/> <net-attr name="netmask" value="255.255.254.0"/> <net-attr name="model" value="virtio"/> <net-attr name="dhcp_server" value="false"/> <net-attr name="allow_dhcp_spoofing" value="false"/> <net-attr name="allow_ip_spoofing" value="true"/> <net-attr name="allow_mac_spoofing" value="false"/> <net-attr name="allow_restricted_traffic" value="false"/> <net-attr name="allow_unfiltered_promisc" value="false"/> <net-attr name="primary" value="true"/> </network> <rctl name="zone.max-physical-memory"> <rctl-value priv="privileged" limit="4294967296" action="deny"/> </rctl> <rctl name="zone.max-locked-memory"> <rctl-value priv="privileged" limit="4294967296" action="deny"/> </rctl> <rctl name="zone.max-swap"> <rctl-value priv="privileged" limit="4294967296" action="deny"/> </rctl> <attr name="ram" type="string" value="3072"/> <attr name="vm-autoboot" type="string" value="true"/> </zone> Sysinfo: { "Live Image": "20140717T041004Z", "System Type": "SunOS", "Boot Time": "1405744711", "Manufacturer": "Supermicro", "Product": "X9DRi-LN4+/X9DR3-LN4+", "Serial Number": "0123456789", "SKU Number": "To be filled by O.E.M.", "HW Version": "0123456789", "HW Family": "To be filled by O.E.M.", "Setup": "", "VM Capable": true, "CPU Type": "Intel(R) Xeon(R) CPU E5-2640 0 @ 2.50GHz", "CPU Virtualization": "vmx", "CPU Physical Cores": 2, "UUID": "00000000-0000-0000-0000-00259085462c", "Hostname": "00-25-90-85-46-2c", "CPU Total Cores": 24, "MiB of Memory": "131039", "Zpool": "zones", "Zpool Disks": "c0t5000C5002FF03DB1d0,c0t55CD2E404B4F0D8Dd0,c10t5000CCA038009085d0,c10t5000CCA03800A41Dd0,c10t5000CCA03800B6E1d0,c10t5000CCA03800B7C1d0,c10t5000CCA03800B835d0,c10t5000CCA03800B91Dd0,c10t5000CCA03800BA65d0,c10t5000CCA041D69C31d0,c10t5000CCA041D7E979d0,c1t5000CCA04181C0F9d0,c2t5000CCA04174501Dd0,c3t5000CCA04181C3F9d0,c4t5000CCA04181C3A5d0,c5t5000CCA04181BEDDd0,c6t5000CCA04181BF9Dd0,c7t5000CCA04173EC79d0", "Zpool Profile": "mirror", "Zpool Creation": 1393123572, "Zpool Size in GiB": 6446, "Disks": { "c0t5000C5002FF03DB1d0": {"Size in GB": 120}, "c0t55CD2E404B4F0D8Dd0": {"Size in GB": 120}, "c10t5000CCA038009085d0": {"Size in GB": 600}, "c10t5000CCA03800A41Dd0": {"Size in GB": 600}, "c10t5000CCA03800B6E1d0": {"Size in GB": 600}, "c10t5000CCA03800B7C1d0": {"Size in GB": 600}, "c10t5000CCA03800B835d0": {"Size in GB": 600}, "c10t5000CCA03800B91Dd0": {"Size in GB": 600}, "c10t5000CCA03800BA65d0": {"Size in GB": 600}, "c10t5000CCA041D69C31d0": {"Size in GB": 600}, "c10t5000CCA041D7E979d0": {"Size in GB": 600}, "c1t5000CCA04181C0F9d0": {"Size in GB": 600}, "c2t5000CCA04174501Dd0": {"Size in GB": 600}, "c3t5000CCA04181C3F9d0": {"Size in GB": 600}, "c4t5000CCA04181C3A5d0": {"Size in GB": 600}, "c5t5000CCA04181BEDDd0": {"Size in GB": 600}, "c6t5000CCA04181BF9Dd0": {"Size in GB": 600}, "c7t5000CCA04173EC79d0": {"Size in GB": 600} }, "Boot Parameters": { "console": "text", "root_shadow": “stuff!", "smartos": "true" }, "Network Interfaces": { "igb0": {"MAC Address": "00:25:90:85:46:2c", "ip4addr": "", "Link Status": "up", "NIC Names": []}, "igb1": {"MAC Address": "00:25:90:85:46:2d", "ip4addr": "", "Link Status": "up", "NIC Names": []}, "igb2": {"MAC Address": "00:25:90:85:46:2e", "ip4addr": "", "Link Status": "up", "NIC Names": []}, "igb3": {"MAC Address": "00:25:90:85:46:2f", "ip4addr": "", "Link Status": "up", "NIC Names": []}, "aggr0": {"MAC Address": "00:25:90:85:46:2c", "ip4addr": "10.4.3.3", "Link Status": "up", "NIC Names": ["admin"]} }, "Virtual Network Interfaces": { }, "Link Aggregations": { "aggr0": {"LACP mode": "active", "Interfaces": ["igb0", "igb1", "igb2", "igb3"]} } } Vmadm get: { "zonename": "122a2197-9889-42b3-9e00-71baface6f59", "autoboot": true, "brand": "kvm", "limit_priv": "default,-file_link_any,-net_access,-proc_fork,-proc_info,-proc_session", "v": 1, "create_timestamp": "2014-06-06T13:37:15.492Z", "cpu_shares": 100, "zfs_io_priority": 100, "max_lwps": 2000, "billing_id": "00000000-0000-0000-0000-000000000000", "owner_uuid": "00000000-0000-0000-0000-000000000000", "hostname": “veryverysleepykvm", "resolvers": [ "10.4.2.1", "10.4.2.7" ], "alias": "veryverysleepykvm", "vcpus": 2, "vnc_port": 37422, "disks": [ { "path": "/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk0", "boot": true, "model": "virtio", "media": "disk", "zfs_filesystem": "zones/122a2197-9889-42b3-9e00-71baface6f59-disk0", "zpool": "zones", "size": 102400, "compression": "off", "refreservation": 102400, "block_size": 8192 }, { "path": "/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk1", "boot": false, "model": "virtio", "media": "disk", "zfs_filesystem": "zones/122a2197-9889-42b3-9e00-71baface6f59-disk1", "zpool": "zones", "size": 307200, "compression": "off", "refreservation": 307200, "block_size": 8192 } ], "nics": [ { "interface": "net0", "mac": "42:7f:f8:fc:90:c8", "nic_tag": "admin", "gateway": "10.4.2.215", "ip": "10.4.3.253", "netmask": "255.255.254.0", "model": "virtio", "dhcp_server": false, "allow_dhcp_spoofing": false, "allow_ip_spoofing": true, "allow_mac_spoofing": false, "allow_restricted_traffic": false, "allow_unfiltered_promisc": false, "primary": true } ], "max_physical_memory": 4096, "max_locked_memory": 4096, "max_swap": 4096, "ram": 3072, "uuid": "122a2197-9889-42b3-9e00-71baface6f59", "zone_state": "running", "zonepath": "/zones/122a2197-9889-42b3-9e00-71baface6f59", "zoneid": 33, "last_modified": "2014-07-19T04:42:22.000Z", "firewall_enabled": false, "server_uuid": "00000000-0000-0000-0000-00259085462c", "platform_buildstamp": "20140717T041004Z", "state": "running", "pid": 20583, "customer_metadata": {}, "internal_metadata": {}, "routes": {}, "tags": {}, "quota": 10, "zfs_root_recsize": 131072, "zfs_filesystem": "zones/122a2197-9889-42b3-9e00-71baface6f59", "zpool": "zones", "snapshots": [] } Vmadm Info: { "status": { "hwsetup": true, "singlestep": false, "running": true }, "version": { "qemu": { "micro": 1, "minor": 14, "major": 0 }, "package": " (qemu-kvm-devel)" }, "chardev": [ { "filename": "unix:/tmp/vm.qmp,server", "label": "qmp" }, { "filename": "unix:/tmp/vm.console,server", "label": "serial0" }, { "filename": "unix:/tmp/vm.ttyb,server", "label": "serial1" } ], "block": [ { "device": "virtio0", "locked": false, "removable": false, "inserted": { "ro": false, "drv": "raw", "encrypted": false, "file": "/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk0" }, "type": "hd" }, { "device": "virtio1", "locked": false, "removable": false, "inserted": { "ro": false, "drv": "raw", "encrypted": false, "file": "/dev/zvol/rdsk/zones/122a2197-9889-42b3-9e00-71baface6f59-disk1" }, "type": "hd" }, { "device": "ide1-cd0", "locked": false, "removable": true, "type": "cdrom" }, { "device": "floppy0", "locked": false, "removable": true, "type": "floppy" }, { "device": "sd0", "locked": false, "removable": true, "type": "floppy" } ], "blockstats": [ { "device": "virtio0", "parent": { "stats": { "wr_highest_offset": 54069242880, "wr_bytes": 72131948544, "wr_operations": 3951364, "rd_bytes": 62590942720, "rd_operations": 2405512 } }, "stats": { "wr_highest_offset": 54069242880, "wr_bytes": 72131948544, "wr_operations": 3951364, "rd_bytes": 62590942720, "rd_operations": 2405512 } }, { "device": "virtio1", "parent": { "stats": { "wr_highest_offset": 302978092544, "wr_bytes": 487860490240, "wr_operations": 3320946, "rd_bytes": 300412063232, "rd_operations": 1279650 } }, "stats": { "wr_highest_offset": 302978092544, "wr_bytes": 487860490240, "wr_operations": 3320946, "rd_bytes": 300412063232, "rd_operations": 1279650 } }, { "device": "ide1-cd0", "stats": { "wr_highest_offset": 0, "wr_bytes": 0, "wr_operations": 0, "rd_bytes": 0, "rd_operations": 0 } }, { "device": "floppy0", "stats": { "wr_highest_offset": 0, "wr_bytes": 0, "wr_operations": 0, "rd_bytes": 0, "rd_operations": 0 } }, { "device": "sd0", "stats": { "wr_highest_offset": 0, "wr_bytes": 0, "wr_operations": 0, "rd_bytes": 0, "rd_operations": 0 } } ], "cpus": [ { "current": true, "CPU": 0, "pc": -8784675928017, "halted": true, "thread_id": 3 }, { "current": false, "CPU": 1, "pc": -8784676019275, "halted": false, "thread_id": 4 } ], "pci": [ { "bus": 0, "devices": [ { "bus": 0, "qdev_id": "", "slot": 0, "class_info": { "class": 1536, "desc": "Host bridge" }, "id": { "device": 32902, "vendor": 4663 }, "function": 0, "regions": [] }, { "bus": 0, "qdev_id": "", "slot": 1, "class_info": { "class": 1537, "desc": "ISA bridge" }, "id": { "device": 32902, "vendor": 28672 }, "function": 0, "regions": [] }, { "bus": 0, "qdev_id": "", "slot": 1, "class_info": { "class": 257, "desc": "IDE controller" }, "id": { "device": 32902, "vendor": 28688 }, "function": 1, "regions": [ { "bar": 4, "size": 16, "address": 49152, "type": "io" } ] }, { "bus": 0, "qdev_id": "", "irq": 11, "slot": 1, "class_info": { "class": 3075, "desc": "USB controller" }, "id": { "device": 32902, "vendor": 28704 }, "function": 2, "regions": [ { "bar": 4, "size": 32, "address": 49184, "type": "io" } ] }, { "bus": 0, "qdev_id": "", "irq": 9, "slot": 1, "class_info": { "class": 1664, "desc": "Bridge" }, "id": { "device": 32902, "vendor": 28947 }, "function": 3, "regions": [] }, { "bus": 0, "qdev_id": "", "slot": 2, "class_info": { "class": 768, "desc": "VGA controller" }, "id": { "device": 4660, "vendor": 4369 }, "function": 0, "regions": [ { "prefetch": true, "mem_type_64": false, "bar": 0, "size": 16777216, "address": 4026531840<tel:4026531840>, "type": "memory" }, { "prefetch": false, "mem_type_64": false, "bar": 6, "size": 65536, "address": -1, "type": "memory" } ] }, { "bus": 0, "qdev_id": "", "irq": 0, "slot": 3, "class_info": { "class": 512, "desc": "Ethernet controller" }, "id": { "device": 6900, "vendor": 4096 }, "function": 0, "regions": [ { "bar": 0, "size": 32, "address": 49216, "type": "io" }, { "prefetch": false, "mem_type_64": false, "bar": 1, "size": 4096, "address": 4043374592<tel:4043374592>, "type": "memory" }, { "prefetch": false, "mem_type_64": false, "bar": 6, "size": 65536, "address": -1, "type": "memory" } ] }, { "bus": 0, "qdev_id": "", "irq": 0, "slot": 4, "class_info": { "class": 256, "desc": "SCSI controller" }, "id": { "device": 6900, "vendor": 4097 }, "function": 0, "regions": [ { "bar": 0, "size": 64, "address": 49280, "type": "io" }, { "prefetch": false, "mem_type_64": false, "bar": 1, "size": 4096, "address": 4043505664<tel:4043505664>, "type": "memory" } ] }, { "bus": 0, "qdev_id": "", "irq": 0, "slot": 5, "class_info": { "class": 256, "desc": "SCSI controller" }, "id": { "device": 6900, "vendor": 4097 }, "function": 0, "regions": [ { "bar": 0, "size": 64, "address": 49344, "type": "io" }, { "prefetch": false, "mem_type_64": false, "bar": 1, "size": 4096, "address": 4043509760<tel:4043509760>, "type": "memory" } ] } ] } ], "kvm": { "enabled": true, "present": true }, "vnc": { "host": "10.4.3.3", "port": 37422, "display": 31522 }, "spice": {} } On 31 Aug 2014, at 6:17 pm, Micky <[email protected]<mailto:[email protected]>> wrote: Can you share the vm json, vmadm info and vmadm get? I'm pretty interested in this issue. The mailing list is active but the Joyent guys don't respond in a reasonable fashion so I don't think debugging any further will help here. What will help here is, how we reproduce this problem and then share more details on the configuration and see if others reported or at least encountered the same sort of issue or not. For instance, I opened up a bug report https://github.com/joyent/smartos-live/issues/320 (also related to Windows VMs). Joyent refused to engage reasonably and the only thing that helped is the way how other users contributed (asked them to do so via IRC), which eventually led me to resolve the issue at my own (related to SLOG). On Sun, Aug 31, 2014 at 11:49 AM, David Finster <[email protected]<mailto:[email protected]>> wrote: And if it is of any use (from a sleeping KVM): > ::status debugging PID 20583 (64-bit) file: /smartdc/bin/qemu-system-x86_64 threading model: native threads status: stopped by debugger, sleeping in pollsys system call > ::stack libc.so.1`__pollsys+0xa() libc.so.1`pselect+0x1cb(18, fffffd7fffdfd4a0, fffffd7fffdfb4a0, fffffd7fffdf94a0, fffffd7fffdf9410, 0) libc.so.1`select+0x5a(18, fffffd7fffdfd4a0, fffffd7fffdfb4a0, fffffd7fffdf94a0, fffffd7fffdf9480) main_loop_wait+0x299(0) kvm_main_loop+0x119() main_loop+0x17() main+0x2da4(30, fffffd7fffdff898, fffffd7fffdffa20) _start+0x6c() > ::stacks THREAD STATE SOBJ COUNT 3 UNPARKED <NONE> 2 kvm_cpu_exec+0x1c kvm_main_loop_cpu+0x69 ap_main_loop+0xe9 libc.so.1`_thrp_setup+0x8a libc.so.1`_lwp_start 2 UNPARKED <NONE> 2 libc.so.1`sigwaitinfo+0x17 sigwait_compat+0x59 libc.so.1`_thrp_setup+0x8a libc.so.1`_lwp_start 1 UNPARKED <NONE> 1 libc.so.1`pselect+0x1cb libc.so.1`select+0x5a main_loop_wait+0x299 kvm_main_loop+0x119 main_loop+0x17 main+0x2da4 _start+0x6c > ::siginfo signal 1 (HUP) code 0 (user generated via kill) errno 0 (Error 0) signal sent from PID -1056859968 (uid -2124672) but I’m not really sure how to debug further. Exiting Dtrace via ::quit woke the KVM back up again. The VNC session that I established prior (that provided no interactivity or GUI or anything) came back to life without reconnecting. On 31 Aug 2014, at 4:22 pm, David Finster via smartos-discuss <[email protected]<mailto:[email protected]>> wrote: I doubt its an IO issue - as IO is very quiet at the moment (weekend): [root@00-25-90-85-46-2c ~]# zpool iostat 1 capacity operations bandwidth pool alloc free read write read write ---------- ----- ----- ----- ----- ----- ----- zones 3.05T 5.64T 36 286 328K 2.53M zones 3.05T 5.64T 0 16 0 299K zones 3.05T 5.64T 0 1.19K 0 7.64M zones 3.05T 5.64T 0 10 7.98K 132K zones 3.05T 5.64T 0 8 0 132K zones 3.05T 5.64T 0 8 0 152K zones 3.05T 5.64T 0 54 0 599K zones 3.05T 5.64T 0 1.07K 0 8.02M zones 3.05T 5.64T 0 2 0 79.8K zones 3.05T 5.64T 0 12 0 195K zones 3.05T 5.64T 0 2 0 83.8K zones 3.05T 5.64T 0 24 0 323K zones 3.05T 5.64T 0 939 0 6.14M zones 3.05T 5.64T 1 45 16.0K 439K zones 3.05T 5.64T 0 17 0 227K zones 3.05T 5.64T 0 15 0 263K zones 3.05T 5.64T 0 12 0 228K zones 3.05T 5.64T 0 1.03K 0 7.61M zones 3.05T 5.64T 0 21 0 259K zones 3.05T 5.64T 0 9 0 148K iostat yielded the following: http://pastebin.com/su5g8M0h - short but that is pretty representative of how it looked over a 2 minute run. Further, I am able to VNC into the KVM machine, but the machine is totally non-responsive. (won’t show any GUI at all). Willing to bet that if I run any of the commands I’ve mentioned previously it’ll wake up again. (Sorry for the double email again Micky - brain melt…) On 31 Aug 2014, at 3:06 pm, David Finster <[email protected]<mailto:[email protected]>> wrote: I’ll see how I go, but it is difficult to know when the issue would have otherwise occurred as it is fairly sporadic/random. If it were a Windows issue, how would running vmadm info/pstack/lockstack cause the machine to begin responding again? I’ve just noticed that one of the machines has gone to sleep again and since it isn’t in production I can keep it down for the time being. If anyone has any debugging suggestions it would be appreciated. On 31 Aug 2014, at 1:39 pm, Micky <[email protected]<mailto:[email protected]>> wrote: Does it happen if you keep an RDP open? Do a ping -t or something to keep the network sockets open on a fresh install or something like that. Seems like a Windows issue. On Sat, Aug 30, 2014 at 12:28 PM, David Finster via smartos-discuss <[email protected]<mailto:[email protected]>> wrote: As an update, the issue occurred again on two different hosts for KVM machines. Running pstack across the qemu process for the first KVM caused it to wake up, but the following was captured: http://pastebin.com/Js20eZPH On the second machine, running plockstat also woke the KVM up, but provided this: http://pastebin.com/pbEt2bnj On 29 Aug 2014, at 1:11 pm, David Finster via smartos-discuss <[email protected]<mailto:[email protected]>> wrote: I should also be more specific in saying that it appears that the VM locks up during this period of sleep. Once woken up, there are no event logs for the period that the machine was unreachable. Network activity isn’t the only thing affected. Thanks, Dave On 29 Aug 2014, at 1:07 pm, David Finster via smartos-discuss <[email protected]<mailto:[email protected]>> wrote: Hi Everyone I’m seeing a weird issue with 3 particular KVM VMs whereby the virtual machine appears to simply stop responding to network traffic (unpingable). The weird thing is that as soon as I jump into the hypervisor and run ‘vmadm info <uuid>’, the machine immediately starts responding to pings. I obtained the stacks of the qemu process, but accidentally ran vmadm info before I went further. Stacks can be found here: http://pastebin.com/8FBxrcJJ Does anyone have any suggestions/further debugging steps? Next time it happens I’ll see if I can obtain lock info and anything else I can think of. The KVM machines are running Server 2012 R2 64-bit (there are other machines also running this OS that aren’t having issues) and the issue is occurring on two separate hosts. Build version is 20140717T041004Z Thanks, Dave smartos-discuss | Archives<https://www.listbox.com/member/archive/184463/=now> [https://www.listbox.com/images/feed-icon-10x10.jpg10f3ec5.jpg?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2ZlZWQtaWNvbi0xMHgxMC5qcGc] <https://www.listbox.com/member/archive/rss/184463/25738179-216c4b5f> | Modify<https://www.listbox.com/member/?&;> Your Subscription [https://www.listbox.com/images/listbox-logo-small.png10f3ec5.png?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2xpc3Rib3gtbG9nby1zbWFsbC5wbmc] <http://www.listbox.com/> smartos-discuss | Archives<https://www.listbox.com/member/archive/184463/=now> [https://www.listbox.com/images/feed-icon-10x10.jpg10f3ec5.jpg?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2ZlZWQtaWNvbi0xMHgxMC5qcGc] <https://www.listbox.com/member/archive/rss/184463/25738179-216c4b5f> | Modify<https://www.listbox.com/member/?&;> Your Subscription [https://www.listbox.com/images/listbox-logo-small.png10f3ec5.png?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2xpc3Rib3gtbG9nby1zbWFsbC5wbmc] <http://www.listbox.com/> smartos-discuss | Archives<https://www.listbox.com/member/archive/184463/=now> [https://www.listbox.com/images/feed-icon-10x10.jpg10f3ec5.jpg?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2ZlZWQtaWNvbi0xMHgxMC5qcGc] <https://www.listbox.com/member/archive/rss/184463/25253051-d319687c> | Modify<https://www.listbox.com/member/?&;> Your Subscription [https://www.listbox.com/images/listbox-logo-small.png10f3ec5.png?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2xpc3Rib3gtbG9nby1zbWFsbC5wbmc] <http://www.listbox.com/> smartos-discuss | Archives<https://www.listbox.com/member/archive/184463/=now> [https://www.listbox.com/images/feed-icon-10x10.jpg10f3ec5.jpg?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2ZlZWQtaWNvbi0xMHgxMC5qcGc] <https://www.listbox.com/member/archive/rss/184463/25738179-216c4b5f> | Modify<https://www.listbox.com/member/?&;> Your Subscription [https://www.listbox.com/images/listbox-logo-small.png10f3ec5.png?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2xpc3Rib3gtbG9nby1zbWFsbC5wbmc] <http://www.listbox.com/> smartos-discuss | Archives<https://www.listbox.com/member/archive/184463/=now> [https://www.listbox.com/images/feed-icon-10x10.jpg10f3ec5.jpg?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2ZlZWQtaWNvbi0xMHgxMC5qcGc] <https://www.listbox.com/member/archive/rss/184463/25738179-216c4b5f> | Modify<https://www.listbox.com/member/?&;> Your Subscription [https://www.listbox.com/images/listbox-logo-small.png10f3ec5.png?uri=aHR0cHM6Ly93d3cubGlzdGJveC5jb20vaW1hZ2VzL2xpc3Rib3gtbG9nby1zbWFsbC5wbmc] <http://www.listbox.com/> ------------------------------------------- smartos-discuss Archives: https://www.listbox.com/member/archive/184463/=now RSS Feed: https://www.listbox.com/member/archive/rss/184463/25769125-55cfbc00 Modify Your Subscription: https://www.listbox.com/member/?member_id=25769125&id_secret=25769125-7688e9fb Powered by Listbox: http://www.listbox.com
