Connection refused is a java exception that usually means that either there's nothing listening on the port or it's firewall'd. Is "master:60000" really what you're expecting to see here? Should be easy to "telnet master 60000" from that machien to really verify that it works as expected.
J-D On Thu, Feb 9, 2012 at 11:19 AM, Tomas Tillery <[email protected]> wrote: > Hello, all > > It's my understanding that it's best practice to colocate thrift on each of > the regionservers, and use load balancing to spread queries between them. > However, I can't seem to get thrift to bind from the regionservers (only > from the hmaster). > > The machines have connectivity between them; regionservers connect and > communicate with the hmaster without any problems. > > I'm running Hadoop 0.20.2-cdh3u2, HBase 0.90.4-cdh3u2. > > Thrift can run succesfully from the HMaster, but not from any of the > datanode/regionservers > > What appears to be the relevant content from > hbase-hbase-thrift-region-1.out below: > > 12/02/09 19:09:17 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 0 of 10 failed; retrying after sleep of 1000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:18 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 1 of 10 failed; retrying after sleep of 1000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:19 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 2 of 10 failed; retrying after sleep of 1000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:20 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 3 of 10 failed; retrying after sleep of 2000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:22 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 4 of 10 failed; retrying after sleep of 2000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:24 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 5 of 10 failed; retrying after sleep of 4000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:28 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 6 of 10 failed; retrying after sleep of 4000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:32 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 7 of 10 failed; retrying after sleep of 8000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:40 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 8 of 10 failed; retrying after sleep of 16000 > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > 12/02/09 19:09:56 INFO client.HConnectionManager$HConnectionImplementation: > getMaster attempt 9 of 10 failed; no more retrying. > java.net.ConnectException: Connection refused > at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method) > at > sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:574) > at > org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:206) > at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:408) > at > org.apache.hadoop.hbase.ipc.HBaseClient$Connection.setupIOstreams(HBaseClient.java:328) > at > org.apache.hadoop.hbase.ipc.HBaseClient.getConnection(HBaseClient.java:883) > at > org.apache.hadoop.hbase.ipc.HBaseClient.call(HBaseClient.java:750) > at > org.apache.hadoop.hbase.ipc.HBaseRPC$Invoker.invoke(HBaseRPC.java:257) > at $Proxy4.getProtocolVersion(Unknown Source) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:419) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:393) > at org.apache.hadoop.hbase.ipc.HBaseRPC.getProxy(HBaseRPC.java:444) > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:360) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927) > Exception in thread "main" > org.apache.hadoop.hbase.MasterNotRunningException: master:60000 > at > org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getMaster(HConnectionManager.java:396) > at > org.apache.hadoop.hbase.client.HBaseAdmin.<init>(HBaseAdmin.java:94) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:194) > at > org.apache.hadoop.hbase.thrift.ThriftServer$HBaseHandler.<init>(ThriftServer.java:188) > at > org.apache.hadoop.hbase.thrift.ThriftServer.doMain(ThriftServer.java:870) > at > org.apache.hadoop.hbase.thrift.ThriftServer.main(ThriftServer.java:927)
