http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e5c3dcd1/devapidocs/src-html/org/apache/hadoop/hbase/ipc/RpcClientImpl.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/ipc/RpcClientImpl.html b/devapidocs/src-html/org/apache/hadoop/hbase/ipc/RpcClientImpl.html index 03ccec3..298a8bd 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/ipc/RpcClientImpl.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/ipc/RpcClientImpl.html @@ -27,88 +27,88 @@ <span class="sourceLineNo">019</span><a name="line.19"></a> <span class="sourceLineNo">020</span>package org.apache.hadoop.hbase.ipc;<a name="line.20"></a> <span class="sourceLineNo">021</span><a name="line.21"></a> -<span class="sourceLineNo">022</span>import com.google.common.annotations.VisibleForTesting;<a name="line.22"></a> -<span class="sourceLineNo">023</span>import com.google.protobuf.Descriptors.MethodDescriptor;<a name="line.23"></a> -<span class="sourceLineNo">024</span>import com.google.protobuf.Message;<a name="line.24"></a> -<span class="sourceLineNo">025</span>import com.google.protobuf.Message.Builder;<a name="line.25"></a> -<span class="sourceLineNo">026</span>import com.google.protobuf.RpcCallback;<a name="line.26"></a> -<span class="sourceLineNo">027</span><a name="line.27"></a> -<span class="sourceLineNo">028</span>import org.apache.commons.logging.Log;<a name="line.28"></a> -<span class="sourceLineNo">029</span>import org.apache.commons.logging.LogFactory;<a name="line.29"></a> -<span class="sourceLineNo">030</span>import org.apache.hadoop.conf.Configuration;<a name="line.30"></a> -<span class="sourceLineNo">031</span>import org.apache.hadoop.hbase.CellScanner;<a name="line.31"></a> -<span class="sourceLineNo">032</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.32"></a> -<span class="sourceLineNo">033</span>import org.apache.hadoop.hbase.HConstants;<a name="line.33"></a> -<span class="sourceLineNo">034</span>import org.apache.hadoop.hbase.ServerName;<a name="line.34"></a> -<span class="sourceLineNo">035</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.35"></a> -<span class="sourceLineNo">036</span>import org.apache.hadoop.hbase.client.MetricsConnection;<a name="line.36"></a> -<span class="sourceLineNo">037</span>import org.apache.hadoop.hbase.codec.Codec;<a name="line.37"></a> -<span class="sourceLineNo">038</span>import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;<a name="line.38"></a> -<span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.protobuf.ProtobufUtil;<a name="line.39"></a> -<span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;<a name="line.40"></a> -<span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta;<a name="line.41"></a> -<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ConnectionHeader;<a name="line.42"></a> -<span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ExceptionResponse;<a name="line.43"></a> -<span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;<a name="line.44"></a> -<span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ResponseHeader;<a name="line.45"></a> -<span class="sourceLineNo">046</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;<a name="line.46"></a> -<span class="sourceLineNo">047</span>import org.apache.hadoop.hbase.protobuf.generated.TracingProtos.RPCTInfo;<a name="line.47"></a> -<span class="sourceLineNo">048</span>import org.apache.hadoop.hbase.security.AuthMethod;<a name="line.48"></a> -<span class="sourceLineNo">049</span>import org.apache.hadoop.hbase.security.HBaseSaslRpcClient;<a name="line.49"></a> -<span class="sourceLineNo">050</span>import org.apache.hadoop.hbase.security.SaslUtil.QualityOfProtection;<a name="line.50"></a> -<span class="sourceLineNo">051</span>import org.apache.hadoop.hbase.security.SecurityInfo;<a name="line.51"></a> -<span class="sourceLineNo">052</span>import org.apache.hadoop.hbase.security.User;<a name="line.52"></a> -<span class="sourceLineNo">053</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.53"></a> -<span class="sourceLineNo">054</span>import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;<a name="line.54"></a> -<span class="sourceLineNo">055</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.55"></a> -<span class="sourceLineNo">056</span>import org.apache.hadoop.hbase.util.ExceptionUtil;<a name="line.56"></a> -<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.57"></a> -<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.util.PoolMap;<a name="line.58"></a> -<span class="sourceLineNo">059</span>import org.apache.hadoop.io.IOUtils;<a name="line.59"></a> -<span class="sourceLineNo">060</span>import org.apache.hadoop.io.Text;<a name="line.60"></a> -<span class="sourceLineNo">061</span>import org.apache.hadoop.io.compress.CompressionCodec;<a name="line.61"></a> -<span class="sourceLineNo">062</span>import org.apache.hadoop.ipc.RemoteException;<a name="line.62"></a> -<span class="sourceLineNo">063</span>import org.apache.hadoop.net.NetUtils;<a name="line.63"></a> -<span class="sourceLineNo">064</span>import org.apache.hadoop.security.SecurityUtil;<a name="line.64"></a> -<span class="sourceLineNo">065</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.65"></a> -<span class="sourceLineNo">066</span>import org.apache.hadoop.security.token.Token;<a name="line.66"></a> -<span class="sourceLineNo">067</span>import org.apache.hadoop.security.token.TokenIdentifier;<a name="line.67"></a> -<span class="sourceLineNo">068</span>import org.apache.hadoop.security.token.TokenSelector;<a name="line.68"></a> -<span class="sourceLineNo">069</span>import org.apache.htrace.Span;<a name="line.69"></a> -<span class="sourceLineNo">070</span>import org.apache.htrace.Trace;<a name="line.70"></a> -<span class="sourceLineNo">071</span>import org.apache.htrace.TraceScope;<a name="line.71"></a> -<span class="sourceLineNo">072</span><a name="line.72"></a> -<span class="sourceLineNo">073</span>import javax.net.SocketFactory;<a name="line.73"></a> -<span class="sourceLineNo">074</span>import javax.security.sasl.SaslException;<a name="line.74"></a> -<span class="sourceLineNo">075</span>import java.io.BufferedInputStream;<a name="line.75"></a> -<span class="sourceLineNo">076</span>import java.io.BufferedOutputStream;<a name="line.76"></a> -<span class="sourceLineNo">077</span>import java.io.Closeable;<a name="line.77"></a> -<span class="sourceLineNo">078</span>import java.io.DataInputStream;<a name="line.78"></a> -<span class="sourceLineNo">079</span>import java.io.DataOutputStream;<a name="line.79"></a> -<span class="sourceLineNo">080</span>import java.io.IOException;<a name="line.80"></a> -<span class="sourceLineNo">081</span>import java.io.InputStream;<a name="line.81"></a> -<span class="sourceLineNo">082</span>import java.io.InterruptedIOException;<a name="line.82"></a> -<span class="sourceLineNo">083</span>import java.io.OutputStream;<a name="line.83"></a> -<span class="sourceLineNo">084</span>import java.net.ConnectException;<a name="line.84"></a> -<span class="sourceLineNo">085</span>import java.net.InetSocketAddress;<a name="line.85"></a> -<span class="sourceLineNo">086</span>import java.net.Socket;<a name="line.86"></a> -<span class="sourceLineNo">087</span>import java.net.SocketAddress;<a name="line.87"></a> -<span class="sourceLineNo">088</span>import java.net.SocketTimeoutException;<a name="line.88"></a> -<span class="sourceLineNo">089</span>import java.net.UnknownHostException;<a name="line.89"></a> -<span class="sourceLineNo">090</span>import java.nio.ByteBuffer;<a name="line.90"></a> -<span class="sourceLineNo">091</span>import java.security.PrivilegedExceptionAction;<a name="line.91"></a> -<span class="sourceLineNo">092</span>import java.util.HashMap;<a name="line.92"></a> -<span class="sourceLineNo">093</span>import java.util.HashSet;<a name="line.93"></a> -<span class="sourceLineNo">094</span>import java.util.Iterator;<a name="line.94"></a> -<span class="sourceLineNo">095</span>import java.util.Map;<a name="line.95"></a> -<span class="sourceLineNo">096</span>import java.util.Map.Entry;<a name="line.96"></a> -<span class="sourceLineNo">097</span>import java.util.Random;<a name="line.97"></a> -<span class="sourceLineNo">098</span>import java.util.Set;<a name="line.98"></a> -<span class="sourceLineNo">099</span>import java.util.concurrent.ArrayBlockingQueue;<a name="line.99"></a> -<span class="sourceLineNo">100</span>import java.util.concurrent.BlockingQueue;<a name="line.100"></a> -<span class="sourceLineNo">101</span>import java.util.concurrent.ConcurrentSkipListMap;<a name="line.101"></a> -<span class="sourceLineNo">102</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.102"></a> -<span class="sourceLineNo">103</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.103"></a> +<span class="sourceLineNo">022</span>import java.io.BufferedInputStream;<a name="line.22"></a> +<span class="sourceLineNo">023</span>import java.io.BufferedOutputStream;<a name="line.23"></a> +<span class="sourceLineNo">024</span>import java.io.Closeable;<a name="line.24"></a> +<span class="sourceLineNo">025</span>import java.io.DataInputStream;<a name="line.25"></a> +<span class="sourceLineNo">026</span>import java.io.DataOutputStream;<a name="line.26"></a> +<span class="sourceLineNo">027</span>import java.io.IOException;<a name="line.27"></a> +<span class="sourceLineNo">028</span>import java.io.InputStream;<a name="line.28"></a> +<span class="sourceLineNo">029</span>import java.io.InterruptedIOException;<a name="line.29"></a> +<span class="sourceLineNo">030</span>import java.io.OutputStream;<a name="line.30"></a> +<span class="sourceLineNo">031</span>import java.net.InetSocketAddress;<a name="line.31"></a> +<span class="sourceLineNo">032</span>import java.net.Socket;<a name="line.32"></a> +<span class="sourceLineNo">033</span>import java.net.SocketAddress;<a name="line.33"></a> +<span class="sourceLineNo">034</span>import java.net.SocketTimeoutException;<a name="line.34"></a> +<span class="sourceLineNo">035</span>import java.net.UnknownHostException;<a name="line.35"></a> +<span class="sourceLineNo">036</span>import java.nio.ByteBuffer;<a name="line.36"></a> +<span class="sourceLineNo">037</span>import java.security.PrivilegedExceptionAction;<a name="line.37"></a> +<span class="sourceLineNo">038</span>import java.util.HashMap;<a name="line.38"></a> +<span class="sourceLineNo">039</span>import java.util.HashSet;<a name="line.39"></a> +<span class="sourceLineNo">040</span>import java.util.Iterator;<a name="line.40"></a> +<span class="sourceLineNo">041</span>import java.util.Map;<a name="line.41"></a> +<span class="sourceLineNo">042</span>import java.util.Map.Entry;<a name="line.42"></a> +<span class="sourceLineNo">043</span>import java.util.Random;<a name="line.43"></a> +<span class="sourceLineNo">044</span>import java.util.Set;<a name="line.44"></a> +<span class="sourceLineNo">045</span>import java.util.concurrent.ArrayBlockingQueue;<a name="line.45"></a> +<span class="sourceLineNo">046</span>import java.util.concurrent.BlockingQueue;<a name="line.46"></a> +<span class="sourceLineNo">047</span>import java.util.concurrent.ConcurrentSkipListMap;<a name="line.47"></a> +<span class="sourceLineNo">048</span>import java.util.concurrent.atomic.AtomicBoolean;<a name="line.48"></a> +<span class="sourceLineNo">049</span>import java.util.concurrent.atomic.AtomicInteger;<a name="line.49"></a> +<span class="sourceLineNo">050</span><a name="line.50"></a> +<span class="sourceLineNo">051</span>import javax.net.SocketFactory;<a name="line.51"></a> +<span class="sourceLineNo">052</span>import javax.security.sasl.SaslException;<a name="line.52"></a> +<span class="sourceLineNo">053</span><a name="line.53"></a> +<span class="sourceLineNo">054</span>import org.apache.commons.logging.Log;<a name="line.54"></a> +<span class="sourceLineNo">055</span>import org.apache.commons.logging.LogFactory;<a name="line.55"></a> +<span class="sourceLineNo">056</span>import org.apache.hadoop.conf.Configuration;<a name="line.56"></a> +<span class="sourceLineNo">057</span>import org.apache.hadoop.hbase.CellScanner;<a name="line.57"></a> +<span class="sourceLineNo">058</span>import org.apache.hadoop.hbase.DoNotRetryIOException;<a name="line.58"></a> +<span class="sourceLineNo">059</span>import org.apache.hadoop.hbase.HConstants;<a name="line.59"></a> +<span class="sourceLineNo">060</span>import org.apache.hadoop.hbase.ServerName;<a name="line.60"></a> +<span class="sourceLineNo">061</span>import org.apache.hadoop.hbase.classification.InterfaceAudience;<a name="line.61"></a> +<span class="sourceLineNo">062</span>import org.apache.hadoop.hbase.client.MetricsConnection;<a name="line.62"></a> +<span class="sourceLineNo">063</span>import org.apache.hadoop.hbase.codec.Codec;<a name="line.63"></a> +<span class="sourceLineNo">064</span>import org.apache.hadoop.hbase.exceptions.ConnectionClosingException;<a name="line.64"></a> +<span class="sourceLineNo">065</span>import org.apache.hadoop.hbase.protobuf.ProtobufUtil;<a name="line.65"></a> +<span class="sourceLineNo">066</span>import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos;<a name="line.66"></a> +<span class="sourceLineNo">067</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta;<a name="line.67"></a> +<span class="sourceLineNo">068</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ConnectionHeader;<a name="line.68"></a> +<span class="sourceLineNo">069</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ExceptionResponse;<a name="line.69"></a> +<span class="sourceLineNo">070</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;<a name="line.70"></a> +<span class="sourceLineNo">071</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ResponseHeader;<a name="line.71"></a> +<span class="sourceLineNo">072</span>import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;<a name="line.72"></a> +<span class="sourceLineNo">073</span>import org.apache.hadoop.hbase.protobuf.generated.TracingProtos.RPCTInfo;<a name="line.73"></a> +<span class="sourceLineNo">074</span>import org.apache.hadoop.hbase.security.AuthMethod;<a name="line.74"></a> +<span class="sourceLineNo">075</span>import org.apache.hadoop.hbase.security.HBaseSaslRpcClient;<a name="line.75"></a> +<span class="sourceLineNo">076</span>import org.apache.hadoop.hbase.security.SaslUtil.QualityOfProtection;<a name="line.76"></a> +<span class="sourceLineNo">077</span>import org.apache.hadoop.hbase.security.SecurityInfo;<a name="line.77"></a> +<span class="sourceLineNo">078</span>import org.apache.hadoop.hbase.security.User;<a name="line.78"></a> +<span class="sourceLineNo">079</span>import org.apache.hadoop.hbase.security.UserProvider;<a name="line.79"></a> +<span class="sourceLineNo">080</span>import org.apache.hadoop.hbase.security.token.AuthenticationTokenSelector;<a name="line.80"></a> +<span class="sourceLineNo">081</span>import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;<a name="line.81"></a> +<span class="sourceLineNo">082</span>import org.apache.hadoop.hbase.util.ExceptionUtil;<a name="line.82"></a> +<span class="sourceLineNo">083</span>import org.apache.hadoop.hbase.util.Pair;<a name="line.83"></a> +<span class="sourceLineNo">084</span>import org.apache.hadoop.hbase.util.PoolMap;<a name="line.84"></a> +<span class="sourceLineNo">085</span>import org.apache.hadoop.io.IOUtils;<a name="line.85"></a> +<span class="sourceLineNo">086</span>import org.apache.hadoop.io.Text;<a name="line.86"></a> +<span class="sourceLineNo">087</span>import org.apache.hadoop.io.compress.CompressionCodec;<a name="line.87"></a> +<span class="sourceLineNo">088</span>import org.apache.hadoop.ipc.RemoteException;<a name="line.88"></a> +<span class="sourceLineNo">089</span>import org.apache.hadoop.net.NetUtils;<a name="line.89"></a> +<span class="sourceLineNo">090</span>import org.apache.hadoop.security.SecurityUtil;<a name="line.90"></a> +<span class="sourceLineNo">091</span>import org.apache.hadoop.security.UserGroupInformation;<a name="line.91"></a> +<span class="sourceLineNo">092</span>import org.apache.hadoop.security.token.Token;<a name="line.92"></a> +<span class="sourceLineNo">093</span>import org.apache.hadoop.security.token.TokenIdentifier;<a name="line.93"></a> +<span class="sourceLineNo">094</span>import org.apache.hadoop.security.token.TokenSelector;<a name="line.94"></a> +<span class="sourceLineNo">095</span>import org.apache.htrace.Span;<a name="line.95"></a> +<span class="sourceLineNo">096</span>import org.apache.htrace.Trace;<a name="line.96"></a> +<span class="sourceLineNo">097</span>import org.apache.htrace.TraceScope;<a name="line.97"></a> +<span class="sourceLineNo">098</span><a name="line.98"></a> +<span class="sourceLineNo">099</span>import com.google.common.annotations.VisibleForTesting;<a name="line.99"></a> +<span class="sourceLineNo">100</span>import com.google.protobuf.Descriptors.MethodDescriptor;<a name="line.100"></a> +<span class="sourceLineNo">101</span>import com.google.protobuf.Message;<a name="line.101"></a> +<span class="sourceLineNo">102</span>import com.google.protobuf.Message.Builder;<a name="line.102"></a> +<span class="sourceLineNo">103</span>import com.google.protobuf.RpcCallback;<a name="line.103"></a> <span class="sourceLineNo">104</span><a name="line.104"></a> <span class="sourceLineNo">105</span>/**<a name="line.105"></a> <span class="sourceLineNo">106</span> * Does RPC against a cluster. Manages connections per regionserver in the cluster.<a name="line.106"></a> @@ -1275,80 +1275,50 @@ <span class="sourceLineNo">1267</span><a name="line.1267"></a> <span class="sourceLineNo">1268</span><a name="line.1268"></a> <span class="sourceLineNo">1269</span> /**<a name="line.1269"></a> -<span class="sourceLineNo">1270</span> * Take an IOException and the address we were trying to connect to<a name="line.1270"></a> -<span class="sourceLineNo">1271</span> * and return an IOException with the input exception as the cause.<a name="line.1271"></a> -<span class="sourceLineNo">1272</span> * The new exception provides the stack trace of the place where<a name="line.1272"></a> -<span class="sourceLineNo">1273</span> * the exception is thrown and some extra diagnostics information.<a name="line.1273"></a> -<span class="sourceLineNo">1274</span> * If the exception is ConnectException or SocketTimeoutException,<a name="line.1274"></a> -<span class="sourceLineNo">1275</span> * return a new one of the same type; Otherwise return an IOException.<a name="line.1275"></a> -<span class="sourceLineNo">1276</span> *<a name="line.1276"></a> -<span class="sourceLineNo">1277</span> * @param addr target address<a name="line.1277"></a> -<span class="sourceLineNo">1278</span> * @param exception the relevant exception<a name="line.1278"></a> -<span class="sourceLineNo">1279</span> * @return an exception to throw<a name="line.1279"></a> -<span class="sourceLineNo">1280</span> */<a name="line.1280"></a> -<span class="sourceLineNo">1281</span> protected IOException wrapException(InetSocketAddress addr,<a name="line.1281"></a> -<span class="sourceLineNo">1282</span> IOException exception) {<a name="line.1282"></a> -<span class="sourceLineNo">1283</span> if (exception instanceof ConnectException) {<a name="line.1283"></a> -<span class="sourceLineNo">1284</span> //connection refused; include the host:port in the error<a name="line.1284"></a> -<span class="sourceLineNo">1285</span> return (ConnectException)new ConnectException(<a name="line.1285"></a> -<span class="sourceLineNo">1286</span> "Call to " + addr + " failed on connection exception: " + exception).initCause(exception);<a name="line.1286"></a> -<span class="sourceLineNo">1287</span> } else if (exception instanceof SocketTimeoutException) {<a name="line.1287"></a> -<span class="sourceLineNo">1288</span> return (SocketTimeoutException)new SocketTimeoutException("Call to " + addr +<a name="line.1288"></a> -<span class="sourceLineNo">1289</span> " failed because " + exception).initCause(exception);<a name="line.1289"></a> -<span class="sourceLineNo">1290</span> } else if (exception instanceof ConnectionClosingException){<a name="line.1290"></a> -<span class="sourceLineNo">1291</span> return (ConnectionClosingException) new ConnectionClosingException(<a name="line.1291"></a> -<span class="sourceLineNo">1292</span> "Call to " + addr + " failed on local exception: " + exception).initCause(exception);<a name="line.1292"></a> -<span class="sourceLineNo">1293</span> } else {<a name="line.1293"></a> -<span class="sourceLineNo">1294</span> return (IOException)new IOException("Call to " + addr + " failed on local exception: " +<a name="line.1294"></a> -<span class="sourceLineNo">1295</span> exception).initCause(exception);<a name="line.1295"></a> -<span class="sourceLineNo">1296</span> }<a name="line.1296"></a> -<span class="sourceLineNo">1297</span> }<a name="line.1297"></a> -<span class="sourceLineNo">1298</span><a name="line.1298"></a> -<span class="sourceLineNo">1299</span> /**<a name="line.1299"></a> -<span class="sourceLineNo">1300</span> * Interrupt the connections to the given ip:port server. This should be called if the server<a name="line.1300"></a> -<span class="sourceLineNo">1301</span> * is known as actually dead. This will not prevent current operation to be retried, and,<a name="line.1301"></a> -<span class="sourceLineNo">1302</span> * depending on their own behavior, they may retry on the same server. This can be a feature,<a name="line.1302"></a> -<span class="sourceLineNo">1303</span> * for example at startup. In any case, they're likely to get connection refused (if the<a name="line.1303"></a> -<span class="sourceLineNo">1304</span> * process died) or no route to host: i.e. their next retries should be faster and with a<a name="line.1304"></a> -<span class="sourceLineNo">1305</span> * safe exception.<a name="line.1305"></a> -<span class="sourceLineNo">1306</span> */<a name="line.1306"></a> -<span class="sourceLineNo">1307</span> @Override<a name="line.1307"></a> -<span class="sourceLineNo">1308</span> public void cancelConnections(ServerName sn) {<a name="line.1308"></a> -<span class="sourceLineNo">1309</span> synchronized (connections) {<a name="line.1309"></a> -<span class="sourceLineNo">1310</span> for (Connection connection : connections.values()) {<a name="line.1310"></a> -<span class="sourceLineNo">1311</span> if (connection.isAlive() &&<a name="line.1311"></a> -<span class="sourceLineNo">1312</span> connection.getRemoteAddress().getPort() == sn.getPort() &&<a name="line.1312"></a> -<span class="sourceLineNo">1313</span> connection.getRemoteAddress().getHostName().equals(sn.getHostname())) {<a name="line.1313"></a> -<span class="sourceLineNo">1314</span> LOG.info("The server on " + sn.toString() +<a name="line.1314"></a> -<span class="sourceLineNo">1315</span> " is dead - stopping the connection " + connection.remoteId);<a name="line.1315"></a> -<span class="sourceLineNo">1316</span> connection.interrupt(); // We're interrupting a Reader. It means we want it to finish.<a name="line.1316"></a> -<span class="sourceLineNo">1317</span> // This will close the connection as well.<a name="line.1317"></a> -<span class="sourceLineNo">1318</span> }<a name="line.1318"></a> -<span class="sourceLineNo">1319</span> }<a name="line.1319"></a> -<span class="sourceLineNo">1320</span> }<a name="line.1320"></a> -<span class="sourceLineNo">1321</span> }<a name="line.1321"></a> -<span class="sourceLineNo">1322</span><a name="line.1322"></a> -<span class="sourceLineNo">1323</span> /**<a name="line.1323"></a> -<span class="sourceLineNo">1324</span> * Get a connection from the pool, or create a new one and add it to the<a name="line.1324"></a> -<span class="sourceLineNo">1325</span> * pool. Connections to a given host/port are reused.<a name="line.1325"></a> -<span class="sourceLineNo">1326</span> */<a name="line.1326"></a> -<span class="sourceLineNo">1327</span> protected Connection getConnection(User ticket, Call call, InetSocketAddress addr)<a name="line.1327"></a> -<span class="sourceLineNo">1328</span> throws IOException {<a name="line.1328"></a> -<span class="sourceLineNo">1329</span> if (!running.get()) throw new StoppedRpcClientException();<a name="line.1329"></a> -<span class="sourceLineNo">1330</span> Connection connection;<a name="line.1330"></a> -<span class="sourceLineNo">1331</span> ConnectionId remoteId =<a name="line.1331"></a> -<span class="sourceLineNo">1332</span> new ConnectionId(ticket, call.md.getService().getName(), addr);<a name="line.1332"></a> -<span class="sourceLineNo">1333</span> synchronized (connections) {<a name="line.1333"></a> -<span class="sourceLineNo">1334</span> connection = connections.get(remoteId);<a name="line.1334"></a> -<span class="sourceLineNo">1335</span> if (connection == null) {<a name="line.1335"></a> -<span class="sourceLineNo">1336</span> connection = createConnection(remoteId, this.codec, this.compressor);<a name="line.1336"></a> -<span class="sourceLineNo">1337</span> connections.put(remoteId, connection);<a name="line.1337"></a> -<span class="sourceLineNo">1338</span> }<a name="line.1338"></a> -<span class="sourceLineNo">1339</span> }<a name="line.1339"></a> -<span class="sourceLineNo">1340</span><a name="line.1340"></a> -<span class="sourceLineNo">1341</span> return connection;<a name="line.1341"></a> -<span class="sourceLineNo">1342</span> }<a name="line.1342"></a> -<span class="sourceLineNo">1343</span>}<a name="line.1343"></a> +<span class="sourceLineNo">1270</span> * Interrupt the connections to the given ip:port server. This should be called if the server<a name="line.1270"></a> +<span class="sourceLineNo">1271</span> * is known as actually dead. This will not prevent current operation to be retried, and,<a name="line.1271"></a> +<span class="sourceLineNo">1272</span> * depending on their own behavior, they may retry on the same server. This can be a feature,<a name="line.1272"></a> +<span class="sourceLineNo">1273</span> * for example at startup. In any case, they're likely to get connection refused (if the<a name="line.1273"></a> +<span class="sourceLineNo">1274</span> * process died) or no route to host: i.e. their next retries should be faster and with a<a name="line.1274"></a> +<span class="sourceLineNo">1275</span> * safe exception.<a name="line.1275"></a> +<span class="sourceLineNo">1276</span> */<a name="line.1276"></a> +<span class="sourceLineNo">1277</span> @Override<a name="line.1277"></a> +<span class="sourceLineNo">1278</span> public void cancelConnections(ServerName sn) {<a name="line.1278"></a> +<span class="sourceLineNo">1279</span> synchronized (connections) {<a name="line.1279"></a> +<span class="sourceLineNo">1280</span> for (Connection connection : connections.values()) {<a name="line.1280"></a> +<span class="sourceLineNo">1281</span> if (connection.isAlive() &&<a name="line.1281"></a> +<span class="sourceLineNo">1282</span> connection.getRemoteAddress().getPort() == sn.getPort() &&<a name="line.1282"></a> +<span class="sourceLineNo">1283</span> connection.getRemoteAddress().getHostName().equals(sn.getHostname())) {<a name="line.1283"></a> +<span class="sourceLineNo">1284</span> LOG.info("The server on " + sn.toString() +<a name="line.1284"></a> +<span class="sourceLineNo">1285</span> " is dead - stopping the connection " + connection.remoteId);<a name="line.1285"></a> +<span class="sourceLineNo">1286</span> connection.interrupt(); // We're interrupting a Reader. It means we want it to finish.<a name="line.1286"></a> +<span class="sourceLineNo">1287</span> // This will close the connection as well.<a name="line.1287"></a> +<span class="sourceLineNo">1288</span> }<a name="line.1288"></a> +<span class="sourceLineNo">1289</span> }<a name="line.1289"></a> +<span class="sourceLineNo">1290</span> }<a name="line.1290"></a> +<span class="sourceLineNo">1291</span> }<a name="line.1291"></a> +<span class="sourceLineNo">1292</span><a name="line.1292"></a> +<span class="sourceLineNo">1293</span> /**<a name="line.1293"></a> +<span class="sourceLineNo">1294</span> * Get a connection from the pool, or create a new one and add it to the<a name="line.1294"></a> +<span class="sourceLineNo">1295</span> * pool. Connections to a given host/port are reused.<a name="line.1295"></a> +<span class="sourceLineNo">1296</span> */<a name="line.1296"></a> +<span class="sourceLineNo">1297</span> protected Connection getConnection(User ticket, Call call, InetSocketAddress addr)<a name="line.1297"></a> +<span class="sourceLineNo">1298</span> throws IOException {<a name="line.1298"></a> +<span class="sourceLineNo">1299</span> if (!running.get()) throw new StoppedRpcClientException();<a name="line.1299"></a> +<span class="sourceLineNo">1300</span> Connection connection;<a name="line.1300"></a> +<span class="sourceLineNo">1301</span> ConnectionId remoteId =<a name="line.1301"></a> +<span class="sourceLineNo">1302</span> new ConnectionId(ticket, call.md.getService().getName(), addr);<a name="line.1302"></a> +<span class="sourceLineNo">1303</span> synchronized (connections) {<a name="line.1303"></a> +<span class="sourceLineNo">1304</span> connection = connections.get(remoteId);<a name="line.1304"></a> +<span class="sourceLineNo">1305</span> if (connection == null) {<a name="line.1305"></a> +<span class="sourceLineNo">1306</span> connection = createConnection(remoteId, this.codec, this.compressor);<a name="line.1306"></a> +<span class="sourceLineNo">1307</span> connections.put(remoteId, connection);<a name="line.1307"></a> +<span class="sourceLineNo">1308</span> }<a name="line.1308"></a> +<span class="sourceLineNo">1309</span> }<a name="line.1309"></a> +<span class="sourceLineNo">1310</span><a name="line.1310"></a> +<span class="sourceLineNo">1311</span> return connection;<a name="line.1311"></a> +<span class="sourceLineNo">1312</span> }<a name="line.1312"></a> +<span class="sourceLineNo">1313</span>}<a name="line.1313"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/e5c3dcd1/devapidocs/src-html/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.BulkHFileVisitor.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.BulkHFileVisitor.html b/devapidocs/src-html/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.BulkHFileVisitor.html index 7c00453..f955098 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.BulkHFileVisitor.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.BulkHFileVisitor.html @@ -911,177 +911,178 @@ <span class="sourceLineNo">903</span> .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf))<a name="line.903"></a> <span class="sourceLineNo">904</span> .withBlockSize(blocksize)<a name="line.904"></a> <span class="sourceLineNo">905</span> .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding())<a name="line.905"></a> -<span class="sourceLineNo">906</span> .build();<a name="line.906"></a> -<span class="sourceLineNo">907</span> halfWriter = new StoreFile.WriterBuilder(conf, cacheConf,<a name="line.907"></a> -<span class="sourceLineNo">908</span> fs)<a name="line.908"></a> -<span class="sourceLineNo">909</span> .withFilePath(outFile)<a name="line.909"></a> -<span class="sourceLineNo">910</span> .withBloomType(bloomFilterType)<a name="line.910"></a> -<span class="sourceLineNo">911</span> .withFileContext(hFileContext)<a name="line.911"></a> -<span class="sourceLineNo">912</span> .build();<a name="line.912"></a> -<span class="sourceLineNo">913</span> HFileScanner scanner = halfReader.getScanner(false, false, false);<a name="line.913"></a> -<span class="sourceLineNo">914</span> scanner.seekTo();<a name="line.914"></a> -<span class="sourceLineNo">915</span> do {<a name="line.915"></a> -<span class="sourceLineNo">916</span> halfWriter.append(scanner.getCell());<a name="line.916"></a> -<span class="sourceLineNo">917</span> } while (scanner.next());<a name="line.917"></a> -<span class="sourceLineNo">918</span><a name="line.918"></a> -<span class="sourceLineNo">919</span> for (Map.Entry<byte[],byte[]> entry : fileInfo.entrySet()) {<a name="line.919"></a> -<span class="sourceLineNo">920</span> if (shouldCopyHFileMetaKey(entry.getKey())) {<a name="line.920"></a> -<span class="sourceLineNo">921</span> halfWriter.appendFileInfo(entry.getKey(), entry.getValue());<a name="line.921"></a> -<span class="sourceLineNo">922</span> }<a name="line.922"></a> -<span class="sourceLineNo">923</span> }<a name="line.923"></a> -<span class="sourceLineNo">924</span> } finally {<a name="line.924"></a> -<span class="sourceLineNo">925</span> if (halfWriter != null) halfWriter.close();<a name="line.925"></a> -<span class="sourceLineNo">926</span> if (halfReader != null) halfReader.close(cacheConf.shouldEvictOnClose());<a name="line.926"></a> -<span class="sourceLineNo">927</span> }<a name="line.927"></a> -<span class="sourceLineNo">928</span> }<a name="line.928"></a> -<span class="sourceLineNo">929</span><a name="line.929"></a> -<span class="sourceLineNo">930</span> private static boolean shouldCopyHFileMetaKey(byte[] key) {<a name="line.930"></a> -<span class="sourceLineNo">931</span> return !HFile.isReservedFileInfoKey(key);<a name="line.931"></a> -<span class="sourceLineNo">932</span> }<a name="line.932"></a> -<span class="sourceLineNo">933</span><a name="line.933"></a> -<span class="sourceLineNo">934</span> /*<a name="line.934"></a> -<span class="sourceLineNo">935</span> * Infers region boundaries for a new table.<a name="line.935"></a> -<span class="sourceLineNo">936</span> * Parameter:<a name="line.936"></a> -<span class="sourceLineNo">937</span> * bdryMap is a map between keys to an integer belonging to {+1, -1}<a name="line.937"></a> -<span class="sourceLineNo">938</span> * If a key is a start key of a file, then it maps to +1<a name="line.938"></a> -<span class="sourceLineNo">939</span> * If a key is an end key of a file, then it maps to -1<a name="line.939"></a> -<span class="sourceLineNo">940</span> * Algo:<a name="line.940"></a> -<span class="sourceLineNo">941</span> * 1) Poll on the keys in order:<a name="line.941"></a> -<span class="sourceLineNo">942</span> * a) Keep adding the mapped values to these keys (runningSum)<a name="line.942"></a> -<span class="sourceLineNo">943</span> * b) Each time runningSum reaches 0, add the start Key from when the runningSum had started to<a name="line.943"></a> -<span class="sourceLineNo">944</span> * a boundary list.<a name="line.944"></a> -<span class="sourceLineNo">945</span> * 2) Return the boundary list.<a name="line.945"></a> -<span class="sourceLineNo">946</span> */<a name="line.946"></a> -<span class="sourceLineNo">947</span> public static byte[][] inferBoundaries(TreeMap<byte[], Integer> bdryMap) {<a name="line.947"></a> -<span class="sourceLineNo">948</span> ArrayList<byte[]> keysArray = new ArrayList<byte[]>();<a name="line.948"></a> -<span class="sourceLineNo">949</span> int runningValue = 0;<a name="line.949"></a> -<span class="sourceLineNo">950</span> byte[] currStartKey = null;<a name="line.950"></a> -<span class="sourceLineNo">951</span> boolean firstBoundary = true;<a name="line.951"></a> -<span class="sourceLineNo">952</span><a name="line.952"></a> -<span class="sourceLineNo">953</span> for (Map.Entry<byte[], Integer> item: bdryMap.entrySet()) {<a name="line.953"></a> -<span class="sourceLineNo">954</span> if (runningValue == 0) currStartKey = item.getKey();<a name="line.954"></a> -<span class="sourceLineNo">955</span> runningValue += item.getValue();<a name="line.955"></a> -<span class="sourceLineNo">956</span> if (runningValue == 0) {<a name="line.956"></a> -<span class="sourceLineNo">957</span> if (!firstBoundary) keysArray.add(currStartKey);<a name="line.957"></a> -<span class="sourceLineNo">958</span> firstBoundary = false;<a name="line.958"></a> -<span class="sourceLineNo">959</span> }<a name="line.959"></a> -<span class="sourceLineNo">960</span> }<a name="line.960"></a> -<span class="sourceLineNo">961</span><a name="line.961"></a> -<span class="sourceLineNo">962</span> return keysArray.toArray(new byte[0][0]);<a name="line.962"></a> -<span class="sourceLineNo">963</span> }<a name="line.963"></a> -<span class="sourceLineNo">964</span><a name="line.964"></a> -<span class="sourceLineNo">965</span> /*<a name="line.965"></a> -<span class="sourceLineNo">966</span> * If the table is created for the first time, then "completebulkload" reads the files twice.<a name="line.966"></a> -<span class="sourceLineNo">967</span> * More modifications necessary if we want to avoid doing it.<a name="line.967"></a> -<span class="sourceLineNo">968</span> */<a name="line.968"></a> -<span class="sourceLineNo">969</span> private void createTable(TableName tableName, String dirPath, Admin admin) throws Exception {<a name="line.969"></a> -<span class="sourceLineNo">970</span> final Path hfofDir = new Path(dirPath);<a name="line.970"></a> -<span class="sourceLineNo">971</span> final FileSystem fs = hfofDir.getFileSystem(getConf());<a name="line.971"></a> -<span class="sourceLineNo">972</span><a name="line.972"></a> -<span class="sourceLineNo">973</span> // Add column families<a name="line.973"></a> -<span class="sourceLineNo">974</span> // Build a set of keys<a name="line.974"></a> -<span class="sourceLineNo">975</span> final HTableDescriptor htd = new HTableDescriptor(tableName);<a name="line.975"></a> -<span class="sourceLineNo">976</span> final TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.976"></a> -<span class="sourceLineNo">977</span> visitBulkHFiles(fs, hfofDir, new BulkHFileVisitor<HColumnDescriptor>() {<a name="line.977"></a> -<span class="sourceLineNo">978</span> @Override<a name="line.978"></a> -<span class="sourceLineNo">979</span> public HColumnDescriptor bulkFamily(final byte[] familyName) {<a name="line.979"></a> -<span class="sourceLineNo">980</span> HColumnDescriptor hcd = new HColumnDescriptor(familyName);<a name="line.980"></a> -<span class="sourceLineNo">981</span> htd.addFamily(hcd);<a name="line.981"></a> -<span class="sourceLineNo">982</span> return hcd;<a name="line.982"></a> -<span class="sourceLineNo">983</span> }<a name="line.983"></a> -<span class="sourceLineNo">984</span> @Override<a name="line.984"></a> -<span class="sourceLineNo">985</span> public void bulkHFile(final HColumnDescriptor hcd, final FileStatus hfileStatus)<a name="line.985"></a> -<span class="sourceLineNo">986</span> throws IOException {<a name="line.986"></a> -<span class="sourceLineNo">987</span> Path hfile = hfileStatus.getPath();<a name="line.987"></a> -<span class="sourceLineNo">988</span> HFile.Reader reader = HFile.createReader(fs, hfile,<a name="line.988"></a> -<span class="sourceLineNo">989</span> new CacheConfig(getConf()), getConf());<a name="line.989"></a> -<span class="sourceLineNo">990</span> try {<a name="line.990"></a> -<span class="sourceLineNo">991</span> if (hcd.getCompressionType() != reader.getFileContext().getCompression()) {<a name="line.991"></a> -<span class="sourceLineNo">992</span> hcd.setCompressionType(reader.getFileContext().getCompression());<a name="line.992"></a> -<span class="sourceLineNo">993</span> LOG.info("Setting compression " + hcd.getCompressionType().name() +<a name="line.993"></a> -<span class="sourceLineNo">994</span> " for family " + hcd.toString());<a name="line.994"></a> -<span class="sourceLineNo">995</span> }<a name="line.995"></a> -<span class="sourceLineNo">996</span> reader.loadFileInfo();<a name="line.996"></a> -<span class="sourceLineNo">997</span> byte[] first = reader.getFirstRowKey();<a name="line.997"></a> -<span class="sourceLineNo">998</span> byte[] last = reader.getLastRowKey();<a name="line.998"></a> -<span class="sourceLineNo">999</span><a name="line.999"></a> -<span class="sourceLineNo">1000</span> LOG.info("Trying to figure out region boundaries hfile=" + hfile +<a name="line.1000"></a> -<span class="sourceLineNo">1001</span> " first=" + Bytes.toStringBinary(first) +<a name="line.1001"></a> -<span class="sourceLineNo">1002</span> " last=" + Bytes.toStringBinary(last));<a name="line.1002"></a> -<span class="sourceLineNo">1003</span><a name="line.1003"></a> -<span class="sourceLineNo">1004</span> // To eventually infer start key-end key boundaries<a name="line.1004"></a> -<span class="sourceLineNo">1005</span> Integer value = map.containsKey(first)? map.get(first):0;<a name="line.1005"></a> -<span class="sourceLineNo">1006</span> map.put(first, value+1);<a name="line.1006"></a> -<span class="sourceLineNo">1007</span><a name="line.1007"></a> -<span class="sourceLineNo">1008</span> value = map.containsKey(last)? map.get(last):0;<a name="line.1008"></a> -<span class="sourceLineNo">1009</span> map.put(last, value-1);<a name="line.1009"></a> -<span class="sourceLineNo">1010</span> } finally {<a name="line.1010"></a> -<span class="sourceLineNo">1011</span> reader.close();<a name="line.1011"></a> -<span class="sourceLineNo">1012</span> }<a name="line.1012"></a> -<span class="sourceLineNo">1013</span> }<a name="line.1013"></a> -<span class="sourceLineNo">1014</span> });<a name="line.1014"></a> -<span class="sourceLineNo">1015</span><a name="line.1015"></a> -<span class="sourceLineNo">1016</span> byte[][] keys = LoadIncrementalHFiles.inferBoundaries(map);<a name="line.1016"></a> -<span class="sourceLineNo">1017</span> admin.createTable(htd, keys);<a name="line.1017"></a> -<span class="sourceLineNo">1018</span><a name="line.1018"></a> -<span class="sourceLineNo">1019</span> LOG.info("Table "+ tableName +" is available!!");<a name="line.1019"></a> -<span class="sourceLineNo">1020</span> }<a name="line.1020"></a> -<span class="sourceLineNo">1021</span><a name="line.1021"></a> -<span class="sourceLineNo">1022</span> @Override<a name="line.1022"></a> -<span class="sourceLineNo">1023</span> public int run(String[] args) throws Exception {<a name="line.1023"></a> -<span class="sourceLineNo">1024</span> if (args.length != 2) {<a name="line.1024"></a> -<span class="sourceLineNo">1025</span> usage();<a name="line.1025"></a> -<span class="sourceLineNo">1026</span> return -1;<a name="line.1026"></a> -<span class="sourceLineNo">1027</span> }<a name="line.1027"></a> -<span class="sourceLineNo">1028</span><a name="line.1028"></a> -<span class="sourceLineNo">1029</span> initialize();<a name="line.1029"></a> -<span class="sourceLineNo">1030</span> try (Connection connection = ConnectionFactory.createConnection(getConf());<a name="line.1030"></a> -<span class="sourceLineNo">1031</span> Admin admin = connection.getAdmin()) {<a name="line.1031"></a> -<span class="sourceLineNo">1032</span> String dirPath = args[0];<a name="line.1032"></a> -<span class="sourceLineNo">1033</span> TableName tableName = TableName.valueOf(args[1]);<a name="line.1033"></a> -<span class="sourceLineNo">1034</span><a name="line.1034"></a> -<span class="sourceLineNo">1035</span> boolean tableExists = admin.tableExists(tableName);<a name="line.1035"></a> -<span class="sourceLineNo">1036</span> if (!tableExists) {<a name="line.1036"></a> -<span class="sourceLineNo">1037</span> if ("yes".equalsIgnoreCase(getConf().get(CREATE_TABLE_CONF_KEY, "yes"))) {<a name="line.1037"></a> -<span class="sourceLineNo">1038</span> this.createTable(tableName, dirPath, admin);<a name="line.1038"></a> -<span class="sourceLineNo">1039</span> } else {<a name="line.1039"></a> -<span class="sourceLineNo">1040</span> String errorMsg = format("Table '%s' does not exist.", tableName);<a name="line.1040"></a> -<span class="sourceLineNo">1041</span> LOG.error(errorMsg);<a name="line.1041"></a> -<span class="sourceLineNo">1042</span> throw new TableNotFoundException(errorMsg);<a name="line.1042"></a> -<span class="sourceLineNo">1043</span> }<a name="line.1043"></a> -<span class="sourceLineNo">1044</span> }<a name="line.1044"></a> -<span class="sourceLineNo">1045</span><a name="line.1045"></a> -<span class="sourceLineNo">1046</span> Path hfofDir = new Path(dirPath);<a name="line.1046"></a> -<span class="sourceLineNo">1047</span><a name="line.1047"></a> -<span class="sourceLineNo">1048</span> try (Table table = connection.getTable(tableName);<a name="line.1048"></a> -<span class="sourceLineNo">1049</span> RegionLocator locator = connection.getRegionLocator(tableName)) {<a name="line.1049"></a> -<span class="sourceLineNo">1050</span> doBulkLoad(hfofDir, admin, table, locator);<a name="line.1050"></a> -<span class="sourceLineNo">1051</span> }<a name="line.1051"></a> -<span class="sourceLineNo">1052</span> }<a name="line.1052"></a> -<span class="sourceLineNo">1053</span><a name="line.1053"></a> -<span class="sourceLineNo">1054</span> return 0;<a name="line.1054"></a> -<span class="sourceLineNo">1055</span> }<a name="line.1055"></a> -<span class="sourceLineNo">1056</span><a name="line.1056"></a> -<span class="sourceLineNo">1057</span> public static void main(String[] args) throws Exception {<a name="line.1057"></a> -<span class="sourceLineNo">1058</span> Configuration conf = HBaseConfiguration.create();<a name="line.1058"></a> -<span class="sourceLineNo">1059</span> int ret = ToolRunner.run(conf, new LoadIncrementalHFiles(), args);<a name="line.1059"></a> -<span class="sourceLineNo">1060</span> System.exit(ret);<a name="line.1060"></a> -<span class="sourceLineNo">1061</span> }<a name="line.1061"></a> -<span class="sourceLineNo">1062</span><a name="line.1062"></a> -<span class="sourceLineNo">1063</span> /**<a name="line.1063"></a> -<span class="sourceLineNo">1064</span> * Called from replication sink, where it manages bulkToken(staging directory) by itself. This is<a name="line.1064"></a> -<span class="sourceLineNo">1065</span> * used only when {@link SecureBulkLoadEndpoint} is configured in hbase.coprocessor.region.classes<a name="line.1065"></a> -<span class="sourceLineNo">1066</span> * property. This directory is used as a temporary directory where all files are initially<a name="line.1066"></a> -<span class="sourceLineNo">1067</span> * copied/moved from user given directory, set all the required file permissions and then from<a name="line.1067"></a> -<span class="sourceLineNo">1068</span> * their it is finally loaded into a table. This should be set only when, one would like to manage<a name="line.1068"></a> -<span class="sourceLineNo">1069</span> * the staging directory by itself. Otherwise this tool will handle this by itself.<a name="line.1069"></a> -<span class="sourceLineNo">1070</span> * @param stagingDir staging directory path<a name="line.1070"></a> -<span class="sourceLineNo">1071</span> */<a name="line.1071"></a> -<span class="sourceLineNo">1072</span> public void setBulkToken(String stagingDir) {<a name="line.1072"></a> -<span class="sourceLineNo">1073</span> this.bulkToken = stagingDir;<a name="line.1073"></a> -<span class="sourceLineNo">1074</span> }<a name="line.1074"></a> -<span class="sourceLineNo">1075</span><a name="line.1075"></a> -<span class="sourceLineNo">1076</span>}<a name="line.1076"></a> +<span class="sourceLineNo">906</span> .withIncludesTags(true)<a name="line.906"></a> +<span class="sourceLineNo">907</span> .build();<a name="line.907"></a> +<span class="sourceLineNo">908</span> halfWriter = new StoreFile.WriterBuilder(conf, cacheConf,<a name="line.908"></a> +<span class="sourceLineNo">909</span> fs)<a name="line.909"></a> +<span class="sourceLineNo">910</span> .withFilePath(outFile)<a name="line.910"></a> +<span class="sourceLineNo">911</span> .withBloomType(bloomFilterType)<a name="line.911"></a> +<span class="sourceLineNo">912</span> .withFileContext(hFileContext)<a name="line.912"></a> +<span class="sourceLineNo">913</span> .build();<a name="line.913"></a> +<span class="sourceLineNo">914</span> HFileScanner scanner = halfReader.getScanner(false, false, false);<a name="line.914"></a> +<span class="sourceLineNo">915</span> scanner.seekTo();<a name="line.915"></a> +<span class="sourceLineNo">916</span> do {<a name="line.916"></a> +<span class="sourceLineNo">917</span> halfWriter.append(scanner.getCell());<a name="line.917"></a> +<span class="sourceLineNo">918</span> } while (scanner.next());<a name="line.918"></a> +<span class="sourceLineNo">919</span><a name="line.919"></a> +<span class="sourceLineNo">920</span> for (Map.Entry<byte[],byte[]> entry : fileInfo.entrySet()) {<a name="line.920"></a> +<span class="sourceLineNo">921</span> if (shouldCopyHFileMetaKey(entry.getKey())) {<a name="line.921"></a> +<span class="sourceLineNo">922</span> halfWriter.appendFileInfo(entry.getKey(), entry.getValue());<a name="line.922"></a> +<span class="sourceLineNo">923</span> }<a name="line.923"></a> +<span class="sourceLineNo">924</span> }<a name="line.924"></a> +<span class="sourceLineNo">925</span> } finally {<a name="line.925"></a> +<span class="sourceLineNo">926</span> if (halfWriter != null) halfWriter.close();<a name="line.926"></a> +<span class="sourceLineNo">927</span> if (halfReader != null) halfReader.close(cacheConf.shouldEvictOnClose());<a name="line.927"></a> +<span class="sourceLineNo">928</span> }<a name="line.928"></a> +<span class="sourceLineNo">929</span> }<a name="line.929"></a> +<span class="sourceLineNo">930</span><a name="line.930"></a> +<span class="sourceLineNo">931</span> private static boolean shouldCopyHFileMetaKey(byte[] key) {<a name="line.931"></a> +<span class="sourceLineNo">932</span> return !HFile.isReservedFileInfoKey(key);<a name="line.932"></a> +<span class="sourceLineNo">933</span> }<a name="line.933"></a> +<span class="sourceLineNo">934</span><a name="line.934"></a> +<span class="sourceLineNo">935</span> /*<a name="line.935"></a> +<span class="sourceLineNo">936</span> * Infers region boundaries for a new table.<a name="line.936"></a> +<span class="sourceLineNo">937</span> * Parameter:<a name="line.937"></a> +<span class="sourceLineNo">938</span> * bdryMap is a map between keys to an integer belonging to {+1, -1}<a name="line.938"></a> +<span class="sourceLineNo">939</span> * If a key is a start key of a file, then it maps to +1<a name="line.939"></a> +<span class="sourceLineNo">940</span> * If a key is an end key of a file, then it maps to -1<a name="line.940"></a> +<span class="sourceLineNo">941</span> * Algo:<a name="line.941"></a> +<span class="sourceLineNo">942</span> * 1) Poll on the keys in order:<a name="line.942"></a> +<span class="sourceLineNo">943</span> * a) Keep adding the mapped values to these keys (runningSum)<a name="line.943"></a> +<span class="sourceLineNo">944</span> * b) Each time runningSum reaches 0, add the start Key from when the runningSum had started to<a name="line.944"></a> +<span class="sourceLineNo">945</span> * a boundary list.<a name="line.945"></a> +<span class="sourceLineNo">946</span> * 2) Return the boundary list.<a name="line.946"></a> +<span class="sourceLineNo">947</span> */<a name="line.947"></a> +<span class="sourceLineNo">948</span> public static byte[][] inferBoundaries(TreeMap<byte[], Integer> bdryMap) {<a name="line.948"></a> +<span class="sourceLineNo">949</span> ArrayList<byte[]> keysArray = new ArrayList<byte[]>();<a name="line.949"></a> +<span class="sourceLineNo">950</span> int runningValue = 0;<a name="line.950"></a> +<span class="sourceLineNo">951</span> byte[] currStartKey = null;<a name="line.951"></a> +<span class="sourceLineNo">952</span> boolean firstBoundary = true;<a name="line.952"></a> +<span class="sourceLineNo">953</span><a name="line.953"></a> +<span class="sourceLineNo">954</span> for (Map.Entry<byte[], Integer> item: bdryMap.entrySet()) {<a name="line.954"></a> +<span class="sourceLineNo">955</span> if (runningValue == 0) currStartKey = item.getKey();<a name="line.955"></a> +<span class="sourceLineNo">956</span> runningValue += item.getValue();<a name="line.956"></a> +<span class="sourceLineNo">957</span> if (runningValue == 0) {<a name="line.957"></a> +<span class="sourceLineNo">958</span> if (!firstBoundary) keysArray.add(currStartKey);<a name="line.958"></a> +<span class="sourceLineNo">959</span> firstBoundary = false;<a name="line.959"></a> +<span class="sourceLineNo">960</span> }<a name="line.960"></a> +<span class="sourceLineNo">961</span> }<a name="line.961"></a> +<span class="sourceLineNo">962</span><a name="line.962"></a> +<span class="sourceLineNo">963</span> return keysArray.toArray(new byte[0][0]);<a name="line.963"></a> +<span class="sourceLineNo">964</span> }<a name="line.964"></a> +<span class="sourceLineNo">965</span><a name="line.965"></a> +<span class="sourceLineNo">966</span> /*<a name="line.966"></a> +<span class="sourceLineNo">967</span> * If the table is created for the first time, then "completebulkload" reads the files twice.<a name="line.967"></a> +<span class="sourceLineNo">968</span> * More modifications necessary if we want to avoid doing it.<a name="line.968"></a> +<span class="sourceLineNo">969</span> */<a name="line.969"></a> +<span class="sourceLineNo">970</span> private void createTable(TableName tableName, String dirPath, Admin admin) throws Exception {<a name="line.970"></a> +<span class="sourceLineNo">971</span> final Path hfofDir = new Path(dirPath);<a name="line.971"></a> +<span class="sourceLineNo">972</span> final FileSystem fs = hfofDir.getFileSystem(getConf());<a name="line.972"></a> +<span class="sourceLineNo">973</span><a name="line.973"></a> +<span class="sourceLineNo">974</span> // Add column families<a name="line.974"></a> +<span class="sourceLineNo">975</span> // Build a set of keys<a name="line.975"></a> +<span class="sourceLineNo">976</span> final HTableDescriptor htd = new HTableDescriptor(tableName);<a name="line.976"></a> +<span class="sourceLineNo">977</span> final TreeMap<byte[], Integer> map = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);<a name="line.977"></a> +<span class="sourceLineNo">978</span> visitBulkHFiles(fs, hfofDir, new BulkHFileVisitor<HColumnDescriptor>() {<a name="line.978"></a> +<span class="sourceLineNo">979</span> @Override<a name="line.979"></a> +<span class="sourceLineNo">980</span> public HColumnDescriptor bulkFamily(final byte[] familyName) {<a name="line.980"></a> +<span class="sourceLineNo">981</span> HColumnDescriptor hcd = new HColumnDescriptor(familyName);<a name="line.981"></a> +<span class="sourceLineNo">982</span> htd.addFamily(hcd);<a name="line.982"></a> +<span class="sourceLineNo">983</span> return hcd;<a name="line.983"></a> +<span class="sourceLineNo">984</span> }<a name="line.984"></a> +<span class="sourceLineNo">985</span> @Override<a name="line.985"></a> +<span class="sourceLineNo">986</span> public void bulkHFile(final HColumnDescriptor hcd, final FileStatus hfileStatus)<a name="line.986"></a> +<span class="sourceLineNo">987</span> throws IOException {<a name="line.987"></a> +<span class="sourceLineNo">988</span> Path hfile = hfileStatus.getPath();<a name="line.988"></a> +<span class="sourceLineNo">989</span> HFile.Reader reader = HFile.createReader(fs, hfile,<a name="line.989"></a> +<span class="sourceLineNo">990</span> new CacheConfig(getConf()), getConf());<a name="line.990"></a> +<span class="sourceLineNo">991</span> try {<a name="line.991"></a> +<span class="sourceLineNo">992</span> if (hcd.getCompressionType() != reader.getFileContext().getCompression()) {<a name="line.992"></a> +<span class="sourceLineNo">993</span> hcd.setCompressionType(reader.getFileContext().getCompression());<a name="line.993"></a> +<span class="sourceLineNo">994</span> LOG.info("Setting compression " + hcd.getCompressionType().name() +<a name="line.994"></a> +<span class="sourceLineNo">995</span> " for family " + hcd.toString());<a name="line.995"></a> +<span class="sourceLineNo">996</span> }<a name="line.996"></a> +<span class="sourceLineNo">997</span> reader.loadFileInfo();<a name="line.997"></a> +<span class="sourceLineNo">998</span> byte[] first = reader.getFirstRowKey();<a name="line.998"></a> +<span class="sourceLineNo">999</span> byte[] last = reader.getLastRowKey();<a name="line.999"></a> +<span class="sourceLineNo">1000</span><a name="line.1000"></a> +<span class="sourceLineNo">1001</span> LOG.info("Trying to figure out region boundaries hfile=" + hfile +<a name="line.1001"></a> +<span class="sourceLineNo">1002</span> " first=" + Bytes.toStringBinary(first) +<a name="line.1002"></a> +<span class="sourceLineNo">1003</span> " last=" + Bytes.toStringBinary(last));<a name="line.1003"></a> +<span class="sourceLineNo">1004</span><a name="line.1004"></a> +<span class="sourceLineNo">1005</span> // To eventually infer start key-end key boundaries<a name="line.1005"></a> +<span class="sourceLineNo">1006</span> Integer value = map.containsKey(first)? map.get(first):0;<a name="line.1006"></a> +<span class="sourceLineNo">1007</span> map.put(first, value+1);<a name="line.1007"></a> +<span class="sourceLineNo">1008</span><a name="line.1008"></a> +<span class="sourceLineNo">1009</span> value = map.containsKey(last)? map.get(last):0;<a name="line.1009"></a> +<span class="sourceLineNo">1010</span> map.put(last, value-1);<a name="line.1010"></a> +<span class="sourceLineNo">1011</span> } finally {<a name="line.1011"></a> +<span class="sourceLineNo">1012</span> reader.close();<a name="line.1012"></a> +<span class="sourceLineNo">1013</span> }<a name="line.1013"></a> +<span class="sourceLineNo">1014</span> }<a name="line.1014"></a> +<span class="sourceLineNo">1015</span> });<a name="line.1015"></a> +<span class="sourceLineNo">1016</span><a name="line.1016"></a> +<span class="sourceLineNo">1017</span> byte[][] keys = LoadIncrementalHFiles.inferBoundaries(map);<a name="line.1017"></a> +<span class="sourceLineNo">1018</span> admin.createTable(htd, keys);<a name="line.1018"></a> +<span class="sourceLineNo">1019</span><a name="line.1019"></a> +<span class="sourceLineNo">1020</span> LOG.info("Table "+ tableName +" is available!!");<a name="line.1020"></a> +<span class="sourceLineNo">1021</span> }<a name="line.1021"></a> +<span class="sourceLineNo">1022</span><a name="line.1022"></a> +<span class="sourceLineNo">1023</span> @Override<a name="line.1023"></a> +<span class="sourceLineNo">1024</span> public int run(String[] args) throws Exception {<a name="line.1024"></a> +<span class="sourceLineNo">1025</span> if (args.length != 2) {<a name="line.1025"></a> +<span class="sourceLineNo">1026</span> usage();<a name="line.1026"></a> +<span class="sourceLineNo">1027</span> return -1;<a name="line.1027"></a> +<span class="sourceLineNo">1028</span> }<a name="line.1028"></a> +<span class="sourceLineNo">1029</span><a name="line.1029"></a> +<span class="sourceLineNo">1030</span> initialize();<a name="line.1030"></a> +<span class="sourceLineNo">1031</span> try (Connection connection = ConnectionFactory.createConnection(getConf());<a name="line.1031"></a> +<span class="sourceLineNo">1032</span> Admin admin = connection.getAdmin()) {<a name="line.1032"></a> +<span class="sourceLineNo">1033</span> String dirPath = args[0];<a name="line.1033"></a> +<span class="sourceLineNo">1034</span> TableName tableName = TableName.valueOf(args[1]);<a name="line.1034"></a> +<span class="sourceLineNo">1035</span><a name="line.1035"></a> +<span class="sourceLineNo">1036</span> boolean tableExists = admin.tableExists(tableName);<a name="line.1036"></a> +<span class="sourceLineNo">1037</span> if (!tableExists) {<a name="line.1037"></a> +<span class="sourceLineNo">1038</span> if ("yes".equalsIgnoreCase(getConf().get(CREATE_TABLE_CONF_KEY, "yes"))) {<a name="line.1038"></a> +<span class="sourceLineNo">1039</span> this.createTable(tableName, dirPath, admin);<a name="line.1039"></a> +<span class="sourceLineNo">1040</span> } else {<a name="line.1040"></a> +<span class="sourceLineNo">1041</span> String errorMsg = format("Table '%s' does not exist.", tableName);<a name="line.1041"></a> +<span class="sourceLineNo">1042</span> LOG.error(errorMsg);<a name="line.1042"></a> +<span class="sourceLineNo">1043</span> throw new TableNotFoundException(errorMsg);<a name="line.1043"></a> +<span class="sourceLineNo">1044</span> }<a name="line.1044"></a> +<span class="sourceLineNo">1045</span> }<a name="line.1045"></a> +<span class="sourceLineNo">1046</span><a name="line.1046"></a> +<span class="sourceLineNo">1047</span> Path hfofDir = new Path(dirPath);<a name="line.1047"></a> +<span class="sourceLineNo">1048</span><a name="line.1048"></a> +<span class="sourceLineNo">1049</span> try (Table table = connection.getTable(tableName);<a name="line.1049"></a> +<span class="sourceLineNo">1050</span> RegionLocator locator = connection.getRegionLocator(tableName)) {<a name="line.1050"></a> +<span class="sourceLineNo">1051</span> doBulkLoad(hfofDir, admin, table, locator);<a name="line.1051"></a> +<span class="sourceLineNo">1052</span> }<a name="line.1052"></a> +<span class="sourceLineNo">1053</span> }<a name="line.1053"></a> +<span class="sourceLineNo">1054</span><a name="line.1054"></a> +<span class="sourceLineNo">1055</span> return 0;<a name="line.1055"></a> +<span class="sourceLineNo">1056</span> }<a name="line.1056"></a> +<span class="sourceLineNo">1057</span><a name="line.1057"></a> +<span class="sourceLineNo">1058</span> public static void main(String[] args) throws Exception {<a name="line.1058"></a> +<span class="sourceLineNo">1059</span> Configuration conf = HBaseConfiguration.create();<a name="line.1059"></a> +<span class="sourceLineNo">1060</span> int ret = ToolRunner.run(conf, new LoadIncrementalHFiles(), args);<a name="line.1060"></a> +<span class="sourceLineNo">1061</span> System.exit(ret);<a name="line.1061"></a> +<span class="sourceLineNo">1062</span> }<a name="line.1062"></a> +<span class="sourceLineNo">1063</span><a name="line.1063"></a> +<span class="sourceLineNo">1064</span> /**<a name="line.1064"></a> +<span class="sourceLineNo">1065</span> * Called from replication sink, where it manages bulkToken(staging directory) by itself. This is<a name="line.1065"></a> +<span class="sourceLineNo">1066</span> * used only when {@link SecureBulkLoadEndpoint} is configured in hbase.coprocessor.region.classes<a name="line.1066"></a> +<span class="sourceLineNo">1067</span> * property. This directory is used as a temporary directory where all files are initially<a name="line.1067"></a> +<span class="sourceLineNo">1068</span> * copied/moved from user given directory, set all the required file permissions and then from<a name="line.1068"></a> +<span class="sourceLineNo">1069</span> * their it is finally loaded into a table. This should be set only when, one would like to manage<a name="line.1069"></a> +<span class="sourceLineNo">1070</span> * the staging directory by itself. Otherwise this tool will handle this by itself.<a name="line.1070"></a> +<span class="sourceLineNo">1071</span> * @param stagingDir staging directory path<a name="line.1071"></a> +<span class="sourceLineNo">1072</span> */<a name="line.1072"></a> +<span class="sourceLineNo">1073</span> public void setBulkToken(String stagingDir) {<a name="line.1073"></a> +<span class="sourceLineNo">1074</span> this.bulkToken = stagingDir;<a name="line.1074"></a> +<span class="sourceLineNo">1075</span> }<a name="line.1075"></a> +<span class="sourceLineNo">1076</span><a name="line.1076"></a> +<span class="sourceLineNo">1077</span>}<a name="line.1077"></a>
