http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html index a5873b7..9ed2eb5 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/RegionSplitPolicy.html @@ -41,115 +41,116 @@ <span class="sourceLineNo">033</span><a name="line.33"></a> <span class="sourceLineNo">034</span>/**<a name="line.34"></a> <span class="sourceLineNo">035</span> * A split policy determines when a region should be split.<a name="line.35"></a> -<span class="sourceLineNo">036</span> * @see IncreasingToUpperBoundRegionSplitPolicy Default split policy since<a name="line.36"></a> -<span class="sourceLineNo">037</span> * 0.94.0<a name="line.37"></a> -<span class="sourceLineNo">038</span> * @see ConstantSizeRegionSplitPolicy Default split policy before 0.94.0<a name="line.38"></a> -<span class="sourceLineNo">039</span> */<a name="line.39"></a> -<span class="sourceLineNo">040</span>@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)<a name="line.40"></a> -<span class="sourceLineNo">041</span>public abstract class RegionSplitPolicy extends Configured {<a name="line.41"></a> -<span class="sourceLineNo">042</span> private static final Class<? extends RegionSplitPolicy><a name="line.42"></a> -<span class="sourceLineNo">043</span> DEFAULT_SPLIT_POLICY_CLASS = IncreasingToUpperBoundRegionSplitPolicy.class;<a name="line.43"></a> -<span class="sourceLineNo">044</span><a name="line.44"></a> -<span class="sourceLineNo">045</span> /**<a name="line.45"></a> -<span class="sourceLineNo">046</span> * The region configured for this split policy.<a name="line.46"></a> -<span class="sourceLineNo">047</span> */<a name="line.47"></a> -<span class="sourceLineNo">048</span> protected HRegion region;<a name="line.48"></a> -<span class="sourceLineNo">049</span><a name="line.49"></a> -<span class="sourceLineNo">050</span> /**<a name="line.50"></a> -<span class="sourceLineNo">051</span> * Upon construction, this method will be called with the region<a name="line.51"></a> -<span class="sourceLineNo">052</span> * to be governed. It will be called once and only once.<a name="line.52"></a> -<span class="sourceLineNo">053</span> */<a name="line.53"></a> -<span class="sourceLineNo">054</span> protected void configureForRegion(HRegion region) {<a name="line.54"></a> -<span class="sourceLineNo">055</span> Preconditions.checkState(<a name="line.55"></a> -<span class="sourceLineNo">056</span> this.region == null,<a name="line.56"></a> -<span class="sourceLineNo">057</span> "Policy already configured for region {}",<a name="line.57"></a> -<span class="sourceLineNo">058</span> this.region);<a name="line.58"></a> -<span class="sourceLineNo">059</span><a name="line.59"></a> -<span class="sourceLineNo">060</span> this.region = region;<a name="line.60"></a> -<span class="sourceLineNo">061</span> }<a name="line.61"></a> -<span class="sourceLineNo">062</span><a name="line.62"></a> -<span class="sourceLineNo">063</span> /**<a name="line.63"></a> -<span class="sourceLineNo">064</span> * @return true if the specified region should be split.<a name="line.64"></a> -<span class="sourceLineNo">065</span> */<a name="line.65"></a> -<span class="sourceLineNo">066</span> protected abstract boolean shouldSplit();<a name="line.66"></a> -<span class="sourceLineNo">067</span><a name="line.67"></a> -<span class="sourceLineNo">068</span> /**<a name="line.68"></a> -<span class="sourceLineNo">069</span> * @return the key at which the region should be split, or null<a name="line.69"></a> -<span class="sourceLineNo">070</span> * if it cannot be split. This will only be called if shouldSplit<a name="line.70"></a> -<span class="sourceLineNo">071</span> * previously returned true.<a name="line.71"></a> -<span class="sourceLineNo">072</span> */<a name="line.72"></a> -<span class="sourceLineNo">073</span> protected byte[] getSplitPoint() {<a name="line.73"></a> -<span class="sourceLineNo">074</span> byte[] explicitSplitPoint = this.region.getExplicitSplitPoint();<a name="line.74"></a> -<span class="sourceLineNo">075</span> if (explicitSplitPoint != null) {<a name="line.75"></a> -<span class="sourceLineNo">076</span> return explicitSplitPoint;<a name="line.76"></a> -<span class="sourceLineNo">077</span> }<a name="line.77"></a> -<span class="sourceLineNo">078</span> List<Store> stores = region.getStores();<a name="line.78"></a> -<span class="sourceLineNo">079</span><a name="line.79"></a> -<span class="sourceLineNo">080</span> byte[] splitPointFromLargestStore = null;<a name="line.80"></a> -<span class="sourceLineNo">081</span> long largestStoreSize = 0;<a name="line.81"></a> -<span class="sourceLineNo">082</span> for (Store s : stores) {<a name="line.82"></a> -<span class="sourceLineNo">083</span> byte[] splitPoint = s.getSplitPoint();<a name="line.83"></a> -<span class="sourceLineNo">084</span> // Store also returns null if it has references as way of indicating it is not splittable<a name="line.84"></a> -<span class="sourceLineNo">085</span> long storeSize = s.getSize();<a name="line.85"></a> -<span class="sourceLineNo">086</span> if (splitPoint != null && largestStoreSize < storeSize) {<a name="line.86"></a> -<span class="sourceLineNo">087</span> splitPointFromLargestStore = splitPoint;<a name="line.87"></a> -<span class="sourceLineNo">088</span> largestStoreSize = storeSize;<a name="line.88"></a> -<span class="sourceLineNo">089</span> }<a name="line.89"></a> -<span class="sourceLineNo">090</span> }<a name="line.90"></a> -<span class="sourceLineNo">091</span><a name="line.91"></a> -<span class="sourceLineNo">092</span> return splitPointFromLargestStore;<a name="line.92"></a> -<span class="sourceLineNo">093</span> }<a name="line.93"></a> -<span class="sourceLineNo">094</span><a name="line.94"></a> -<span class="sourceLineNo">095</span> /**<a name="line.95"></a> -<span class="sourceLineNo">096</span> * Create the RegionSplitPolicy configured for the given table.<a name="line.96"></a> -<span class="sourceLineNo">097</span> * @param region<a name="line.97"></a> -<span class="sourceLineNo">098</span> * @param conf<a name="line.98"></a> -<span class="sourceLineNo">099</span> * @return a RegionSplitPolicy<a name="line.99"></a> -<span class="sourceLineNo">100</span> * @throws IOException<a name="line.100"></a> -<span class="sourceLineNo">101</span> */<a name="line.101"></a> -<span class="sourceLineNo">102</span> public static RegionSplitPolicy create(HRegion region,<a name="line.102"></a> -<span class="sourceLineNo">103</span> Configuration conf) throws IOException {<a name="line.103"></a> -<span class="sourceLineNo">104</span> Class<? extends RegionSplitPolicy> clazz = getSplitPolicyClass(<a name="line.104"></a> -<span class="sourceLineNo">105</span> region.getTableDescriptor(), conf);<a name="line.105"></a> -<span class="sourceLineNo">106</span> RegionSplitPolicy policy = ReflectionUtils.newInstance(clazz, conf);<a name="line.106"></a> -<span class="sourceLineNo">107</span> policy.configureForRegion(region);<a name="line.107"></a> -<span class="sourceLineNo">108</span> return policy;<a name="line.108"></a> -<span class="sourceLineNo">109</span> }<a name="line.109"></a> -<span class="sourceLineNo">110</span><a name="line.110"></a> -<span class="sourceLineNo">111</span> public static Class<? extends RegionSplitPolicy> getSplitPolicyClass(<a name="line.111"></a> -<span class="sourceLineNo">112</span> TableDescriptor htd, Configuration conf) throws IOException {<a name="line.112"></a> -<span class="sourceLineNo">113</span> String className = htd.getRegionSplitPolicyClassName();<a name="line.113"></a> -<span class="sourceLineNo">114</span> if (className == null) {<a name="line.114"></a> -<span class="sourceLineNo">115</span> className = conf.get(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,<a name="line.115"></a> -<span class="sourceLineNo">116</span> DEFAULT_SPLIT_POLICY_CLASS.getName());<a name="line.116"></a> -<span class="sourceLineNo">117</span> }<a name="line.117"></a> -<span class="sourceLineNo">118</span><a name="line.118"></a> -<span class="sourceLineNo">119</span> try {<a name="line.119"></a> -<span class="sourceLineNo">120</span> Class<? extends RegionSplitPolicy> clazz =<a name="line.120"></a> -<span class="sourceLineNo">121</span> Class.forName(className).asSubclass(RegionSplitPolicy.class);<a name="line.121"></a> -<span class="sourceLineNo">122</span> return clazz;<a name="line.122"></a> -<span class="sourceLineNo">123</span> } catch (Exception e) {<a name="line.123"></a> -<span class="sourceLineNo">124</span> throw new IOException(<a name="line.124"></a> -<span class="sourceLineNo">125</span> "Unable to load configured region split policy '" +<a name="line.125"></a> -<span class="sourceLineNo">126</span> className + "' for table '" + htd.getTableName() + "'",<a name="line.126"></a> -<span class="sourceLineNo">127</span> e);<a name="line.127"></a> -<span class="sourceLineNo">128</span> }<a name="line.128"></a> -<span class="sourceLineNo">129</span> }<a name="line.129"></a> -<span class="sourceLineNo">130</span><a name="line.130"></a> -<span class="sourceLineNo">131</span> /**<a name="line.131"></a> -<span class="sourceLineNo">132</span> * In {@link HRegionFileSystem#splitStoreFile(org.apache.hadoop.hbase.HRegionInfo, String,<a name="line.132"></a> -<span class="sourceLineNo">133</span> * StoreFile, byte[], boolean, RegionSplitPolicy)} we are not creating the split reference<a name="line.133"></a> -<span class="sourceLineNo">134</span> * if split row not lies in the StoreFile range. But in some use cases we may need to create<a name="line.134"></a> -<span class="sourceLineNo">135</span> * the split reference even when the split row not lies in the range. This method can be used<a name="line.135"></a> -<span class="sourceLineNo">136</span> * to decide, whether to skip the the StoreFile range check or not.<a name="line.136"></a> -<span class="sourceLineNo">137</span> * @return whether to skip the StoreFile range check or not<a name="line.137"></a> -<span class="sourceLineNo">138</span> * @param familyName<a name="line.138"></a> -<span class="sourceLineNo">139</span> * @return whether to skip the StoreFile range check or not<a name="line.139"></a> -<span class="sourceLineNo">140</span> */<a name="line.140"></a> -<span class="sourceLineNo">141</span> protected boolean skipStoreFileRangeCheck(String familyName) {<a name="line.141"></a> -<span class="sourceLineNo">142</span> return false;<a name="line.142"></a> -<span class="sourceLineNo">143</span> }<a name="line.143"></a> -<span class="sourceLineNo">144</span>}<a name="line.144"></a> +<span class="sourceLineNo">036</span> * @see SteppingSplitPolicy Default split policy since 2.0.0<a name="line.36"></a> +<span class="sourceLineNo">037</span> * @see IncreasingToUpperBoundRegionSplitPolicy Default split policy since<a name="line.37"></a> +<span class="sourceLineNo">038</span> * 0.94.0<a name="line.38"></a> +<span class="sourceLineNo">039</span> * @see ConstantSizeRegionSplitPolicy Default split policy before 0.94.0<a name="line.39"></a> +<span class="sourceLineNo">040</span> */<a name="line.40"></a> +<span class="sourceLineNo">041</span>@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)<a name="line.41"></a> +<span class="sourceLineNo">042</span>public abstract class RegionSplitPolicy extends Configured {<a name="line.42"></a> +<span class="sourceLineNo">043</span> private static final Class<? extends RegionSplitPolicy><a name="line.43"></a> +<span class="sourceLineNo">044</span> DEFAULT_SPLIT_POLICY_CLASS = SteppingSplitPolicy.class;<a name="line.44"></a> +<span class="sourceLineNo">045</span><a name="line.45"></a> +<span class="sourceLineNo">046</span> /**<a name="line.46"></a> +<span class="sourceLineNo">047</span> * The region configured for this split policy.<a name="line.47"></a> +<span class="sourceLineNo">048</span> */<a name="line.48"></a> +<span class="sourceLineNo">049</span> protected HRegion region;<a name="line.49"></a> +<span class="sourceLineNo">050</span><a name="line.50"></a> +<span class="sourceLineNo">051</span> /**<a name="line.51"></a> +<span class="sourceLineNo">052</span> * Upon construction, this method will be called with the region<a name="line.52"></a> +<span class="sourceLineNo">053</span> * to be governed. It will be called once and only once.<a name="line.53"></a> +<span class="sourceLineNo">054</span> */<a name="line.54"></a> +<span class="sourceLineNo">055</span> protected void configureForRegion(HRegion region) {<a name="line.55"></a> +<span class="sourceLineNo">056</span> Preconditions.checkState(<a name="line.56"></a> +<span class="sourceLineNo">057</span> this.region == null,<a name="line.57"></a> +<span class="sourceLineNo">058</span> "Policy already configured for region {}",<a name="line.58"></a> +<span class="sourceLineNo">059</span> this.region);<a name="line.59"></a> +<span class="sourceLineNo">060</span><a name="line.60"></a> +<span class="sourceLineNo">061</span> this.region = region;<a name="line.61"></a> +<span class="sourceLineNo">062</span> }<a name="line.62"></a> +<span class="sourceLineNo">063</span><a name="line.63"></a> +<span class="sourceLineNo">064</span> /**<a name="line.64"></a> +<span class="sourceLineNo">065</span> * @return true if the specified region should be split.<a name="line.65"></a> +<span class="sourceLineNo">066</span> */<a name="line.66"></a> +<span class="sourceLineNo">067</span> protected abstract boolean shouldSplit();<a name="line.67"></a> +<span class="sourceLineNo">068</span><a name="line.68"></a> +<span class="sourceLineNo">069</span> /**<a name="line.69"></a> +<span class="sourceLineNo">070</span> * @return the key at which the region should be split, or null<a name="line.70"></a> +<span class="sourceLineNo">071</span> * if it cannot be split. This will only be called if shouldSplit<a name="line.71"></a> +<span class="sourceLineNo">072</span> * previously returned true.<a name="line.72"></a> +<span class="sourceLineNo">073</span> */<a name="line.73"></a> +<span class="sourceLineNo">074</span> protected byte[] getSplitPoint() {<a name="line.74"></a> +<span class="sourceLineNo">075</span> byte[] explicitSplitPoint = this.region.getExplicitSplitPoint();<a name="line.75"></a> +<span class="sourceLineNo">076</span> if (explicitSplitPoint != null) {<a name="line.76"></a> +<span class="sourceLineNo">077</span> return explicitSplitPoint;<a name="line.77"></a> +<span class="sourceLineNo">078</span> }<a name="line.78"></a> +<span class="sourceLineNo">079</span> List<Store> stores = region.getStores();<a name="line.79"></a> +<span class="sourceLineNo">080</span><a name="line.80"></a> +<span class="sourceLineNo">081</span> byte[] splitPointFromLargestStore = null;<a name="line.81"></a> +<span class="sourceLineNo">082</span> long largestStoreSize = 0;<a name="line.82"></a> +<span class="sourceLineNo">083</span> for (Store s : stores) {<a name="line.83"></a> +<span class="sourceLineNo">084</span> byte[] splitPoint = s.getSplitPoint();<a name="line.84"></a> +<span class="sourceLineNo">085</span> // Store also returns null if it has references as way of indicating it is not splittable<a name="line.85"></a> +<span class="sourceLineNo">086</span> long storeSize = s.getSize();<a name="line.86"></a> +<span class="sourceLineNo">087</span> if (splitPoint != null && largestStoreSize < storeSize) {<a name="line.87"></a> +<span class="sourceLineNo">088</span> splitPointFromLargestStore = splitPoint;<a name="line.88"></a> +<span class="sourceLineNo">089</span> largestStoreSize = storeSize;<a name="line.89"></a> +<span class="sourceLineNo">090</span> }<a name="line.90"></a> +<span class="sourceLineNo">091</span> }<a name="line.91"></a> +<span class="sourceLineNo">092</span><a name="line.92"></a> +<span class="sourceLineNo">093</span> return splitPointFromLargestStore;<a name="line.93"></a> +<span class="sourceLineNo">094</span> }<a name="line.94"></a> +<span class="sourceLineNo">095</span><a name="line.95"></a> +<span class="sourceLineNo">096</span> /**<a name="line.96"></a> +<span class="sourceLineNo">097</span> * Create the RegionSplitPolicy configured for the given table.<a name="line.97"></a> +<span class="sourceLineNo">098</span> * @param region<a name="line.98"></a> +<span class="sourceLineNo">099</span> * @param conf<a name="line.99"></a> +<span class="sourceLineNo">100</span> * @return a RegionSplitPolicy<a name="line.100"></a> +<span class="sourceLineNo">101</span> * @throws IOException<a name="line.101"></a> +<span class="sourceLineNo">102</span> */<a name="line.102"></a> +<span class="sourceLineNo">103</span> public static RegionSplitPolicy create(HRegion region,<a name="line.103"></a> +<span class="sourceLineNo">104</span> Configuration conf) throws IOException {<a name="line.104"></a> +<span class="sourceLineNo">105</span> Class<? extends RegionSplitPolicy> clazz = getSplitPolicyClass(<a name="line.105"></a> +<span class="sourceLineNo">106</span> region.getTableDescriptor(), conf);<a name="line.106"></a> +<span class="sourceLineNo">107</span> RegionSplitPolicy policy = ReflectionUtils.newInstance(clazz, conf);<a name="line.107"></a> +<span class="sourceLineNo">108</span> policy.configureForRegion(region);<a name="line.108"></a> +<span class="sourceLineNo">109</span> return policy;<a name="line.109"></a> +<span class="sourceLineNo">110</span> }<a name="line.110"></a> +<span class="sourceLineNo">111</span><a name="line.111"></a> +<span class="sourceLineNo">112</span> public static Class<? extends RegionSplitPolicy> getSplitPolicyClass(<a name="line.112"></a> +<span class="sourceLineNo">113</span> TableDescriptor htd, Configuration conf) throws IOException {<a name="line.113"></a> +<span class="sourceLineNo">114</span> String className = htd.getRegionSplitPolicyClassName();<a name="line.114"></a> +<span class="sourceLineNo">115</span> if (className == null) {<a name="line.115"></a> +<span class="sourceLineNo">116</span> className = conf.get(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,<a name="line.116"></a> +<span class="sourceLineNo">117</span> DEFAULT_SPLIT_POLICY_CLASS.getName());<a name="line.117"></a> +<span class="sourceLineNo">118</span> }<a name="line.118"></a> +<span class="sourceLineNo">119</span><a name="line.119"></a> +<span class="sourceLineNo">120</span> try {<a name="line.120"></a> +<span class="sourceLineNo">121</span> Class<? extends RegionSplitPolicy> clazz =<a name="line.121"></a> +<span class="sourceLineNo">122</span> Class.forName(className).asSubclass(RegionSplitPolicy.class);<a name="line.122"></a> +<span class="sourceLineNo">123</span> return clazz;<a name="line.123"></a> +<span class="sourceLineNo">124</span> } catch (Exception e) {<a name="line.124"></a> +<span class="sourceLineNo">125</span> throw new IOException(<a name="line.125"></a> +<span class="sourceLineNo">126</span> "Unable to load configured region split policy '" +<a name="line.126"></a> +<span class="sourceLineNo">127</span> className + "' for table '" + htd.getTableName() + "'",<a name="line.127"></a> +<span class="sourceLineNo">128</span> e);<a name="line.128"></a> +<span class="sourceLineNo">129</span> }<a name="line.129"></a> +<span class="sourceLineNo">130</span> }<a name="line.130"></a> +<span class="sourceLineNo">131</span><a name="line.131"></a> +<span class="sourceLineNo">132</span> /**<a name="line.132"></a> +<span class="sourceLineNo">133</span> * In {@link HRegionFileSystem#splitStoreFile(org.apache.hadoop.hbase.HRegionInfo, String,<a name="line.133"></a> +<span class="sourceLineNo">134</span> * StoreFile, byte[], boolean, RegionSplitPolicy)} we are not creating the split reference<a name="line.134"></a> +<span class="sourceLineNo">135</span> * if split row not lies in the StoreFile range. But in some use cases we may need to create<a name="line.135"></a> +<span class="sourceLineNo">136</span> * the split reference even when the split row not lies in the range. This method can be used<a name="line.136"></a> +<span class="sourceLineNo">137</span> * to decide, whether to skip the the StoreFile range check or not.<a name="line.137"></a> +<span class="sourceLineNo">138</span> * @return whether to skip the StoreFile range check or not<a name="line.138"></a> +<span class="sourceLineNo">139</span> * @param familyName<a name="line.139"></a> +<span class="sourceLineNo">140</span> * @return whether to skip the StoreFile range check or not<a name="line.140"></a> +<span class="sourceLineNo">141</span> */<a name="line.141"></a> +<span class="sourceLineNo">142</span> protected boolean skipStoreFileRangeCheck(String familyName) {<a name="line.142"></a> +<span class="sourceLineNo">143</span> return false;<a name="line.143"></a> +<span class="sourceLineNo">144</span> }<a name="line.144"></a> +<span class="sourceLineNo">145</span>}<a name="line.145"></a>
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferEventHandler.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferEventHandler.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferEventHandler.html index 8a6a93f..4ce36a7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferEventHandler.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferEventHandler.html @@ -224,7 +224,7 @@ <span class="sourceLineNo">216</span> FSUtils.getDefaultReplication(fs, this.walDir));<a name="line.216"></a> <span class="sourceLineNo">217</span> this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",<a name="line.217"></a> <span class="sourceLineNo">218</span> 5);<a name="line.218"></a> -<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 0);<a name="line.219"></a> +<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);<a name="line.219"></a> <span class="sourceLineNo">220</span><a name="line.220"></a> <span class="sourceLineNo">221</span> // rollWriter sets this.hdfs_out if it can.<a name="line.221"></a> <span class="sourceLineNo">222</span> rollWriter();<a name="line.222"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferExceptionHandler.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferExceptionHandler.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferExceptionHandler.html index 8a6a93f..4ce36a7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferExceptionHandler.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.RingBufferExceptionHandler.html @@ -224,7 +224,7 @@ <span class="sourceLineNo">216</span> FSUtils.getDefaultReplication(fs, this.walDir));<a name="line.216"></a> <span class="sourceLineNo">217</span> this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",<a name="line.217"></a> <span class="sourceLineNo">218</span> 5);<a name="line.218"></a> -<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 0);<a name="line.219"></a> +<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);<a name="line.219"></a> <span class="sourceLineNo">220</span><a name="line.220"></a> <span class="sourceLineNo">221</span> // rollWriter sets this.hdfs_out if it can.<a name="line.221"></a> <span class="sourceLineNo">222</span> rollWriter();<a name="line.222"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SafePointZigZagLatch.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SafePointZigZagLatch.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SafePointZigZagLatch.html index 8a6a93f..4ce36a7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SafePointZigZagLatch.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SafePointZigZagLatch.html @@ -224,7 +224,7 @@ <span class="sourceLineNo">216</span> FSUtils.getDefaultReplication(fs, this.walDir));<a name="line.216"></a> <span class="sourceLineNo">217</span> this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",<a name="line.217"></a> <span class="sourceLineNo">218</span> 5);<a name="line.218"></a> -<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 0);<a name="line.219"></a> +<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);<a name="line.219"></a> <span class="sourceLineNo">220</span><a name="line.220"></a> <span class="sourceLineNo">221</span> // rollWriter sets this.hdfs_out if it can.<a name="line.221"></a> <span class="sourceLineNo">222</span> rollWriter();<a name="line.222"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html index 8a6a93f..4ce36a7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.SyncRunner.html @@ -224,7 +224,7 @@ <span class="sourceLineNo">216</span> FSUtils.getDefaultReplication(fs, this.walDir));<a name="line.216"></a> <span class="sourceLineNo">217</span> this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",<a name="line.217"></a> <span class="sourceLineNo">218</span> 5);<a name="line.218"></a> -<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 0);<a name="line.219"></a> +<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);<a name="line.219"></a> <span class="sourceLineNo">220</span><a name="line.220"></a> <span class="sourceLineNo">221</span> // rollWriter sets this.hdfs_out if it can.<a name="line.221"></a> <span class="sourceLineNo">222</span> rollWriter();<a name="line.222"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html index 8a6a93f..4ce36a7 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/FSHLog.html @@ -224,7 +224,7 @@ <span class="sourceLineNo">216</span> FSUtils.getDefaultReplication(fs, this.walDir));<a name="line.216"></a> <span class="sourceLineNo">217</span> this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",<a name="line.217"></a> <span class="sourceLineNo">218</span> 5);<a name="line.218"></a> -<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 0);<a name="line.219"></a> +<span class="sourceLineNo">219</span> this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);<a name="line.219"></a> <span class="sourceLineNo">220</span><a name="line.220"></a> <span class="sourceLineNo">221</span> // rollWriter sets this.hdfs_out if it can.<a name="line.221"></a> <span class="sourceLineNo">222</span> rollWriter();<a name="line.222"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrContext.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrContext.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrContext.html index 2daec52..a4a5d2c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrContext.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrContext.html @@ -47,7 +47,7 @@ <span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;<a name="line.39"></a> <span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;<a name="line.40"></a> <span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader.Builder;<a name="line.41"></a> -<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey;<a name="line.42"></a> +<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALEdit;<a name="line.42"></a> <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer;<a name="line.43"></a> <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.44"></a> <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.wal.WAL.Entry;<a name="line.45"></a> @@ -341,7 +341,7 @@ <span class="sourceLineNo">333</span> }<a name="line.333"></a> <span class="sourceLineNo">334</span> return false;<a name="line.334"></a> <span class="sourceLineNo">335</span> }<a name="line.335"></a> -<span class="sourceLineNo">336</span> WALKey.Builder builder = WALKey.newBuilder();<a name="line.336"></a> +<span class="sourceLineNo">336</span> WALEdit.Builder builder = WALEdit.newBuilder();<a name="line.336"></a> <span class="sourceLineNo">337</span> long size = 0;<a name="line.337"></a> <span class="sourceLineNo">338</span> try {<a name="line.338"></a> <span class="sourceLineNo">339</span> long available = -1;<a name="line.339"></a> @@ -371,86 +371,87 @@ <span class="sourceLineNo">363</span> throw new EOFException("Partial PB while reading WAL, " +<a name="line.363"></a> <span class="sourceLineNo">364</span> "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos());<a name="line.364"></a> <span class="sourceLineNo">365</span> }<a name="line.365"></a> -<span class="sourceLineNo">366</span> WALKey walKey = builder.build();<a name="line.366"></a> +<span class="sourceLineNo">366</span> WALEdit walKey = builder.build();<a name="line.366"></a> <span class="sourceLineNo">367</span> entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor);<a name="line.367"></a> <span class="sourceLineNo">368</span> if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) {<a name="line.368"></a> <span class="sourceLineNo">369</span> if (LOG.isTraceEnabled()) {<a name="line.369"></a> -<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());<a name="line.370"></a> -<span class="sourceLineNo">371</span> }<a name="line.371"></a> -<span class="sourceLineNo">372</span> continue;<a name="line.372"></a> -<span class="sourceLineNo">373</span> }<a name="line.373"></a> -<span class="sourceLineNo">374</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.374"></a> -<span class="sourceLineNo">375</span> long posBefore = this.inputStream.getPos();<a name="line.375"></a> -<span class="sourceLineNo">376</span> try {<a name="line.376"></a> -<span class="sourceLineNo">377</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.377"></a> -<span class="sourceLineNo">378</span> if (expectedCells != actualCells) {<a name="line.378"></a> -<span class="sourceLineNo">379</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.379"></a> -<span class="sourceLineNo">380</span> }<a name="line.380"></a> -<span class="sourceLineNo">381</span> } catch (Exception ex) {<a name="line.381"></a> -<span class="sourceLineNo">382</span> String posAfterStr = "<unknown>";<a name="line.382"></a> -<span class="sourceLineNo">383</span> try {<a name="line.383"></a> -<span class="sourceLineNo">384</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.384"></a> -<span class="sourceLineNo">385</span> } catch (Throwable t) {<a name="line.385"></a> -<span class="sourceLineNo">386</span> if (LOG.isTraceEnabled()) {<a name="line.386"></a> -<span class="sourceLineNo">387</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.387"></a> -<span class="sourceLineNo">388</span> }<a name="line.388"></a> -<span class="sourceLineNo">389</span> }<a name="line.389"></a> -<span class="sourceLineNo">390</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.390"></a> -<span class="sourceLineNo">391</span> + posBefore + " and read up to " + posAfterStr;<a name="line.391"></a> -<span class="sourceLineNo">392</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.392"></a> -<span class="sourceLineNo">393</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.393"></a> -<span class="sourceLineNo">394</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.394"></a> -<span class="sourceLineNo">395</span> }<a name="line.395"></a> -<span class="sourceLineNo">396</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.396"></a> -<span class="sourceLineNo">397</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.397"></a> -<span class="sourceLineNo">398</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.398"></a> -<span class="sourceLineNo">399</span> + this.walEditsStopOffset);<a name="line.399"></a> -<span class="sourceLineNo">400</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.400"></a> -<span class="sourceLineNo">401</span> }<a name="line.401"></a> -<span class="sourceLineNo">402</span> } catch (EOFException eof) {<a name="line.402"></a> -<span class="sourceLineNo">403</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.403"></a> -<span class="sourceLineNo">404</span> if (originalPosition < 0) {<a name="line.404"></a> -<span class="sourceLineNo">405</span> if (LOG.isTraceEnabled()) {<a name="line.405"></a> -<span class="sourceLineNo">406</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.406"></a> -<span class="sourceLineNo">407</span> }<a name="line.407"></a> -<span class="sourceLineNo">408</span> throw eof;<a name="line.408"></a> -<span class="sourceLineNo">409</span> }<a name="line.409"></a> -<span class="sourceLineNo">410</span> // Else restore our position to original location in hope that next time through we will<a name="line.410"></a> -<span class="sourceLineNo">411</span> // read successfully.<a name="line.411"></a> -<span class="sourceLineNo">412</span> if (LOG.isTraceEnabled()) {<a name="line.412"></a> -<span class="sourceLineNo">413</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.413"></a> -<span class="sourceLineNo">414</span> }<a name="line.414"></a> -<span class="sourceLineNo">415</span> seekOnFs(originalPosition);<a name="line.415"></a> -<span class="sourceLineNo">416</span> return false;<a name="line.416"></a> -<span class="sourceLineNo">417</span> }<a name="line.417"></a> -<span class="sourceLineNo">418</span> return true;<a name="line.418"></a> -<span class="sourceLineNo">419</span> }<a name="line.419"></a> -<span class="sourceLineNo">420</span> }<a name="line.420"></a> -<span class="sourceLineNo">421</span><a name="line.421"></a> -<span class="sourceLineNo">422</span> private IOException extractHiddenEof(Exception ex) {<a name="line.422"></a> -<span class="sourceLineNo">423</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.423"></a> -<span class="sourceLineNo">424</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.424"></a> -<span class="sourceLineNo">425</span> IOException ioEx = null;<a name="line.425"></a> -<span class="sourceLineNo">426</span> if (ex instanceof EOFException) {<a name="line.426"></a> -<span class="sourceLineNo">427</span> return (EOFException)ex;<a name="line.427"></a> -<span class="sourceLineNo">428</span> } else if (ex instanceof IOException) {<a name="line.428"></a> -<span class="sourceLineNo">429</span> ioEx = (IOException)ex;<a name="line.429"></a> -<span class="sourceLineNo">430</span> } else if (ex instanceof RuntimeException<a name="line.430"></a> -<span class="sourceLineNo">431</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.431"></a> -<span class="sourceLineNo">432</span> ioEx = (IOException)ex.getCause();<a name="line.432"></a> -<span class="sourceLineNo">433</span> }<a name="line.433"></a> -<span class="sourceLineNo">434</span> if (ioEx != null) {<a name="line.434"></a> -<span class="sourceLineNo">435</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.435"></a> -<span class="sourceLineNo">436</span> return null;<a name="line.436"></a> -<span class="sourceLineNo">437</span> }<a name="line.437"></a> -<span class="sourceLineNo">438</span> return null;<a name="line.438"></a> -<span class="sourceLineNo">439</span> }<a name="line.439"></a> -<span class="sourceLineNo">440</span><a name="line.440"></a> -<span class="sourceLineNo">441</span> @Override<a name="line.441"></a> -<span class="sourceLineNo">442</span> protected void seekOnFs(long pos) throws IOException {<a name="line.442"></a> -<span class="sourceLineNo">443</span> this.inputStream.seek(pos);<a name="line.443"></a> -<span class="sourceLineNo">444</span> }<a name="line.444"></a> -<span class="sourceLineNo">445</span>}<a name="line.445"></a> +<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +<a name="line.370"></a> +<span class="sourceLineNo">371</span> this.inputStream.getPos());<a name="line.371"></a> +<span class="sourceLineNo">372</span> }<a name="line.372"></a> +<span class="sourceLineNo">373</span> continue;<a name="line.373"></a> +<span class="sourceLineNo">374</span> }<a name="line.374"></a> +<span class="sourceLineNo">375</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.375"></a> +<span class="sourceLineNo">376</span> long posBefore = this.inputStream.getPos();<a name="line.376"></a> +<span class="sourceLineNo">377</span> try {<a name="line.377"></a> +<span class="sourceLineNo">378</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.378"></a> +<span class="sourceLineNo">379</span> if (expectedCells != actualCells) {<a name="line.379"></a> +<span class="sourceLineNo">380</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.380"></a> +<span class="sourceLineNo">381</span> }<a name="line.381"></a> +<span class="sourceLineNo">382</span> } catch (Exception ex) {<a name="line.382"></a> +<span class="sourceLineNo">383</span> String posAfterStr = "<unknown>";<a name="line.383"></a> +<span class="sourceLineNo">384</span> try {<a name="line.384"></a> +<span class="sourceLineNo">385</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.385"></a> +<span class="sourceLineNo">386</span> } catch (Throwable t) {<a name="line.386"></a> +<span class="sourceLineNo">387</span> if (LOG.isTraceEnabled()) {<a name="line.387"></a> +<span class="sourceLineNo">388</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.388"></a> +<span class="sourceLineNo">389</span> }<a name="line.389"></a> +<span class="sourceLineNo">390</span> }<a name="line.390"></a> +<span class="sourceLineNo">391</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.391"></a> +<span class="sourceLineNo">392</span> + posBefore + " and read up to " + posAfterStr;<a name="line.392"></a> +<span class="sourceLineNo">393</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.393"></a> +<span class="sourceLineNo">394</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.394"></a> +<span class="sourceLineNo">395</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.395"></a> +<span class="sourceLineNo">396</span> }<a name="line.396"></a> +<span class="sourceLineNo">397</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.397"></a> +<span class="sourceLineNo">398</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.398"></a> +<span class="sourceLineNo">399</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.399"></a> +<span class="sourceLineNo">400</span> + this.walEditsStopOffset);<a name="line.400"></a> +<span class="sourceLineNo">401</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.401"></a> +<span class="sourceLineNo">402</span> }<a name="line.402"></a> +<span class="sourceLineNo">403</span> } catch (EOFException eof) {<a name="line.403"></a> +<span class="sourceLineNo">404</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.404"></a> +<span class="sourceLineNo">405</span> if (originalPosition < 0) {<a name="line.405"></a> +<span class="sourceLineNo">406</span> if (LOG.isTraceEnabled()) {<a name="line.406"></a> +<span class="sourceLineNo">407</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.407"></a> +<span class="sourceLineNo">408</span> }<a name="line.408"></a> +<span class="sourceLineNo">409</span> throw eof;<a name="line.409"></a> +<span class="sourceLineNo">410</span> }<a name="line.410"></a> +<span class="sourceLineNo">411</span> // Else restore our position to original location in hope that next time through we will<a name="line.411"></a> +<span class="sourceLineNo">412</span> // read successfully.<a name="line.412"></a> +<span class="sourceLineNo">413</span> if (LOG.isTraceEnabled()) {<a name="line.413"></a> +<span class="sourceLineNo">414</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.414"></a> +<span class="sourceLineNo">415</span> }<a name="line.415"></a> +<span class="sourceLineNo">416</span> seekOnFs(originalPosition);<a name="line.416"></a> +<span class="sourceLineNo">417</span> return false;<a name="line.417"></a> +<span class="sourceLineNo">418</span> }<a name="line.418"></a> +<span class="sourceLineNo">419</span> return true;<a name="line.419"></a> +<span class="sourceLineNo">420</span> }<a name="line.420"></a> +<span class="sourceLineNo">421</span> }<a name="line.421"></a> +<span class="sourceLineNo">422</span><a name="line.422"></a> +<span class="sourceLineNo">423</span> private IOException extractHiddenEof(Exception ex) {<a name="line.423"></a> +<span class="sourceLineNo">424</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.424"></a> +<span class="sourceLineNo">425</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.425"></a> +<span class="sourceLineNo">426</span> IOException ioEx = null;<a name="line.426"></a> +<span class="sourceLineNo">427</span> if (ex instanceof EOFException) {<a name="line.427"></a> +<span class="sourceLineNo">428</span> return (EOFException)ex;<a name="line.428"></a> +<span class="sourceLineNo">429</span> } else if (ex instanceof IOException) {<a name="line.429"></a> +<span class="sourceLineNo">430</span> ioEx = (IOException)ex;<a name="line.430"></a> +<span class="sourceLineNo">431</span> } else if (ex instanceof RuntimeException<a name="line.431"></a> +<span class="sourceLineNo">432</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.432"></a> +<span class="sourceLineNo">433</span> ioEx = (IOException)ex.getCause();<a name="line.433"></a> +<span class="sourceLineNo">434</span> }<a name="line.434"></a> +<span class="sourceLineNo">435</span> if (ioEx != null) {<a name="line.435"></a> +<span class="sourceLineNo">436</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.436"></a> +<span class="sourceLineNo">437</span> return null;<a name="line.437"></a> +<span class="sourceLineNo">438</span> }<a name="line.438"></a> +<span class="sourceLineNo">439</span> return null;<a name="line.439"></a> +<span class="sourceLineNo">440</span> }<a name="line.440"></a> +<span class="sourceLineNo">441</span><a name="line.441"></a> +<span class="sourceLineNo">442</span> @Override<a name="line.442"></a> +<span class="sourceLineNo">443</span> protected void seekOnFs(long pos) throws IOException {<a name="line.443"></a> +<span class="sourceLineNo">444</span> this.inputStream.seek(pos);<a name="line.444"></a> +<span class="sourceLineNo">445</span> }<a name="line.445"></a> +<span class="sourceLineNo">446</span>}<a name="line.446"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html index 2daec52..a4a5d2c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.WALHdrResult.html @@ -47,7 +47,7 @@ <span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;<a name="line.39"></a> <span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;<a name="line.40"></a> <span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader.Builder;<a name="line.41"></a> -<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey;<a name="line.42"></a> +<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALEdit;<a name="line.42"></a> <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer;<a name="line.43"></a> <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.44"></a> <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.wal.WAL.Entry;<a name="line.45"></a> @@ -341,7 +341,7 @@ <span class="sourceLineNo">333</span> }<a name="line.333"></a> <span class="sourceLineNo">334</span> return false;<a name="line.334"></a> <span class="sourceLineNo">335</span> }<a name="line.335"></a> -<span class="sourceLineNo">336</span> WALKey.Builder builder = WALKey.newBuilder();<a name="line.336"></a> +<span class="sourceLineNo">336</span> WALEdit.Builder builder = WALEdit.newBuilder();<a name="line.336"></a> <span class="sourceLineNo">337</span> long size = 0;<a name="line.337"></a> <span class="sourceLineNo">338</span> try {<a name="line.338"></a> <span class="sourceLineNo">339</span> long available = -1;<a name="line.339"></a> @@ -371,86 +371,87 @@ <span class="sourceLineNo">363</span> throw new EOFException("Partial PB while reading WAL, " +<a name="line.363"></a> <span class="sourceLineNo">364</span> "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos());<a name="line.364"></a> <span class="sourceLineNo">365</span> }<a name="line.365"></a> -<span class="sourceLineNo">366</span> WALKey walKey = builder.build();<a name="line.366"></a> +<span class="sourceLineNo">366</span> WALEdit walKey = builder.build();<a name="line.366"></a> <span class="sourceLineNo">367</span> entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor);<a name="line.367"></a> <span class="sourceLineNo">368</span> if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) {<a name="line.368"></a> <span class="sourceLineNo">369</span> if (LOG.isTraceEnabled()) {<a name="line.369"></a> -<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());<a name="line.370"></a> -<span class="sourceLineNo">371</span> }<a name="line.371"></a> -<span class="sourceLineNo">372</span> continue;<a name="line.372"></a> -<span class="sourceLineNo">373</span> }<a name="line.373"></a> -<span class="sourceLineNo">374</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.374"></a> -<span class="sourceLineNo">375</span> long posBefore = this.inputStream.getPos();<a name="line.375"></a> -<span class="sourceLineNo">376</span> try {<a name="line.376"></a> -<span class="sourceLineNo">377</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.377"></a> -<span class="sourceLineNo">378</span> if (expectedCells != actualCells) {<a name="line.378"></a> -<span class="sourceLineNo">379</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.379"></a> -<span class="sourceLineNo">380</span> }<a name="line.380"></a> -<span class="sourceLineNo">381</span> } catch (Exception ex) {<a name="line.381"></a> -<span class="sourceLineNo">382</span> String posAfterStr = "<unknown>";<a name="line.382"></a> -<span class="sourceLineNo">383</span> try {<a name="line.383"></a> -<span class="sourceLineNo">384</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.384"></a> -<span class="sourceLineNo">385</span> } catch (Throwable t) {<a name="line.385"></a> -<span class="sourceLineNo">386</span> if (LOG.isTraceEnabled()) {<a name="line.386"></a> -<span class="sourceLineNo">387</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.387"></a> -<span class="sourceLineNo">388</span> }<a name="line.388"></a> -<span class="sourceLineNo">389</span> }<a name="line.389"></a> -<span class="sourceLineNo">390</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.390"></a> -<span class="sourceLineNo">391</span> + posBefore + " and read up to " + posAfterStr;<a name="line.391"></a> -<span class="sourceLineNo">392</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.392"></a> -<span class="sourceLineNo">393</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.393"></a> -<span class="sourceLineNo">394</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.394"></a> -<span class="sourceLineNo">395</span> }<a name="line.395"></a> -<span class="sourceLineNo">396</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.396"></a> -<span class="sourceLineNo">397</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.397"></a> -<span class="sourceLineNo">398</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.398"></a> -<span class="sourceLineNo">399</span> + this.walEditsStopOffset);<a name="line.399"></a> -<span class="sourceLineNo">400</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.400"></a> -<span class="sourceLineNo">401</span> }<a name="line.401"></a> -<span class="sourceLineNo">402</span> } catch (EOFException eof) {<a name="line.402"></a> -<span class="sourceLineNo">403</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.403"></a> -<span class="sourceLineNo">404</span> if (originalPosition < 0) {<a name="line.404"></a> -<span class="sourceLineNo">405</span> if (LOG.isTraceEnabled()) {<a name="line.405"></a> -<span class="sourceLineNo">406</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.406"></a> -<span class="sourceLineNo">407</span> }<a name="line.407"></a> -<span class="sourceLineNo">408</span> throw eof;<a name="line.408"></a> -<span class="sourceLineNo">409</span> }<a name="line.409"></a> -<span class="sourceLineNo">410</span> // Else restore our position to original location in hope that next time through we will<a name="line.410"></a> -<span class="sourceLineNo">411</span> // read successfully.<a name="line.411"></a> -<span class="sourceLineNo">412</span> if (LOG.isTraceEnabled()) {<a name="line.412"></a> -<span class="sourceLineNo">413</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.413"></a> -<span class="sourceLineNo">414</span> }<a name="line.414"></a> -<span class="sourceLineNo">415</span> seekOnFs(originalPosition);<a name="line.415"></a> -<span class="sourceLineNo">416</span> return false;<a name="line.416"></a> -<span class="sourceLineNo">417</span> }<a name="line.417"></a> -<span class="sourceLineNo">418</span> return true;<a name="line.418"></a> -<span class="sourceLineNo">419</span> }<a name="line.419"></a> -<span class="sourceLineNo">420</span> }<a name="line.420"></a> -<span class="sourceLineNo">421</span><a name="line.421"></a> -<span class="sourceLineNo">422</span> private IOException extractHiddenEof(Exception ex) {<a name="line.422"></a> -<span class="sourceLineNo">423</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.423"></a> -<span class="sourceLineNo">424</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.424"></a> -<span class="sourceLineNo">425</span> IOException ioEx = null;<a name="line.425"></a> -<span class="sourceLineNo">426</span> if (ex instanceof EOFException) {<a name="line.426"></a> -<span class="sourceLineNo">427</span> return (EOFException)ex;<a name="line.427"></a> -<span class="sourceLineNo">428</span> } else if (ex instanceof IOException) {<a name="line.428"></a> -<span class="sourceLineNo">429</span> ioEx = (IOException)ex;<a name="line.429"></a> -<span class="sourceLineNo">430</span> } else if (ex instanceof RuntimeException<a name="line.430"></a> -<span class="sourceLineNo">431</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.431"></a> -<span class="sourceLineNo">432</span> ioEx = (IOException)ex.getCause();<a name="line.432"></a> -<span class="sourceLineNo">433</span> }<a name="line.433"></a> -<span class="sourceLineNo">434</span> if (ioEx != null) {<a name="line.434"></a> -<span class="sourceLineNo">435</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.435"></a> -<span class="sourceLineNo">436</span> return null;<a name="line.436"></a> -<span class="sourceLineNo">437</span> }<a name="line.437"></a> -<span class="sourceLineNo">438</span> return null;<a name="line.438"></a> -<span class="sourceLineNo">439</span> }<a name="line.439"></a> -<span class="sourceLineNo">440</span><a name="line.440"></a> -<span class="sourceLineNo">441</span> @Override<a name="line.441"></a> -<span class="sourceLineNo">442</span> protected void seekOnFs(long pos) throws IOException {<a name="line.442"></a> -<span class="sourceLineNo">443</span> this.inputStream.seek(pos);<a name="line.443"></a> -<span class="sourceLineNo">444</span> }<a name="line.444"></a> -<span class="sourceLineNo">445</span>}<a name="line.445"></a> +<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +<a name="line.370"></a> +<span class="sourceLineNo">371</span> this.inputStream.getPos());<a name="line.371"></a> +<span class="sourceLineNo">372</span> }<a name="line.372"></a> +<span class="sourceLineNo">373</span> continue;<a name="line.373"></a> +<span class="sourceLineNo">374</span> }<a name="line.374"></a> +<span class="sourceLineNo">375</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.375"></a> +<span class="sourceLineNo">376</span> long posBefore = this.inputStream.getPos();<a name="line.376"></a> +<span class="sourceLineNo">377</span> try {<a name="line.377"></a> +<span class="sourceLineNo">378</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.378"></a> +<span class="sourceLineNo">379</span> if (expectedCells != actualCells) {<a name="line.379"></a> +<span class="sourceLineNo">380</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.380"></a> +<span class="sourceLineNo">381</span> }<a name="line.381"></a> +<span class="sourceLineNo">382</span> } catch (Exception ex) {<a name="line.382"></a> +<span class="sourceLineNo">383</span> String posAfterStr = "<unknown>";<a name="line.383"></a> +<span class="sourceLineNo">384</span> try {<a name="line.384"></a> +<span class="sourceLineNo">385</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.385"></a> +<span class="sourceLineNo">386</span> } catch (Throwable t) {<a name="line.386"></a> +<span class="sourceLineNo">387</span> if (LOG.isTraceEnabled()) {<a name="line.387"></a> +<span class="sourceLineNo">388</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.388"></a> +<span class="sourceLineNo">389</span> }<a name="line.389"></a> +<span class="sourceLineNo">390</span> }<a name="line.390"></a> +<span class="sourceLineNo">391</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.391"></a> +<span class="sourceLineNo">392</span> + posBefore + " and read up to " + posAfterStr;<a name="line.392"></a> +<span class="sourceLineNo">393</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.393"></a> +<span class="sourceLineNo">394</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.394"></a> +<span class="sourceLineNo">395</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.395"></a> +<span class="sourceLineNo">396</span> }<a name="line.396"></a> +<span class="sourceLineNo">397</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.397"></a> +<span class="sourceLineNo">398</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.398"></a> +<span class="sourceLineNo">399</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.399"></a> +<span class="sourceLineNo">400</span> + this.walEditsStopOffset);<a name="line.400"></a> +<span class="sourceLineNo">401</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.401"></a> +<span class="sourceLineNo">402</span> }<a name="line.402"></a> +<span class="sourceLineNo">403</span> } catch (EOFException eof) {<a name="line.403"></a> +<span class="sourceLineNo">404</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.404"></a> +<span class="sourceLineNo">405</span> if (originalPosition < 0) {<a name="line.405"></a> +<span class="sourceLineNo">406</span> if (LOG.isTraceEnabled()) {<a name="line.406"></a> +<span class="sourceLineNo">407</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.407"></a> +<span class="sourceLineNo">408</span> }<a name="line.408"></a> +<span class="sourceLineNo">409</span> throw eof;<a name="line.409"></a> +<span class="sourceLineNo">410</span> }<a name="line.410"></a> +<span class="sourceLineNo">411</span> // Else restore our position to original location in hope that next time through we will<a name="line.411"></a> +<span class="sourceLineNo">412</span> // read successfully.<a name="line.412"></a> +<span class="sourceLineNo">413</span> if (LOG.isTraceEnabled()) {<a name="line.413"></a> +<span class="sourceLineNo">414</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.414"></a> +<span class="sourceLineNo">415</span> }<a name="line.415"></a> +<span class="sourceLineNo">416</span> seekOnFs(originalPosition);<a name="line.416"></a> +<span class="sourceLineNo">417</span> return false;<a name="line.417"></a> +<span class="sourceLineNo">418</span> }<a name="line.418"></a> +<span class="sourceLineNo">419</span> return true;<a name="line.419"></a> +<span class="sourceLineNo">420</span> }<a name="line.420"></a> +<span class="sourceLineNo">421</span> }<a name="line.421"></a> +<span class="sourceLineNo">422</span><a name="line.422"></a> +<span class="sourceLineNo">423</span> private IOException extractHiddenEof(Exception ex) {<a name="line.423"></a> +<span class="sourceLineNo">424</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.424"></a> +<span class="sourceLineNo">425</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.425"></a> +<span class="sourceLineNo">426</span> IOException ioEx = null;<a name="line.426"></a> +<span class="sourceLineNo">427</span> if (ex instanceof EOFException) {<a name="line.427"></a> +<span class="sourceLineNo">428</span> return (EOFException)ex;<a name="line.428"></a> +<span class="sourceLineNo">429</span> } else if (ex instanceof IOException) {<a name="line.429"></a> +<span class="sourceLineNo">430</span> ioEx = (IOException)ex;<a name="line.430"></a> +<span class="sourceLineNo">431</span> } else if (ex instanceof RuntimeException<a name="line.431"></a> +<span class="sourceLineNo">432</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.432"></a> +<span class="sourceLineNo">433</span> ioEx = (IOException)ex.getCause();<a name="line.433"></a> +<span class="sourceLineNo">434</span> }<a name="line.434"></a> +<span class="sourceLineNo">435</span> if (ioEx != null) {<a name="line.435"></a> +<span class="sourceLineNo">436</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.436"></a> +<span class="sourceLineNo">437</span> return null;<a name="line.437"></a> +<span class="sourceLineNo">438</span> }<a name="line.438"></a> +<span class="sourceLineNo">439</span> return null;<a name="line.439"></a> +<span class="sourceLineNo">440</span> }<a name="line.440"></a> +<span class="sourceLineNo">441</span><a name="line.441"></a> +<span class="sourceLineNo">442</span> @Override<a name="line.442"></a> +<span class="sourceLineNo">443</span> protected void seekOnFs(long pos) throws IOException {<a name="line.443"></a> +<span class="sourceLineNo">444</span> this.inputStream.seek(pos);<a name="line.444"></a> +<span class="sourceLineNo">445</span> }<a name="line.445"></a> +<span class="sourceLineNo">446</span>}<a name="line.446"></a> http://git-wip-us.apache.org/repos/asf/hbase-site/blob/539471a7/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.html b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.html index 2daec52..a4a5d2c 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/regionserver/wal/ProtobufLogReader.html @@ -47,7 +47,7 @@ <span class="sourceLineNo">039</span>import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;<a name="line.39"></a> <span class="sourceLineNo">040</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos;<a name="line.40"></a> <span class="sourceLineNo">041</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALHeader.Builder;<a name="line.41"></a> -<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALKey;<a name="line.42"></a> +<span class="sourceLineNo">042</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALEdit;<a name="line.42"></a> <span class="sourceLineNo">043</span>import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.WALTrailer;<a name="line.43"></a> <span class="sourceLineNo">044</span>import org.apache.hadoop.hbase.util.Bytes;<a name="line.44"></a> <span class="sourceLineNo">045</span>import org.apache.hadoop.hbase.wal.WAL.Entry;<a name="line.45"></a> @@ -341,7 +341,7 @@ <span class="sourceLineNo">333</span> }<a name="line.333"></a> <span class="sourceLineNo">334</span> return false;<a name="line.334"></a> <span class="sourceLineNo">335</span> }<a name="line.335"></a> -<span class="sourceLineNo">336</span> WALKey.Builder builder = WALKey.newBuilder();<a name="line.336"></a> +<span class="sourceLineNo">336</span> WALEdit.Builder builder = WALEdit.newBuilder();<a name="line.336"></a> <span class="sourceLineNo">337</span> long size = 0;<a name="line.337"></a> <span class="sourceLineNo">338</span> try {<a name="line.338"></a> <span class="sourceLineNo">339</span> long available = -1;<a name="line.339"></a> @@ -371,86 +371,87 @@ <span class="sourceLineNo">363</span> throw new EOFException("Partial PB while reading WAL, " +<a name="line.363"></a> <span class="sourceLineNo">364</span> "probably an unexpected EOF, ignoring. current offset=" + this.inputStream.getPos());<a name="line.364"></a> <span class="sourceLineNo">365</span> }<a name="line.365"></a> -<span class="sourceLineNo">366</span> WALKey walKey = builder.build();<a name="line.366"></a> +<span class="sourceLineNo">366</span> WALEdit walKey = builder.build();<a name="line.366"></a> <span class="sourceLineNo">367</span> entry.getKey().readFieldsFromPb(walKey, this.byteStringUncompressor);<a name="line.367"></a> <span class="sourceLineNo">368</span> if (!walKey.hasFollowingKvCount() || 0 == walKey.getFollowingKvCount()) {<a name="line.368"></a> <span class="sourceLineNo">369</span> if (LOG.isTraceEnabled()) {<a name="line.369"></a> -<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" + this.inputStream.getPos());<a name="line.370"></a> -<span class="sourceLineNo">371</span> }<a name="line.371"></a> -<span class="sourceLineNo">372</span> continue;<a name="line.372"></a> -<span class="sourceLineNo">373</span> }<a name="line.373"></a> -<span class="sourceLineNo">374</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.374"></a> -<span class="sourceLineNo">375</span> long posBefore = this.inputStream.getPos();<a name="line.375"></a> -<span class="sourceLineNo">376</span> try {<a name="line.376"></a> -<span class="sourceLineNo">377</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.377"></a> -<span class="sourceLineNo">378</span> if (expectedCells != actualCells) {<a name="line.378"></a> -<span class="sourceLineNo">379</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.379"></a> -<span class="sourceLineNo">380</span> }<a name="line.380"></a> -<span class="sourceLineNo">381</span> } catch (Exception ex) {<a name="line.381"></a> -<span class="sourceLineNo">382</span> String posAfterStr = "<unknown>";<a name="line.382"></a> -<span class="sourceLineNo">383</span> try {<a name="line.383"></a> -<span class="sourceLineNo">384</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.384"></a> -<span class="sourceLineNo">385</span> } catch (Throwable t) {<a name="line.385"></a> -<span class="sourceLineNo">386</span> if (LOG.isTraceEnabled()) {<a name="line.386"></a> -<span class="sourceLineNo">387</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.387"></a> -<span class="sourceLineNo">388</span> }<a name="line.388"></a> -<span class="sourceLineNo">389</span> }<a name="line.389"></a> -<span class="sourceLineNo">390</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.390"></a> -<span class="sourceLineNo">391</span> + posBefore + " and read up to " + posAfterStr;<a name="line.391"></a> -<span class="sourceLineNo">392</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.392"></a> -<span class="sourceLineNo">393</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.393"></a> -<span class="sourceLineNo">394</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.394"></a> -<span class="sourceLineNo">395</span> }<a name="line.395"></a> -<span class="sourceLineNo">396</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.396"></a> -<span class="sourceLineNo">397</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.397"></a> -<span class="sourceLineNo">398</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.398"></a> -<span class="sourceLineNo">399</span> + this.walEditsStopOffset);<a name="line.399"></a> -<span class="sourceLineNo">400</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.400"></a> -<span class="sourceLineNo">401</span> }<a name="line.401"></a> -<span class="sourceLineNo">402</span> } catch (EOFException eof) {<a name="line.402"></a> -<span class="sourceLineNo">403</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.403"></a> -<span class="sourceLineNo">404</span> if (originalPosition < 0) {<a name="line.404"></a> -<span class="sourceLineNo">405</span> if (LOG.isTraceEnabled()) {<a name="line.405"></a> -<span class="sourceLineNo">406</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.406"></a> -<span class="sourceLineNo">407</span> }<a name="line.407"></a> -<span class="sourceLineNo">408</span> throw eof;<a name="line.408"></a> -<span class="sourceLineNo">409</span> }<a name="line.409"></a> -<span class="sourceLineNo">410</span> // Else restore our position to original location in hope that next time through we will<a name="line.410"></a> -<span class="sourceLineNo">411</span> // read successfully.<a name="line.411"></a> -<span class="sourceLineNo">412</span> if (LOG.isTraceEnabled()) {<a name="line.412"></a> -<span class="sourceLineNo">413</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.413"></a> -<span class="sourceLineNo">414</span> }<a name="line.414"></a> -<span class="sourceLineNo">415</span> seekOnFs(originalPosition);<a name="line.415"></a> -<span class="sourceLineNo">416</span> return false;<a name="line.416"></a> -<span class="sourceLineNo">417</span> }<a name="line.417"></a> -<span class="sourceLineNo">418</span> return true;<a name="line.418"></a> -<span class="sourceLineNo">419</span> }<a name="line.419"></a> -<span class="sourceLineNo">420</span> }<a name="line.420"></a> -<span class="sourceLineNo">421</span><a name="line.421"></a> -<span class="sourceLineNo">422</span> private IOException extractHiddenEof(Exception ex) {<a name="line.422"></a> -<span class="sourceLineNo">423</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.423"></a> -<span class="sourceLineNo">424</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.424"></a> -<span class="sourceLineNo">425</span> IOException ioEx = null;<a name="line.425"></a> -<span class="sourceLineNo">426</span> if (ex instanceof EOFException) {<a name="line.426"></a> -<span class="sourceLineNo">427</span> return (EOFException)ex;<a name="line.427"></a> -<span class="sourceLineNo">428</span> } else if (ex instanceof IOException) {<a name="line.428"></a> -<span class="sourceLineNo">429</span> ioEx = (IOException)ex;<a name="line.429"></a> -<span class="sourceLineNo">430</span> } else if (ex instanceof RuntimeException<a name="line.430"></a> -<span class="sourceLineNo">431</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.431"></a> -<span class="sourceLineNo">432</span> ioEx = (IOException)ex.getCause();<a name="line.432"></a> -<span class="sourceLineNo">433</span> }<a name="line.433"></a> -<span class="sourceLineNo">434</span> if (ioEx != null) {<a name="line.434"></a> -<span class="sourceLineNo">435</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.435"></a> -<span class="sourceLineNo">436</span> return null;<a name="line.436"></a> -<span class="sourceLineNo">437</span> }<a name="line.437"></a> -<span class="sourceLineNo">438</span> return null;<a name="line.438"></a> -<span class="sourceLineNo">439</span> }<a name="line.439"></a> -<span class="sourceLineNo">440</span><a name="line.440"></a> -<span class="sourceLineNo">441</span> @Override<a name="line.441"></a> -<span class="sourceLineNo">442</span> protected void seekOnFs(long pos) throws IOException {<a name="line.442"></a> -<span class="sourceLineNo">443</span> this.inputStream.seek(pos);<a name="line.443"></a> -<span class="sourceLineNo">444</span> }<a name="line.444"></a> -<span class="sourceLineNo">445</span>}<a name="line.445"></a> +<span class="sourceLineNo">370</span> LOG.trace("WALKey has no KVs that follow it; trying the next one. current offset=" +<a name="line.370"></a> +<span class="sourceLineNo">371</span> this.inputStream.getPos());<a name="line.371"></a> +<span class="sourceLineNo">372</span> }<a name="line.372"></a> +<span class="sourceLineNo">373</span> continue;<a name="line.373"></a> +<span class="sourceLineNo">374</span> }<a name="line.374"></a> +<span class="sourceLineNo">375</span> int expectedCells = walKey.getFollowingKvCount();<a name="line.375"></a> +<span class="sourceLineNo">376</span> long posBefore = this.inputStream.getPos();<a name="line.376"></a> +<span class="sourceLineNo">377</span> try {<a name="line.377"></a> +<span class="sourceLineNo">378</span> int actualCells = entry.getEdit().readFromCells(cellDecoder, expectedCells);<a name="line.378"></a> +<span class="sourceLineNo">379</span> if (expectedCells != actualCells) {<a name="line.379"></a> +<span class="sourceLineNo">380</span> throw new EOFException("Only read " + actualCells); // other info added in catch<a name="line.380"></a> +<span class="sourceLineNo">381</span> }<a name="line.381"></a> +<span class="sourceLineNo">382</span> } catch (Exception ex) {<a name="line.382"></a> +<span class="sourceLineNo">383</span> String posAfterStr = "<unknown>";<a name="line.383"></a> +<span class="sourceLineNo">384</span> try {<a name="line.384"></a> +<span class="sourceLineNo">385</span> posAfterStr = this.inputStream.getPos() + "";<a name="line.385"></a> +<span class="sourceLineNo">386</span> } catch (Throwable t) {<a name="line.386"></a> +<span class="sourceLineNo">387</span> if (LOG.isTraceEnabled()) {<a name="line.387"></a> +<span class="sourceLineNo">388</span> LOG.trace("Error getting pos for error message - ignoring", t);<a name="line.388"></a> +<span class="sourceLineNo">389</span> }<a name="line.389"></a> +<span class="sourceLineNo">390</span> }<a name="line.390"></a> +<span class="sourceLineNo">391</span> String message = " while reading " + expectedCells + " WAL KVs; started reading at "<a name="line.391"></a> +<span class="sourceLineNo">392</span> + posBefore + " and read up to " + posAfterStr;<a name="line.392"></a> +<span class="sourceLineNo">393</span> IOException realEofEx = extractHiddenEof(ex);<a name="line.393"></a> +<span class="sourceLineNo">394</span> throw (EOFException) new EOFException("EOF " + message).<a name="line.394"></a> +<span class="sourceLineNo">395</span> initCause(realEofEx != null ? realEofEx : ex);<a name="line.395"></a> +<span class="sourceLineNo">396</span> }<a name="line.396"></a> +<span class="sourceLineNo">397</span> if (trailerPresent && this.inputStream.getPos() > this.walEditsStopOffset) {<a name="line.397"></a> +<span class="sourceLineNo">398</span> LOG.error("Read WALTrailer while reading WALEdits. wal: " + this.path<a name="line.398"></a> +<span class="sourceLineNo">399</span> + ", inputStream.getPos(): " + this.inputStream.getPos() + ", walEditsStopOffset: "<a name="line.399"></a> +<span class="sourceLineNo">400</span> + this.walEditsStopOffset);<a name="line.400"></a> +<span class="sourceLineNo">401</span> throw new EOFException("Read WALTrailer while reading WALEdits");<a name="line.401"></a> +<span class="sourceLineNo">402</span> }<a name="line.402"></a> +<span class="sourceLineNo">403</span> } catch (EOFException eof) {<a name="line.403"></a> +<span class="sourceLineNo">404</span> // If originalPosition is < 0, it is rubbish and we cannot use it (probably local fs)<a name="line.404"></a> +<span class="sourceLineNo">405</span> if (originalPosition < 0) {<a name="line.405"></a> +<span class="sourceLineNo">406</span> if (LOG.isTraceEnabled()) {<a name="line.406"></a> +<span class="sourceLineNo">407</span> LOG.trace("Encountered a malformed edit, but can't seek back to last good position because originalPosition is negative. last offset=" + this.inputStream.getPos(), eof);<a name="line.407"></a> +<span class="sourceLineNo">408</span> }<a name="line.408"></a> +<span class="sourceLineNo">409</span> throw eof;<a name="line.409"></a> +<span class="sourceLineNo">410</span> }<a name="line.410"></a> +<span class="sourceLineNo">411</span> // Else restore our position to original location in hope that next time through we will<a name="line.411"></a> +<span class="sourceLineNo">412</span> // read successfully.<a name="line.412"></a> +<span class="sourceLineNo">413</span> if (LOG.isTraceEnabled()) {<a name="line.413"></a> +<span class="sourceLineNo">414</span> LOG.trace("Encountered a malformed edit, seeking back to last good position in file, from "+ inputStream.getPos()+" to " + originalPosition, eof);<a name="line.414"></a> +<span class="sourceLineNo">415</span> }<a name="line.415"></a> +<span class="sourceLineNo">416</span> seekOnFs(originalPosition);<a name="line.416"></a> +<span class="sourceLineNo">417</span> return false;<a name="line.417"></a> +<span class="sourceLineNo">418</span> }<a name="line.418"></a> +<span class="sourceLineNo">419</span> return true;<a name="line.419"></a> +<span class="sourceLineNo">420</span> }<a name="line.420"></a> +<span class="sourceLineNo">421</span> }<a name="line.421"></a> +<span class="sourceLineNo">422</span><a name="line.422"></a> +<span class="sourceLineNo">423</span> private IOException extractHiddenEof(Exception ex) {<a name="line.423"></a> +<span class="sourceLineNo">424</span> // There are two problems we are dealing with here. Hadoop stream throws generic exception<a name="line.424"></a> +<span class="sourceLineNo">425</span> // for EOF, not EOFException; and scanner further hides it inside RuntimeException.<a name="line.425"></a> +<span class="sourceLineNo">426</span> IOException ioEx = null;<a name="line.426"></a> +<span class="sourceLineNo">427</span> if (ex instanceof EOFException) {<a name="line.427"></a> +<span class="sourceLineNo">428</span> return (EOFException)ex;<a name="line.428"></a> +<span class="sourceLineNo">429</span> } else if (ex instanceof IOException) {<a name="line.429"></a> +<span class="sourceLineNo">430</span> ioEx = (IOException)ex;<a name="line.430"></a> +<span class="sourceLineNo">431</span> } else if (ex instanceof RuntimeException<a name="line.431"></a> +<span class="sourceLineNo">432</span> && ex.getCause() != null && ex.getCause() instanceof IOException) {<a name="line.432"></a> +<span class="sourceLineNo">433</span> ioEx = (IOException)ex.getCause();<a name="line.433"></a> +<span class="sourceLineNo">434</span> }<a name="line.434"></a> +<span class="sourceLineNo">435</span> if (ioEx != null) {<a name="line.435"></a> +<span class="sourceLineNo">436</span> if (ioEx.getMessage().contains("EOF")) return ioEx;<a name="line.436"></a> +<span class="sourceLineNo">437</span> return null;<a name="line.437"></a> +<span class="sourceLineNo">438</span> }<a name="line.438"></a> +<span class="sourceLineNo">439</span> return null;<a name="line.439"></a> +<span class="sourceLineNo">440</span> }<a name="line.440"></a> +<span class="sourceLineNo">441</span><a name="line.441"></a> +<span class="sourceLineNo">442</span> @Override<a name="line.442"></a> +<span class="sourceLineNo">443</span> protected void seekOnFs(long pos) throws IOException {<a name="line.443"></a> +<span class="sourceLineNo">444</span> this.inputStream.seek(pos);<a name="line.444"></a> +<span class="sourceLineNo">445</span> }<a name="line.445"></a> +<span class="sourceLineNo">446</span>}<a name="line.446"></a>
