http://git-wip-us.apache.org/repos/asf/hbase-site/blob/4f8b8424/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.html b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.html index 1124f8b..0c29054 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.html @@ -1111,587 +1111,592 @@ <span class="sourceLineNo">1103</span> blockStream.write(midKeyMetadata);<a name="line.1103"></a> <span class="sourceLineNo">1104</span> blockWriter.writeHeaderAndData(out);<a name="line.1104"></a> <span class="sourceLineNo">1105</span> if (cacheConf != null) {<a name="line.1105"></a> -<span class="sourceLineNo">1106</span> HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);<a name="line.1106"></a> -<span class="sourceLineNo">1107</span> cacheConf.getBlockCache().cacheBlock(new BlockCacheKey(nameForCaching,<a name="line.1107"></a> -<span class="sourceLineNo">1108</span> rootLevelIndexPos, true, blockForCaching.getBlockType()), blockForCaching);<a name="line.1108"></a> -<span class="sourceLineNo">1109</span> }<a name="line.1109"></a> -<span class="sourceLineNo">1110</span> }<a name="line.1110"></a> -<span class="sourceLineNo">1111</span><a name="line.1111"></a> -<span class="sourceLineNo">1112</span> // Add root index block size<a name="line.1112"></a> -<span class="sourceLineNo">1113</span> totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();<a name="line.1113"></a> -<span class="sourceLineNo">1114</span> totalBlockUncompressedSize +=<a name="line.1114"></a> -<span class="sourceLineNo">1115</span> blockWriter.getUncompressedSizeWithoutHeader();<a name="line.1115"></a> -<span class="sourceLineNo">1116</span><a name="line.1116"></a> -<span class="sourceLineNo">1117</span> if (LOG.isTraceEnabled()) {<a name="line.1117"></a> -<span class="sourceLineNo">1118</span> LOG.trace("Wrote a " + numLevels + "-level index with root level at pos "<a name="line.1118"></a> -<span class="sourceLineNo">1119</span> + rootLevelIndexPos + ", " + rootChunk.getNumEntries()<a name="line.1119"></a> -<span class="sourceLineNo">1120</span> + " root-level entries, " + totalNumEntries + " total entries, "<a name="line.1120"></a> -<span class="sourceLineNo">1121</span> + StringUtils.humanReadableInt(this.totalBlockOnDiskSize) +<a name="line.1121"></a> -<span class="sourceLineNo">1122</span> " on-disk size, "<a name="line.1122"></a> -<span class="sourceLineNo">1123</span> + StringUtils.humanReadableInt(totalBlockUncompressedSize) +<a name="line.1123"></a> -<span class="sourceLineNo">1124</span> " total uncompressed size.");<a name="line.1124"></a> -<span class="sourceLineNo">1125</span> }<a name="line.1125"></a> -<span class="sourceLineNo">1126</span> return rootLevelIndexPos;<a name="line.1126"></a> -<span class="sourceLineNo">1127</span> }<a name="line.1127"></a> -<span class="sourceLineNo">1128</span><a name="line.1128"></a> -<span class="sourceLineNo">1129</span> /**<a name="line.1129"></a> -<span class="sourceLineNo">1130</span> * Writes the block index data as a single level only. Does not do any<a name="line.1130"></a> -<span class="sourceLineNo">1131</span> * block framing.<a name="line.1131"></a> -<span class="sourceLineNo">1132</span> *<a name="line.1132"></a> -<span class="sourceLineNo">1133</span> * @param out the buffered output stream to write the index to. Typically a<a name="line.1133"></a> -<span class="sourceLineNo">1134</span> * stream writing into an {@link HFile} block.<a name="line.1134"></a> -<span class="sourceLineNo">1135</span> * @param description a short description of the index being written. Used<a name="line.1135"></a> -<span class="sourceLineNo">1136</span> * in a log message.<a name="line.1136"></a> -<span class="sourceLineNo">1137</span> * @throws IOException<a name="line.1137"></a> -<span class="sourceLineNo">1138</span> */<a name="line.1138"></a> -<span class="sourceLineNo">1139</span> public void writeSingleLevelIndex(DataOutput out, String description)<a name="line.1139"></a> -<span class="sourceLineNo">1140</span> throws IOException {<a name="line.1140"></a> -<span class="sourceLineNo">1141</span> expectNumLevels(1);<a name="line.1141"></a> -<span class="sourceLineNo">1142</span><a name="line.1142"></a> -<span class="sourceLineNo">1143</span> if (!singleLevelOnly)<a name="line.1143"></a> -<span class="sourceLineNo">1144</span> throw new IOException("Single-level mode is turned off");<a name="line.1144"></a> -<span class="sourceLineNo">1145</span><a name="line.1145"></a> -<span class="sourceLineNo">1146</span> if (rootChunk.getNumEntries() > 0)<a name="line.1146"></a> -<span class="sourceLineNo">1147</span> throw new IOException("Root-level entries already added in " +<a name="line.1147"></a> -<span class="sourceLineNo">1148</span> "single-level mode");<a name="line.1148"></a> -<span class="sourceLineNo">1149</span><a name="line.1149"></a> -<span class="sourceLineNo">1150</span> rootChunk = curInlineChunk;<a name="line.1150"></a> -<span class="sourceLineNo">1151</span> curInlineChunk = new BlockIndexChunk();<a name="line.1151"></a> -<span class="sourceLineNo">1152</span><a name="line.1152"></a> -<span class="sourceLineNo">1153</span> if (LOG.isTraceEnabled()) {<a name="line.1153"></a> -<span class="sourceLineNo">1154</span> LOG.trace("Wrote a single-level " + description + " index with "<a name="line.1154"></a> -<span class="sourceLineNo">1155</span> + rootChunk.getNumEntries() + " entries, " + rootChunk.getRootSize()<a name="line.1155"></a> -<span class="sourceLineNo">1156</span> + " bytes");<a name="line.1156"></a> -<span class="sourceLineNo">1157</span> }<a name="line.1157"></a> -<span class="sourceLineNo">1158</span> rootChunk.writeRoot(out);<a name="line.1158"></a> -<span class="sourceLineNo">1159</span> }<a name="line.1159"></a> -<span class="sourceLineNo">1160</span><a name="line.1160"></a> -<span class="sourceLineNo">1161</span> /**<a name="line.1161"></a> -<span class="sourceLineNo">1162</span> * Split the current level of the block index into intermediate index<a name="line.1162"></a> -<span class="sourceLineNo">1163</span> * blocks of permitted size and write those blocks to disk. Return the next<a name="line.1163"></a> -<span class="sourceLineNo">1164</span> * level of the block index referencing those intermediate-level blocks.<a name="line.1164"></a> -<span class="sourceLineNo">1165</span> *<a name="line.1165"></a> -<span class="sourceLineNo">1166</span> * @param out<a name="line.1166"></a> -<span class="sourceLineNo">1167</span> * @param currentLevel the current level of the block index, such as the a<a name="line.1167"></a> -<span class="sourceLineNo">1168</span> * chunk referencing all leaf-level index blocks<a name="line.1168"></a> -<span class="sourceLineNo">1169</span> * @return the parent level block index, which becomes the root index after<a name="line.1169"></a> -<span class="sourceLineNo">1170</span> * a few (usually zero) iterations<a name="line.1170"></a> -<span class="sourceLineNo">1171</span> * @throws IOException<a name="line.1171"></a> -<span class="sourceLineNo">1172</span> */<a name="line.1172"></a> -<span class="sourceLineNo">1173</span> private BlockIndexChunk writeIntermediateLevel(FSDataOutputStream out,<a name="line.1173"></a> -<span class="sourceLineNo">1174</span> BlockIndexChunk currentLevel) throws IOException {<a name="line.1174"></a> -<span class="sourceLineNo">1175</span> // Entries referencing intermediate-level blocks we are about to create.<a name="line.1175"></a> -<span class="sourceLineNo">1176</span> BlockIndexChunk parent = new BlockIndexChunk();<a name="line.1176"></a> -<span class="sourceLineNo">1177</span><a name="line.1177"></a> -<span class="sourceLineNo">1178</span> // The current intermediate-level block index chunk.<a name="line.1178"></a> -<span class="sourceLineNo">1179</span> BlockIndexChunk curChunk = new BlockIndexChunk();<a name="line.1179"></a> -<span class="sourceLineNo">1180</span><a name="line.1180"></a> -<span class="sourceLineNo">1181</span> for (int i = 0; i < currentLevel.getNumEntries(); ++i) {<a name="line.1181"></a> -<span class="sourceLineNo">1182</span> curChunk.add(currentLevel.getBlockKey(i),<a name="line.1182"></a> -<span class="sourceLineNo">1183</span> currentLevel.getBlockOffset(i), currentLevel.getOnDiskDataSize(i));<a name="line.1183"></a> -<span class="sourceLineNo">1184</span><a name="line.1184"></a> -<span class="sourceLineNo">1185</span> // HBASE-16288: We have to have at least minIndexNumEntries(16) items in the index so that<a name="line.1185"></a> -<span class="sourceLineNo">1186</span> // we won't end up with too-many levels for a index with very large rowKeys. Also, if the<a name="line.1186"></a> -<span class="sourceLineNo">1187</span> // first key is larger than maxChunkSize this will cause infinite recursion.<a name="line.1187"></a> -<span class="sourceLineNo">1188</span> if (i >= minIndexNumEntries && curChunk.getRootSize() >= maxChunkSize) {<a name="line.1188"></a> -<span class="sourceLineNo">1189</span> writeIntermediateBlock(out, parent, curChunk);<a name="line.1189"></a> -<span class="sourceLineNo">1190</span> }<a name="line.1190"></a> -<span class="sourceLineNo">1191</span> }<a name="line.1191"></a> -<span class="sourceLineNo">1192</span><a name="line.1192"></a> -<span class="sourceLineNo">1193</span> if (curChunk.getNumEntries() > 0) {<a name="line.1193"></a> -<span class="sourceLineNo">1194</span> writeIntermediateBlock(out, parent, curChunk);<a name="line.1194"></a> -<span class="sourceLineNo">1195</span> }<a name="line.1195"></a> -<span class="sourceLineNo">1196</span><a name="line.1196"></a> -<span class="sourceLineNo">1197</span> return parent;<a name="line.1197"></a> -<span class="sourceLineNo">1198</span> }<a name="line.1198"></a> -<span class="sourceLineNo">1199</span><a name="line.1199"></a> -<span class="sourceLineNo">1200</span> private void writeIntermediateBlock(FSDataOutputStream out,<a name="line.1200"></a> -<span class="sourceLineNo">1201</span> BlockIndexChunk parent, BlockIndexChunk curChunk) throws IOException {<a name="line.1201"></a> -<span class="sourceLineNo">1202</span> long beginOffset = out.getPos();<a name="line.1202"></a> -<span class="sourceLineNo">1203</span> DataOutputStream dos = blockWriter.startWriting(<a name="line.1203"></a> -<span class="sourceLineNo">1204</span> BlockType.INTERMEDIATE_INDEX);<a name="line.1204"></a> -<span class="sourceLineNo">1205</span> curChunk.writeNonRoot(dos);<a name="line.1205"></a> -<span class="sourceLineNo">1206</span> byte[] curFirstKey = curChunk.getBlockKey(0);<a name="line.1206"></a> -<span class="sourceLineNo">1207</span> blockWriter.writeHeaderAndData(out);<a name="line.1207"></a> -<span class="sourceLineNo">1208</span><a name="line.1208"></a> -<span class="sourceLineNo">1209</span> if (getCacheOnWrite()) {<a name="line.1209"></a> -<span class="sourceLineNo">1210</span> HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);<a name="line.1210"></a> -<span class="sourceLineNo">1211</span> cacheConf.getBlockCache().cacheBlock(new BlockCacheKey(nameForCaching,<a name="line.1211"></a> -<span class="sourceLineNo">1212</span> beginOffset, true, blockForCaching.getBlockType()), blockForCaching);<a name="line.1212"></a> -<span class="sourceLineNo">1213</span> }<a name="line.1213"></a> -<span class="sourceLineNo">1214</span><a name="line.1214"></a> -<span class="sourceLineNo">1215</span> // Add intermediate index block size<a name="line.1215"></a> -<span class="sourceLineNo">1216</span> totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();<a name="line.1216"></a> -<span class="sourceLineNo">1217</span> totalBlockUncompressedSize +=<a name="line.1217"></a> -<span class="sourceLineNo">1218</span> blockWriter.getUncompressedSizeWithoutHeader();<a name="line.1218"></a> +<span class="sourceLineNo">1106</span> cacheConf.getBlockCache().ifPresent(cache -> {<a name="line.1106"></a> +<span class="sourceLineNo">1107</span> HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);<a name="line.1107"></a> +<span class="sourceLineNo">1108</span> cache.cacheBlock(new BlockCacheKey(nameForCaching, rootLevelIndexPos, true,<a name="line.1108"></a> +<span class="sourceLineNo">1109</span> blockForCaching.getBlockType()), blockForCaching);<a name="line.1109"></a> +<span class="sourceLineNo">1110</span> });<a name="line.1110"></a> +<span class="sourceLineNo">1111</span> }<a name="line.1111"></a> +<span class="sourceLineNo">1112</span> }<a name="line.1112"></a> +<span class="sourceLineNo">1113</span><a name="line.1113"></a> +<span class="sourceLineNo">1114</span> // Add root index block size<a name="line.1114"></a> +<span class="sourceLineNo">1115</span> totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();<a name="line.1115"></a> +<span class="sourceLineNo">1116</span> totalBlockUncompressedSize +=<a name="line.1116"></a> +<span class="sourceLineNo">1117</span> blockWriter.getUncompressedSizeWithoutHeader();<a name="line.1117"></a> +<span class="sourceLineNo">1118</span><a name="line.1118"></a> +<span class="sourceLineNo">1119</span> if (LOG.isTraceEnabled()) {<a name="line.1119"></a> +<span class="sourceLineNo">1120</span> LOG.trace("Wrote a " + numLevels + "-level index with root level at pos "<a name="line.1120"></a> +<span class="sourceLineNo">1121</span> + rootLevelIndexPos + ", " + rootChunk.getNumEntries()<a name="line.1121"></a> +<span class="sourceLineNo">1122</span> + " root-level entries, " + totalNumEntries + " total entries, "<a name="line.1122"></a> +<span class="sourceLineNo">1123</span> + StringUtils.humanReadableInt(this.totalBlockOnDiskSize) +<a name="line.1123"></a> +<span class="sourceLineNo">1124</span> " on-disk size, "<a name="line.1124"></a> +<span class="sourceLineNo">1125</span> + StringUtils.humanReadableInt(totalBlockUncompressedSize) +<a name="line.1125"></a> +<span class="sourceLineNo">1126</span> " total uncompressed size.");<a name="line.1126"></a> +<span class="sourceLineNo">1127</span> }<a name="line.1127"></a> +<span class="sourceLineNo">1128</span> return rootLevelIndexPos;<a name="line.1128"></a> +<span class="sourceLineNo">1129</span> }<a name="line.1129"></a> +<span class="sourceLineNo">1130</span><a name="line.1130"></a> +<span class="sourceLineNo">1131</span> /**<a name="line.1131"></a> +<span class="sourceLineNo">1132</span> * Writes the block index data as a single level only. Does not do any<a name="line.1132"></a> +<span class="sourceLineNo">1133</span> * block framing.<a name="line.1133"></a> +<span class="sourceLineNo">1134</span> *<a name="line.1134"></a> +<span class="sourceLineNo">1135</span> * @param out the buffered output stream to write the index to. Typically a<a name="line.1135"></a> +<span class="sourceLineNo">1136</span> * stream writing into an {@link HFile} block.<a name="line.1136"></a> +<span class="sourceLineNo">1137</span> * @param description a short description of the index being written. Used<a name="line.1137"></a> +<span class="sourceLineNo">1138</span> * in a log message.<a name="line.1138"></a> +<span class="sourceLineNo">1139</span> * @throws IOException<a name="line.1139"></a> +<span class="sourceLineNo">1140</span> */<a name="line.1140"></a> +<span class="sourceLineNo">1141</span> public void writeSingleLevelIndex(DataOutput out, String description)<a name="line.1141"></a> +<span class="sourceLineNo">1142</span> throws IOException {<a name="line.1142"></a> +<span class="sourceLineNo">1143</span> expectNumLevels(1);<a name="line.1143"></a> +<span class="sourceLineNo">1144</span><a name="line.1144"></a> +<span class="sourceLineNo">1145</span> if (!singleLevelOnly)<a name="line.1145"></a> +<span class="sourceLineNo">1146</span> throw new IOException("Single-level mode is turned off");<a name="line.1146"></a> +<span class="sourceLineNo">1147</span><a name="line.1147"></a> +<span class="sourceLineNo">1148</span> if (rootChunk.getNumEntries() > 0)<a name="line.1148"></a> +<span class="sourceLineNo">1149</span> throw new IOException("Root-level entries already added in " +<a name="line.1149"></a> +<span class="sourceLineNo">1150</span> "single-level mode");<a name="line.1150"></a> +<span class="sourceLineNo">1151</span><a name="line.1151"></a> +<span class="sourceLineNo">1152</span> rootChunk = curInlineChunk;<a name="line.1152"></a> +<span class="sourceLineNo">1153</span> curInlineChunk = new BlockIndexChunk();<a name="line.1153"></a> +<span class="sourceLineNo">1154</span><a name="line.1154"></a> +<span class="sourceLineNo">1155</span> if (LOG.isTraceEnabled()) {<a name="line.1155"></a> +<span class="sourceLineNo">1156</span> LOG.trace("Wrote a single-level " + description + " index with "<a name="line.1156"></a> +<span class="sourceLineNo">1157</span> + rootChunk.getNumEntries() + " entries, " + rootChunk.getRootSize()<a name="line.1157"></a> +<span class="sourceLineNo">1158</span> + " bytes");<a name="line.1158"></a> +<span class="sourceLineNo">1159</span> }<a name="line.1159"></a> +<span class="sourceLineNo">1160</span> rootChunk.writeRoot(out);<a name="line.1160"></a> +<span class="sourceLineNo">1161</span> }<a name="line.1161"></a> +<span class="sourceLineNo">1162</span><a name="line.1162"></a> +<span class="sourceLineNo">1163</span> /**<a name="line.1163"></a> +<span class="sourceLineNo">1164</span> * Split the current level of the block index into intermediate index<a name="line.1164"></a> +<span class="sourceLineNo">1165</span> * blocks of permitted size and write those blocks to disk. Return the next<a name="line.1165"></a> +<span class="sourceLineNo">1166</span> * level of the block index referencing those intermediate-level blocks.<a name="line.1166"></a> +<span class="sourceLineNo">1167</span> *<a name="line.1167"></a> +<span class="sourceLineNo">1168</span> * @param out<a name="line.1168"></a> +<span class="sourceLineNo">1169</span> * @param currentLevel the current level of the block index, such as the a<a name="line.1169"></a> +<span class="sourceLineNo">1170</span> * chunk referencing all leaf-level index blocks<a name="line.1170"></a> +<span class="sourceLineNo">1171</span> * @return the parent level block index, which becomes the root index after<a name="line.1171"></a> +<span class="sourceLineNo">1172</span> * a few (usually zero) iterations<a name="line.1172"></a> +<span class="sourceLineNo">1173</span> * @throws IOException<a name="line.1173"></a> +<span class="sourceLineNo">1174</span> */<a name="line.1174"></a> +<span class="sourceLineNo">1175</span> private BlockIndexChunk writeIntermediateLevel(FSDataOutputStream out,<a name="line.1175"></a> +<span class="sourceLineNo">1176</span> BlockIndexChunk currentLevel) throws IOException {<a name="line.1176"></a> +<span class="sourceLineNo">1177</span> // Entries referencing intermediate-level blocks we are about to create.<a name="line.1177"></a> +<span class="sourceLineNo">1178</span> BlockIndexChunk parent = new BlockIndexChunk();<a name="line.1178"></a> +<span class="sourceLineNo">1179</span><a name="line.1179"></a> +<span class="sourceLineNo">1180</span> // The current intermediate-level block index chunk.<a name="line.1180"></a> +<span class="sourceLineNo">1181</span> BlockIndexChunk curChunk = new BlockIndexChunk();<a name="line.1181"></a> +<span class="sourceLineNo">1182</span><a name="line.1182"></a> +<span class="sourceLineNo">1183</span> for (int i = 0; i < currentLevel.getNumEntries(); ++i) {<a name="line.1183"></a> +<span class="sourceLineNo">1184</span> curChunk.add(currentLevel.getBlockKey(i),<a name="line.1184"></a> +<span class="sourceLineNo">1185</span> currentLevel.getBlockOffset(i), currentLevel.getOnDiskDataSize(i));<a name="line.1185"></a> +<span class="sourceLineNo">1186</span><a name="line.1186"></a> +<span class="sourceLineNo">1187</span> // HBASE-16288: We have to have at least minIndexNumEntries(16) items in the index so that<a name="line.1187"></a> +<span class="sourceLineNo">1188</span> // we won't end up with too-many levels for a index with very large rowKeys. Also, if the<a name="line.1188"></a> +<span class="sourceLineNo">1189</span> // first key is larger than maxChunkSize this will cause infinite recursion.<a name="line.1189"></a> +<span class="sourceLineNo">1190</span> if (i >= minIndexNumEntries && curChunk.getRootSize() >= maxChunkSize) {<a name="line.1190"></a> +<span class="sourceLineNo">1191</span> writeIntermediateBlock(out, parent, curChunk);<a name="line.1191"></a> +<span class="sourceLineNo">1192</span> }<a name="line.1192"></a> +<span class="sourceLineNo">1193</span> }<a name="line.1193"></a> +<span class="sourceLineNo">1194</span><a name="line.1194"></a> +<span class="sourceLineNo">1195</span> if (curChunk.getNumEntries() > 0) {<a name="line.1195"></a> +<span class="sourceLineNo">1196</span> writeIntermediateBlock(out, parent, curChunk);<a name="line.1196"></a> +<span class="sourceLineNo">1197</span> }<a name="line.1197"></a> +<span class="sourceLineNo">1198</span><a name="line.1198"></a> +<span class="sourceLineNo">1199</span> return parent;<a name="line.1199"></a> +<span class="sourceLineNo">1200</span> }<a name="line.1200"></a> +<span class="sourceLineNo">1201</span><a name="line.1201"></a> +<span class="sourceLineNo">1202</span> private void writeIntermediateBlock(FSDataOutputStream out,<a name="line.1202"></a> +<span class="sourceLineNo">1203</span> BlockIndexChunk parent, BlockIndexChunk curChunk) throws IOException {<a name="line.1203"></a> +<span class="sourceLineNo">1204</span> long beginOffset = out.getPos();<a name="line.1204"></a> +<span class="sourceLineNo">1205</span> DataOutputStream dos = blockWriter.startWriting(<a name="line.1205"></a> +<span class="sourceLineNo">1206</span> BlockType.INTERMEDIATE_INDEX);<a name="line.1206"></a> +<span class="sourceLineNo">1207</span> curChunk.writeNonRoot(dos);<a name="line.1207"></a> +<span class="sourceLineNo">1208</span> byte[] curFirstKey = curChunk.getBlockKey(0);<a name="line.1208"></a> +<span class="sourceLineNo">1209</span> blockWriter.writeHeaderAndData(out);<a name="line.1209"></a> +<span class="sourceLineNo">1210</span><a name="line.1210"></a> +<span class="sourceLineNo">1211</span> if (getCacheOnWrite()) {<a name="line.1211"></a> +<span class="sourceLineNo">1212</span> cacheConf.getBlockCache().ifPresent(cache -> {<a name="line.1212"></a> +<span class="sourceLineNo">1213</span> HFileBlock blockForCaching = blockWriter.getBlockForCaching(cacheConf);<a name="line.1213"></a> +<span class="sourceLineNo">1214</span> cache.cacheBlock(<a name="line.1214"></a> +<span class="sourceLineNo">1215</span> new BlockCacheKey(nameForCaching, beginOffset, true, blockForCaching.getBlockType()),<a name="line.1215"></a> +<span class="sourceLineNo">1216</span> blockForCaching);<a name="line.1216"></a> +<span class="sourceLineNo">1217</span> });<a name="line.1217"></a> +<span class="sourceLineNo">1218</span> }<a name="line.1218"></a> <span class="sourceLineNo">1219</span><a name="line.1219"></a> -<span class="sourceLineNo">1220</span> // OFFSET is the beginning offset the chunk of block index entries.<a name="line.1220"></a> -<span class="sourceLineNo">1221</span> // SIZE is the total byte size of the chunk of block index entries<a name="line.1221"></a> -<span class="sourceLineNo">1222</span> // + the secondary index size<a name="line.1222"></a> -<span class="sourceLineNo">1223</span> // FIRST_KEY is the first key in the chunk of block index<a name="line.1223"></a> -<span class="sourceLineNo">1224</span> // entries.<a name="line.1224"></a> -<span class="sourceLineNo">1225</span> parent.add(curFirstKey, beginOffset,<a name="line.1225"></a> -<span class="sourceLineNo">1226</span> blockWriter.getOnDiskSizeWithHeader());<a name="line.1226"></a> -<span class="sourceLineNo">1227</span><a name="line.1227"></a> -<span class="sourceLineNo">1228</span> // clear current block index chunk<a name="line.1228"></a> -<span class="sourceLineNo">1229</span> curChunk.clear();<a name="line.1229"></a> -<span class="sourceLineNo">1230</span> curFirstKey = null;<a name="line.1230"></a> -<span class="sourceLineNo">1231</span> }<a name="line.1231"></a> +<span class="sourceLineNo">1220</span> // Add intermediate index block size<a name="line.1220"></a> +<span class="sourceLineNo">1221</span> totalBlockOnDiskSize += blockWriter.getOnDiskSizeWithoutHeader();<a name="line.1221"></a> +<span class="sourceLineNo">1222</span> totalBlockUncompressedSize +=<a name="line.1222"></a> +<span class="sourceLineNo">1223</span> blockWriter.getUncompressedSizeWithoutHeader();<a name="line.1223"></a> +<span class="sourceLineNo">1224</span><a name="line.1224"></a> +<span class="sourceLineNo">1225</span> // OFFSET is the beginning offset the chunk of block index entries.<a name="line.1225"></a> +<span class="sourceLineNo">1226</span> // SIZE is the total byte size of the chunk of block index entries<a name="line.1226"></a> +<span class="sourceLineNo">1227</span> // + the secondary index size<a name="line.1227"></a> +<span class="sourceLineNo">1228</span> // FIRST_KEY is the first key in the chunk of block index<a name="line.1228"></a> +<span class="sourceLineNo">1229</span> // entries.<a name="line.1229"></a> +<span class="sourceLineNo">1230</span> parent.add(curFirstKey, beginOffset,<a name="line.1230"></a> +<span class="sourceLineNo">1231</span> blockWriter.getOnDiskSizeWithHeader());<a name="line.1231"></a> <span class="sourceLineNo">1232</span><a name="line.1232"></a> -<span class="sourceLineNo">1233</span> /**<a name="line.1233"></a> -<span class="sourceLineNo">1234</span> * @return how many block index entries there are in the root level<a name="line.1234"></a> -<span class="sourceLineNo">1235</span> */<a name="line.1235"></a> -<span class="sourceLineNo">1236</span> public final int getNumRootEntries() {<a name="line.1236"></a> -<span class="sourceLineNo">1237</span> return rootChunk.getNumEntries();<a name="line.1237"></a> -<span class="sourceLineNo">1238</span> }<a name="line.1238"></a> -<span class="sourceLineNo">1239</span><a name="line.1239"></a> -<span class="sourceLineNo">1240</span> /**<a name="line.1240"></a> -<span class="sourceLineNo">1241</span> * @return the number of levels in this block index.<a name="line.1241"></a> -<span class="sourceLineNo">1242</span> */<a name="line.1242"></a> -<span class="sourceLineNo">1243</span> public int getNumLevels() {<a name="line.1243"></a> -<span class="sourceLineNo">1244</span> return numLevels;<a name="line.1244"></a> -<span class="sourceLineNo">1245</span> }<a name="line.1245"></a> -<span class="sourceLineNo">1246</span><a name="line.1246"></a> -<span class="sourceLineNo">1247</span> private void expectNumLevels(int expectedNumLevels) {<a name="line.1247"></a> -<span class="sourceLineNo">1248</span> if (numLevels != expectedNumLevels) {<a name="line.1248"></a> -<span class="sourceLineNo">1249</span> throw new IllegalStateException("Number of block index levels is "<a name="line.1249"></a> -<span class="sourceLineNo">1250</span> + numLevels + "but is expected to be " + expectedNumLevels);<a name="line.1250"></a> -<span class="sourceLineNo">1251</span> }<a name="line.1251"></a> -<span class="sourceLineNo">1252</span> }<a name="line.1252"></a> -<span class="sourceLineNo">1253</span><a name="line.1253"></a> -<span class="sourceLineNo">1254</span> /**<a name="line.1254"></a> -<span class="sourceLineNo">1255</span> * Whether there is an inline block ready to be written. In general, we<a name="line.1255"></a> -<span class="sourceLineNo">1256</span> * write an leaf-level index block as an inline block as soon as its size<a name="line.1256"></a> -<span class="sourceLineNo">1257</span> * as serialized in the non-root format reaches a certain threshold.<a name="line.1257"></a> -<span class="sourceLineNo">1258</span> */<a name="line.1258"></a> -<span class="sourceLineNo">1259</span> @Override<a name="line.1259"></a> -<span class="sourceLineNo">1260</span> public boolean shouldWriteBlock(boolean closing) {<a name="line.1260"></a> -<span class="sourceLineNo">1261</span> if (singleLevelOnly) {<a name="line.1261"></a> -<span class="sourceLineNo">1262</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1262"></a> -<span class="sourceLineNo">1263</span> }<a name="line.1263"></a> -<span class="sourceLineNo">1264</span><a name="line.1264"></a> -<span class="sourceLineNo">1265</span> if (curInlineChunk == null) {<a name="line.1265"></a> -<span class="sourceLineNo">1266</span> throw new IllegalStateException("curInlineChunk is null; has shouldWriteBlock been " +<a name="line.1266"></a> -<span class="sourceLineNo">1267</span> "called with closing=true and then called again?");<a name="line.1267"></a> +<span class="sourceLineNo">1233</span> // clear current block index chunk<a name="line.1233"></a> +<span class="sourceLineNo">1234</span> curChunk.clear();<a name="line.1234"></a> +<span class="sourceLineNo">1235</span> curFirstKey = null;<a name="line.1235"></a> +<span class="sourceLineNo">1236</span> }<a name="line.1236"></a> +<span class="sourceLineNo">1237</span><a name="line.1237"></a> +<span class="sourceLineNo">1238</span> /**<a name="line.1238"></a> +<span class="sourceLineNo">1239</span> * @return how many block index entries there are in the root level<a name="line.1239"></a> +<span class="sourceLineNo">1240</span> */<a name="line.1240"></a> +<span class="sourceLineNo">1241</span> public final int getNumRootEntries() {<a name="line.1241"></a> +<span class="sourceLineNo">1242</span> return rootChunk.getNumEntries();<a name="line.1242"></a> +<span class="sourceLineNo">1243</span> }<a name="line.1243"></a> +<span class="sourceLineNo">1244</span><a name="line.1244"></a> +<span class="sourceLineNo">1245</span> /**<a name="line.1245"></a> +<span class="sourceLineNo">1246</span> * @return the number of levels in this block index.<a name="line.1246"></a> +<span class="sourceLineNo">1247</span> */<a name="line.1247"></a> +<span class="sourceLineNo">1248</span> public int getNumLevels() {<a name="line.1248"></a> +<span class="sourceLineNo">1249</span> return numLevels;<a name="line.1249"></a> +<span class="sourceLineNo">1250</span> }<a name="line.1250"></a> +<span class="sourceLineNo">1251</span><a name="line.1251"></a> +<span class="sourceLineNo">1252</span> private void expectNumLevels(int expectedNumLevels) {<a name="line.1252"></a> +<span class="sourceLineNo">1253</span> if (numLevels != expectedNumLevels) {<a name="line.1253"></a> +<span class="sourceLineNo">1254</span> throw new IllegalStateException("Number of block index levels is "<a name="line.1254"></a> +<span class="sourceLineNo">1255</span> + numLevels + "but is expected to be " + expectedNumLevels);<a name="line.1255"></a> +<span class="sourceLineNo">1256</span> }<a name="line.1256"></a> +<span class="sourceLineNo">1257</span> }<a name="line.1257"></a> +<span class="sourceLineNo">1258</span><a name="line.1258"></a> +<span class="sourceLineNo">1259</span> /**<a name="line.1259"></a> +<span class="sourceLineNo">1260</span> * Whether there is an inline block ready to be written. In general, we<a name="line.1260"></a> +<span class="sourceLineNo">1261</span> * write an leaf-level index block as an inline block as soon as its size<a name="line.1261"></a> +<span class="sourceLineNo">1262</span> * as serialized in the non-root format reaches a certain threshold.<a name="line.1262"></a> +<span class="sourceLineNo">1263</span> */<a name="line.1263"></a> +<span class="sourceLineNo">1264</span> @Override<a name="line.1264"></a> +<span class="sourceLineNo">1265</span> public boolean shouldWriteBlock(boolean closing) {<a name="line.1265"></a> +<span class="sourceLineNo">1266</span> if (singleLevelOnly) {<a name="line.1266"></a> +<span class="sourceLineNo">1267</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1267"></a> <span class="sourceLineNo">1268</span> }<a name="line.1268"></a> <span class="sourceLineNo">1269</span><a name="line.1269"></a> -<span class="sourceLineNo">1270</span> if (curInlineChunk.getNumEntries() == 0) {<a name="line.1270"></a> -<span class="sourceLineNo">1271</span> return false;<a name="line.1271"></a> -<span class="sourceLineNo">1272</span> }<a name="line.1272"></a> -<span class="sourceLineNo">1273</span><a name="line.1273"></a> -<span class="sourceLineNo">1274</span> // We do have some entries in the current inline chunk.<a name="line.1274"></a> -<span class="sourceLineNo">1275</span> if (closing) {<a name="line.1275"></a> -<span class="sourceLineNo">1276</span> if (rootChunk.getNumEntries() == 0) {<a name="line.1276"></a> -<span class="sourceLineNo">1277</span> // We did not add any leaf-level blocks yet. Instead of creating a<a name="line.1277"></a> -<span class="sourceLineNo">1278</span> // leaf level with one block, move these entries to the root level.<a name="line.1278"></a> -<span class="sourceLineNo">1279</span><a name="line.1279"></a> -<span class="sourceLineNo">1280</span> expectNumLevels(1);<a name="line.1280"></a> -<span class="sourceLineNo">1281</span> rootChunk = curInlineChunk;<a name="line.1281"></a> -<span class="sourceLineNo">1282</span> curInlineChunk = null; // Disallow adding any more index entries.<a name="line.1282"></a> -<span class="sourceLineNo">1283</span> return false;<a name="line.1283"></a> -<span class="sourceLineNo">1284</span> }<a name="line.1284"></a> -<span class="sourceLineNo">1285</span><a name="line.1285"></a> -<span class="sourceLineNo">1286</span> return true;<a name="line.1286"></a> -<span class="sourceLineNo">1287</span> } else {<a name="line.1287"></a> -<span class="sourceLineNo">1288</span> return curInlineChunk.getNonRootSize() >= maxChunkSize;<a name="line.1288"></a> -<span class="sourceLineNo">1289</span> }<a name="line.1289"></a> -<span class="sourceLineNo">1290</span> }<a name="line.1290"></a> -<span class="sourceLineNo">1291</span><a name="line.1291"></a> -<span class="sourceLineNo">1292</span> /**<a name="line.1292"></a> -<span class="sourceLineNo">1293</span> * Write out the current inline index block. Inline blocks are non-root<a name="line.1293"></a> -<span class="sourceLineNo">1294</span> * blocks, so the non-root index format is used.<a name="line.1294"></a> -<span class="sourceLineNo">1295</span> *<a name="line.1295"></a> -<span class="sourceLineNo">1296</span> * @param out<a name="line.1296"></a> -<span class="sourceLineNo">1297</span> */<a name="line.1297"></a> -<span class="sourceLineNo">1298</span> @Override<a name="line.1298"></a> -<span class="sourceLineNo">1299</span> public void writeInlineBlock(DataOutput out) throws IOException {<a name="line.1299"></a> -<span class="sourceLineNo">1300</span> if (singleLevelOnly)<a name="line.1300"></a> -<span class="sourceLineNo">1301</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1301"></a> -<span class="sourceLineNo">1302</span><a name="line.1302"></a> -<span class="sourceLineNo">1303</span> // Write the inline block index to the output stream in the non-root<a name="line.1303"></a> -<span class="sourceLineNo">1304</span> // index block format.<a name="line.1304"></a> -<span class="sourceLineNo">1305</span> curInlineChunk.writeNonRoot(out);<a name="line.1305"></a> -<span class="sourceLineNo">1306</span><a name="line.1306"></a> -<span class="sourceLineNo">1307</span> // Save the first key of the inline block so that we can add it to the<a name="line.1307"></a> -<span class="sourceLineNo">1308</span> // parent-level index.<a name="line.1308"></a> -<span class="sourceLineNo">1309</span> firstKey = curInlineChunk.getBlockKey(0);<a name="line.1309"></a> -<span class="sourceLineNo">1310</span><a name="line.1310"></a> -<span class="sourceLineNo">1311</span> // Start a new inline index block<a name="line.1311"></a> -<span class="sourceLineNo">1312</span> curInlineChunk.clear();<a name="line.1312"></a> -<span class="sourceLineNo">1313</span> }<a name="line.1313"></a> -<span class="sourceLineNo">1314</span><a name="line.1314"></a> -<span class="sourceLineNo">1315</span> /**<a name="line.1315"></a> -<span class="sourceLineNo">1316</span> * Called after an inline block has been written so that we can add an<a name="line.1316"></a> -<span class="sourceLineNo">1317</span> * entry referring to that block to the parent-level index.<a name="line.1317"></a> -<span class="sourceLineNo">1318</span> */<a name="line.1318"></a> -<span class="sourceLineNo">1319</span> @Override<a name="line.1319"></a> -<span class="sourceLineNo">1320</span> public void blockWritten(long offset, int onDiskSize, int uncompressedSize) {<a name="line.1320"></a> -<span class="sourceLineNo">1321</span> // Add leaf index block size<a name="line.1321"></a> -<span class="sourceLineNo">1322</span> totalBlockOnDiskSize += onDiskSize;<a name="line.1322"></a> -<span class="sourceLineNo">1323</span> totalBlockUncompressedSize += uncompressedSize;<a name="line.1323"></a> -<span class="sourceLineNo">1324</span><a name="line.1324"></a> -<span class="sourceLineNo">1325</span> if (singleLevelOnly)<a name="line.1325"></a> -<span class="sourceLineNo">1326</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1326"></a> -<span class="sourceLineNo">1327</span><a name="line.1327"></a> -<span class="sourceLineNo">1328</span> if (firstKey == null) {<a name="line.1328"></a> -<span class="sourceLineNo">1329</span> throw new IllegalStateException("Trying to add second-level index " +<a name="line.1329"></a> -<span class="sourceLineNo">1330</span> "entry with offset=" + offset + " and onDiskSize=" + onDiskSize +<a name="line.1330"></a> -<span class="sourceLineNo">1331</span> "but the first key was not set in writeInlineBlock");<a name="line.1331"></a> -<span class="sourceLineNo">1332</span> }<a name="line.1332"></a> -<span class="sourceLineNo">1333</span><a name="line.1333"></a> -<span class="sourceLineNo">1334</span> if (rootChunk.getNumEntries() == 0) {<a name="line.1334"></a> -<span class="sourceLineNo">1335</span> // We are writing the first leaf block, so increase index level.<a name="line.1335"></a> -<span class="sourceLineNo">1336</span> expectNumLevels(1);<a name="line.1336"></a> -<span class="sourceLineNo">1337</span> numLevels = 2;<a name="line.1337"></a> -<span class="sourceLineNo">1338</span> }<a name="line.1338"></a> -<span class="sourceLineNo">1339</span><a name="line.1339"></a> -<span class="sourceLineNo">1340</span> // Add another entry to the second-level index. Include the number of<a name="line.1340"></a> -<span class="sourceLineNo">1341</span> // entries in all previous leaf-level chunks for mid-key calculation.<a name="line.1341"></a> -<span class="sourceLineNo">1342</span> rootChunk.add(firstKey, offset, onDiskSize, totalNumEntries);<a name="line.1342"></a> -<span class="sourceLineNo">1343</span> firstKey = null;<a name="line.1343"></a> -<span class="sourceLineNo">1344</span> }<a name="line.1344"></a> -<span class="sourceLineNo">1345</span><a name="line.1345"></a> -<span class="sourceLineNo">1346</span> @Override<a name="line.1346"></a> -<span class="sourceLineNo">1347</span> public BlockType getInlineBlockType() {<a name="line.1347"></a> -<span class="sourceLineNo">1348</span> return BlockType.LEAF_INDEX;<a name="line.1348"></a> +<span class="sourceLineNo">1270</span> if (curInlineChunk == null) {<a name="line.1270"></a> +<span class="sourceLineNo">1271</span> throw new IllegalStateException("curInlineChunk is null; has shouldWriteBlock been " +<a name="line.1271"></a> +<span class="sourceLineNo">1272</span> "called with closing=true and then called again?");<a name="line.1272"></a> +<span class="sourceLineNo">1273</span> }<a name="line.1273"></a> +<span class="sourceLineNo">1274</span><a name="line.1274"></a> +<span class="sourceLineNo">1275</span> if (curInlineChunk.getNumEntries() == 0) {<a name="line.1275"></a> +<span class="sourceLineNo">1276</span> return false;<a name="line.1276"></a> +<span class="sourceLineNo">1277</span> }<a name="line.1277"></a> +<span class="sourceLineNo">1278</span><a name="line.1278"></a> +<span class="sourceLineNo">1279</span> // We do have some entries in the current inline chunk.<a name="line.1279"></a> +<span class="sourceLineNo">1280</span> if (closing) {<a name="line.1280"></a> +<span class="sourceLineNo">1281</span> if (rootChunk.getNumEntries() == 0) {<a name="line.1281"></a> +<span class="sourceLineNo">1282</span> // We did not add any leaf-level blocks yet. Instead of creating a<a name="line.1282"></a> +<span class="sourceLineNo">1283</span> // leaf level with one block, move these entries to the root level.<a name="line.1283"></a> +<span class="sourceLineNo">1284</span><a name="line.1284"></a> +<span class="sourceLineNo">1285</span> expectNumLevels(1);<a name="line.1285"></a> +<span class="sourceLineNo">1286</span> rootChunk = curInlineChunk;<a name="line.1286"></a> +<span class="sourceLineNo">1287</span> curInlineChunk = null; // Disallow adding any more index entries.<a name="line.1287"></a> +<span class="sourceLineNo">1288</span> return false;<a name="line.1288"></a> +<span class="sourceLineNo">1289</span> }<a name="line.1289"></a> +<span class="sourceLineNo">1290</span><a name="line.1290"></a> +<span class="sourceLineNo">1291</span> return true;<a name="line.1291"></a> +<span class="sourceLineNo">1292</span> } else {<a name="line.1292"></a> +<span class="sourceLineNo">1293</span> return curInlineChunk.getNonRootSize() >= maxChunkSize;<a name="line.1293"></a> +<span class="sourceLineNo">1294</span> }<a name="line.1294"></a> +<span class="sourceLineNo">1295</span> }<a name="line.1295"></a> +<span class="sourceLineNo">1296</span><a name="line.1296"></a> +<span class="sourceLineNo">1297</span> /**<a name="line.1297"></a> +<span class="sourceLineNo">1298</span> * Write out the current inline index block. Inline blocks are non-root<a name="line.1298"></a> +<span class="sourceLineNo">1299</span> * blocks, so the non-root index format is used.<a name="line.1299"></a> +<span class="sourceLineNo">1300</span> *<a name="line.1300"></a> +<span class="sourceLineNo">1301</span> * @param out<a name="line.1301"></a> +<span class="sourceLineNo">1302</span> */<a name="line.1302"></a> +<span class="sourceLineNo">1303</span> @Override<a name="line.1303"></a> +<span class="sourceLineNo">1304</span> public void writeInlineBlock(DataOutput out) throws IOException {<a name="line.1304"></a> +<span class="sourceLineNo">1305</span> if (singleLevelOnly)<a name="line.1305"></a> +<span class="sourceLineNo">1306</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1306"></a> +<span class="sourceLineNo">1307</span><a name="line.1307"></a> +<span class="sourceLineNo">1308</span> // Write the inline block index to the output stream in the non-root<a name="line.1308"></a> +<span class="sourceLineNo">1309</span> // index block format.<a name="line.1309"></a> +<span class="sourceLineNo">1310</span> curInlineChunk.writeNonRoot(out);<a name="line.1310"></a> +<span class="sourceLineNo">1311</span><a name="line.1311"></a> +<span class="sourceLineNo">1312</span> // Save the first key of the inline block so that we can add it to the<a name="line.1312"></a> +<span class="sourceLineNo">1313</span> // parent-level index.<a name="line.1313"></a> +<span class="sourceLineNo">1314</span> firstKey = curInlineChunk.getBlockKey(0);<a name="line.1314"></a> +<span class="sourceLineNo">1315</span><a name="line.1315"></a> +<span class="sourceLineNo">1316</span> // Start a new inline index block<a name="line.1316"></a> +<span class="sourceLineNo">1317</span> curInlineChunk.clear();<a name="line.1317"></a> +<span class="sourceLineNo">1318</span> }<a name="line.1318"></a> +<span class="sourceLineNo">1319</span><a name="line.1319"></a> +<span class="sourceLineNo">1320</span> /**<a name="line.1320"></a> +<span class="sourceLineNo">1321</span> * Called after an inline block has been written so that we can add an<a name="line.1321"></a> +<span class="sourceLineNo">1322</span> * entry referring to that block to the parent-level index.<a name="line.1322"></a> +<span class="sourceLineNo">1323</span> */<a name="line.1323"></a> +<span class="sourceLineNo">1324</span> @Override<a name="line.1324"></a> +<span class="sourceLineNo">1325</span> public void blockWritten(long offset, int onDiskSize, int uncompressedSize) {<a name="line.1325"></a> +<span class="sourceLineNo">1326</span> // Add leaf index block size<a name="line.1326"></a> +<span class="sourceLineNo">1327</span> totalBlockOnDiskSize += onDiskSize;<a name="line.1327"></a> +<span class="sourceLineNo">1328</span> totalBlockUncompressedSize += uncompressedSize;<a name="line.1328"></a> +<span class="sourceLineNo">1329</span><a name="line.1329"></a> +<span class="sourceLineNo">1330</span> if (singleLevelOnly)<a name="line.1330"></a> +<span class="sourceLineNo">1331</span> throw new UnsupportedOperationException(INLINE_BLOCKS_NOT_ALLOWED);<a name="line.1331"></a> +<span class="sourceLineNo">1332</span><a name="line.1332"></a> +<span class="sourceLineNo">1333</span> if (firstKey == null) {<a name="line.1333"></a> +<span class="sourceLineNo">1334</span> throw new IllegalStateException("Trying to add second-level index " +<a name="line.1334"></a> +<span class="sourceLineNo">1335</span> "entry with offset=" + offset + " and onDiskSize=" + onDiskSize +<a name="line.1335"></a> +<span class="sourceLineNo">1336</span> "but the first key was not set in writeInlineBlock");<a name="line.1336"></a> +<span class="sourceLineNo">1337</span> }<a name="line.1337"></a> +<span class="sourceLineNo">1338</span><a name="line.1338"></a> +<span class="sourceLineNo">1339</span> if (rootChunk.getNumEntries() == 0) {<a name="line.1339"></a> +<span class="sourceLineNo">1340</span> // We are writing the first leaf block, so increase index level.<a name="line.1340"></a> +<span class="sourceLineNo">1341</span> expectNumLevels(1);<a name="line.1341"></a> +<span class="sourceLineNo">1342</span> numLevels = 2;<a name="line.1342"></a> +<span class="sourceLineNo">1343</span> }<a name="line.1343"></a> +<span class="sourceLineNo">1344</span><a name="line.1344"></a> +<span class="sourceLineNo">1345</span> // Add another entry to the second-level index. Include the number of<a name="line.1345"></a> +<span class="sourceLineNo">1346</span> // entries in all previous leaf-level chunks for mid-key calculation.<a name="line.1346"></a> +<span class="sourceLineNo">1347</span> rootChunk.add(firstKey, offset, onDiskSize, totalNumEntries);<a name="line.1347"></a> +<span class="sourceLineNo">1348</span> firstKey = null;<a name="line.1348"></a> <span class="sourceLineNo">1349</span> }<a name="line.1349"></a> <span class="sourceLineNo">1350</span><a name="line.1350"></a> -<span class="sourceLineNo">1351</span> /**<a name="line.1351"></a> -<span class="sourceLineNo">1352</span> * Add one index entry to the current leaf-level block. When the leaf-level<a name="line.1352"></a> -<span class="sourceLineNo">1353</span> * block gets large enough, it will be flushed to disk as an inline block.<a name="line.1353"></a> -<span class="sourceLineNo">1354</span> *<a name="line.1354"></a> -<span class="sourceLineNo">1355</span> * @param firstKey the first key of the data block<a name="line.1355"></a> -<span class="sourceLineNo">1356</span> * @param blockOffset the offset of the data block<a name="line.1356"></a> -<span class="sourceLineNo">1357</span> * @param blockDataSize the on-disk size of the data block ({@link HFile}<a name="line.1357"></a> -<span class="sourceLineNo">1358</span> * format version 2), or the uncompressed size of the data block (<a name="line.1358"></a> -<span class="sourceLineNo">1359</span> * {@link HFile} format version 1).<a name="line.1359"></a> -<span class="sourceLineNo">1360</span> */<a name="line.1360"></a> -<span class="sourceLineNo">1361</span> public void addEntry(byte[] firstKey, long blockOffset, int blockDataSize) {<a name="line.1361"></a> -<span class="sourceLineNo">1362</span> curInlineChunk.add(firstKey, blockOffset, blockDataSize);<a name="line.1362"></a> -<span class="sourceLineNo">1363</span> ++totalNumEntries;<a name="line.1363"></a> -<span class="sourceLineNo">1364</span> }<a name="line.1364"></a> -<span class="sourceLineNo">1365</span><a name="line.1365"></a> -<span class="sourceLineNo">1366</span> /**<a name="line.1366"></a> -<span class="sourceLineNo">1367</span> * @throws IOException if we happened to write a multi-level index.<a name="line.1367"></a> -<span class="sourceLineNo">1368</span> */<a name="line.1368"></a> -<span class="sourceLineNo">1369</span> public void ensureSingleLevel() throws IOException {<a name="line.1369"></a> -<span class="sourceLineNo">1370</span> if (numLevels > 1) {<a name="line.1370"></a> -<span class="sourceLineNo">1371</span> throw new IOException ("Wrote a " + numLevels + "-level index with " +<a name="line.1371"></a> -<span class="sourceLineNo">1372</span> rootChunk.getNumEntries() + " root-level entries, but " +<a name="line.1372"></a> -<span class="sourceLineNo">1373</span> "this is expected to be a single-level block index.");<a name="line.1373"></a> -<span class="sourceLineNo">1374</span> }<a name="line.1374"></a> -<span class="sourceLineNo">1375</span> }<a name="line.1375"></a> -<span class="sourceLineNo">1376</span><a name="line.1376"></a> -<span class="sourceLineNo">1377</span> /**<a name="line.1377"></a> -<span class="sourceLineNo">1378</span> * @return true if we are using cache-on-write. This is configured by the<a name="line.1378"></a> -<span class="sourceLineNo">1379</span> * caller of the constructor by either passing a valid block cache<a name="line.1379"></a> -<span class="sourceLineNo">1380</span> * or null.<a name="line.1380"></a> -<span class="sourceLineNo">1381</span> */<a name="line.1381"></a> -<span class="sourceLineNo">1382</span> @Override<a name="line.1382"></a> -<span class="sourceLineNo">1383</span> public boolean getCacheOnWrite() {<a name="line.1383"></a> -<span class="sourceLineNo">1384</span> return cacheConf != null && cacheConf.shouldCacheIndexesOnWrite();<a name="line.1384"></a> -<span class="sourceLineNo">1385</span> }<a name="line.1385"></a> -<span class="sourceLineNo">1386</span><a name="line.1386"></a> -<span class="sourceLineNo">1387</span> /**<a name="line.1387"></a> -<span class="sourceLineNo">1388</span> * The total uncompressed size of the root index block, intermediate-level<a name="line.1388"></a> -<span class="sourceLineNo">1389</span> * index blocks, and leaf-level index blocks.<a name="line.1389"></a> -<span class="sourceLineNo">1390</span> *<a name="line.1390"></a> -<span class="sourceLineNo">1391</span> * @return the total uncompressed size of all index blocks<a name="line.1391"></a> -<span class="sourceLineNo">1392</span> */<a name="line.1392"></a> -<span class="sourceLineNo">1393</span> public long getTotalUncompressedSize() {<a name="line.1393"></a> -<span class="sourceLineNo">1394</span> return totalBlockUncompressedSize;<a name="line.1394"></a> -<span class="sourceLineNo">1395</span> }<a name="line.1395"></a> -<span class="sourceLineNo">1396</span><a name="line.1396"></a> -<span class="sourceLineNo">1397</span> }<a name="line.1397"></a> -<span class="sourceLineNo">1398</span><a name="line.1398"></a> -<span class="sourceLineNo">1399</span> /**<a name="line.1399"></a> -<span class="sourceLineNo">1400</span> * A single chunk of the block index in the process of writing. The data in<a name="line.1400"></a> -<span class="sourceLineNo">1401</span> * this chunk can become a leaf-level, intermediate-level, or root index<a name="line.1401"></a> -<span class="sourceLineNo">1402</span> * block.<a name="line.1402"></a> -<span class="sourceLineNo">1403</span> */<a name="line.1403"></a> -<span class="sourceLineNo">1404</span> static class BlockIndexChunk {<a name="line.1404"></a> -<span class="sourceLineNo">1405</span><a name="line.1405"></a> -<span class="sourceLineNo">1406</span> /** First keys of the key range corresponding to each index entry. */<a name="line.1406"></a> -<span class="sourceLineNo">1407</span> private final List<byte[]> blockKeys = new ArrayList<>();<a name="line.1407"></a> -<span class="sourceLineNo">1408</span><a name="line.1408"></a> -<span class="sourceLineNo">1409</span> /** Block offset in backing stream. */<a name="line.1409"></a> -<span class="sourceLineNo">1410</span> private final List<Long> blockOffsets = new ArrayList<>();<a name="line.1410"></a> -<span class="sourceLineNo">1411</span><a name="line.1411"></a> -<span class="sourceLineNo">1412</span> /** On-disk data sizes of lower-level data or index blocks. */<a name="line.1412"></a> -<span class="sourceLineNo">1413</span> private final List<Integer> onDiskDataSizes = new ArrayList<>();<a name="line.1413"></a> -<span class="sourceLineNo">1414</span><a name="line.1414"></a> -<span class="sourceLineNo">1415</span> /**<a name="line.1415"></a> -<span class="sourceLineNo">1416</span> * The cumulative number of sub-entries, i.e. entries on deeper-level block<a name="line.1416"></a> -<span class="sourceLineNo">1417</span> * index entries. numSubEntriesAt[i] is the number of sub-entries in the<a name="line.1417"></a> -<span class="sourceLineNo">1418</span> * blocks corresponding to this chunk's entries #0 through #i inclusively.<a name="line.1418"></a> -<span class="sourceLineNo">1419</span> */<a name="line.1419"></a> -<span class="sourceLineNo">1420</span> private final List<Long> numSubEntriesAt = new ArrayList<>();<a name="line.1420"></a> -<span class="sourceLineNo">1421</span><a name="line.1421"></a> -<span class="sourceLineNo">1422</span> /**<a name="line.1422"></a> -<span class="sourceLineNo">1423</span> * The offset of the next entry to be added, relative to the end of the<a name="line.1423"></a> -<span class="sourceLineNo">1424</span> * "secondary index" in the "non-root" format representation of this index<a name="line.1424"></a> -<span class="sourceLineNo">1425</span> * chunk. This is the next value to be added to the secondary index.<a name="line.1425"></a> -<span class="sourceLineNo">1426</span> */<a name="line.1426"></a> -<span class="sourceLineNo">1427</span> private int curTotalNonRootEntrySize = 0;<a name="line.1427"></a> -<span class="sourceLineNo">1428</span><a name="line.1428"></a> -<span class="sourceLineNo">1429</span> /**<a name="line.1429"></a> -<span class="sourceLineNo">1430</span> * The accumulated size of this chunk if stored in the root index format.<a name="line.1430"></a> +<span class="sourceLineNo">1351</span> @Override<a name="line.1351"></a> +<span class="sourceLineNo">1352</span> public BlockType getInlineBlockType() {<a name="line.1352"></a> +<span class="sourceLineNo">1353</span> return BlockType.LEAF_INDEX;<a name="line.1353"></a> +<span class="sourceLineNo">1354</span> }<a name="line.1354"></a> +<span class="sourceLineNo">1355</span><a name="line.1355"></a> +<span class="sourceLineNo">1356</span> /**<a name="line.1356"></a> +<span class="sourceLineNo">1357</span> * Add one index entry to the current leaf-level block. When the leaf-level<a name="line.1357"></a> +<span class="sourceLineNo">1358</span> * block gets large enough, it will be flushed to disk as an inline block.<a name="line.1358"></a> +<span class="sourceLineNo">1359</span> *<a name="line.1359"></a> +<span class="sourceLineNo">1360</span> * @param firstKey the first key of the data block<a name="line.1360"></a> +<span class="sourceLineNo">1361</span> * @param blockOffset the offset of the data block<a name="line.1361"></a> +<span class="sourceLineNo">1362</span> * @param blockDataSize the on-disk size of the data block ({@link HFile}<a name="line.1362"></a> +<span class="sourceLineNo">1363</span> * format version 2), or the uncompressed size of the data block (<a name="line.1363"></a> +<span class="sourceLineNo">1364</span> * {@link HFile} format version 1).<a name="line.1364"></a> +<span class="sourceLineNo">1365</span> */<a name="line.1365"></a> +<span class="sourceLineNo">1366</span> public void addEntry(byte[] firstKey, long blockOffset, int blockDataSize) {<a name="line.1366"></a> +<span class="sourceLineNo">1367</span> curInlineChunk.add(firstKey, blockOffset, blockDataSize);<a name="line.1367"></a> +<span class="sourceLineNo">1368</span> ++totalNumEntries;<a name="line.1368"></a> +<span class="sourceLineNo">1369</span> }<a name="line.1369"></a> +<span class="sourceLineNo">1370</span><a name="line.1370"></a> +<span class="sourceLineNo">1371</span> /**<a name="line.1371"></a> +<span class="sourceLineNo">1372</span> * @throws IOException if we happened to write a multi-level index.<a name="line.1372"></a> +<span class="sourceLineNo">1373</span> */<a name="line.1373"></a> +<span class="sourceLineNo">1374</span> public void ensureSingleLevel() throws IOException {<a name="line.1374"></a> +<span class="sourceLineNo">1375</span> if (numLevels > 1) {<a name="line.1375"></a> +<span class="sourceLineNo">1376</span> throw new IOException ("Wrote a " + numLevels + "-level index with " +<a name="line.1376"></a> +<span class="sourceLineNo">1377</span> rootChunk.getNumEntries() + " root-level entries, but " +<a name="line.1377"></a> +<span class="sourceLineNo">1378</span> "this is expected to be a single-level block index.");<a name="line.1378"></a> +<span class="sourceLineNo">1379</span> }<a name="line.1379"></a> +<span class="sourceLineNo">1380</span> }<a name="line.1380"></a> +<span class="sourceLineNo">1381</span><a name="line.1381"></a> +<span class="sourceLineNo">1382</span> /**<a name="line.1382"></a> +<span class="sourceLineNo">1383</span> * @return true if we are using cache-on-write. This is configured by the<a name="line.1383"></a> +<span class="sourceLineNo">1384</span> * caller of the constructor by either passing a valid block cache<a name="line.1384"></a> +<span class="sourceLineNo">1385</span> * or null.<a name="line.1385"></a> +<span class="sourceLineNo">1386</span> */<a name="line.1386"></a> +<span class="sourceLineNo">1387</span> @Override<a name="line.1387"></a> +<span class="sourceLineNo">1388</span> public boolean getCacheOnWrite() {<a name="line.1388"></a> +<span class="sourceLineNo">1389</span> return cacheConf != null && cacheConf.shouldCacheIndexesOnWrite();<a name="line.1389"></a> +<span class="sourceLineNo">1390</span> }<a name="line.1390"></a> +<span class="sourceLineNo">1391</span><a name="line.1391"></a> +<span class="sourceLineNo">1392</span> /**<a name="line.1392"></a> +<span class="sourceLineNo">1393</span> * The total uncompressed size of the root index block, intermediate-level<a name="line.1393"></a> +<span class="sourceLineNo">1394</span> * index blocks, and leaf-level index blocks.<a name="line.1394"></a> +<span class="sourceLineNo">1395</span> *<a name="line.1395"></a> +<span class="sourceLineNo">1396</span> * @return the total uncompressed size of all index blocks<a name="line.1396"></a> +<span class="sourceLineNo">1397</span> */<a name="line.1397"></a> +<span class="sourceLineNo">1398</span> public long getTotalUncompressedSize() {<a name="line.1398"></a> +<span class="sourceLineNo">1399</span> return totalBlockUncompressedSize;<a name="line.1399"></a> +<span class="sourceLineNo">1400</span> }<a name="line.1400"></a> +<span class="sourceLineNo">1401</span><a name="line.1401"></a> +<span class="sourceLineNo">1402</span> }<a name="line.1402"></a> +<span class="sourceLineNo">1403</span><a name="line.1403"></a> +<span class="sourceLineNo">1404</span> /**<a name="line.1404"></a> +<span class="sourceLineNo">1405</span> * A single chunk of the block index in the process of writing. The data in<a name="line.1405"></a> +<span class="sourceLineNo">1406</span> * this chunk can become a leaf-level, intermediate-level, or root index<a name="line.1406"></a> +<span class="sourceLineNo">1407</span> * block.<a name="line.1407"></a> +<span class="sourceLineNo">1408</span> */<a name="line.1408"></a> +<span class="sourceLineNo">1409</span> static class BlockIndexChunk {<a name="line.1409"></a> +<span class="sourceLineNo">1410</span><a name="line.1410"></a> +<span class="sourceLineNo">1411</span> /** First keys of the key range corresponding to each index entry. */<a name="line.1411"></a> +<span class="sourceLineNo">1412</span> private final List<byte[]> blockKeys = new ArrayList<>();<a name="line.1412"></a> +<span class="sourceLineNo">1413</span><a name="line.1413"></a> +<span class="sourceLineNo">1414</span> /** Block offset in backing stream. */<a name="line.1414"></a> +<span class="sourceLineNo">1415</span> private final List<Long> blockOffsets = new ArrayList<>();<a name="line.1415"></a> +<span class="sourceLineNo">1416</span><a name="line.1416"></a> +<span class="sourceLineNo">1417</span> /** On-disk data sizes of lower-level data or index blocks. */<a name="line.1417"></a> +<span class="sourceLineNo">1418</span> private final List<Integer> onDiskDataSizes = new ArrayList<>();<a name="line.1418"></a> +<span class="sourceLineNo">1419</span><a name="line.1419"></a> +<span class="sourceLineNo">1420</span> /**<a name="line.1420"></a> +<span class="sourceLineNo">1421</span> * The cumulative number of sub-entries, i.e. entries on deeper-level block<a name="line.1421"></a> +<span class="sourceLineNo">1422</span> * index entries. numSubEntriesAt[i] is the number of sub-entries in the<a name="line.1422"></a> +<span class="sourceLineNo">1423</span> * blocks corresponding to this chunk's entries #0 through #i inclusively.<a name="line.1423"></a> +<span class="sourceLineNo">1424</span> */<a name="line.1424"></a> +<span class="sourceLineNo">1425</span> private final List<Long> numSubEntriesAt = new ArrayList<>();<a name="line.1425"></a> +<span class="sourceLineNo">1426</span><a name="line.1426"></a> +<span class="sourceLineNo">1427</span> /**<a name="line.1427"></a> +<span class="sourceLineNo">1428</span> * The offset of the next entry to be added, relative to the end of the<a name="line.1428"></a> +<span class="sourceLineNo">1429</span> * "secondary index" in the "non-root" format representation of this index<a name="line.1429"></a> +<span class="sourceLineNo">1430</span> * chunk. This is the next value to be added to the secondary index.<a name="line.1430"></a> <span class="sourceLineNo">1431</span> */<a name="line.1431"></a> -<span class="sourceLineNo">1432</span> private int curTotalRootSize = 0;<a name="line.1432"></a> +<span class="sourceLineNo">1432</span> private int curTotalNonRootEntrySize = 0;<a name="line.1432"></a> <span class="sourceLineNo">1433</span><a name="line.1433"></a> <span class="sourceLineNo">1434</span> /**<a name="line.1434"></a> -<span class="sourceLineNo">1435</span> * The "secondary index" used for binary search over variable-length<a name="line.1435"></a> -<span class="sourceLineNo">1436</span> * records in a "non-root" format block. These offsets are relative to the<a name="line.1436"></a> -<span class="sourceLineNo">1437</span> * end of this secondary index.<a name="line.1437"></a> -<span class="sourceLineNo">1438</span> */<a name="line.1438"></a> -<span class="sourceLineNo">1439</span> private final List<Integer> secondaryIndexOffsetMarks = new ArrayList<>();<a name="line.1439"></a> -<span class="sourceLineNo">1440</span><a name="line.1440"></a> -<span class="sourceLineNo">1441</span> /**<a name="line.1441"></a> -<span class="sourceLineNo">1442</span> * Adds a new entry to this block index chunk.<a name="line.1442"></a> -<span class="sourceLineNo">1443</span> *<a name="line.1443"></a> -<span class="sourceLineNo">1444</span> * @param firstKey the first key in the block pointed to by this entry<a name="line.1444"></a> -<span class="sourceLineNo">1445</span> * @param blockOffset the offset of the next-level block pointed to by this<a name="line.1445"></a> -<span class="sourceLineNo">1446</span> * entry<a name="line.1446"></a> -<span class="sourceLineNo">1447</span> * @param onDiskDataSize the on-disk data of the block pointed to by this<a name="line.1447"></a> -<span class="sourceLineNo">1448</span> * entry, including header size<a name="line.1448"></a> -<span class="sourceLineNo">1449</span> * @param curTotalNumSubEntries if this chunk is the root index chunk under<a name="line.1449"></a> -<span class="sourceLineNo">1450</span> * construction, this specifies the current total number of<a name="line.1450"></a> -<span class="sourceLineNo">1451</span> * sub-entries in all leaf-level chunks, including the one<a name="line.1451"></a> -<span class="sourceLineNo">1452</span> * corresponding to the second-level entry being added.<a name="line.1452"></a> -<span class="sourceLineNo">1453</span> */<a name="line.1453"></a> -<span class="sourceLineNo">1454</span> void add(byte[] firstKey, long blockOffset, int onDiskDataSize,<a name="line.1454"></a> -<span class="sourceLineNo">1455</span> long curTotalNumSubEntries) {<a name="line.1455"></a> -<span class="sourceLineNo">1456</span> // Record the offset for the secondary index<a name="line.1456"></a> -<span class="sourceLineNo">1457</span> secondaryIndexOffsetMarks.add(curTotalNonRootEntrySize);<a name="line.1457"></a> -<span class="sourceLineNo">1458</span> curTotalNonRootEntrySize += SECONDARY_INDEX_ENTRY_OVERHEAD<a name="line.1458"></a> -<span class="sourceLineNo">1459</span> + firstKey.length;<a name="line.1459"></a> -<span class="sourceLineNo">1460</span><a name="line.1460"></a> -<span class="sourceLineNo">1461</span> curTotalRootSize += Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT<a name="line.1461"></a> -<span class="sourceLineNo">1462</span> + WritableUtils.getVIntSize(firstKey.length) + firstKey.length;<a name="line.1462"></a> -<span class="sourceLineNo">1463</span><a name="line.1463"></a> -<span class="sourceLineNo">1464</span> blockKeys.add(firstKey);<a name="line.1464"></a> -<span class="sourceLineNo">1465</span> blockOffsets.add(blockOffset);<a name="line.1465"></a> -<span class="sourceLineNo">1466</span> onDiskDataSizes.add(onDiskDataSize);<a name="line.1466"></a> -<span class="sourceLineNo">1467</span><a name="line.1467"></a> -<span class="sourceLineNo">1468</span> if (curTotalNumSubEntries != -1) {<a name="line.1468"></a> -<span class="sourceLineNo">1469</span> numSubEntriesAt.add(curTotalNumSubEntries);<a name="line.1469"></a> -<span class="sourceLineNo">1470</span><a name="line.1470"></a> -<span class="sourceLineNo">1471</span> // Make sure the parallel arrays are in sync.<a name="line.1471"></a> -<span class="sourceLineNo">1472</span> if (numSubEntriesAt.size() != blockKeys.size()) {<a name="line.1472"></a> -<span class="sourceLineNo">1473</span> throw new IllegalStateException("Only have key/value count " +<a name="line.1473"></a> -<span class="sourceLineNo">1474</span> "stats for " + numSubEntriesAt.size() + " block index " +<a name="line.1474"></a> -<span class="sourceLineNo">1475</span> "entries out of " + blockKeys.size());<a name="line.1475"></a> -<span class="sourceLineNo">1476</span> }<a name="line.1476"></a> -<span class="sourceLineNo">1477</span> }<a name="line.1477"></a> -<span class="sourceLineNo">1478</span> }<a name="line.1478"></a> -<span class="sourceLineNo">1479</span><a name="line.1479"></a> -<span class="sourceLineNo">1480</span> /**<a name="line.1480"></a> -<span class="sourceLineNo">1481</span> * The same as {@link #add(byte[], long, int, long)} but does not take the<a name="line.1481"></a> -<span class="sourceLineNo">1482</span> * key/value into account. Used for single-level indexes.<a name="line.1482"></a> -<span class="sourceLineNo">1483</span> *<a name="line.1483"></a> -<span class="sourceLineNo">1484</span> * @see #add(byte[], long, int, long)<a name="line.1484"></a> -<span class="sourceLineNo">1485</span> */<a name="line.1485"></a> -<span class="sourceLineNo">1486</span> public void add(byte[] firstKey, long blockOffset, int onDiskDataSize) {<a name="line.1486"></a> -<span class="sourceLineNo">1487</span> add(firstKey, blockOffset, onDiskDataSize, -1);<a name="line.1487"></a> -<span class="sourceLineNo">1488</span> }<a name="line.1488"></a> -<span class="sourceLineNo">1489</span><a name="line.1489"></a> -<span class="sourceLineNo">1490</span> public void clear() {<a name="line.1490"></a> -<span class="sourceLineNo">1491</span> blockKeys.clear();<a name="line.1491"></a> -<span class="sourceLineNo">1492</span> blockOffsets.clear();<a name="line.1492"></a> -<span class="sourceLineNo">1493</span> onDiskDataSizes.clear();<a name="line.1493"></a> -<span class="sourceLineNo">1494</span> secondaryIndexOffsetMarks.clear();<a name="line.1494"></a> -<span class="sourceLineNo">1495</span> numSubEntriesAt.clear();<a name="line.1495"></a> -<span class="sourceLineNo">1496</span> curTotalNonRootEntrySize = 0;<a name="line.1496"></a> -<span class="sourceLineNo">1497</span> curTotalRootSize = 0;<a name="line.1497"></a> -<span class="sourceLineNo">1498</span> }<a name="line.1498"></a> -<span class="sourceLineNo">1499</span><a name="line.1499"></a> -<span class="sourceLineNo">1500</span> /**<a name="line.1500"></a> -<span class="sourceLineNo">1501</span> * Finds the entry corresponding to the deeper-level index block containing<a name="line.1501"></a> -<span class="sourceLineNo">1502</span> * the given deeper-level entry (a "sub-entry"), assuming a global 0-based<a name="line.1502"></a> -<span class="sourceLineNo">1503</span> * ordering of sub-entries.<a name="line.1503"></a> -<span class="sourceLineNo">1504</span> *<a name="line.1504"></a> -<span class="sourceLineNo">1505</span> * <p><a name="line.1505"></a> -<span class="sourceLineNo">1506</span> * <i> Implementation note. </i> We are looking for i such that<a name="line.1506"></a> -<span class="sourceLineNo">1507</span> * numSubEntriesAt[i - 1] <= k < numSubEntriesAt[i], because a deeper-level<a name="line.1507"></a> -<span class="sourceLineNo">1508</span> * block #i (0-based) contains sub-entries # numSubEntriesAt[i - 1]'th<a name="line.1508"></a> -<span class="sourceLineNo">1509</span> * through numSubEntriesAt[i] - 1, assuming a global 0-based ordering of<a name="line.1509"></a> -<span class="sourceLineNo">1510</span> * sub-entries. i is by definition the insertion point of k in<a name="line.1510"></a> -<span class="sourceLineNo">1511</span> * numSubEntriesAt.<a name="line.1511"></a> -<span class="sourceLineNo">1512</span> *<a name="line.1512"></a> -<span class="sourceLineNo">1513</span> * @param k sub-entry index, from 0 to the total number sub-entries - 1<a name="line.1513"></a> -<span class="sourceLineNo">1514</span> * @return the 0-based index of the entry corresponding to the given<a name="line.1514"></a> -<span class="sourceLineNo">1515</span> * sub-entry<a name="line.1515"></a> -<span class="sourceLineNo">1516</span> */<a name="line.1516"></a> -<span class="sourceLineNo">1517</span> public int getEntryBySubEntry(long k) {<a name="line.1517"></a> -<span class="sourceLineNo">1518</span> // We define mid-key as the key corresponding to k'th sub-entry<a name="line.1518"></a> -<span class="sourceLineNo">1519</span> // (0-based).<a name="line.1519"></a> -<span class="sourceLineNo">1520</span><a name="line.1520"></a> -<span class="sourceLineNo">1521</span> int i = Collections.binarySearch(numSubEntriesAt, k);<a name="line.1521"></a> -<span class="sourceLineNo">1522</span><a name="line.1522"></a> -<span class="sourceLineNo">1523</span> // Exact match: cumulativeWeight[i] = k. This means chunks #0 through<a name="line.1523"></a> -<span class="sourceLineNo">1524</span> // #i contain exactly k sub-entries, and the sub-entry #k (0-based)<a name="line.1524"></a> -<span class="sourceLineNo">1525</span> // is in the (i + 1)'th chunk.<a name="line.1525"></a> -<span class="sourceLineNo">1526</span> if (i >= 0)<a name="line.1526"></a> -<span class="sourceLineNo">1527</span> return i + 1;<a name="line.1527"></a> -<span class="sourceLineNo">1528</span><a name="line.1528"></a> -<span class="sourceLineNo">1529</span> // Inexact match. Return the insertion point.<a name="line.1529"></a> -<span class="sourceLineNo">1530</span> return -i - 1;<a name="line.1530"></a> -<span class="sourceLineNo">1531</span> }<a name="line.1531"></a> -<span class="sourceLineNo">1532</span><a name="line.1532"></a> -<span class="sourceLineNo">1533</span> /**<a name="line.1533"></a> -<span class="sourceLineNo">1534</span> * Used when writing the root block index of a multi-level block index.<a name="line.1534"></a> -<span class="sourceLineNo">1535</span> * Serializes additional information allowing to efficiently identify the<a name="line.1535"></a> -<span class="sourceLineNo">1536</span> * mid-key.<a name="line.1536"></a> -<span class="sourceLineNo">1537</span> *<a name="line.1537"></a> -<span class="sourceLineNo">1538</span> * @return a few serialized fields for finding the mid-key<a name="line.1538"></a> -<span class="sourceLineNo">1539</span> * @throws IOException if could not create metadata for computing mid-key<a name="line.1539"></a> -<span class="sourceLineNo">1540</span> */<a name="line.1540"></a> -<span class="sourceLineNo">1541</span> public byte[] getMidKeyMetadata() throws IOException {<a name="line.1541"></a> -<span class="sourceLineNo">1542</span> ByteArrayOutputStream baos = new ByteArrayOutputStream(<a name="line.1542"></a> -<span class="sourceLineNo">1543</span> MID_KEY_METADATA_SIZE);<a name="line.1543"></a> -<span class="sourceLineNo">1544</span> DataOutputStream baosDos = new DataOutputStream(baos);<a name="line.1544"></a> -<span class="sourceLineNo">1545</span> long totalNumSubEntries = numSubEntriesAt.get(blockKeys.size() - 1);<a name="line.1545"></a> -<span class="sourceLineNo">1546</span> if (totalNumSubEntries == 0) {<a name="line.1546"></a> -<span class="sourceLineNo">1547</span> throw new IOException("No leaf-level entries, mid-key unavailable");<a name="line.1547"></a> -<span class="sourceLineNo">1548</span> }<a name="line.1548"></a> -<span class="sourceLineNo">1549</span> long midKeySubEntry = (totalNumSubEntries - 1) / 2;<a name="line.1549"></a> -<span class="sourceLineNo">1550</span> int midKeyEntry = getEntryBySubEntry(midKeySubEntry);<a name="line.1550"></a> -<span class="sourceLineNo">1551</span><a name="line.1551"></a> -<span class="sourceLineNo">1552</span> baosDos.writeLong(blockOffsets.get(midKeyEntry));<a name="line.1552"></a> -<span class="sourceLineNo">1553</span> baosDos.writeInt(onDiskDataSizes.get(midKeyEntry));<a name="line.1553"></a> -<span class="sourceLineNo">1554</span><a name="line.1554"></a> -<span class="sourceLineNo">1555</span> long numSubEntriesBefore = midKeyEntry > 0<a name="line.1555"></a> -<span class="sourceLineNo">1556</span> ? numSubEntriesAt.get(midKeyEntry - 1) : 0;<a name="line.1556"></a> -<span class="sourceLineNo">1557</span> long subEntryWithinEntry = midKeySubEntry - numSubEntriesBefore;<a name="line.1557"></a> -<span class="sourceLineNo">1558</span> if (subEntryWithinEntry < 0 || subEntryWithinEntry > Integer.MAX_VALUE)<a name="line.1558"></a> -<span class="sourceLineNo">1559</span> {<a name="line.1559"></a> -<span class="sourceLineNo">1560</span> throw new IOException("Could not identify mid-key index within the "<a name="line.1560"></a> -<span class="sourceLineNo">1561</span> + "leaf-level block containing mid-key: out of range ("<a name="line.1561"></a> -<span class="sourceLineNo">1562</span> + subEntryWithinEntry + ", numSubEntriesBefore="<a name="line.1562"></a> -<span class="sourceLineNo">1563</span> + numSubEntriesBefore + ", midKeySubEntry=" + midKeySubEntry<a name="line.1563"></a> -<span class="sourceLineNo">1564</span> + ")");<a name="line.1564"></a> -<span class="sourceLineNo">1565</span> }<a name="line.1565"></a> -<span class="sourceLineNo">1566</span><a name="line.1566"></a> -<span class="sourceLineNo">1567</span> baosDos.writeInt((int) subEntryWithinEntry);<a name="line.1567"></a> -<span class="sourceLineNo">1568</span><a name="line.1568"></a> -<span class="sourceLineNo">1569</span> if (baosDos.size() != MID_KEY_METADATA_SIZE) {<a name="line.1569"></a> -<span class="sourceLineNo">1570</span> throw new IOException("Could not write mid-key metadata: size=" +<a name="line.1570"></a> -<span class="sourceLineNo">1571</span> baosDos.size() + ", correct size: " + MID_KEY_METADATA_SIZE);<a name="line.1571"></a> -<span class="sourceLineNo">1572</span> }<a name="line.1572"></a> +<span class="sourceLineNo">1435</span> * The accumulated size of this chunk if stored in the root index format.<a name="line.1435"></a> +<span class="sourceLineNo">1436</span> */<a name="line.1436"></a> +<span class="sourceLineNo">1437</span> private int curTotalRootSize = 0;<a name="line.1437"></a> +<span class="sourceLineNo">1438</span><a name="line.1438"></a> +<span class="sourceLineNo">1439</span> /**<a name="line.1439"></a> +<span class="sourceLineNo">1440</span> * The "secondary index" used for binary search over variable-length<a name="line.1440"></a> +<span class="sourceLineNo">1441</span> * records in a "non-root" format block. These offsets are relative to the<a name="line.1441"></a> +<span class="sourceLineNo">1442</span> * end of this secondary index.<a name="line.1442"></a> +<span class="sourceLineNo">1443</span> */<a name="line.1443"></a> +<span class="sourceLineNo">1444</span> private final List<Integer> secondaryIndexOffsetMarks = new ArrayList<>();<a name="line.1444"></a> +<span class="sourceLineNo">1445</span><a name="line.1445"></a> +<span class="sourceLineNo">1446</span> /**<a name="line.1446"></a> +<span class="sourceLineNo">1447</span> * Adds a new entry to this block index chunk.<a name="line.1447"></a> +<span class="sourceLineNo">1448</span> *<a name="line.1448"></a> +<span class="sourceLineNo">1449</span> * @param firstKey the first key in the block pointed to by this entry<a name="line.1449"></a> +<span class="sourceLineNo">1450</span> * @param blockOffset the offset of the next-level block pointed to by this<a name="line.1450"></a> +<span class="sourceLineNo">1451</span> * entry<a name="line.1451"></a> +<span class="sourceLineNo">1452</span> * @param onDiskDataSize the on-disk data of the block pointed to by this<a name="line.1452"></a> +<span class="sourceLineNo">1453</span> * entry, including header size<a name="line.1453"></a> +<span class="sourceLineNo">1454</span> * @param curTotalNumSubEntries if this chunk is the root index chunk under<a name="line.1454"></a> +<span class="sourceLineNo">1455</span> * construction, this specifies the current total number of<a name="line.1455"></a> +<span class="sourceLineNo">1456</span> * sub-entries in all leaf-level chunks, including the one<a name="line.1456"></a> +<span class="sourceLineNo">1457</span> * corresponding to the second-level entry being added.<a name="line.1457"></a> +<span class="sourceLineNo">1458</span> */<a name="line.1458"></a> +<span class="sourceLineNo">1459</span> void add(byte[] firstKey, long blockOffset, int onDiskDataSize,<a name="line.1459"></a> +<span class="sourceLineNo">1460</span> long curTotalNumSubEntries) {<a name="line.1460"></a> +<span class="sourceLineNo">1461</span> // Record the offset for the secondary index<a name="line.1461"></a> +<span class="sourceLineNo">1462</span> secondaryIndexOffsetMarks.add(curTotalNonRootEntrySize);<a name="line.1462"></a> +<span class="sourceLineNo">1463</span> curTotalNonRootEntrySize += SECONDARY_INDEX_ENTRY_OVERHEAD<a name="line.1463"></a> +<span class="sourceLineNo">1464</span> + firstKey.length;<a name="line.1464"></a> +<span class="sourceLineNo">1465</span><a name="line.1465"></a> +<span class="sourceLineNo">1466</span> curTotalRootSize += Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT<a name="line.1466"></a> +<span class="sourceLineNo">1467</span> + WritableUtils.getVIntSize(firstKey.length) + firstKey.length;<a name="line.1467"></a> +<span class="sourceLineNo">1468</span><a name="line.1468"></a> +<span class="sourceLineNo">1469</span> blockKeys.add(firstKey);<a name="line.1469"></a> +<span class="sourceLineNo">1470</span> blockOffsets.add(blockOffset);<a name="line.1470"></a> +<span class="sourceLineNo">1471</span> onDiskDataSizes.add(onDiskDataSize);<a name="line.1471"></a> +<span class="sourceLineNo">1472</span><a name="line.1472"></a> +<span class="sourceLineNo">1473</span> if (curTotalNumSubEntries != -1) {<a name="line.1473"></a> +<span class="sourceLineNo">1474</span> numSubEntriesAt.add(curTotalNumSubEntries);<a name="line.1474"></a> +<span class="sourceLineNo">1475</span><a name="line.1475"></a> +<span class="sourceLineNo">1476</span> // Make sure the parallel arrays are in sync.<a name="line.1476"></a> +<span class="sourceLineNo">1477</span> if (numSubEntriesAt.size() != blockKeys.size()) {<a name="line.1477"></a> +<span class="sourceLineNo">1478</span> throw new IllegalStateException("Only have key/value count " +<a name="line.1478"></a> +<span class="sourceLineNo">1479</span> "stats for " + numSubEntriesAt.size() + " block index " +<a name="line.1479"></a> +<span class="sourceLineNo">1480</span> "entries out of " + blockKeys.size());<a name="line.1480"></a> +<span class="sourceLineNo">1481</span> }<a name="line.1481"></a> +<span class="sourceLineNo">1482</span> }<a name="line.1482"></a> +<span class="sourceLineNo">1483</span> }<a name="line.1483"></a> +<span class="sourceLineNo">1484</span><a name="line.1484"></a> +<span class="sourceLineNo">1485</span> /**<a name="line.1485"></a> +<span class="sourceLineNo">1486</span> * The same as {@link #add(byte[], long, int, long)} but does not take the<a name="line.1486"></a> +<span class="sourceLineNo">1487</span> * key/value into account. Used for single-level indexes.<a name="line.1487"></a> +<span class="sourceLineNo">1488</span> *<a name="line.1488"></a> +<span class="sourceLineNo">1489</span> * @see #add(byte[], long, int, long)<a name="line.1489"></a> +<span class="sourceLineNo">1490</span> */<a name="line.1490"></a> +<span class="sourceLineNo">1491</span> public void add(byte[] firstKey, long blockOffset, int onDiskDataSize) {<a name="line.1491"></a> +<span class="sourceLineNo">1492</span> add(firstKey, blockOffset, onDiskDataSize, -1);<a name="line.1492"></a> +<span class="sourceLineNo">1493</span> }<a name="line.1493"></a> +<span class="sourceLineNo">1494</span><a name="line.1494"></a> +<span class="sourceLineNo">1495</span> public void clear() {<a name="line.1495"></a> +<span class="sourceLineNo">1496</span> blockKeys.clear();<a name="line.1496"></a> +<span class="sourceLineNo">1497</span> blockOffsets.clear();<a name="line.1497"></a> +<span class="sourceLineNo">1498</span> onDiskDataSizes.clear();<a name="line.1498"></a> +<span class="sourceLineNo">1499</span> secondaryIndexOffsetMarks.clear();<a name="line.1499"></a> +<span class="sourceLineNo">1500</span> numSubEntriesAt.clear();<a name="line.1500"></a> +<span class="sourceLineNo">1501</span> curTotalNonRootEntrySize = 0;<a name="line.1501"></a> +<span class="sourceLineNo">1502</span> curTotalRootSize = 0;<a name="line.1502"></a> +<span class="sourceLineNo">1503</span> }<a name="line.1503"></a> +<span class="sourceLineNo">1504</span><a name="line.1504"></a> +<span class="sourceLineNo">1505</span> /**<a name="line.1505"></a> +<span class="sourceLineNo">1506</span> * Finds the entry corresponding to the deeper-level index block containing<a name="line.1506"></a> +<span class="sourceLineNo">1507</span> * the given deeper-level entry (a "sub-entry"), assuming a global 0-based<a name="line.1507"></a> +<span class="sourceLineNo">1508</span> * ordering of sub-entries.<a name="line.1508"></a> +<span class="sourceLineNo">1509</span> *<a name="line.1509"></a> +<span class="sourceLineNo">1510</span> * <p><a name="line.1510"></a> +<span class="sourceLineNo">1511</span> * <i> Implementation note. </i> We are looking for i such that<a name="line.1511"></a> +<span class="sourceLineNo">1512</span> * numSubEntriesAt[i - 1] <= k < numSubEntriesAt[i], because a deeper-level<a name="line.1512"></a> +<span class="sourceLineNo">1513</span> * block #i (0-based) contains sub-entries # numSubEntriesAt[i - 1]'th<a name="line.1513"></a> +<span class="sourceLineNo">1514</span> * through numSubEntriesAt[i] - 1, assuming a global 0-based ordering of<a name="line.1514"></a> +<span class="sourceLineNo">1515</span> * sub-entries. i is by definition the insertion point of k in<a name="line.1515"></a> +<span class="sourceLineNo">1516</span> * numSubEntriesAt.<a name="line.1516"></a> +<span class="sourceLineNo">1517</span> *<a name="line.1517"></a> +<span class="sourceLineNo">1518</span> * @param k sub-entry index, from 0 to the total number sub-entries - 1<a name="line.1518"></a> +<span class="sourceLineNo">1519</span> * @return the 0-based index of the entry corresponding to the given<a name="line.1519"></a> +<span class="sourceLineNo">1520</span> * sub-entry<a name="line.1520"></a> +<span class="sourceLineNo">1521</span> */<
<TRUNCATED>
