Repository: phoenix Updated Branches: refs/heads/calcite c3613fd2b -> c8c33a918
http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java index 291cb6d..43fe7f3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfo.java @@ -22,51 +22,53 @@ import java.util.List; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.SizedUtil; + +import com.google.common.primitives.Longs; /** * A class that holds the guidePosts of a region and also allows combining the * guidePosts of different regions when the GuidePostsInfo is formed for a table. */ public class GuidePostsInfo { + public final static GuidePostsInfo EMPTY_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(), + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0) { + @Override + public int getEstimatedSize() { + return 0; + } + }; + public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(), + new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0) { + @Override + public int getEstimatedSize() { + return 0; + } + }; /** * the total number of guidePosts for the table combining all the guidePosts per region per cf. */ - private ImmutableBytesWritable guidePosts; - + private final ImmutableBytesWritable guidePosts; /** * Maximum length of a guidePost collected */ - private int maxLength; - - public final static GuidePostsInfo NO_GUIDEPOST = new GuidePostsInfo(new ArrayList<Long>(), - new ImmutableBytesWritable(ByteUtil.EMPTY_BYTE_ARRAY), new ArrayList<Long>(), 0, 0); - - public int getMaxLength() { - return maxLength; - } - + private final int maxLength; /** * Number of guidePosts */ - private int guidePostsCount; - + private final int guidePostsCount; /** * The rowCounts of each guidePost traversed */ - private List<Long> rowCounts; - + private final long[] rowCounts; /** * The bytecounts of each guidePost traversed */ - private List<Long> byteCounts; - - public List<Long> getRowCounts() { - return rowCounts; - } - - public List<Long> getByteCounts() { - return byteCounts; - } + private final long[] byteCounts; + /** + * Estimate of byte size of this instance + */ + private final int estimatedSize; /** * Constructor that creates GuidePostsInfo per region @@ -87,8 +89,16 @@ public class GuidePostsInfo { this.guidePosts = new ImmutableBytesWritable(guidePosts); this.maxLength = maxLength; this.guidePostsCount = guidePostsCount; - this.rowCounts = rowCounts; - this.byteCounts = byteCounts; + this.rowCounts = Longs.toArray(rowCounts); + this.byteCounts = Longs.toArray(byteCounts); + int estimatedSize = SizedUtil.OBJECT_SIZE + + SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE + guidePosts.getLength() // guidePosts + + SizedUtil.INT_SIZE // maxLength + + SizedUtil.INT_SIZE // guidePostsCount + + SizedUtil.ARRAY_SIZE + this.rowCounts.length * SizedUtil.LONG_SIZE // rowCounts + + SizedUtil.ARRAY_SIZE + this.byteCounts.length * SizedUtil.LONG_SIZE // byteCounts + + SizedUtil.INT_SIZE; // estimatedSize + this.estimatedSize = estimatedSize; } public ImmutableBytesWritable getGuidePosts() { @@ -98,5 +108,20 @@ public class GuidePostsInfo { public int getGuidePostsCount() { return guidePostsCount; } + + public int getMaxLength() { + return maxLength; + } + public long[] getRowCounts() { + return rowCounts; + } + + public long[] getByteCounts() { + return byteCounts; + } + + public int getEstimatedSize() { + return estimatedSize; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java index 435f30a..246ef6c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsInfoBuilder.java @@ -51,6 +51,10 @@ public class GuidePostsInfoBuilder { private List<Long> rowCounts = new ArrayList<Long>(); private List<Long> byteCounts = new ArrayList<Long>(); + public boolean isEmpty() { + return rowCounts.size() == 0; + } + public List<Long> getRowCounts() { return rowCounts; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java new file mode 100644 index 0000000..3bde602 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/GuidePostsKey.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.schema.stats; + +import java.util.Arrays; + +import javax.annotation.Nonnull; + +import org.apache.hadoop.hbase.util.Bytes; + +import com.google.common.base.Preconditions; + +/** + * + * Key for the client-side caching of the guideposts information + * + */ +public final class GuidePostsKey { + private final int hashCode; + @Nonnull private final byte[] physicalName; + @Nonnull private final byte[] columnFamily; + + public GuidePostsKey(byte[] physicalName, byte[] columnFamily) { + Preconditions.checkNotNull(physicalName); + Preconditions.checkNotNull(columnFamily); + this.physicalName = physicalName; + this.columnFamily = columnFamily; + this.hashCode = computeHashCode(); + } + + public byte[] getPhysicalName() { + return physicalName; + } + + public byte[] getColumnFamily() { + return columnFamily; + } + + @Override + public int hashCode() { + return hashCode; + } + + private int computeHashCode() { + final int prime = 31; + int result = 1; + result = prime * result + Arrays.hashCode(columnFamily); + result = prime * result + Arrays.hashCode(physicalName); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + GuidePostsKey other = (GuidePostsKey)obj; + if (other.hashCode != this.hashCode) return false; + if (!Arrays.equals(columnFamily, other.columnFamily)) return false; + if (!Arrays.equals(physicalName, other.physicalName)) return false; + return true; + } + + @Override + public String toString() { + return "GuidePostsKey[physicalName=" + Bytes.toStringBinary(physicalName) + + ",columnFamily=" + Bytes.toStringBinary(columnFamily) + "]"; + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java index a16a48a..74d1710 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.Region; @@ -44,7 +45,7 @@ public class NoOpStatisticsCollector implements StatisticsCollector { } @Override - public void updateStatistic(Region region) { + public void updateStatistic(Region region, Scan scan) { // No-op } http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java deleted file mode 100644 index f297b3b..0000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStats.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.schema.stats; - -import java.util.SortedMap; - -import com.google.common.collect.ImmutableSortedMap; - - -/* - * The table is defined on the client side, but it is populated on the server side. The client should not populate any data to the - * statistics object. - */ -public interface PTableStats { - public static final PTableStats EMPTY_STATS = new PTableStats() { - @Override - public SortedMap<byte[], GuidePostsInfo> getGuidePosts() { - return ImmutableSortedMap.of(); - } - - @Override - public int getEstimatedSize() { - return 0; - } - - @Override - public long getTimestamp() { - return DefaultStatisticsCollector.NO_TIMESTAMP; - } - }; - - /** - * TODO: Change from TreeMap to Map - * Returns a tree map of the guide posts collected against a column family - * @return - */ - SortedMap<byte[], GuidePostsInfo> getGuidePosts(); - - int getEstimatedSize(); - - long getTimestamp(); -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java deleted file mode 100644 index 5eb02fd..0000000 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/PTableStatsImpl.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.phoenix.schema.stats; - -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.EOFException; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import org.apache.hadoop.hbase.io.ImmutableBytesWritable; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.coprocessor.MetaDataProtocol; -import org.apache.phoenix.util.Closeables; -import org.apache.phoenix.util.PrefixByteCodec; -import org.apache.phoenix.util.PrefixByteDecoder; -import org.apache.phoenix.util.SizedUtil; - -import com.sun.istack.NotNull; - - /** - * Implementation for PTableStats. - */ -public class PTableStatsImpl implements PTableStats { - private final SortedMap<byte[], GuidePostsInfo> guidePosts; - private final int estimatedSize; - private final long timeStamp; - - public PTableStatsImpl() { - this(new TreeMap<byte[], GuidePostsInfo>(Bytes.BYTES_COMPARATOR), MetaDataProtocol.MIN_TABLE_TIMESTAMP); - } - - public PTableStatsImpl(@NotNull SortedMap<byte[], GuidePostsInfo> guidePosts, long timeStamp) { - this.guidePosts = guidePosts; - this.timeStamp = timeStamp; - int estimatedSize = SizedUtil.OBJECT_SIZE + SizedUtil.INT_SIZE + SizedUtil.sizeOfTreeMap(guidePosts.size()); - for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) { - byte[] cf = entry.getKey(); - estimatedSize += SizedUtil.ARRAY_SIZE + cf.length; - estimatedSize += entry.getValue().getGuidePosts().getLength(); - estimatedSize += SizedUtil.LONG_SIZE; - estimatedSize += SizedUtil.INT_SIZE; - estimatedSize += SizedUtil.INT_SIZE; - } - this.estimatedSize = estimatedSize; - } - - @Override - public SortedMap<byte[], GuidePostsInfo> getGuidePosts() { - return guidePosts; - } - - @Override - public String toString() { - StringBuilder buf = new StringBuilder(); - - buf.append("PTableStats ["); - for (Map.Entry<byte[], GuidePostsInfo> entry : guidePosts.entrySet()) { - buf.append(Bytes.toStringBinary(entry.getKey())); - buf.append(":("); - ImmutableBytesWritable keys = entry.getValue().getGuidePosts(); - ByteArrayInputStream stream = new ByteArrayInputStream(keys.get(), keys.getOffset(), keys.getLength()); - try { - if (keys.getLength() != 0) { - DataInput input = new DataInputStream(stream); - PrefixByteDecoder decoder = new PrefixByteDecoder(entry.getValue().getMaxLength()); - try { - while (true) { - ImmutableBytesWritable ptr = PrefixByteCodec.decode(decoder, input); - buf.append(Bytes.toStringBinary(ptr.get())); - buf.append(","); - } - } catch (EOFException e) { // Ignore as this signifies we're done - - } finally { - Closeables.closeQuietly(stream); - } - buf.setLength(buf.length() - 1); - } - buf.append(")"); - } finally { - Closeables.closeQuietly(stream); - } - } - buf.append("]"); - return buf.toString(); - } - - @Override - public int getEstimatedSize() { - return estimatedSize; - } - - @Override - public long getTimestamp() { - return timeStamp; - } -} http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 43ec6c7..60e83a8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.Region; @@ -42,9 +43,9 @@ public interface StatisticsCollector extends Closeable { long getMaxTimeStamp(); /** - * Write the collected statistics for the given region. + * Write the collected statistics for the given region over the scan provided. */ - void updateStatistic(Region region); + void updateStatistic(Region region, Scan scan); /** * Collect statistics for the given list of cells. This method can be called multiple times http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index 685c24e..07b412f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -75,11 +75,11 @@ public class StatisticsUtil { /** Number of parts in our complex key */ protected static final int NUM_KEY_PARTS = 3; - public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, byte[] guidePostStartKey) { + public static byte[] getRowKey(byte[] table, ImmutableBytesWritable fam, byte[] guidePostStartKey) { return getRowKey(table, fam, new ImmutableBytesWritable(guidePostStartKey,0,guidePostStartKey.length)); } - public static byte[] getRowKey(byte[] table, ImmutableBytesPtr fam, ImmutableBytesWritable guidePostStartKey) { + public static byte[] getRowKey(byte[] table, ImmutableBytesWritable fam, ImmutableBytesWritable guidePostStartKey) { // always starts with the source table int guidePostLength = guidePostStartKey.getLength(); boolean hasGuidePost = guidePostLength > 0; @@ -97,21 +97,36 @@ public class StatisticsUtil { return rowKey; } - private static byte[] getKey(byte[] table, ImmutableBytesPtr fam) { + private static byte[] getStartKey(byte[] table, ImmutableBytesWritable fam) { + return getKey(table, fam, false); + } + + private static byte[] getEndKey(byte[] table, ImmutableBytesWritable fam) { + byte[] key = getKey(table, fam, true); + ByteUtil.nextKey(key, key.length); + return key; + } + + private static byte[] getKey(byte[] table, ImmutableBytesWritable fam, boolean terminateWithSeparator) { // always starts with the source table and column family - byte[] rowKey = new byte[table.length + fam.getLength() + 1]; + byte[] rowKey = new byte[table.length + fam.getLength() + 1 + (terminateWithSeparator ? 1 : 0)]; int offset = 0; System.arraycopy(table, 0, rowKey, offset, table.length); offset += table.length; rowKey[offset++] = QueryConstants.SEPARATOR_BYTE; // assumes stats table columns not DESC System.arraycopy(fam.get(), fam.getOffset(), rowKey, offset, fam.getLength()); offset += fam.getLength(); + if (terminateWithSeparator) { + rowKey[offset] = QueryConstants.SEPARATOR_BYTE; + } return rowKey; } - private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) { - if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { return getRowKey(tableNameBytes, cf, key); } - key = ByteUtil.concat(getKey(tableNameBytes, cf), QueryConstants.SEPARATOR_BYTE_ARRAY); + private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesWritable cf, boolean nextKey) { + if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { + return getRowKey(tableNameBytes, cf, key); + } + key = getKey(tableNameBytes, cf, nextKey); if (nextKey) { ByteUtil.nextKey(key, key.length); } @@ -121,8 +136,10 @@ public class StatisticsUtil { public static List<Result> readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, ImmutableBytesPtr cf, byte[] startKey, byte[] stopKey, long clientTimeStamp) throws IOException { List<Result> statsForRegion = new ArrayList<Result>(); - Scan s = MetaDataUtil.newTableRowsScan(getAdjustedKey(startKey, tableNameBytes, cf, false), - getAdjustedKey(stopKey, tableNameBytes, cf, true), MetaDataProtocol.MIN_TABLE_TIMESTAMP, + Scan s = MetaDataUtil.newTableRowsScan( + getAdjustedKey(startKey, tableNameBytes, cf, false), + getAdjustedKey(stopKey, tableNameBytes, cf, true), + MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp); s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); ResultScanner scanner = null; @@ -140,40 +157,27 @@ public class StatisticsUtil { return statsForRegion; } - public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp) + public static GuidePostsInfo readStatistics(HTableInterface statsHTable, GuidePostsKey key, long clientTimeStamp) throws IOException { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - Scan s = MetaDataUtil.newTableRowsScan(tableNameBytes, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp); + ptr.set(key.getColumnFamily()); + byte[] tableNameBytes = key.getPhysicalName(); + byte[] startKey = getStartKey(tableNameBytes, ptr); + byte[] endKey = getEndKey(tableNameBytes, ptr); + Scan s = MetaDataUtil.newTableRowsScan(startKey, endKey, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp); s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES); s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES); s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES); - ResultScanner scanner = null; - long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; - TreeMap<byte[], GuidePostsInfoBuilder> guidePostsInfoWriterPerCf = new TreeMap<byte[], GuidePostsInfoBuilder>(Bytes.BYTES_COMPARATOR); - try { - scanner = statsHTable.getScanner(s); + GuidePostsInfoBuilder guidePostsInfoWriter = new GuidePostsInfoBuilder(); + Cell current = null; + try (ResultScanner scanner = statsHTable.getScanner(s)) { Result result = null; while ((result = scanner.next()) != null) { CellScanner cellScanner = result.cellScanner(); long rowCount = 0; long byteCount = 0; - byte[] cfName = null; - int tableNameLength; - int cfOffset; - int cfLength; - boolean valuesSet = false; - // Only the two cells with quals GUIDE_POSTS_ROW_COUNT_BYTES and GUIDE_POSTS_BYTES would be retrieved - while (cellScanner.advance()) { - Cell current = cellScanner.current(); - if (!valuesSet) { - tableNameLength = tableNameBytes.length + 1; - cfOffset = current.getRowOffset() + tableNameLength; - cfLength = getVarCharLength(current.getRowArray(), cfOffset, - current.getRowLength() - tableNameLength); - ptr.set(current.getRowArray(), cfOffset, cfLength); - valuesSet = true; - } - cfName = ByteUtil.copyKeyBytesIfNecessary(ptr); + while (cellScanner.advance()) { + current = cellScanner.current(); if (Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(), current.getQualifierLength(), PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, 0, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES.length)) { @@ -185,28 +189,22 @@ public class StatisticsUtil { byteCount = PLong.INSTANCE.getCodec().decodeLong(current.getValueArray(), current.getValueOffset(), SortOrder.getDefault()); } - if (current.getTimestamp() > timeStamp) { - timeStamp = current.getTimestamp(); - } } - if (cfName != null) { + if (current != null) { + int tableNameLength = tableNameBytes.length + 1; + int cfOffset = current.getRowOffset() + tableNameLength; + int cfLength = getVarCharLength(current.getRowArray(), cfOffset, + current.getRowLength() - tableNameLength); + ptr.set(current.getRowArray(), cfOffset, cfLength); + byte[] cfName = ByteUtil.copyKeyBytesIfNecessary(ptr); byte[] newGPStartKey = getGuidePostsInfoFromRowKey(tableNameBytes, cfName, result.getRow()); - GuidePostsInfoBuilder guidePostsInfoWriter = guidePostsInfoWriterPerCf.get(cfName); - if (guidePostsInfoWriter == null) { - guidePostsInfoWriter = new GuidePostsInfoBuilder(); - guidePostsInfoWriterPerCf.put(cfName, guidePostsInfoWriter); - } guidePostsInfoWriter.addGuidePosts(newGPStartKey, byteCount, rowCount); } } - if (!guidePostsInfoWriterPerCf.isEmpty()) { return new PTableStatsImpl( - getGuidePostsPerCf(guidePostsInfoWriterPerCf), timeStamp); } - } finally { - if (scanner != null) { - scanner.close(); - } } - return PTableStats.EMPTY_STATS; + // We write a row with an empty KeyValue in the case that stats were generated but without enough data + // for any guideposts. If we have no rows, it means stats were never generated. + return current == null ? GuidePostsInfo.NO_GUIDEPOST : guidePostsInfoWriter.isEmpty() ? GuidePostsInfo.EMPTY_GUIDEPOST : guidePostsInfoWriter.build(); } private static SortedMap<byte[], GuidePostsInfo> getGuidePostsPerCf( http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java index b40d34c..fb8d664 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsWriter.java @@ -135,8 +135,8 @@ public class StatisticsWriter implements Closeable { } GuidePostsInfo gps = tracker.getGuidePosts(cfKey); if (gps != null) { - List<Long> byteCounts = gps.getByteCounts(); - List<Long> rowCounts = gps.getRowCounts(); + long[] byteCounts = gps.getByteCounts(); + long[] rowCounts = gps.getRowCounts(); ImmutableBytesWritable keys = gps.getGuidePosts(); boolean hasGuidePosts = keys.getLength() > 0; if (hasGuidePosts) { @@ -146,7 +146,7 @@ public class StatisticsWriter implements Closeable { PrefixByteDecoder decoder = new PrefixByteDecoder(gps.getMaxLength()); do { ImmutableBytesWritable ptr = decoder.decode(input); - addGuidepost(cfKey, mutations, ptr, byteCounts.get(guidePostCount), rowCounts.get(guidePostCount), timeStamp); + addGuidepost(cfKey, mutations, ptr, byteCounts[guidePostCount], rowCounts[guidePostCount], timeStamp); guidePostCount++; } while (decoder != null); } catch (EOFException e) { // Ignore as this signifies we're done http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index 52346a5..7acd30b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -639,4 +639,8 @@ public class MetaDataUtil { return cfPtr.getLength() >= QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length && Bytes.compareTo(cfPtr.get(), cfPtr.getOffset(), QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES, 0, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length) == 0; } + + public static boolean isLocalIndexFamily(byte[] cf) { + return Bytes.startsWith(cf, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java index ad99514..9ba7f6d 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/filter/SkipScanBigFilterTest.java @@ -24,7 +24,6 @@ import java.sql.DriverManager; import java.sql.PreparedStatement; import java.util.Map; import java.util.Properties; -import java.util.SortedMap; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; @@ -32,7 +31,6 @@ import org.apache.phoenix.end2end.Shadower; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseConnectionlessQueryTest; -import org.apache.phoenix.query.ConnectionQueryServicesImpl; import org.apache.phoenix.query.ConnectionlessQueryServicesImpl; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; @@ -40,7 +38,7 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.stats.GuidePostsInfo; import org.apache.phoenix.schema.stats.GuidePostsInfoBuilder; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; @@ -653,29 +651,10 @@ public class SkipScanBigFilterTest extends BaseConnectionlessQueryTest { gpWriter.addGuidePosts(gp, 1000); } GuidePostsInfo info = gpWriter.build(); - final SortedMap<byte[], GuidePostsInfo> gpMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); - gpMap.put(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, info); - PTableStats stats = new PTableStats() { - - @Override - public SortedMap<byte[], GuidePostsInfo> getGuidePosts() { - return gpMap; - } - - @Override - public int getEstimatedSize() { - return 10000; - } - - @Override - public long getTimestamp() { - return table.getTimeStamp()+1; - } - }; PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class); pConn.addTable(table, System.currentTimeMillis()); ((ConnectionlessQueryServicesImpl) pConn.getQueryServices()) - .addTableStats(table.getName().getBytesPtr(), stats); + .addTableStats(new GuidePostsKey(table.getName().getBytes(), QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES), info); String query = "SELECT count(1) cnt,\n" + " coalesce(SUM(impressions), 0.0) AS \"impressions\",\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java index b4e01a7..7f84219 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/query/PhoenixStatsCacheRemovalListenerTest.java @@ -19,7 +19,7 @@ package org.apache.phoenix.query; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import org.apache.phoenix.query.TableStatsCache.PhoenixStatsCacheRemovalListener; +import org.apache.phoenix.query.GuidePostsCache.PhoenixStatsCacheRemovalListener; import org.junit.Test; import com.google.common.cache.RemovalCause; http://git-wip-us.apache.org/repos/asf/phoenix/blob/eaf99f23/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java index de5af46..03469e2 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java @@ -52,6 +52,7 @@ import java.util.Properties; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -105,11 +106,12 @@ import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PLongColumn; import org.apache.phoenix.schema.PName; +import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.TableRef; import org.apache.phoenix.schema.stats.GuidePostsInfo; -import org.apache.phoenix.schema.stats.PTableStats; +import org.apache.phoenix.schema.stats.GuidePostsKey; import org.apache.phoenix.schema.tuple.Tuple; import org.apache.phoenix.schema.types.PDataType; @@ -595,9 +597,9 @@ public class TestUtil { pstmt.execute(); TableRef tableRef = pstmt.getQueryPlan().getTableRef(); PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class); - long scn = null == pconn.getSCN() ? Long.MAX_VALUE : pconn.getSCN(); - PTableStats tableStats = pconn.getQueryServices().getTableStats(tableRef.getTable().getName().getBytes(), scn); - return tableStats.getGuidePosts().values(); + PTable table = tableRef.getTable(); + GuidePostsInfo info = pconn.getQueryServices().getTableStats(new GuidePostsKey(table.getName().getBytes(), SchemaUtil.getEmptyColumnFamily(table))); + return Collections.singletonList(info); } public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException { @@ -793,5 +795,23 @@ public class TestUtil { public static void createTransactionalTable(Connection conn, String tableName) throws SQLException { conn.createStatement().execute("create table " + tableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true"); } + + public static void dumpTable(HTableInterface table) throws IOException { + System.out.println("************ dumping " + table + " **************"); + Scan s = new Scan(); + try (ResultScanner scanner = table.getScanner(s)) { + Result result = null; + while ((result = scanner.next()) != null) { + CellScanner cellScanner = result.cellScanner(); + Cell current = null; + while (cellScanner.advance()) { + current = cellScanner.current(); + System.out.println(current); + } + } + } + System.out.println("-----------------------------------------------"); + } + }
