[01/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.1 03c2bc2cd -> d6a82e29f


http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
index 089c5f1..4571115 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
@@ -95,8 +95,8 @@ public class ConnectionlessTest {
 "entity_history_id char(12) not null,\n" + 
 "created_by varchar,\n" + 
 "created_date date\n" +
-"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) " +
-(saltBuckets == null ? "" : (PhoenixDatabaseMetaData.SALT_BUCKETS + 
"=" + saltBuckets));
+"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) COLUMN_ENCODED_BYTES=4 " +
+(saltBuckets == null ? "" : " , " + 
(PhoenixDatabaseMetaData.SALT_BUCKETS + "=" + saltBuckets));
 Properties props = new Properties();
 Connection conn = DriverManager.getConnection(getUrl(), props);
 PreparedStatement statement = conn.prepareStatement(dmlStmt);
@@ -141,31 +141,29 @@ public class ConnectionlessTest {
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
-assertTrue(iterator.hasNext());
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
 private static void assertRow2(Iterator iterator, byte[] 
expectedRowKey2) {
 KeyValue kv;
-assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
 @Test

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
new file mode 100644
index 000..1052184
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static 

[08/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index fde403c..8595eda 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -368,6 +368,10 @@ public enum SQLExceptionCode {
 CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW(1134, "XCL34", "Altering this table 
property on a view is not allowed"),
 
 IMMUTABLE_TABLE_PROPERTY_INVALID(1135, "XCL35", "IMMUTABLE table property 
cannot be used with CREATE IMMUTABLE TABLE statement "),
+
+MAX_COLUMNS_EXCEEDED(1136, "XCL36", "The number of columns exceed the 
maximum supported by the table's qualifier encoding scheme"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES(1137, "XCL37", 
"If IMMUTABLE_STORAGE_SCHEME property is not set to ONE_CELL_PER_COLUMN 
COLUMN_ENCODED_BYTES cannot be 0"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE(1138, "XCL38", 
"IMMUTABLE_STORAGE_SCHEME property cannot be changed from/to 
ONE_CELL_PER_COLUMN "),
 
 /**
  * Implementation defined class. Phoenix internal error. (errorcode 20, 
sqlstate INT).

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 76dec2f..f6010ac 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -65,11 +65,13 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.trace.TracingIterator;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.SQLCloseable;
@@ -313,10 +315,6 @@ public abstract class BaseQueryPlan implements QueryPlan {
 // project is not present in the index then we need to skip this 
plan.
 if (!dataColumns.isEmpty()) {
 // Set data columns to be join back from data table.
-serializeDataTableColumnsToJoin(scan, dataColumns);
-KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
-// Set key value schema of the data columns.
-serializeSchemaIntoScan(scan, schema);
 PTable parentTable = context.getCurrentTable().getTable();
 String parentSchemaName = 
parentTable.getParentSchemaName().getString();
 String parentTableName = 
parentTable.getParentTableName().getString();
@@ -327,6 +325,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
 FACTORY.namedTable(null, 
TableName.create(parentSchemaName, parentTableName)),
 
context.getConnection()).resolveTable(parentSchemaName, parentTableName);
 PTable dataTable = dataTableRef.getTable();
+// Set data columns to be join back from data table.
+serializeDataTableColumnsToJoin(scan, dataColumns, dataTable);
+KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
+// Set key value schema of the data columns.
+serializeSchemaIntoScan(scan, schema);
+
 // Set index maintainer of the local index.
 serializeIndexMaintainerIntoScan(scan, dataTable);
 // Set view constants if exists.
@@ -373,7 +377,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
 }
 ImmutableBytesWritable ptr = new ImmutableBytesWritable();
 IndexMaintainer.serialize(dataTable, ptr, indexes, 
context.getConnection());
-scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
+scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
 if (dataTable.isTransactional()) {
 

[12/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 1399f6c..f37d09b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -26,337 +26,110 @@ import static org.junit.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
-import java.sql.Date;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.List;
 import java.util.Properties;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessor.PhoenixTransactionalProcessor;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.StringUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.apache.tephra.TxConstants;
-import org.junit.Ignore;
 import org.junit.Test;
 
-import com.google.common.collect.Lists;
+public class TransactionIT  extends ParallelStatsDisabledIT {
 
-public class TransactionIT extends ParallelStatsDisabledIT {
-
 @Test
-public void testReadOwnWrites() throws Exception {
-String transTableName = generateUniqueName();
-String fullTableName = INDEX_DATA_SCHEMA + 
QueryConstants.NAME_SEPARATOR + transTableName;
-String selectSql = "SELECT * FROM "+ fullTableName;
-try (Connection conn = DriverManager.getConnection(getUrl())) {
-TestUtil.createTransactionalTable(conn, fullTableName);
-conn.setAutoCommit(false);
-ResultSet rs = conn.createStatement().executeQuery(selectSql);
-assertFalse(rs.next());
-
-String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, 
char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
-PreparedStatement stmt = conn.prepareStatement(upsert);
-// upsert two rows
-TestUtil.setRowKeyColumns(stmt, 1);
-stmt.execute();
-TestUtil.setRowKeyColumns(stmt, 2);
-stmt.execute();
-
-// verify rows can be read even though commit has not been called
-rs = conn.createStatement().executeQuery(selectSql);
-TestUtil.validateRowKeyColumns(rs, 1);
-TestUtil.validateRowKeyColumns(rs, 2);
-assertFalse(rs.next());
-
-conn.commit();
-
-// verify rows can be read after commit
-rs = conn.createStatement().executeQuery(selectSql);
-TestUtil.validateRowKeyColumns(rs, 1);
-TestUtil.validateRowKeyColumns(rs, 2);
-assertFalse(rs.next());
-}
+public void testReCreateTxnTableAfterDroppingExistingNonTxnTable() throws 
SQLException {
+String tableName = generateUniqueName();
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+conn.setAutoCommit(false);
+Statement stmt = conn.createStatement();
+stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 
VARCHAR, v2 VARCHAR)");
+stmt.execute("DROP TABLE " + tableName);
+stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 
VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+stmt.execute("CREATE INDEX " + tableName + "_IDX ON " + tableName + " 
(v1) INCLUDE(v2)");
+assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new 
PTableKey(null, tableName)).isTransactional());
+assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new 
PTableKey(null,  tableName + "_IDX")).isTransactional());
 }
 
 @Test
-public void testTxnClosedCorrecty() throws Exception {
-String transTableName = 

[06/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index e499e29..9f944c7 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -70,7 +70,6 @@ import 
org.apache.phoenix.hbase.index.covered.update.IndexedColumnGroup;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.write.IndexWriter;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
@@ -304,8 +303,16 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 for (ColumnReference ref : mutableColumns) {
 scan.addColumn(ref.getFamily(), ref.getQualifier());
 }
+/*
+ * Indexes inherit the storage scheme of the data table which 
means all the indexes have the same
+ * storage scheme and empty key value qualifier. Note that 
this assumption would be broken if we start
+ * supporting new indexes over existing data tables to have a 
different storage scheme than the data
+ * table.
+ */
+byte[] emptyKeyValueQualifier = 
indexMaintainers.get(0).getEmptyKeyValueQualifier();
+
 // Project empty key value column
-
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
QueryConstants.EMPTY_COLUMN_BYTES);
+
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
emptyKeyValueQualifier);
 ScanRanges scanRanges = 
ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, 
Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, 
KeyRange.EVERYTHING_RANGE, null, true, -1);
 scanRanges.initializeScan(scan);
 TableName tableName = 
env.getRegion().getRegionInfo().getTable();
@@ -356,7 +363,8 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 Map 
mutationsToFindPreviousValue) throws IOException {
 if (scanner != null) {
 Result result;
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0)
+.getDataEmptyKeyValueCF(), 
indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 // Process existing data table rows by removing the old index row 
and adding the new index row
 while ((result = scanner.next()) != null) {
 Mutation m = mutationsToFindPreviousValue.remove(new 
ImmutableBytesPtr(result.getRow()));
@@ -384,7 +392,7 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 // to generate point delete markers for all index rows that were 
added. We don't have Tephra
 // manage index rows in change sets because we don't want to be 
hit with the additional
 // memory hit and do not need to do conflict detection on index 
rows.
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 while ((result = scanner.next()) != null) {
 Mutation m = mutations.remove(new 
ImmutableBytesPtr(result.getRow()));
 // Sort by timestamp, type, cf, cq so we can process in time 
batches from oldest to newest

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 51587f1..d563bc2 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ 

[15/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
index b9fa15b..7f64fee 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
@@ -21,7 +21,6 @@ import static org.apache.phoenix.util.TestUtil.A_VALUE;
 import static org.apache.phoenix.util.TestUtil.B_VALUE;
 import static org.apache.phoenix.util.TestUtil.C_VALUE;
 import static org.apache.phoenix.util.TestUtil.E_VALUE;
-import static org.apache.phoenix.util.TestUtil.ROW3;
 import static org.apache.phoenix.util.TestUtil.ROW5;
 import static org.apache.phoenix.util.TestUtil.ROW6;
 import static org.apache.phoenix.util.TestUtil.ROW7;
@@ -35,9 +34,7 @@ import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -50,15 +47,14 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 
 @RunWith(Parameterized.class)
 public class GroupByIT extends BaseQueryIT {
 
-public GroupByIT(String indexDDL) {
-super(indexDDL);
+public GroupByIT(String indexDDL, boolean mutable, boolean columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Parameters(name="GroupByIT_{index}") // name is used by failsafe as file 
name in reports
@@ -74,71 +70,9 @@ public class GroupByIT extends BaseQueryIT {
BaseQueryIT.doSetup(props);
 }
 
-@SuppressWarnings("unchecked")
-@Test
-public void testGroupByCondition() throws Exception {
-Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 20));
-Connection conn = DriverManager.getConnection(getUrl(), props);
-PreparedStatement statement = conn.prepareStatement("SELECT count(*) 
FROM aTable WHERE organization_id=? GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-ResultSet rs = statement.executeQuery();
-assertValueEqualsResultSet(rs, Arrays.asList(1L,8L));
-try {
-statement = conn.prepareStatement("SELECT count(*),a_integer=6 
FROM aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is 
null) GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-List expectedResults = Lists.newArrayList(
-Arrays.asList(1L,false),
-Arrays.asList(1L,true));
-assertValuesEqualsResultSet(rs, expectedResults);
-} finally {
-conn.close();
-}
-
-
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 40));
-conn = DriverManager.getConnection(getUrl(), props);
-try {
-statement = conn.prepareStatement("UPSERT into 
aTable(organization_id,entity_id,a_integer) values(?,?,null)");
-statement.setString(1, tenantId);
-statement.setString(2, ROW3);
-statement.executeUpdate();
-conn.commit();
-} finally {
-conn.close();
-}
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 60));
-conn = DriverManager.getConnection(getUrl(), props);
-statement = conn.prepareStatement("SELECT count(*) FROM aTable WHERE 
organization_id=? GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-assertValueEqualsResultSet(rs, Arrays.asList(1L,1L,7L));
-statement = conn.prepareStatement("SELECT a_integer, entity_id FROM 
aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is null)");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-List expectedResults = Lists.newArrayList(
-Arrays.asList(null,ROW3),
-Arrays.asList(5,ROW5),
-Arrays.asList(6,ROW6));
-assertValuesEqualsResultSet(rs, expectedResults);
-try {
-statement = conn.prepareStatement("SELECT count(*),a_integer=6 
FROM aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is 
null) GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-expectedResults = Lists.newArrayList(
-Arrays.asList(1L,null),

[14/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index d3bbe23..1084f14 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -62,15 +62,15 @@ import org.junit.Test;
  */
 public class QueryIT extends BaseQueryIT {
 
-public QueryIT(String indexDDL) {
-super(indexDDL);
+public QueryIT(String indexDDL, boolean mutable, boolean columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Test
 public void testIntFilter() throws Exception {
 String updateStmt = 
-"upsert into " +
-"ATABLE(" +
+"upsert into " + tableName +
+" (" +
 "ORGANIZATION_ID, " +
 "ENTITY_ID, " +
 "A_INTEGER) " +
@@ -89,10 +89,10 @@ public class QueryIT extends BaseQueryIT {
 url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 
6);
 props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 upsertConn = DriverManager.getConnection(url, props);
-analyzeTable(upsertConn, "ATABLE");
+analyzeTable(upsertConn, tableName);
 upsertConn.close();
 
-String query = "SELECT entity_id FROM aTable WHERE organization_id=? 
and a_integer >= ?";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer >= ?";
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
 PreparedStatement statement = conn.prepareStatement(query);
@@ -101,19 +101,19 @@ public class QueryIT extends BaseQueryIT {
 ResultSet rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW7, ROW8, 
ROW9));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer < 2";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer < 2";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW1, ROW4));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer <= 2";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer <= 2";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW1, ROW2, 
ROW4));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer >=9";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer >=9";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
@@ -124,13 +124,8 @@ public class QueryIT extends BaseQueryIT {
 }
 
 @Test
-public void testEmptyStringValue() throws Exception {
-testNoStringValue("");
-}
-
-@Test
 public void testToDateOnString() throws Exception { // TODO: test more 
conversion combinations
-String query = "SELECT a_string FROM aTable WHERE organization_id=? 
and a_integer = 5";
+String query = "SELECT a_string FROM " + tableName + " WHERE 
organization_id=? and a_integer = 5";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -151,7 +146,7 @@ public class QueryIT extends BaseQueryIT {
 
 @Test
 public void testColumnOnBothSides() throws Exception {
-String query = "SELECT entity_id FROM aTable WHERE organization_id=? 
and a_string = b_string";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_string = b_string";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -167,50 +162,9 @@ public class QueryIT extends BaseQueryIT {
 }
 }
 
-private void testNoStringValue(String value) throws Exception {
-String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 10);
-Properties props = 

[02/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
--
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java 
b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 4a9cb57..3e2c9b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -18,6 +18,8 @@
 package org.apache.phoenix.util;
 
 import static 
org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER;
 import static org.apache.phoenix.util.PhoenixRuntime.getTable;
 
 import java.io.ByteArrayInputStream;
@@ -28,6 +30,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 
 import org.apache.hadoop.hbase.Cell;
@@ -67,10 +70,12 @@ import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
 import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.UpdateIndexStateRequest;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState.RowMutationState;
 import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
 import org.apache.phoenix.expression.RowKeyColumnExpression;
+import org.apache.phoenix.expression.SingleCellColumnExpression;
 import org.apache.phoenix.expression.visitor.RowKeyExpressionVisitor;
 import org.apache.phoenix.hbase.index.ValueGetter;
 import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
@@ -93,9 +98,12 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnFamily;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.tuple.ResultTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PBinary;
@@ -190,6 +198,11 @@ public class IndexUtil {
 : QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX + 
dataColumnFamilyName;
 }
 
+public static byte[] getLocalIndexColumnFamily(byte[] 
dataColumnFamilyBytes) {
+String dataCF = Bytes.toString(dataColumnFamilyBytes);
+return getLocalIndexColumnFamily(dataCF).getBytes();
+}
+
 public static PColumn getDataColumn(PTable dataTable, String 
indexColumnName) {
 int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP);
 if (pos < 0) {
@@ -209,7 +222,7 @@ public class IndexUtil {
 throw new IllegalArgumentException("Could not find column family 
\"" +  indexColumnName.substring(0, pos) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
 try {
-return family.getColumn(indexColumnName.substring(pos+1));
+return 
family.getPColumnForColumnName(indexColumnName.substring(pos+1));
 } catch (ColumnNotFoundException e) {
 throw new IllegalArgumentException("Could not find column \"" +  
indexColumnName.substring(pos+1) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
@@ -236,10 +249,11 @@ public class IndexUtil {
 
 private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) {
 byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table);
+byte[] emptyKeyValueQualifier = 
EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
 return (Bytes.compareTo(emptyKeyValueCF, 0, emptyKeyValueCF.length, 
ref.getFamilyWritable()
 .get(), ref.getFamilyWritable().getOffset(), 
ref.getFamilyWritable().getLength()) == 0 && Bytes
-.compareTo(QueryConstants.EMPTY_COLUMN_BYTES, 0,
-QueryConstants.EMPTY_COLUMN_BYTES.length, 
ref.getQualifierWritable().get(), ref
+.compareTo(emptyKeyValueQualifier, 0,
+emptyKeyValueQualifier.length, 
ref.getQualifierWritable().get(), ref
 .getQualifierWritable().getOffset(), 
ref.getQualifierWritable()
 .getLength()) == 0);
 }
@@ -271,10 +285,10 @@ public class IndexUtil {
 }
 
 public static List generateIndexData(final PTable table, 

[11/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 4d3c0cf..32e9f68 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 package org.apache.phoenix.compile;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -24,7 +25,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.parse.AliasedNode;
 import org.apache.phoenix.parse.ColumnParseNode;
 import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -43,11 +43,13 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.ProjectedColumn;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -120,7 +122,7 @@ public class TupleProjectionCompiler {
 PColumn sourceColumn = table.getPKColumns().get(i);
 ColumnRef sourceColumnRef = new ColumnRef(tableRef, 
sourceColumn.getPosition());
 PColumn column = new ProjectedColumn(sourceColumn.getName(), 
sourceColumn.getFamilyName(), 
-position++, sourceColumn.isNullable(), sourceColumnRef);
+position++, sourceColumn.isNullable(), sourceColumnRef, 
null);
 projectedColumns.add(column);
 }
 for (PColumn sourceColumn : table.getColumns()) {
@@ -132,18 +134,18 @@ public class TupleProjectionCompiler {
 && 
!families.contains(sourceColumn.getFamilyName().getString()))
 continue;
 PColumn column = new ProjectedColumn(sourceColumn.getName(), 
sourceColumn.getFamilyName(), 
-position++, sourceColumn.isNullable(), sourceColumnRef);
+position++, sourceColumn.isNullable(), sourceColumnRef, 
sourceColumn.getColumnQualifierBytes());
 projectedColumns.add(column);
 // Wildcard or FamilyWildcard will be handled by 
ProjectionCompiler.
 if (!isWildcard && 
!families.contains(sourceColumn.getFamilyName())) {
-
context.getScan().addColumn(sourceColumn.getFamilyName().getBytes(), 
sourceColumn.getName().getBytes());
+   EncodedColumnsUtil.setColumns(column, table, context.getScan());
 }
 }
 // add LocalIndexDataColumnRef
 for (LocalIndexDataColumnRef sourceColumnRef : 
visitor.localIndexColumnRefSet) {
 PColumn column = new 
ProjectedColumn(sourceColumnRef.getColumn().getName(), 
 sourceColumnRef.getColumn().getFamilyName(), position++, 
-sourceColumnRef.getColumn().isNullable(), sourceColumnRef);
+sourceColumnRef.getColumn().isNullable(), sourceColumnRef, 
sourceColumnRef.getColumn().getColumnQualifierBytes());
 projectedColumns.add(column);
 }
 
@@ -154,9 +156,9 @@ public class TupleProjectionCompiler {
 null, null, table.isWALDisabled(), table.isMultiTenant(), 
table.getStoreNulls(), table.getViewType(),
 table.getViewIndexId(),
 table.getIndexType(), table.rowKeyOrderOptimizable(), 
table.isTransactional(), table.getUpdateCacheFrequency(), 
-table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
table.getAutoPartitionSeqName(), table.isAppendOnlySchema());
+table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), 
table.getImmutableStorageScheme(), table.getEncodingScheme(), 
table.getEncodedCQCounter());
 }
-
+
 public static PTable createProjectedTable(TableRef tableRef, 
List sourceColumnRefs, boolean retainPKColumns) throws SQLException {
 PTable table = tableRef.getTable();
 boolean hasSaltingColumn = retainPKColumns && table.getBucketNum() != 
null;
@@ -169,20 +171,23 @@ public class TupleProjectionCompiler {
 String aliasedName = 

[16/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 211145e..6b57148 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -54,22 +54,41 @@ import org.junit.runners.Parameterized.Parameters;
 public class AlterTableWithViewsIT extends ParallelStatsDisabledIT {
 
 private final boolean isMultiTenant;
+private final boolean columnEncoded;
 
 private final String TENANT_SPECIFIC_URL1 = getUrl() + ';' + 
TENANT_ID_ATTRIB + "=tenant1";
 private final String TENANT_SPECIFIC_URL2 = getUrl() + ';' + 
TENANT_ID_ATTRIB + "=tenant2";
 
-public AlterTableWithViewsIT(boolean isMultiTenant) {
+public AlterTableWithViewsIT(boolean isMultiTenant, boolean columnEncoded) 
{
 this.isMultiTenant = isMultiTenant;
+this.columnEncoded = columnEncoded;
 }
 
-@Parameters(name="AlterTableWithViewsIT_multiTenant={0}") // name is used 
by failsafe as file name in reports
-public static Collection data() {
-return Arrays.asList(false, true);
+@Parameters(name="AlterTableWithViewsIT_multiTenant={0}, 
columnEncoded={1}") // name is used by failsafe as file name in reports
+public static Collection data() {
+return Arrays.asList(new Boolean[][] { 
+{ false, false }, { false, true },
+{ true, false }, { true, true } });
 }
-   
+
 private String generateDDL(String format) {
+return generateDDL("", format);
+}
+
+private String generateDDL(String options, String format) {
+StringBuilder optionsBuilder = new StringBuilder(options);
+if (!columnEncoded) {
+if (optionsBuilder.length()!=0)
+optionsBuilder.append(",");
+optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+}
+if (isMultiTenant) {
+if (optionsBuilder.length()!=0)
+optionsBuilder.append(",");
+optionsBuilder.append("MULTI_TENANT=true");
+}
 return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT 
NULL, " : "",
-isMultiTenant ? "TENANT_ID, " : "", isMultiTenant ? 
"MULTI_TENANT=true" : "");
+isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
 }
 
 @Test
@@ -92,7 +111,7 @@ public class AlterTableWithViewsIT extends 
ParallelStatsDisabledIT {
 
 // adding a new pk column and a new regular column
 conn.createStatement().execute("ALTER TABLE " + tableName + " ADD 
COL3 varchar(10) PRIMARY KEY, COL4 integer");
-assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 
5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "COL3", 
"COL4");
+assertTableDefinition(conn, tableName, PTableType.TABLE, null, 
columnEncoded ? 2 : 1, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", 
"COL1", "COL2", "COL3", "COL4");
 assertTableDefinition(conn, viewOfTable, PTableType.VIEW, 
tableName, 1, 7, 5, "ID", "COL1", "COL2", "COL3", "COL4", "VIEW_COL1", 
"VIEW_COL2");
 } 
 }
@@ -109,28 +128,27 @@ public class AlterTableWithViewsIT extends 
ParallelStatsDisabledIT {
 + " COL1 integer NOT NULL,"
 + " COL2 bigint NOT NULL,"
 + " CONSTRAINT NAME_PK PRIMARY KEY (%s ID, COL1, 
COL2)"
-+ " ) UPDATE_CACHE_FREQUENCY=15 "
-+ (isMultiTenant ? ",%s" : "%s");
-conn.createStatement().execute(generateDDL(ddlFormat));
++ " ) %s ";
+
conn.createStatement().execute(generateDDL("UPDATE_CACHE_FREQUENCY=2", 
ddlFormat));
 viewConn.createStatement().execute("CREATE VIEW " + viewOfTable1 + 
" ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 VARCHAR ) AS SELECT * FROM " + 
tableName);
 viewConn.createStatement().execute("CREATE VIEW " + viewOfTable2 + 
" ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 VARCHAR ) AS SELECT * FROM " + 
tableName);
-viewConn.createStatement().execute("ALTER VIEW " + viewOfTable2 + 
" SET UPDATE_CACHE_FREQUENCY = 5");
+viewConn.createStatement().execute("ALTER VIEW " + viewOfTable2 + 
" SET UPDATE_CACHE_FREQUENCY = 1");
 
 PhoenixConnection phoenixConn = 
conn.unwrap(PhoenixConnection.class);
 PTable table = phoenixConn.getTable(new PTableKey(null, 
tableName));
 PName tenantId = isMultiTenant ? 

[03/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
new file mode 100644
index 000..5a5b355
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema.tuple;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME;
+
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+
+/**
+ * List implementation that provides indexed based look up when the cell 
column qualifiers are positive numbers. 
+ * These qualifiers are generated by using one of the column qualifier 
encoding schemes specified in {@link ImmutableStorageScheme}. 
+ * The api methods in this list assume that the caller wants to see
+ * and add only non null elements in the list. 
+ * 
+ * Please note that this implementation doesn't implement all the optional 
methods of the 
+ * {@link List} interface. Such unsupported methods could violate the basic 
invariance of the list that every cell with
+ * an encoded column qualifier has a fixed position in the list.
+ * 
+ * 
+ * An important performance characteristic of this list is that doing look up 
on the basis of index via {@link #get(int)}
+ * is an O(n) operation. This makes iterating through the list using {@link 
#get(int)} an O(n^2) operation.
+ * Instead, for iterating through the list, one should use the iterators 
created through {@link #iterator()} or 
+ * {@link #listIterator()}. Do note that getting an element using {@link 
#getCellForColumnQualifier(int)} is an O(1) operation
+ * and should generally be the way for accessing elements in the list.
+ *  
+ */
+@NotThreadSafe
+public class EncodedColumnQualiferCellsList implements List {
+
+private int minQualifier;
+private int maxQualifier;
+private int nonReservedRangeOffset;
+private final Cell[] array;
+private int numNonNullElements;
+private int firstNonNullElementIdx = -1;
+private static final int RESERVED_RANGE_SIZE = 
ENCODED_CQ_COUNTER_INITIAL_VALUE - ENCODED_EMPTY_COLUMN_NAME;
+// Used by iterators to figure out if the list was structurally modified.
+private int modCount = 0;
+private final QualifierEncodingScheme encodingScheme;
+
+public EncodedColumnQualiferCellsList(int minQ, int maxQ, 
QualifierEncodingScheme encodingScheme) {
+checkArgument(minQ <= maxQ, "Invalid arguments. Min: " + minQ
++ ". Max: " + maxQ);
+this.minQualifier = minQ;
+this.maxQualifier = maxQ;
+int size = 0;
+if (maxQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = RESERVED_RANGE_SIZE;
+} else if (minQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = (maxQ - minQ + 1);
+} else {
+size = RESERVED_RANGE_SIZE + (maxQ - minQ + 1);
+}
+this.array = new Cell[size];
+this.nonReservedRangeOffset = minQ > ENCODED_CQ_COUNTER_INITIAL_VALUE 
? minQ  - ENCODED_CQ_COUNTER_INITIAL_VALUE : 0;
+this.encodingScheme = encodingScheme;
+}
+
+@Override
+public int size() {
+return numNonNullElements;
+}
+
+@Override
+public boolean isEmpty() {
+return numNonNullElements == 0;
+}
+
+@Override
+public 

[17/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

PHOENIX-1598 Encode column names to save space and improve performance(Samarth 
Jain and Thomas D'Silva)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d6a82e29
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d6a82e29
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d6a82e29

Branch: refs/heads/4.x-HBase-1.1
Commit: d6a82e29fce471bd47c42802ab4ac9548acf60ae
Parents: 03c2bc2
Author: Samarth 
Authored: Sun Feb 26 22:22:03 2017 -0800
Committer: Samarth 
Committed: Sun Feb 26 22:22:03 2017 -0800

--
 .../phoenix/end2end/AggregateQueryIT.java   |   74 +-
 .../AlterMultiTenantTableWithViewsIT.java   |   25 +-
 .../apache/phoenix/end2end/AlterTableIT.java|  491 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |  133 +-
 .../org/apache/phoenix/end2end/ArrayIT.java |   28 +
 .../org/apache/phoenix/end2end/BaseJoinIT.java  |4 +-
 .../org/apache/phoenix/end2end/BaseQueryIT.java |   99 +-
 .../apache/phoenix/end2end/CaseStatementIT.java |   28 +-
 .../apache/phoenix/end2end/CastAndCoerceIT.java |   34 +-
 .../end2end/ClientTimeArithmeticQueryIT.java|   76 +-
 .../end2end/ColumnEncodedBytesPropIT.java   |   95 +
 .../end2end/CountDistinctCompressionIT.java |2 +-
 .../apache/phoenix/end2end/CreateTableIT.java   |   57 +
 .../org/apache/phoenix/end2end/DateTimeIT.java  |2 +-
 .../phoenix/end2end/DefaultColumnValueIT.java   |1 +
 .../apache/phoenix/end2end/DerivedTableIT.java  |2 +-
 .../apache/phoenix/end2end/DistinctCountIT.java |4 +-
 .../apache/phoenix/end2end/DynamicColumnIT.java |   63 +
 .../phoenix/end2end/ExtendedQueryExecIT.java|8 +-
 .../apache/phoenix/end2end/FunkyNamesIT.java|2 +-
 .../org/apache/phoenix/end2end/GroupByIT.java   |  162 +-
 .../phoenix/end2end/ImmutableTablePropIT.java   |  130 -
 .../end2end/ImmutableTablePropertiesIT.java |  189 +
 .../apache/phoenix/end2end/MutableQueryIT.java  |  424 ++
 .../phoenix/end2end/NativeHBaseTypesIT.java |2 +-
 .../org/apache/phoenix/end2end/NotQueryIT.java  |   28 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |2 -
 .../apache/phoenix/end2end/PercentileIT.java|4 +-
 .../phoenix/end2end/PhoenixRuntimeIT.java   |4 +-
 .../phoenix/end2end/PointInTimeQueryIT.java |   78 +-
 .../phoenix/end2end/ProductMetricsIT.java   |2 +-
 .../end2end/QueryDatabaseMetaDataIT.java|   16 +-
 .../org/apache/phoenix/end2end/QueryIT.java |  112 +-
 .../phoenix/end2end/ReadIsolationLevelIT.java   |2 +-
 .../phoenix/end2end/RowValueConstructorIT.java  |   36 +-
 .../org/apache/phoenix/end2end/ScanQueryIT.java |   93 +-
 .../phoenix/end2end/StatsCollectorIT.java   |  124 +-
 .../apache/phoenix/end2end/StoreNullsIT.java|  310 +-
 .../phoenix/end2end/StoreNullsPropIT.java   |   51 +
 ...SysTableNamespaceMappedStatsCollectorIT.java |4 +-
 .../java/org/apache/phoenix/end2end/TopNIT.java |6 +-
 .../apache/phoenix/end2end/UpsertSelectIT.java  |   10 +-
 .../apache/phoenix/end2end/UpsertValuesIT.java  |   51 +-
 .../phoenix/end2end/UserDefinedFunctionsIT.java |3 +-
 .../phoenix/end2end/VariableLengthPKIT.java |   38 +-
 .../phoenix/end2end/index/DropMetadataIT.java   |   13 +-
 .../phoenix/end2end/index/ImmutableIndexIT.java |   20 +-
 .../end2end/index/IndexExpressionIT.java|   28 +-
 .../apache/phoenix/end2end/index/IndexIT.java   |   58 +-
 .../phoenix/end2end/index/IndexTestUtil.java|   11 +-
 .../end2end/index/MutableIndexFailureIT.java|2 +
 .../phoenix/end2end/index/MutableIndexIT.java   |   21 +-
 .../phoenix/end2end/salted/SaltedTableIT.java   |2 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |  518 ++
 .../org/apache/phoenix/tx/TransactionIT.java|  589 +-
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |   42 +-
 .../apache/phoenix/cache/ServerCacheClient.java |2 +
 .../org/apache/phoenix/cache/TenantCache.java   |2 +-
 .../apache/phoenix/cache/TenantCacheImpl.java   |4 +-
 .../phoenix/compile/CreateTableCompiler.java|   10 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |2 +-
 .../phoenix/compile/ExpressionCompiler.java |   18 +-
 .../apache/phoenix/compile/FromCompiler.java|   54 +-
 .../apache/phoenix/compile/JoinCompiler.java|   15 +-
 .../phoenix/compile/ListJarsQueryPlan.java  |6 +-
 .../apache/phoenix/compile/PostDDLCompiler.java |   11 +-
 .../compile/PostLocalIndexDDLCompiler.java  |9 +-
 .../phoenix/compile/ProjectionCompiler.java |   35 +-
 .../apache/phoenix/compile/TraceQueryPlan.java  |4 +-
 .../compile/TupleProjectionCompiler.java|   31 +-
 .../apache/phoenix/compile/UnionCompiler.java   |7 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |6 +-
 

[07/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
new file mode 100644
index 000..00e662f
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.NoSuchElementException;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.KeyValueColumnExpression;
+import org.apache.phoenix.expression.visitor.ExpressionVisitor;
+import 
org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import org.apache.phoenix.schema.tuple.BaseTuple;
+
+/**
+ * Filter used for tables that use number based column qualifiers generated by 
one of the encoding schemes in
+ * {@link QualifierEncodingScheme}. Because the qualifiers are number based, 
instead of using a map of cells to track
+ * the columns that have been found, we can use an array of cells where the 
index into the array would be derived by the
+ * number based column qualifier. See {@link EncodedCQIncrementalResultTuple}. 
Using this filter helps us to directly
+ * seek to the next row when the column qualifier that we have encountered is 
greater than the maxQualifier that we
+ * expect. This helps in speeding up the queries filtering on key value 
columns.
+ */
+public class MultiEncodedCQKeyValueComparisonFilter extends 
BooleanExpressionFilter {
+// Smallest qualifier for the columns that are being projected and 
filtered on
+private int minQualifier;
+
+// Largest qualifier for the columns that are being projected and filtered 
on
+private int maxQualifier;
+
+private QualifierEncodingScheme encodingScheme;
+
+// Smallest qualifier for the columns in where expression
+private int whereExpressionMinQualifier;
+
+// Largest qualifier for the columns in where expression
+private int whereExpressionMaxQualifier;
+
+private FilteredKeyValueHolder filteredKeyValues;
+
+// BitSet to track the qualifiers in where expression that we expect to 
find while filtering a row
+private BitSet whereExpressionQualifiers;
+
+// Set to track the column families of the columns in where expression
+private TreeSet cfSet;
+
+// Boolean that tells us whether the result of expression evaluation as 
and when we filter key values in a row
+private Boolean matchedColumn;
+
+// Tuple used to store the relevant key values found while filtering a row
+private EncodedCQIncrementalResultTuple inputTuple = new 
EncodedCQIncrementalResultTuple();
+
+// Member variable to cache the size of whereExpressionQualifiers
+private int expectedCardinality;
+
+private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
+
+public MultiEncodedCQKeyValueComparisonFilter() {}
+
+public MultiEncodedCQKeyValueComparisonFilter(Expression expression, 
QualifierEncodingScheme scheme) {
+super(expression);
+checkArgument(scheme != NON_ENCODED_QUALIFIERS, "Filter can only be 
used for encoded 

[10/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
index b201c8e..139a69c 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
@@ -36,7 +36,7 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
  */
 public interface ServerCachingProtocol {
 public static interface ServerCacheFactory extends Writable {
-public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] 
txState, MemoryChunk chunk) throws SQLException;
+public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] 
txState, MemoryChunk chunk, boolean useProtoForIndexMaintainer) throws 
SQLException;
 }
 /**
  * Add the cache to the region server cache.  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index db3c792..c5854d3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -101,7 +101,10 @@ import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
 import org.apache.phoenix.schema.stats.StatisticsCollector;
 import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
+import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList;
 import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
+import org.apache.phoenix.schema.tuple.PositionBasedMultiKeyValueTuple;
+import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PBinary;
 import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
@@ -109,6 +112,7 @@ import org.apache.phoenix.schema.types.PDouble;
 import org.apache.phoenix.schema.types.PFloat;
 import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.KeyValueUtil;
 import org.apache.phoenix.util.LogUtil;
@@ -181,9 +185,9 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 
 private void commitBatch(Region region, List mutations, byte[] 
indexUUID, long blockingMemstoreSize,
-byte[] indexMaintainersPtr, byte[] txState) throws IOException {
+byte[] indexMaintainersPtr, byte[] txState, boolean useIndexProto) 
throws IOException {
 if (indexMaintainersPtr != null) {
-mutations.get(0).setAttribute(PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
+mutations.get(0).setAttribute(useIndexProto ? 
PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
 }
 
 if (txState != null) {
@@ -213,13 +217,13 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 
 private void commitBatchWithHTable(HTable table, Region region, 
List mutations, byte[] indexUUID,
-long blockingMemstoreSize, byte[] indexMaintainersPtr, byte[] 
txState) throws IOException {
+long blockingMemstoreSize, byte[] indexMaintainersPtr, byte[] 
txState, boolean useIndexProto) throws IOException {
 
 if (indexUUID != null) {
 // Need to add indexMaintainers for each mutation as table.batch 
can be distributed across servers
 for (Mutation m : mutations) {
 if (indexMaintainersPtr != null) {
-m.setAttribute(PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
+m.setAttribute(useIndexProto ? 
PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
 }
 if (txState != null) {
 m.setAttribute(BaseScannerRegionObserver.TX_STATE, 
txState);
@@ -328,8 +332,13 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 values = new byte[projectedTable.getPKColumns().size()][];
 }
-byte[] localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
-List indexMaintainers 

[13/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
index f5905ee..d36e0fe 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
@@ -89,7 +89,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT {
 private void testUpsertSelect(boolean createIndex) throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 
 ensureTableCreated(getUrl(), CUSTOM_ENTITY_DATA_FULL_NAME, 
CUSTOM_ENTITY_DATA_FULL_NAME, ts-1);
 String indexName = "IDX1";
@@ -210,7 +210,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 public void testUpsertSelectEmptyPKColumn() throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1)); // Execute at timestamp 1
@@ -386,7 +386,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 private void testUpsertSelectForAgg(boolean autoCommit) throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1)); // Execute at timestamp 1
@@ -462,7 +462,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), 
PInteger.INSTANCE.toBytes(2),
 PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)};
 long ts = nextTimestamp();
-ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2);
+ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2, 
null);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1));
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -602,7 +602,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), 
PInteger.INSTANCE.toBytes(2),
 PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)};
 long ts = nextTimestamp();
-ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2);
+ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2, 
null);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1));
 Connection conn = DriverManager.getConnection(getUrl(), props);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
index 64935d2..11df167 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
@@ -35,15 +35,26 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Time;
 import java.sql.Timestamp;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
 import java.util.Properties;
 
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.compile.QueryPlan;
 import 

[05/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6a82e29/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index cc2b5b9..c8179e8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -35,6 +35,8 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
@@ -42,9 +44,11 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
@@ -85,9 +89,14 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 import static 
org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
 import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_UPDATE_STATS_ASYNC;
+import static org.apache.phoenix.schema.PTable.EncodedCQCounter.NULL_COUNTER;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
 import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
 import static org.apache.phoenix.schema.PTableType.TABLE;
 import static org.apache.phoenix.schema.PTableType.VIEW;
@@ -187,8 +196,12 @@ import 
org.apache.phoenix.query.ConnectionQueryServices.Feature;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.LinkType;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.QualifierOutOfRangeException;
 import org.apache.phoenix.schema.PTable.ViewType;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
 import org.apache.phoenix.schema.types.PDataType;
@@ -200,6 +213,7 @@ import org.apache.phoenix.schema.types.PUnsignedLong;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.MetaDataUtil;
@@ -215,6 +229,7 @@ import org.apache.tephra.TxConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
@@ -259,8 +274,10 @@ 

[13/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
index f5905ee..d36e0fe 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java
@@ -89,7 +89,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT {
 private void testUpsertSelect(boolean createIndex) throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 
 ensureTableCreated(getUrl(), CUSTOM_ENTITY_DATA_FULL_NAME, 
CUSTOM_ENTITY_DATA_FULL_NAME, ts-1);
 String indexName = "IDX1";
@@ -210,7 +210,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 public void testUpsertSelectEmptyPKColumn() throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1)); // Execute at timestamp 1
@@ -386,7 +386,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 private void testUpsertSelectForAgg(boolean autoCommit) throws Exception {
 long ts = nextTimestamp();
 String tenantId = getOrganizationId();
-initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl());
+initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), 
null, ts-1, getUrl(), null);
 ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1)); // Execute at timestamp 1
@@ -462,7 +462,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), 
PInteger.INSTANCE.toBytes(2),
 PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)};
 long ts = nextTimestamp();
-ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2);
+ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2, 
null);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1));
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -602,7 +602,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT 
{
 byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), 
PInteger.INSTANCE.toBytes(2),
 PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)};
 long ts = nextTimestamp();
-ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2);
+ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2, 
null);
 Properties props = new Properties();
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 1));
 Connection conn = DriverManager.getConnection(getUrl(), props);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
index 64935d2..11df167 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java
@@ -35,15 +35,26 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.sql.Time;
 import java.sql.Timestamp;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
 import java.util.Properties;
 
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.compile.QueryPlan;
 import 

[03/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
new file mode 100644
index 000..5a5b355
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema.tuple;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME;
+
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+
+/**
+ * List implementation that provides indexed based look up when the cell 
column qualifiers are positive numbers. 
+ * These qualifiers are generated by using one of the column qualifier 
encoding schemes specified in {@link ImmutableStorageScheme}. 
+ * The api methods in this list assume that the caller wants to see
+ * and add only non null elements in the list. 
+ * 
+ * Please note that this implementation doesn't implement all the optional 
methods of the 
+ * {@link List} interface. Such unsupported methods could violate the basic 
invariance of the list that every cell with
+ * an encoded column qualifier has a fixed position in the list.
+ * 
+ * 
+ * An important performance characteristic of this list is that doing look up 
on the basis of index via {@link #get(int)}
+ * is an O(n) operation. This makes iterating through the list using {@link 
#get(int)} an O(n^2) operation.
+ * Instead, for iterating through the list, one should use the iterators 
created through {@link #iterator()} or 
+ * {@link #listIterator()}. Do note that getting an element using {@link 
#getCellForColumnQualifier(int)} is an O(1) operation
+ * and should generally be the way for accessing elements in the list.
+ *  
+ */
+@NotThreadSafe
+public class EncodedColumnQualiferCellsList implements List {
+
+private int minQualifier;
+private int maxQualifier;
+private int nonReservedRangeOffset;
+private final Cell[] array;
+private int numNonNullElements;
+private int firstNonNullElementIdx = -1;
+private static final int RESERVED_RANGE_SIZE = 
ENCODED_CQ_COUNTER_INITIAL_VALUE - ENCODED_EMPTY_COLUMN_NAME;
+// Used by iterators to figure out if the list was structurally modified.
+private int modCount = 0;
+private final QualifierEncodingScheme encodingScheme;
+
+public EncodedColumnQualiferCellsList(int minQ, int maxQ, 
QualifierEncodingScheme encodingScheme) {
+checkArgument(minQ <= maxQ, "Invalid arguments. Min: " + minQ
++ ". Max: " + maxQ);
+this.minQualifier = minQ;
+this.maxQualifier = maxQ;
+int size = 0;
+if (maxQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = RESERVED_RANGE_SIZE;
+} else if (minQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = (maxQ - minQ + 1);
+} else {
+size = RESERVED_RANGE_SIZE + (maxQ - minQ + 1);
+}
+this.array = new Cell[size];
+this.nonReservedRangeOffset = minQ > ENCODED_CQ_COUNTER_INITIAL_VALUE 
? minQ  - ENCODED_CQ_COUNTER_INITIAL_VALUE : 0;
+this.encodingScheme = encodingScheme;
+}
+
+@Override
+public int size() {
+return numNonNullElements;
+}
+
+@Override
+public boolean isEmpty() {
+return numNonNullElements == 0;
+}
+
+@Override
+public 

[08/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index fde403c..8595eda 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -368,6 +368,10 @@ public enum SQLExceptionCode {
 CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW(1134, "XCL34", "Altering this table 
property on a view is not allowed"),
 
 IMMUTABLE_TABLE_PROPERTY_INVALID(1135, "XCL35", "IMMUTABLE table property 
cannot be used with CREATE IMMUTABLE TABLE statement "),
+
+MAX_COLUMNS_EXCEEDED(1136, "XCL36", "The number of columns exceed the 
maximum supported by the table's qualifier encoding scheme"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES(1137, "XCL37", 
"If IMMUTABLE_STORAGE_SCHEME property is not set to ONE_CELL_PER_COLUMN 
COLUMN_ENCODED_BYTES cannot be 0"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE(1138, "XCL38", 
"IMMUTABLE_STORAGE_SCHEME property cannot be changed from/to 
ONE_CELL_PER_COLUMN "),
 
 /**
  * Implementation defined class. Phoenix internal error. (errorcode 20, 
sqlstate INT).

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 76dec2f..f6010ac 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -65,11 +65,13 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.trace.TracingIterator;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.SQLCloseable;
@@ -313,10 +315,6 @@ public abstract class BaseQueryPlan implements QueryPlan {
 // project is not present in the index then we need to skip this 
plan.
 if (!dataColumns.isEmpty()) {
 // Set data columns to be join back from data table.
-serializeDataTableColumnsToJoin(scan, dataColumns);
-KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
-// Set key value schema of the data columns.
-serializeSchemaIntoScan(scan, schema);
 PTable parentTable = context.getCurrentTable().getTable();
 String parentSchemaName = 
parentTable.getParentSchemaName().getString();
 String parentTableName = 
parentTable.getParentTableName().getString();
@@ -327,6 +325,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
 FACTORY.namedTable(null, 
TableName.create(parentSchemaName, parentTableName)),
 
context.getConnection()).resolveTable(parentSchemaName, parentTableName);
 PTable dataTable = dataTableRef.getTable();
+// Set data columns to be join back from data table.
+serializeDataTableColumnsToJoin(scan, dataColumns, dataTable);
+KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
+// Set key value schema of the data columns.
+serializeSchemaIntoScan(scan, schema);
+
 // Set index maintainer of the local index.
 serializeIndexMaintainerIntoScan(scan, dataTable);
 // Set view constants if exists.
@@ -373,7 +377,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
 }
 ImmutableBytesWritable ptr = new ImmutableBytesWritable();
 IndexMaintainer.serialize(dataTable, ptr, indexes, 
context.getConnection());
-scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
+scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
 if (dataTable.isTransactional()) {
 

[09/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
index 5ee1dfb..3b8984a 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
@@ -649,6 +649,4947 @@ public final class ServerCachingProtos {
 // @@protoc_insertion_point(class_scope:ImmutableBytesWritable)
   }
 
+  public interface ColumnReferenceOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// required bytes family = 1;
+/**
+ * required bytes family = 1;
+ */
+boolean hasFamily();
+/**
+ * required bytes family = 1;
+ */
+com.google.protobuf.ByteString getFamily();
+
+// required bytes qualifier = 2;
+/**
+ * required bytes qualifier = 2;
+ */
+boolean hasQualifier();
+/**
+ * required bytes qualifier = 2;
+ */
+com.google.protobuf.ByteString getQualifier();
+  }
+  /**
+   * Protobuf type {@code ColumnReference}
+   */
+  public static final class ColumnReference extends
+  com.google.protobuf.GeneratedMessage
+  implements ColumnReferenceOrBuilder {
+// Use ColumnReference.newBuilder() to construct.
+private ColumnReference(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private ColumnReference(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final ColumnReference defaultInstance;
+public static ColumnReference getDefaultInstance() {
+  return defaultInstance;
+}
+
+public ColumnReference getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private ColumnReference(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 10: {
+  bitField0_ |= 0x0001;
+  family_ = input.readBytes();
+  break;
+}
+case 18: {
+  bitField0_ |= 0x0002;
+  qualifier_ = input.readBytes();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.internal_static_ColumnReference_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.internal_static_ColumnReference_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ColumnReference.class,
 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ColumnReference.Builder.class);
+}
+
+public static com.google.protobuf.Parser PARSER =
+new com.google.protobuf.AbstractParser() {
+  public ColumnReference parsePartialFrom(
+  com.google.protobuf.CodedInputStream input,
+  com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+  throws 

[04/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
--
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 01e8afe..91a41a3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -17,13 +17,32 @@
  */
 package org.apache.phoenix.schema;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
+import static 
org.apache.phoenix.util.EncodedColumnsUtil.isReservedColumnQualifier;
+
+import java.io.DataOutputStream;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
 
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PArrayDataTypeDecoder;
+import org.apache.phoenix.schema.types.PArrayDataTypeEncoder;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.util.TrustedByteArrayOutputStream;
+
+import com.google.common.annotations.VisibleForTesting;
 
 
 /**
@@ -129,7 +148,7 @@ public interface PTable extends PMetaDataEntity {
  * Link from a view to its parent table
  */
 PARENT_TABLE((byte)3);
-
+
 private final byte[] byteValue;
 private final byte serializedValue;
 
@@ -153,6 +172,318 @@ public interface PTable extends PMetaDataEntity {
 return LinkType.values()[serializedValue-1];
 }
 }
+
+public enum ImmutableStorageScheme implements 
ColumnValueEncoderDecoderSupplier {
+ONE_CELL_PER_COLUMN((byte)1) {
+@Override
+public ColumnValueEncoder getEncoder(int numElements) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public ColumnValueDecoder getDecoder() {
+throw new UnsupportedOperationException();
+}
+},
+// stores a single cell per column family that contains all serialized 
column values
+SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2) {
+@Override
+public ColumnValueEncoder getEncoder(int numElements) {
+PDataType type = PVarbinary.INSTANCE;
+int estimatedSize = PArrayDataType.estimateSize(numElements, 
type);
+TrustedByteArrayOutputStream byteStream = new 
TrustedByteArrayOutputStream(estimatedSize);
+DataOutputStream oStream = new DataOutputStream(byteStream);
+return new PArrayDataTypeEncoder(byteStream, oStream, 
numElements, type, SortOrder.ASC, false, 
PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+}
+
+@Override
+public ColumnValueDecoder getDecoder() {
+return new PArrayDataTypeDecoder();
+}
+};
+
+private final byte serializedValue;
+
+private ImmutableStorageScheme(byte serializedValue) {
+this.serializedValue = serializedValue;
+}
+
+public byte getSerializedMetadataValue() {
+return this.serializedValue;
+}
+
+public static ImmutableStorageScheme fromSerializedValue(byte 
serializedValue) {
+if (serializedValue < 1 || serializedValue > 
ImmutableStorageScheme.values().length) {
+return null;
+}
+return ImmutableStorageScheme.values()[serializedValue-1];
+}
+
+}
+
+interface ColumnValueEncoderDecoderSupplier {
+ColumnValueEncoder getEncoder(int numElements);
+ColumnValueDecoder getDecoder();
+}
+
+public enum QualifierEncodingScheme implements QualifierEncoderDecoder {
+NON_ENCODED_QUALIFIERS((byte)0, null) {
+@Override
+public byte[] encode(int value) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public int decode(byte[] bytes) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public int decode(byte[] bytes, int offset, int length) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public String toString() {
+return 

[15/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
index b9fa15b..7f64fee 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
@@ -21,7 +21,6 @@ import static org.apache.phoenix.util.TestUtil.A_VALUE;
 import static org.apache.phoenix.util.TestUtil.B_VALUE;
 import static org.apache.phoenix.util.TestUtil.C_VALUE;
 import static org.apache.phoenix.util.TestUtil.E_VALUE;
-import static org.apache.phoenix.util.TestUtil.ROW3;
 import static org.apache.phoenix.util.TestUtil.ROW5;
 import static org.apache.phoenix.util.TestUtil.ROW6;
 import static org.apache.phoenix.util.TestUtil.ROW7;
@@ -35,9 +34,7 @@ import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.Statement;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -50,15 +47,14 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
 
 @RunWith(Parameterized.class)
 public class GroupByIT extends BaseQueryIT {
 
-public GroupByIT(String indexDDL) {
-super(indexDDL);
+public GroupByIT(String indexDDL, boolean mutable, boolean columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Parameters(name="GroupByIT_{index}") // name is used by failsafe as file 
name in reports
@@ -74,71 +70,9 @@ public class GroupByIT extends BaseQueryIT {
BaseQueryIT.doSetup(props);
 }
 
-@SuppressWarnings("unchecked")
-@Test
-public void testGroupByCondition() throws Exception {
-Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 20));
-Connection conn = DriverManager.getConnection(getUrl(), props);
-PreparedStatement statement = conn.prepareStatement("SELECT count(*) 
FROM aTable WHERE organization_id=? GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-ResultSet rs = statement.executeQuery();
-assertValueEqualsResultSet(rs, Arrays.asList(1L,8L));
-try {
-statement = conn.prepareStatement("SELECT count(*),a_integer=6 
FROM aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is 
null) GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-List expectedResults = Lists.newArrayList(
-Arrays.asList(1L,false),
-Arrays.asList(1L,true));
-assertValuesEqualsResultSet(rs, expectedResults);
-} finally {
-conn.close();
-}
-
-
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 40));
-conn = DriverManager.getConnection(getUrl(), props);
-try {
-statement = conn.prepareStatement("UPSERT into 
aTable(organization_id,entity_id,a_integer) values(?,?,null)");
-statement.setString(1, tenantId);
-statement.setString(2, ROW3);
-statement.executeUpdate();
-conn.commit();
-} finally {
-conn.close();
-}
-props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 60));
-conn = DriverManager.getConnection(getUrl(), props);
-statement = conn.prepareStatement("SELECT count(*) FROM aTable WHERE 
organization_id=? GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-assertValueEqualsResultSet(rs, Arrays.asList(1L,1L,7L));
-statement = conn.prepareStatement("SELECT a_integer, entity_id FROM 
aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is null)");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-List expectedResults = Lists.newArrayList(
-Arrays.asList(null,ROW3),
-Arrays.asList(5,ROW5),
-Arrays.asList(6,ROW6));
-assertValuesEqualsResultSet(rs, expectedResults);
-try {
-statement = conn.prepareStatement("SELECT count(*),a_integer=6 
FROM aTable WHERE organization_id=? and (a_integer IN (5,6) or a_integer is 
null) GROUP BY a_integer=6");
-statement.setString(1, tenantId);
-rs = statement.executeQuery();
-expectedResults = Lists.newArrayList(
-Arrays.asList(1L,null),

[17/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

PHOENIX-1598 Encode column names to save space and improve performance


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3c7ff99b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3c7ff99b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3c7ff99b

Branch: refs/heads/master
Commit: 3c7ff99bfb958774c3e2ba5d3714ccfc46bd2367
Parents: 05c37a9
Author: Samarth 
Authored: Sun Feb 26 22:14:26 2017 -0800
Committer: Samarth 
Committed: Sun Feb 26 22:14:26 2017 -0800

--
 .../phoenix/end2end/AggregateQueryIT.java   |   74 +-
 .../AlterMultiTenantTableWithViewsIT.java   |   25 +-
 .../apache/phoenix/end2end/AlterTableIT.java|  491 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |  133 +-
 .../org/apache/phoenix/end2end/ArrayIT.java |   28 +
 .../org/apache/phoenix/end2end/BaseJoinIT.java  |4 +-
 .../org/apache/phoenix/end2end/BaseQueryIT.java |   99 +-
 .../apache/phoenix/end2end/CaseStatementIT.java |   28 +-
 .../apache/phoenix/end2end/CastAndCoerceIT.java |   34 +-
 .../end2end/ClientTimeArithmeticQueryIT.java|   76 +-
 .../end2end/ColumnEncodedBytesPropIT.java   |   95 +
 .../end2end/CountDistinctCompressionIT.java |2 +-
 .../apache/phoenix/end2end/CreateTableIT.java   |   57 +
 .../org/apache/phoenix/end2end/DateTimeIT.java  |2 +-
 .../phoenix/end2end/DefaultColumnValueIT.java   |1 +
 .../apache/phoenix/end2end/DerivedTableIT.java  |2 +-
 .../apache/phoenix/end2end/DistinctCountIT.java |4 +-
 .../apache/phoenix/end2end/DynamicColumnIT.java |   63 +
 .../phoenix/end2end/ExtendedQueryExecIT.java|8 +-
 .../apache/phoenix/end2end/FunkyNamesIT.java|2 +-
 .../org/apache/phoenix/end2end/GroupByIT.java   |  162 +-
 .../phoenix/end2end/ImmutableTablePropIT.java   |  130 -
 .../end2end/ImmutableTablePropertiesIT.java |  189 +
 .../apache/phoenix/end2end/MutableQueryIT.java  |  424 ++
 .../phoenix/end2end/NativeHBaseTypesIT.java |2 +-
 .../org/apache/phoenix/end2end/NotQueryIT.java  |   28 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |2 -
 .../apache/phoenix/end2end/PercentileIT.java|4 +-
 .../phoenix/end2end/PhoenixRuntimeIT.java   |4 +-
 .../phoenix/end2end/PointInTimeQueryIT.java |   78 +-
 .../phoenix/end2end/ProductMetricsIT.java   |2 +-
 .../end2end/QueryDatabaseMetaDataIT.java|   16 +-
 .../org/apache/phoenix/end2end/QueryIT.java |  112 +-
 .../phoenix/end2end/ReadIsolationLevelIT.java   |2 +-
 .../phoenix/end2end/RowValueConstructorIT.java  |   36 +-
 .../org/apache/phoenix/end2end/ScanQueryIT.java |   93 +-
 .../phoenix/end2end/StatsCollectorIT.java   |  124 +-
 .../apache/phoenix/end2end/StoreNullsIT.java|  310 +-
 .../phoenix/end2end/StoreNullsPropIT.java   |   51 +
 ...SysTableNamespaceMappedStatsCollectorIT.java |4 +-
 .../java/org/apache/phoenix/end2end/TopNIT.java |6 +-
 .../apache/phoenix/end2end/UpsertSelectIT.java  |   10 +-
 .../apache/phoenix/end2end/UpsertValuesIT.java  |   51 +-
 .../phoenix/end2end/UserDefinedFunctionsIT.java |3 +-
 .../phoenix/end2end/VariableLengthPKIT.java |   38 +-
 .../phoenix/end2end/index/DropMetadataIT.java   |   13 +-
 .../phoenix/end2end/index/ImmutableIndexIT.java |   20 +-
 .../end2end/index/IndexExpressionIT.java|   28 +-
 .../apache/phoenix/end2end/index/IndexIT.java   |   58 +-
 .../phoenix/end2end/index/IndexTestUtil.java|   11 +-
 .../end2end/index/MutableIndexFailureIT.java|2 +
 .../phoenix/end2end/index/MutableIndexIT.java   |   21 +-
 .../phoenix/end2end/salted/SaltedTableIT.java   |2 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |  518 ++
 .../org/apache/phoenix/tx/TransactionIT.java|  589 +-
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |   42 +-
 .../apache/phoenix/cache/ServerCacheClient.java |2 +
 .../org/apache/phoenix/cache/TenantCache.java   |2 +-
 .../apache/phoenix/cache/TenantCacheImpl.java   |4 +-
 .../phoenix/compile/CreateTableCompiler.java|   10 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |2 +-
 .../phoenix/compile/ExpressionCompiler.java |   18 +-
 .../apache/phoenix/compile/FromCompiler.java|   54 +-
 .../apache/phoenix/compile/JoinCompiler.java|   15 +-
 .../phoenix/compile/ListJarsQueryPlan.java  |6 +-
 .../apache/phoenix/compile/PostDDLCompiler.java |   11 +-
 .../compile/PostLocalIndexDDLCompiler.java  |9 +-
 .../phoenix/compile/ProjectionCompiler.java |   35 +-
 .../apache/phoenix/compile/TraceQueryPlan.java  |4 +-
 .../compile/TupleProjectionCompiler.java|   31 +-
 .../apache/phoenix/compile/UnionCompiler.java   |7 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |6 +-
 .../apache/phoenix/compile/WhereCompiler.java   |   24 +-
 

[02/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
--
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java 
b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 4a9cb57..3e2c9b5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -18,6 +18,8 @@
 package org.apache.phoenix.util;
 
 import static 
org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER;
 import static org.apache.phoenix.util.PhoenixRuntime.getTable;
 
 import java.io.ByteArrayInputStream;
@@ -28,6 +30,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 
 import org.apache.hadoop.hbase.Cell;
@@ -67,10 +70,12 @@ import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
 import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.UpdateIndexStateRequest;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState.RowMutationState;
 import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
 import org.apache.phoenix.expression.RowKeyColumnExpression;
+import org.apache.phoenix.expression.SingleCellColumnExpression;
 import org.apache.phoenix.expression.visitor.RowKeyExpressionVisitor;
 import org.apache.phoenix.hbase.index.ValueGetter;
 import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
@@ -93,9 +98,12 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnFamily;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.tuple.ResultTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PBinary;
@@ -190,6 +198,11 @@ public class IndexUtil {
 : QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX + 
dataColumnFamilyName;
 }
 
+public static byte[] getLocalIndexColumnFamily(byte[] 
dataColumnFamilyBytes) {
+String dataCF = Bytes.toString(dataColumnFamilyBytes);
+return getLocalIndexColumnFamily(dataCF).getBytes();
+}
+
 public static PColumn getDataColumn(PTable dataTable, String 
indexColumnName) {
 int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP);
 if (pos < 0) {
@@ -209,7 +222,7 @@ public class IndexUtil {
 throw new IllegalArgumentException("Could not find column family 
\"" +  indexColumnName.substring(0, pos) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
 try {
-return family.getColumn(indexColumnName.substring(pos+1));
+return 
family.getPColumnForColumnName(indexColumnName.substring(pos+1));
 } catch (ColumnNotFoundException e) {
 throw new IllegalArgumentException("Could not find column \"" +  
indexColumnName.substring(pos+1) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
@@ -236,10 +249,11 @@ public class IndexUtil {
 
 private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) {
 byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table);
+byte[] emptyKeyValueQualifier = 
EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
 return (Bytes.compareTo(emptyKeyValueCF, 0, emptyKeyValueCF.length, 
ref.getFamilyWritable()
 .get(), ref.getFamilyWritable().getOffset(), 
ref.getFamilyWritable().getLength()) == 0 && Bytes
-.compareTo(QueryConstants.EMPTY_COLUMN_BYTES, 0,
-QueryConstants.EMPTY_COLUMN_BYTES.length, 
ref.getQualifierWritable().get(), ref
+.compareTo(emptyKeyValueQualifier, 0,
+emptyKeyValueQualifier.length, 
ref.getQualifierWritable().get(), ref
 .getQualifierWritable().getOffset(), 
ref.getQualifierWritable()
 .getLength()) == 0);
 }
@@ -271,10 +285,10 @@ public class IndexUtil {
 }
 
 public static List generateIndexData(final PTable table, 

[12/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
index 1399f6c..f37d09b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java
@@ -26,337 +26,110 @@ import static org.junit.Assert.fail;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
-import java.sql.Date;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-import java.util.List;
 import java.util.Properties;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessor.PhoenixTransactionalProcessor;
 import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryConstants;
-import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
-import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.StringUtil;
 import org.apache.phoenix.util.TestUtil;
 import org.apache.tephra.TxConstants;
-import org.junit.Ignore;
 import org.junit.Test;
 
-import com.google.common.collect.Lists;
+public class TransactionIT  extends ParallelStatsDisabledIT {
 
-public class TransactionIT extends ParallelStatsDisabledIT {
-
 @Test
-public void testReadOwnWrites() throws Exception {
-String transTableName = generateUniqueName();
-String fullTableName = INDEX_DATA_SCHEMA + 
QueryConstants.NAME_SEPARATOR + transTableName;
-String selectSql = "SELECT * FROM "+ fullTableName;
-try (Connection conn = DriverManager.getConnection(getUrl())) {
-TestUtil.createTransactionalTable(conn, fullTableName);
-conn.setAutoCommit(false);
-ResultSet rs = conn.createStatement().executeQuery(selectSql);
-assertFalse(rs.next());
-
-String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, 
char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
-PreparedStatement stmt = conn.prepareStatement(upsert);
-// upsert two rows
-TestUtil.setRowKeyColumns(stmt, 1);
-stmt.execute();
-TestUtil.setRowKeyColumns(stmt, 2);
-stmt.execute();
-
-// verify rows can be read even though commit has not been called
-rs = conn.createStatement().executeQuery(selectSql);
-TestUtil.validateRowKeyColumns(rs, 1);
-TestUtil.validateRowKeyColumns(rs, 2);
-assertFalse(rs.next());
-
-conn.commit();
-
-// verify rows can be read after commit
-rs = conn.createStatement().executeQuery(selectSql);
-TestUtil.validateRowKeyColumns(rs, 1);
-TestUtil.validateRowKeyColumns(rs, 2);
-assertFalse(rs.next());
-}
+public void testReCreateTxnTableAfterDroppingExistingNonTxnTable() throws 
SQLException {
+String tableName = generateUniqueName();
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+Connection conn = DriverManager.getConnection(getUrl(), props);
+conn.setAutoCommit(false);
+Statement stmt = conn.createStatement();
+stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 
VARCHAR, v2 VARCHAR)");
+stmt.execute("DROP TABLE " + tableName);
+stmt.execute("CREATE TABLE " + tableName + "(k VARCHAR PRIMARY KEY, v1 
VARCHAR, v2 VARCHAR) TRANSACTIONAL=true");
+stmt.execute("CREATE INDEX " + tableName + "_IDX ON " + tableName + " 
(v1) INCLUDE(v2)");
+assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new 
PTableKey(null, tableName)).isTransactional());
+assertTrue(conn.unwrap(PhoenixConnection.class).getTable(new 
PTableKey(null,  tableName + "_IDX")).isTransactional());
 }
 
 @Test
-public void testTxnClosedCorrecty() throws Exception {
-String transTableName = 

[10/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
index b201c8e..139a69c 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/ServerCachingProtocol.java
@@ -36,7 +36,7 @@ import org.apache.phoenix.memory.MemoryManager.MemoryChunk;
  */
 public interface ServerCachingProtocol {
 public static interface ServerCacheFactory extends Writable {
-public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] 
txState, MemoryChunk chunk) throws SQLException;
+public Closeable newCache(ImmutableBytesWritable cachePtr, byte[] 
txState, MemoryChunk chunk, boolean useProtoForIndexMaintainer) throws 
SQLException;
 }
 /**
  * Add the cache to the region server cache.  

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index db3c792..c5854d3 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -101,7 +101,10 @@ import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker;
 import org.apache.phoenix.schema.stats.StatisticsCollector;
 import org.apache.phoenix.schema.stats.StatisticsCollectorFactory;
+import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList;
 import org.apache.phoenix.schema.tuple.MultiKeyValueTuple;
+import org.apache.phoenix.schema.tuple.PositionBasedMultiKeyValueTuple;
+import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PBinary;
 import org.apache.phoenix.schema.types.PChar;
 import org.apache.phoenix.schema.types.PDataType;
@@ -109,6 +112,7 @@ import org.apache.phoenix.schema.types.PDouble;
 import org.apache.phoenix.schema.types.PFloat;
 import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.KeyValueUtil;
 import org.apache.phoenix.util.LogUtil;
@@ -181,9 +185,9 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 
 private void commitBatch(Region region, List mutations, byte[] 
indexUUID, long blockingMemstoreSize,
-byte[] indexMaintainersPtr, byte[] txState) throws IOException {
+byte[] indexMaintainersPtr, byte[] txState, boolean useIndexProto) 
throws IOException {
 if (indexMaintainersPtr != null) {
-mutations.get(0).setAttribute(PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
+mutations.get(0).setAttribute(useIndexProto ? 
PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
 }
 
 if (txState != null) {
@@ -213,13 +217,13 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 
 private void commitBatchWithHTable(HTable table, Region region, 
List mutations, byte[] indexUUID,
-long blockingMemstoreSize, byte[] indexMaintainersPtr, byte[] 
txState) throws IOException {
+long blockingMemstoreSize, byte[] indexMaintainersPtr, byte[] 
txState, boolean useIndexProto) throws IOException {
 
 if (indexUUID != null) {
 // Need to add indexMaintainers for each mutation as table.batch 
can be distributed across servers
 for (Mutation m : mutations) {
 if (indexMaintainersPtr != null) {
-m.setAttribute(PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
+m.setAttribute(useIndexProto ? 
PhoenixIndexCodec.INDEX_PROTO_MD : PhoenixIndexCodec.INDEX_MD, 
indexMaintainersPtr);
 }
 if (txState != null) {
 m.setAttribute(BaseScannerRegionObserver.TX_STATE, 
txState);
@@ -328,8 +332,13 @@ public class UngroupedAggregateRegionObserver extends 
BaseScannerRegionObserver
 }
 values = new byte[projectedTable.getPKColumns().size()][];
 }
-byte[] localIndexBytes = scan.getAttribute(LOCAL_INDEX_BUILD);
-List indexMaintainers 

[14/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
--
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index d3bbe23..1084f14 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -62,15 +62,15 @@ import org.junit.Test;
  */
 public class QueryIT extends BaseQueryIT {
 
-public QueryIT(String indexDDL) {
-super(indexDDL);
+public QueryIT(String indexDDL, boolean mutable, boolean columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Test
 public void testIntFilter() throws Exception {
 String updateStmt = 
-"upsert into " +
-"ATABLE(" +
+"upsert into " + tableName +
+" (" +
 "ORGANIZATION_ID, " +
 "ENTITY_ID, " +
 "A_INTEGER) " +
@@ -89,10 +89,10 @@ public class QueryIT extends BaseQueryIT {
 url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 
6);
 props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 upsertConn = DriverManager.getConnection(url, props);
-analyzeTable(upsertConn, "ATABLE");
+analyzeTable(upsertConn, tableName);
 upsertConn.close();
 
-String query = "SELECT entity_id FROM aTable WHERE organization_id=? 
and a_integer >= ?";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer >= ?";
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
 PreparedStatement statement = conn.prepareStatement(query);
@@ -101,19 +101,19 @@ public class QueryIT extends BaseQueryIT {
 ResultSet rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW7, ROW8, 
ROW9));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer < 2";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer < 2";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW1, ROW4));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer <= 2";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer <= 2";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
 assertValueEqualsResultSet(rs, Arrays.asList(ROW1, ROW2, 
ROW4));
 
-query = "SELECT entity_id FROM aTable WHERE organization_id=? and 
a_integer >=9";
+query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_integer >=9";
 statement = conn.prepareStatement(query);
 statement.setString(1, tenantId);
 rs = statement.executeQuery();
@@ -124,13 +124,8 @@ public class QueryIT extends BaseQueryIT {
 }
 
 @Test
-public void testEmptyStringValue() throws Exception {
-testNoStringValue("");
-}
-
-@Test
 public void testToDateOnString() throws Exception { // TODO: test more 
conversion combinations
-String query = "SELECT a_string FROM aTable WHERE organization_id=? 
and a_integer = 5";
+String query = "SELECT a_string FROM " + tableName + " WHERE 
organization_id=? and a_integer = 5";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -151,7 +146,7 @@ public class QueryIT extends BaseQueryIT {
 
 @Test
 public void testColumnOnBothSides() throws Exception {
-String query = "SELECT entity_id FROM aTable WHERE organization_id=? 
and a_string = b_string";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and a_string = b_string";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -167,50 +162,9 @@ public class QueryIT extends BaseQueryIT {
 }
 }
 
-private void testNoStringValue(String value) throws Exception {
-String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 10);
-Properties props = 

[07/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
new file mode 100644
index 000..00e662f
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.NoSuchElementException;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.KeyValueColumnExpression;
+import org.apache.phoenix.expression.visitor.ExpressionVisitor;
+import 
org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import org.apache.phoenix.schema.tuple.BaseTuple;
+
+/**
+ * Filter used for tables that use number based column qualifiers generated by 
one of the encoding schemes in
+ * {@link QualifierEncodingScheme}. Because the qualifiers are number based, 
instead of using a map of cells to track
+ * the columns that have been found, we can use an array of cells where the 
index into the array would be derived by the
+ * number based column qualifier. See {@link EncodedCQIncrementalResultTuple}. 
Using this filter helps us to directly
+ * seek to the next row when the column qualifier that we have encountered is 
greater than the maxQualifier that we
+ * expect. This helps in speeding up the queries filtering on key value 
columns.
+ */
+public class MultiEncodedCQKeyValueComparisonFilter extends 
BooleanExpressionFilter {
+// Smallest qualifier for the columns that are being projected and 
filtered on
+private int minQualifier;
+
+// Largest qualifier for the columns that are being projected and filtered 
on
+private int maxQualifier;
+
+private QualifierEncodingScheme encodingScheme;
+
+// Smallest qualifier for the columns in where expression
+private int whereExpressionMinQualifier;
+
+// Largest qualifier for the columns in where expression
+private int whereExpressionMaxQualifier;
+
+private FilteredKeyValueHolder filteredKeyValues;
+
+// BitSet to track the qualifiers in where expression that we expect to 
find while filtering a row
+private BitSet whereExpressionQualifiers;
+
+// Set to track the column families of the columns in where expression
+private TreeSet cfSet;
+
+// Boolean that tells us whether the result of expression evaluation as 
and when we filter key values in a row
+private Boolean matchedColumn;
+
+// Tuple used to store the relevant key values found while filtering a row
+private EncodedCQIncrementalResultTuple inputTuple = new 
EncodedCQIncrementalResultTuple();
+
+// Member variable to cache the size of whereExpressionQualifiers
+private int expectedCardinality;
+
+private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
+
+public MultiEncodedCQKeyValueComparisonFilter() {}
+
+public MultiEncodedCQKeyValueComparisonFilter(Expression expression, 
QualifierEncodingScheme scheme) {
+super(expression);
+checkArgument(scheme != NON_ENCODED_QUALIFIERS, "Filter can only be 
used for encoded 

[16/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
index 211145e..6b57148 100644
--- 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java
@@ -54,22 +54,41 @@ import org.junit.runners.Parameterized.Parameters;
 public class AlterTableWithViewsIT extends ParallelStatsDisabledIT {
 
 private final boolean isMultiTenant;
+private final boolean columnEncoded;
 
 private final String TENANT_SPECIFIC_URL1 = getUrl() + ';' + 
TENANT_ID_ATTRIB + "=tenant1";
 private final String TENANT_SPECIFIC_URL2 = getUrl() + ';' + 
TENANT_ID_ATTRIB + "=tenant2";
 
-public AlterTableWithViewsIT(boolean isMultiTenant) {
+public AlterTableWithViewsIT(boolean isMultiTenant, boolean columnEncoded) 
{
 this.isMultiTenant = isMultiTenant;
+this.columnEncoded = columnEncoded;
 }
 
-@Parameters(name="AlterTableWithViewsIT_multiTenant={0}") // name is used 
by failsafe as file name in reports
-public static Collection data() {
-return Arrays.asList(false, true);
+@Parameters(name="AlterTableWithViewsIT_multiTenant={0}, 
columnEncoded={1}") // name is used by failsafe as file name in reports
+public static Collection data() {
+return Arrays.asList(new Boolean[][] { 
+{ false, false }, { false, true },
+{ true, false }, { true, true } });
 }
-   
+
 private String generateDDL(String format) {
+return generateDDL("", format);
+}
+
+private String generateDDL(String options, String format) {
+StringBuilder optionsBuilder = new StringBuilder(options);
+if (!columnEncoded) {
+if (optionsBuilder.length()!=0)
+optionsBuilder.append(",");
+optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+}
+if (isMultiTenant) {
+if (optionsBuilder.length()!=0)
+optionsBuilder.append(",");
+optionsBuilder.append("MULTI_TENANT=true");
+}
 return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT 
NULL, " : "",
-isMultiTenant ? "TENANT_ID, " : "", isMultiTenant ? 
"MULTI_TENANT=true" : "");
+isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
 }
 
 @Test
@@ -92,7 +111,7 @@ public class AlterTableWithViewsIT extends 
ParallelStatsDisabledIT {
 
 // adding a new pk column and a new regular column
 conn.createStatement().execute("ALTER TABLE " + tableName + " ADD 
COL3 varchar(10) PRIMARY KEY, COL4 integer");
-assertTableDefinition(conn, tableName, PTableType.TABLE, null, 1, 
5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", "COL1", "COL2", "COL3", 
"COL4");
+assertTableDefinition(conn, tableName, PTableType.TABLE, null, 
columnEncoded ? 2 : 1, 5, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, "ID", 
"COL1", "COL2", "COL3", "COL4");
 assertTableDefinition(conn, viewOfTable, PTableType.VIEW, 
tableName, 1, 7, 5, "ID", "COL1", "COL2", "COL3", "COL4", "VIEW_COL1", 
"VIEW_COL2");
 } 
 }
@@ -109,28 +128,27 @@ public class AlterTableWithViewsIT extends 
ParallelStatsDisabledIT {
 + " COL1 integer NOT NULL,"
 + " COL2 bigint NOT NULL,"
 + " CONSTRAINT NAME_PK PRIMARY KEY (%s ID, COL1, 
COL2)"
-+ " ) UPDATE_CACHE_FREQUENCY=15 "
-+ (isMultiTenant ? ",%s" : "%s");
-conn.createStatement().execute(generateDDL(ddlFormat));
++ " ) %s ";
+
conn.createStatement().execute(generateDDL("UPDATE_CACHE_FREQUENCY=2", 
ddlFormat));
 viewConn.createStatement().execute("CREATE VIEW " + viewOfTable1 + 
" ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 VARCHAR ) AS SELECT * FROM " + 
tableName);
 viewConn.createStatement().execute("CREATE VIEW " + viewOfTable2 + 
" ( VIEW_COL1 DECIMAL(10,2), VIEW_COL2 VARCHAR ) AS SELECT * FROM " + 
tableName);
-viewConn.createStatement().execute("ALTER VIEW " + viewOfTable2 + 
" SET UPDATE_CACHE_FREQUENCY = 5");
+viewConn.createStatement().execute("ALTER VIEW " + viewOfTable2 + 
" SET UPDATE_CACHE_FREQUENCY = 1");
 
 PhoenixConnection phoenixConn = 
conn.unwrap(PhoenixConnection.class);
 PTable table = phoenixConn.getTable(new PTableKey(null, 
tableName));
 PName tenantId = isMultiTenant ? 

[01/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

Repository: phoenix
Updated Branches:
  refs/heads/master 05c37a915 -> 3c7ff99bf


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
index 089c5f1..4571115 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
@@ -95,8 +95,8 @@ public class ConnectionlessTest {
 "entity_history_id char(12) not null,\n" + 
 "created_by varchar,\n" + 
 "created_date date\n" +
-"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) " +
-(saltBuckets == null ? "" : (PhoenixDatabaseMetaData.SALT_BUCKETS + 
"=" + saltBuckets));
+"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) COLUMN_ENCODED_BYTES=4 " +
+(saltBuckets == null ? "" : " , " + 
(PhoenixDatabaseMetaData.SALT_BUCKETS + "=" + saltBuckets));
 Properties props = new Properties();
 Connection conn = DriverManager.getConnection(getUrl(), props);
 PreparedStatement statement = conn.prepareStatement(dmlStmt);
@@ -141,31 +141,29 @@ public class ConnectionlessTest {
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
-assertTrue(iterator.hasNext());
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
 private static void assertRow2(Iterator iterator, byte[] 
expectedRowKey2) {
 KeyValue kv;
-assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
 @Test

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
new file mode 100644
index 000..1052184
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static 

[11/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 4d3c0cf..32e9f68 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 package org.apache.phoenix.compile;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -24,7 +25,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
-import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.parse.AliasedNode;
 import org.apache.phoenix.parse.ColumnParseNode;
 import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -43,11 +43,13 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.ProjectedColumn;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.SchemaUtil;
 
@@ -120,7 +122,7 @@ public class TupleProjectionCompiler {
 PColumn sourceColumn = table.getPKColumns().get(i);
 ColumnRef sourceColumnRef = new ColumnRef(tableRef, 
sourceColumn.getPosition());
 PColumn column = new ProjectedColumn(sourceColumn.getName(), 
sourceColumn.getFamilyName(), 
-position++, sourceColumn.isNullable(), sourceColumnRef);
+position++, sourceColumn.isNullable(), sourceColumnRef, 
null);
 projectedColumns.add(column);
 }
 for (PColumn sourceColumn : table.getColumns()) {
@@ -132,18 +134,18 @@ public class TupleProjectionCompiler {
 && 
!families.contains(sourceColumn.getFamilyName().getString()))
 continue;
 PColumn column = new ProjectedColumn(sourceColumn.getName(), 
sourceColumn.getFamilyName(), 
-position++, sourceColumn.isNullable(), sourceColumnRef);
+position++, sourceColumn.isNullable(), sourceColumnRef, 
sourceColumn.getColumnQualifierBytes());
 projectedColumns.add(column);
 // Wildcard or FamilyWildcard will be handled by 
ProjectionCompiler.
 if (!isWildcard && 
!families.contains(sourceColumn.getFamilyName())) {
-
context.getScan().addColumn(sourceColumn.getFamilyName().getBytes(), 
sourceColumn.getName().getBytes());
+   EncodedColumnsUtil.setColumns(column, table, context.getScan());
 }
 }
 // add LocalIndexDataColumnRef
 for (LocalIndexDataColumnRef sourceColumnRef : 
visitor.localIndexColumnRefSet) {
 PColumn column = new 
ProjectedColumn(sourceColumnRef.getColumn().getName(), 
 sourceColumnRef.getColumn().getFamilyName(), position++, 
-sourceColumnRef.getColumn().isNullable(), sourceColumnRef);
+sourceColumnRef.getColumn().isNullable(), sourceColumnRef, 
sourceColumnRef.getColumn().getColumnQualifierBytes());
 projectedColumns.add(column);
 }
 
@@ -154,9 +156,9 @@ public class TupleProjectionCompiler {
 null, null, table.isWALDisabled(), table.isMultiTenant(), 
table.getStoreNulls(), table.getViewType(),
 table.getViewIndexId(),
 table.getIndexType(), table.rowKeyOrderOptimizable(), 
table.isTransactional(), table.getUpdateCacheFrequency(), 
-table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
table.getAutoPartitionSeqName(), table.isAppendOnlySchema());
+table.getIndexDisableTimestamp(), table.isNamespaceMapped(), 
table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), 
table.getImmutableStorageScheme(), table.getEncodingScheme(), 
table.getEncodedCQCounter());
 }
-
+
 public static PTable createProjectedTable(TableRef tableRef, 
List sourceColumnRefs, boolean retainPKColumns) throws SQLException {
 PTable table = tableRef.getTable();
 boolean hasSaltingColumn = retainPKColumns && table.getBucketNum() != 
null;
@@ -169,20 +171,23 @@ public class TupleProjectionCompiler {
 String aliasedName = 

[06/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index fdf5498..a41dbf0 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -65,7 +65,6 @@ import 
org.apache.phoenix.hbase.index.covered.update.IndexedColumnGroup;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.write.IndexWriter;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
@@ -244,8 +243,16 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 for (ColumnReference ref : mutableColumns) {
 scan.addColumn(ref.getFamily(), ref.getQualifier());
 }
+/*
+ * Indexes inherit the storage scheme of the data table which 
means all the indexes have the same
+ * storage scheme and empty key value qualifier. Note that 
this assumption would be broken if we start
+ * supporting new indexes over existing data tables to have a 
different storage scheme than the data
+ * table.
+ */
+byte[] emptyKeyValueQualifier = 
indexMaintainers.get(0).getEmptyKeyValueQualifier();
+
 // Project empty key value column
-
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
QueryConstants.EMPTY_COLUMN_BYTES);
+
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
emptyKeyValueQualifier);
 ScanRanges scanRanges = 
ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, 
Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, 
KeyRange.EVERYTHING_RANGE, null, true, -1);
 scanRanges.initializeScan(scan);
 TableName tableName = 
env.getRegion().getRegionInfo().getTable();
@@ -296,7 +303,8 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 Map 
mutationsToFindPreviousValue) throws IOException {
 if (scanner != null) {
 Result result;
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0)
+.getDataEmptyKeyValueCF(), 
indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 // Process existing data table rows by removing the old index row 
and adding the new index row
 while ((result = scanner.next()) != null) {
 Mutation m = mutationsToFindPreviousValue.remove(new 
ImmutableBytesPtr(result.getRow()));
@@ -324,7 +332,7 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 // to generate point delete markers for all index rows that were 
added. We don't have Tephra
 // manage index rows in change sets because we don't want to be 
hit with the additional
 // memory hit and do not need to do conflict detection on index 
rows.
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 while ((result = scanner.next()) != null) {
 Mutation m = mutations.remove(new 
ImmutableBytesPtr(result.getRow()));
 // Sort by timestamp, type, cf, cq so we can process in time 
batches from oldest to newest

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 51587f1..d563bc2 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ 

[05/17] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3c7ff99b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index cc2b5b9..c8179e8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -35,6 +35,8 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
@@ -42,9 +44,11 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
@@ -85,9 +89,14 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 import static 
org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
 import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_UPDATE_STATS_ASYNC;
+import static org.apache.phoenix.schema.PTable.EncodedCQCounter.NULL_COUNTER;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
 import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
 import static org.apache.phoenix.schema.PTableType.TABLE;
 import static org.apache.phoenix.schema.PTableType.VIEW;
@@ -187,8 +196,12 @@ import 
org.apache.phoenix.query.ConnectionQueryServices.Feature;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.LinkType;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.QualifierOutOfRangeException;
 import org.apache.phoenix.schema.PTable.ViewType;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
 import org.apache.phoenix.schema.types.PDataType;
@@ -200,6 +213,7 @@ import org.apache.phoenix.schema.types.PUnsignedLong;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.MetaDataUtil;
@@ -215,6 +229,7 @@ import org.apache.tephra.TxConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Strings;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
@@ -259,8 +274,10 @@ 

Apache-Phoenix | 4.x-HBase-0.98 | Build Successful

[Apache Jenkins Server]

4.x-HBase-0.98 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf?p=phoenix.git;a=shortlog;h=refs/heads/4.x-HBase-0.98

Compiled Artifacts https://builds.apache.org/job/Phoenix-4.x-HBase-0.98/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-4.x-HBase-0.98/lastCompletedBuild/testReport/

Changes
[samarth] PHOENIX-1598 Encode column names to save space and improve



Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout

[10/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
index b8b8b2f..b7ca46f 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/PTableProtos.java
@@ -269,6 +269,16 @@ public final class PTableProtos {
  * optional bool isDynamic = 14;
  */
 boolean getIsDynamic();
+
+// optional bytes columnQualifierBytes = 15;
+/**
+ * optional bytes columnQualifierBytes = 15;
+ */
+boolean hasColumnQualifierBytes();
+/**
+ * optional bytes columnQualifierBytes = 15;
+ */
+com.google.protobuf.ByteString getColumnQualifierBytes();
   }
   /**
* Protobuf type {@code PColumn}
@@ -391,6 +401,11 @@ public final class PTableProtos {
   isDynamic_ = input.readBool();
   break;
 }
+case 122: {
+  bitField0_ |= 0x4000;
+  columnQualifierBytes_ = input.readBytes();
+  break;
+}
   }
 }
   } catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -709,6 +724,22 @@ public final class PTableProtos {
   return isDynamic_;
 }
 
+// optional bytes columnQualifierBytes = 15;
+public static final int COLUMNQUALIFIERBYTES_FIELD_NUMBER = 15;
+private com.google.protobuf.ByteString columnQualifierBytes_;
+/**
+ * optional bytes columnQualifierBytes = 15;
+ */
+public boolean hasColumnQualifierBytes() {
+  return ((bitField0_ & 0x4000) == 0x4000);
+}
+/**
+ * optional bytes columnQualifierBytes = 15;
+ */
+public com.google.protobuf.ByteString getColumnQualifierBytes() {
+  return columnQualifierBytes_;
+}
+
 private void initFields() {
   columnNameBytes_ = com.google.protobuf.ByteString.EMPTY;
   familyNameBytes_ = com.google.protobuf.ByteString.EMPTY;
@@ -724,6 +755,7 @@ public final class PTableProtos {
   expression_ = "";
   isRowTimestamp_ = false;
   isDynamic_ = false;
+  columnQualifierBytes_ = com.google.protobuf.ByteString.EMPTY;
 }
 private byte memoizedIsInitialized = -1;
 public final boolean isInitialized() {
@@ -799,6 +831,9 @@ public final class PTableProtos {
   if (((bitField0_ & 0x2000) == 0x2000)) {
 output.writeBool(14, isDynamic_);
   }
+  if (((bitField0_ & 0x4000) == 0x4000)) {
+output.writeBytes(15, columnQualifierBytes_);
+  }
   getUnknownFields().writeTo(output);
 }
 
@@ -864,6 +899,10 @@ public final class PTableProtos {
 size += com.google.protobuf.CodedOutputStream
   .computeBoolSize(14, isDynamic_);
   }
+  if (((bitField0_ & 0x4000) == 0x4000)) {
+size += com.google.protobuf.CodedOutputStream
+  .computeBytesSize(15, columnQualifierBytes_);
+  }
   size += getUnknownFields().getSerializedSize();
   memoizedSerializedSize = size;
   return size;
@@ -957,6 +996,11 @@ public final class PTableProtos {
 result = result && (getIsDynamic()
 == other.getIsDynamic());
   }
+  result = result && (hasColumnQualifierBytes() == 
other.hasColumnQualifierBytes());
+  if (hasColumnQualifierBytes()) {
+result = result && getColumnQualifierBytes()
+.equals(other.getColumnQualifierBytes());
+  }
   result = result &&
   getUnknownFields().equals(other.getUnknownFields());
   return result;
@@ -1026,6 +1070,10 @@ public final class PTableProtos {
 hash = (37 * hash) + ISDYNAMIC_FIELD_NUMBER;
 hash = (53 * hash) + hashBoolean(getIsDynamic());
   }
+  if (hasColumnQualifierBytes()) {
+hash = (37 * hash) + COLUMNQUALIFIERBYTES_FIELD_NUMBER;
+hash = (53 * hash) + getColumnQualifierBytes().hashCode();
+  }
   hash = (29 * hash) + getUnknownFields().hashCode();
   memoizedHashCode = hash;
   return hash;
@@ -1163,6 +1211,8 @@ public final class PTableProtos {
 bitField0_ = (bitField0_ & ~0x1000);
 isDynamic_ = false;
 bitField0_ = (bitField0_ & ~0x2000);
+columnQualifierBytes_ = com.google.protobuf.ByteString.EMPTY;
+bitField0_ = (bitField0_ & ~0x4000);
 return this;
   }
 
@@ -1247,6 +1297,10 @@ public final class PTableProtos {
   to_bitField0_ |= 0x2000;
 }
 result.isDynamic_ = isDynamic_;
+if (((from_bitField0_ & 0x4000) == 0x4000)) {
+  to_bitField0_ |= 0x4000;
+}
+

[14/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java
index 9b28bad..b0ce8cd 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.end2end;
 import static org.apache.phoenix.util.TestUtil.A_VALUE;
 import static org.apache.phoenix.util.TestUtil.B_VALUE;
 import static org.apache.phoenix.util.TestUtil.C_VALUE;
-import static org.apache.phoenix.util.TestUtil.E_VALUE;
 import static org.apache.phoenix.util.TestUtil.ROW1;
 import static org.apache.phoenix.util.TestUtil.ROW2;
 import static org.apache.phoenix.util.TestUtil.ROW3;
@@ -39,10 +38,8 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Properties;
 import java.util.Set;
 
@@ -53,7 +50,6 @@ import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
 
-import com.google.common.collect.Lists;
 import com.google.common.primitives.Doubles;
 import com.google.common.primitives.Floats;
 
@@ -66,13 +62,13 @@ public class ScanQueryIT extends BaseQueryIT {
 return QueryIT.data();
 }
 
-public ScanQueryIT(String indexDDL) {
-super(indexDDL);
+public ScanQueryIT(String indexDDL, boolean mutable, boolean 
columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Test
 public void testScan() throws Exception {
-String query = "SELECT a_string, /* comment ok? */ b_string FROM 
aTable WHERE ?=organization_id and 5=a_integer";
+String query = "SELECT a_string, /* comment ok? */ b_string FROM " + 
tableName + " WHERE ?=organization_id and 5=a_integer";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -91,7 +87,7 @@ public class ScanQueryIT extends BaseQueryIT {
 
 @Test
 public void testScanByByteValue() throws Exception {
-String query = "SELECT a_string, b_string, a_byte FROM aTable WHERE 
?=organization_id and 1=a_byte";
+String query = "SELECT a_string, b_string, a_byte FROM " + tableName + 
" WHERE ?=organization_id and 1=a_byte";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -111,7 +107,7 @@ public class ScanQueryIT extends BaseQueryIT {
 
 @Test
 public void testScanByShortValue() throws Exception {
-String query = "SELECT a_string, b_string, a_short FROM aTable WHERE 
?=organization_id and 128=a_short";
+String query = "SELECT a_string, b_string, a_short FROM " + tableName 
+ " WHERE ?=organization_id and 128=a_short";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -131,7 +127,7 @@ public class ScanQueryIT extends BaseQueryIT {
 
 @Test
 public void testScanByFloatValue() throws Exception {
-String query = "SELECT a_string, b_string, a_float FROM aTable WHERE 
?=organization_id and ?=a_float";
+String query = "SELECT a_string, b_string, a_float FROM " + tableName 
+ " WHERE ?=organization_id and ?=a_float";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn = DriverManager.getConnection(getUrl(), props);
@@ -152,7 +148,7 @@ public class ScanQueryIT extends BaseQueryIT {
 
 @Test
 public void testScanByUnsignedFloatValue() throws Exception {
-String query = "SELECT a_string, b_string, a_unsigned_float FROM 
aTable WHERE ?=organization_id and ?=a_unsigned_float";
+String query = "SELECT a_string, b_string, a_unsigned_float FROM " + 
tableName + " WHERE ?=organization_id and ?=a_unsigned_float";
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts 
+ 2)); // Execute at timestamp 2
 Connection conn 

[03/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
new file mode 100644
index 000..5a5b355
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/tuple/EncodedColumnQualiferCellsList.java
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema.tuple;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME;
+
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+
+/**
+ * List implementation that provides indexed based look up when the cell 
column qualifiers are positive numbers. 
+ * These qualifiers are generated by using one of the column qualifier 
encoding schemes specified in {@link ImmutableStorageScheme}. 
+ * The api methods in this list assume that the caller wants to see
+ * and add only non null elements in the list. 
+ * 
+ * Please note that this implementation doesn't implement all the optional 
methods of the 
+ * {@link List} interface. Such unsupported methods could violate the basic 
invariance of the list that every cell with
+ * an encoded column qualifier has a fixed position in the list.
+ * 
+ * 
+ * An important performance characteristic of this list is that doing look up 
on the basis of index via {@link #get(int)}
+ * is an O(n) operation. This makes iterating through the list using {@link 
#get(int)} an O(n^2) operation.
+ * Instead, for iterating through the list, one should use the iterators 
created through {@link #iterator()} or 
+ * {@link #listIterator()}. Do note that getting an element using {@link 
#getCellForColumnQualifier(int)} is an O(1) operation
+ * and should generally be the way for accessing elements in the list.
+ *  
+ */
+@NotThreadSafe
+public class EncodedColumnQualiferCellsList implements List {
+
+private int minQualifier;
+private int maxQualifier;
+private int nonReservedRangeOffset;
+private final Cell[] array;
+private int numNonNullElements;
+private int firstNonNullElementIdx = -1;
+private static final int RESERVED_RANGE_SIZE = 
ENCODED_CQ_COUNTER_INITIAL_VALUE - ENCODED_EMPTY_COLUMN_NAME;
+// Used by iterators to figure out if the list was structurally modified.
+private int modCount = 0;
+private final QualifierEncodingScheme encodingScheme;
+
+public EncodedColumnQualiferCellsList(int minQ, int maxQ, 
QualifierEncodingScheme encodingScheme) {
+checkArgument(minQ <= maxQ, "Invalid arguments. Min: " + minQ
++ ". Max: " + maxQ);
+this.minQualifier = minQ;
+this.maxQualifier = maxQ;
+int size = 0;
+if (maxQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = RESERVED_RANGE_SIZE;
+} else if (minQ < ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+size = (maxQ - minQ + 1);
+} else {
+size = RESERVED_RANGE_SIZE + (maxQ - minQ + 1);
+}
+this.array = new Cell[size];
+this.nonReservedRangeOffset = minQ > ENCODED_CQ_COUNTER_INITIAL_VALUE 
? minQ  - ENCODED_CQ_COUNTER_INITIAL_VALUE : 0;
+this.encodingScheme = encodingScheme;
+}
+
+@Override
+public int size() {
+return numNonNullElements;
+}
+
+@Override
+public boolean isEmpty() {
+return numNonNullElements == 0;
+}
+
+@Override
+public 

[17/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
index 73554c9..15f60df 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
@@ -18,6 +18,15 @@
 package org.apache.phoenix.end2end;
 
 import static 
org.apache.hadoop.hbase.HColumnDescriptor.DEFAULT_REPLICATION_SCOPE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
 import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
 import static org.apache.phoenix.util.TestUtil.closeConnection;
 import static org.apache.phoenix.util.TestUtil.closeStatement;
@@ -35,6 +44,8 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Properties;
 
 import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -55,6 +66,8 @@ import org.apache.phoenix.query.BaseTest;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.util.IndexUtil;
@@ -62,6 +75,9 @@ import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
 /**
  *
@@ -74,14 +90,28 @@ import org.junit.Test;
  * or at the end of test class.
  *
  */
+@RunWith(Parameterized.class)
 public class AlterTableIT extends ParallelStatsDisabledIT {
 private String schemaName;
 private String dataTableName;
 private String indexTableName;
 private String localIndexTableName;
+private String viewName;
 private String dataTableFullName;
 private String indexTableFullName;
 private String localIndexTableFullName;
+private String tableDDLOptions;
+private final boolean columnEncoded;
+
+public AlterTableIT(boolean columnEncoded) {
+this.columnEncoded = columnEncoded;
+this.tableDDLOptions = columnEncoded ? "" : "COLUMN_ENCODED_BYTES=0";
+}
+
+@Parameters(name="AlterTableIT_columnEncoded={0}") // name is used by 
failsafe as file name in reports
+public static Collection data() {
+return Arrays.asList( false, true);
+}
 
 @Before
 public void setupTableNames() throws Exception {
@@ -92,6 +122,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
 dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
 indexTableFullName = SchemaUtil.getTableName(schemaName, 
indexTableName);
 localIndexTableFullName = SchemaUtil.getTableName(schemaName, 
localIndexTableName);
+viewName = generateUniqueName();
 }
 
 @Test
@@ -103,7 +134,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
 try {
 String ddl = "CREATE TABLE  " + dataTableFullName +
 "  (a_string varchar not null, a_binary varbinary not 
null, col1 integer" +
-"  CONSTRAINT pk PRIMARY KEY (a_string, a_binary))\n";
+"  CONSTRAINT pk PRIMARY KEY (a_string, a_binary)) " + 
tableDDLOptions;
 createTestTable(getUrl(), ddl);
 
 ddl = "ALTER TABLE " + dataTableFullName + " ADD b_string VARCHAR 
NULL PRIMARY KEY";
@@ -144,7 +175,7 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
 try {
 String ddl = "CREATE TABLE " + dataTableFullName +
 "  (a_string varchar not null, col1 integer" +
-"  CONSTRAINT pk PRIMARY KEY (a_string))\n";
+"  CONSTRAINT pk PRIMARY KEY (a_string)) " + 

[16/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/CaseStatementIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CaseStatementIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CaseStatementIT.java
index 59b0f41..d2b14a9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CaseStatementIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CaseStatementIT.java
@@ -44,6 +44,7 @@ import java.util.Properties;
 
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -54,8 +55,8 @@ import com.google.common.collect.Lists;
 @RunWith(Parameterized.class)
 public class CaseStatementIT extends BaseQueryIT {
 
-public CaseStatementIT(String indexDDL) {
-super(indexDDL);
+public CaseStatementIT(String indexDDL, boolean mutable, boolean 
columnEncoded) {
+super(indexDDL, mutable, columnEncoded);
 }
 
 @Parameters(name="CaseStatementIT_{index}") // name is used by failsafe as 
file name in reports
@@ -65,7 +66,7 @@ public class CaseStatementIT extends BaseQueryIT {
 
 @Test
 public void testSimpleCaseStatement() throws Exception {
-String query = "SELECT CASE a_integer WHEN 1 THEN 'a' WHEN 2 THEN 'b' 
WHEN 3 THEN 'c' ELSE 'd' END, entity_id AS a FROM ATABLE WHERE 
organization_id=? AND a_integer < 6";
+String query = "SELECT CASE a_integer WHEN 1 THEN 'a' WHEN 2 THEN 'b' 
WHEN 3 THEN 'c' ELSE 'd' END, entity_id AS a FROM " + tableName + " WHERE 
organization_id=? AND a_integer < 6";
 String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 5); // Run query at timestamp 5
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 Connection conn = DriverManager.getConnection(url, props);
@@ -88,7 +89,7 @@ public class CaseStatementIT extends BaseQueryIT {
 
 @Test
 public void testMultiCondCaseStatement() throws Exception {
-String query = "SELECT CASE WHEN a_integer <= 2 THEN 1.5 WHEN 
a_integer = 3 THEN 2 WHEN a_integer <= 6 THEN 4.5 ELSE 5 END AS a FROM ATABLE 
WHERE organization_id=?";
+String query = "SELECT CASE WHEN a_integer <= 2 THEN 1.5 WHEN 
a_integer = 3 THEN 2 WHEN a_integer <= 6 THEN 4.5 ELSE 5 END AS a FROM " + 
tableName + " WHERE organization_id=?";
 String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 5); // Run query at timestamp 5
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 Connection conn = DriverManager.getConnection(url, props);
@@ -122,7 +123,7 @@ public class CaseStatementIT extends BaseQueryIT {
 
 @Test
 public void testPartialEvalCaseStatement() throws Exception {
-String query = "SELECT entity_id FROM ATABLE WHERE organization_id=? 
and CASE WHEN 1234 = a_integer THEN 1 WHEN x_integer = 5 THEN 2 ELSE 3 END = 2";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and CASE WHEN 1234 = a_integer THEN 1 WHEN x_integer = 5 THEN 
2 ELSE 3 END = 2";
 String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 5); // Run query at timestamp 5
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 Connection conn = DriverManager.getConnection(url, props);
@@ -140,7 +141,7 @@ public class CaseStatementIT extends BaseQueryIT {
 
 @Test
 public void testFoundIndexOnPartialEvalCaseStatement() throws Exception {
-String query = "SELECT entity_id FROM ATABLE WHERE organization_id=? 
and CASE WHEN a_integer = 1234 THEN 1 WHEN x_integer = 3 THEN y_integer ELSE 3 
END = 300";
+String query = "SELECT entity_id FROM " + tableName + " WHERE 
organization_id=? and CASE WHEN a_integer = 1234 THEN 1 WHEN x_integer = 3 THEN 
y_integer ELSE 3 END = 300";
 String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" 
+ (ts + 5); // Run query at timestamp 5
 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
 Connection conn = DriverManager.getConnection(url, props);
@@ -159,7 +160,7 @@ public class CaseStatementIT extends BaseQueryIT {
 // TODO: we need some tests that have multiple versions of key values
 @Test
 public void testUnfoundMultiColumnCaseStatement() throws Exception {
-String query = "SELECT entity_id, b_string FROM ATABLE WHERE 
organization_id=? and CASE WHEN a_integer = 1234 THEN 1 WHEN a_date < ? THEN 
y_integer WHEN x_integer = 4 THEN 4 ELSE 3 END = 4";
+String query = "SELECT entity_id, b_string FROM " + tableName + " 
WHERE organization_id=? and CASE WHEN a_integer = 1234 THEN 1 WHEN a_date < ? 
THEN y_integer WHEN x_integer = 4 

[02/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
--
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java 
b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index d913abd..8d48204 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -18,6 +18,8 @@
 package org.apache.phoenix.util;
 
 import static 
org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_QUALIFIER;
 
 import java.io.ByteArrayInputStream;
 import java.io.DataInputStream;
@@ -27,6 +29,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.ListIterator;
 import java.util.Map;
 
 import org.apache.hadoop.hbase.Cell;
@@ -39,7 +42,6 @@ import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Durability;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
@@ -52,6 +54,7 @@ import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
 import org.apache.hadoop.hbase.ipc.ServerRpcController;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.WritableUtils;
 import org.apache.phoenix.compile.ColumnResolver;
@@ -66,10 +69,12 @@ import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
 import 
org.apache.phoenix.coprocessor.generated.MetaDataProtos.UpdateIndexStateRequest;
 import org.apache.phoenix.exception.SQLExceptionCode;
 import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.execute.MutationState.RowMutationState;
 import org.apache.phoenix.execute.TupleProjector;
 import org.apache.phoenix.expression.Expression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
 import org.apache.phoenix.expression.RowKeyColumnExpression;
+import org.apache.phoenix.expression.SingleCellColumnExpression;
 import org.apache.phoenix.expression.visitor.RowKeyExpressionVisitor;
 import org.apache.phoenix.hbase.index.ValueGetter;
 import org.apache.phoenix.hbase.index.covered.update.ColumnReference;
@@ -92,9 +97,12 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PColumnFamily;
 import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.tuple.ResultTuple;
 import org.apache.phoenix.schema.tuple.Tuple;
 import org.apache.phoenix.schema.types.PBinary;
@@ -189,6 +197,11 @@ public class IndexUtil {
 : QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX + 
dataColumnFamilyName;
 }
 
+public static byte[] getLocalIndexColumnFamily(byte[] 
dataColumnFamilyBytes) {
+String dataCF = Bytes.toString(dataColumnFamilyBytes);
+return getLocalIndexColumnFamily(dataCF).getBytes();
+}
+
 public static PColumn getDataColumn(PTable dataTable, String 
indexColumnName) {
 int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP);
 if (pos < 0) {
@@ -208,7 +221,7 @@ public class IndexUtil {
 throw new IllegalArgumentException("Could not find column family 
\"" +  indexColumnName.substring(0, pos) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
 try {
-return family.getColumn(indexColumnName.substring(pos+1));
+return 
family.getPColumnForColumnName(indexColumnName.substring(pos+1));
 } catch (ColumnNotFoundException e) {
 throw new IllegalArgumentException("Could not find column \"" +  
indexColumnName.substring(pos+1) + "\" in index column name of \"" + 
indexColumnName + "\"", e);
 }
@@ -235,10 +248,11 @@ public class IndexUtil {
 
 private static boolean isEmptyKeyValue(PTable table, ColumnReference ref) {
 byte[] emptyKeyValueCF = SchemaUtil.getEmptyColumnFamily(table);
+byte[] emptyKeyValueQualifier = 

[04/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
--
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 01e8afe..91a41a3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -17,13 +17,32 @@
  */
 package org.apache.phoenix.schema;
 
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
+import static 
org.apache.phoenix.util.EncodedColumnsUtil.isReservedColumnQualifier;
+
+import java.io.DataOutputStream;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
 
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.hbase.index.util.KeyValueBuilder;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.types.PArrayDataType;
+import org.apache.phoenix.schema.types.PArrayDataTypeDecoder;
+import org.apache.phoenix.schema.types.PArrayDataTypeEncoder;
+import org.apache.phoenix.schema.types.PDataType;
+import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.util.TrustedByteArrayOutputStream;
+
+import com.google.common.annotations.VisibleForTesting;
 
 
 /**
@@ -129,7 +148,7 @@ public interface PTable extends PMetaDataEntity {
  * Link from a view to its parent table
  */
 PARENT_TABLE((byte)3);
-
+
 private final byte[] byteValue;
 private final byte serializedValue;
 
@@ -153,6 +172,318 @@ public interface PTable extends PMetaDataEntity {
 return LinkType.values()[serializedValue-1];
 }
 }
+
+public enum ImmutableStorageScheme implements 
ColumnValueEncoderDecoderSupplier {
+ONE_CELL_PER_COLUMN((byte)1) {
+@Override
+public ColumnValueEncoder getEncoder(int numElements) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public ColumnValueDecoder getDecoder() {
+throw new UnsupportedOperationException();
+}
+},
+// stores a single cell per column family that contains all serialized 
column values
+SINGLE_CELL_ARRAY_WITH_OFFSETS((byte)2) {
+@Override
+public ColumnValueEncoder getEncoder(int numElements) {
+PDataType type = PVarbinary.INSTANCE;
+int estimatedSize = PArrayDataType.estimateSize(numElements, 
type);
+TrustedByteArrayOutputStream byteStream = new 
TrustedByteArrayOutputStream(estimatedSize);
+DataOutputStream oStream = new DataOutputStream(byteStream);
+return new PArrayDataTypeEncoder(byteStream, oStream, 
numElements, type, SortOrder.ASC, false, 
PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION);
+}
+
+@Override
+public ColumnValueDecoder getDecoder() {
+return new PArrayDataTypeDecoder();
+}
+};
+
+private final byte serializedValue;
+
+private ImmutableStorageScheme(byte serializedValue) {
+this.serializedValue = serializedValue;
+}
+
+public byte getSerializedMetadataValue() {
+return this.serializedValue;
+}
+
+public static ImmutableStorageScheme fromSerializedValue(byte 
serializedValue) {
+if (serializedValue < 1 || serializedValue > 
ImmutableStorageScheme.values().length) {
+return null;
+}
+return ImmutableStorageScheme.values()[serializedValue-1];
+}
+
+}
+
+interface ColumnValueEncoderDecoderSupplier {
+ColumnValueEncoder getEncoder(int numElements);
+ColumnValueDecoder getDecoder();
+}
+
+public enum QualifierEncodingScheme implements QualifierEncoderDecoder {
+NON_ENCODED_QUALIFIERS((byte)0, null) {
+@Override
+public byte[] encode(int value) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public int decode(byte[] bytes) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public int decode(byte[] bytes, int offset, int length) {
+throw new UnsupportedOperationException();
+}
+
+@Override
+public String toString() {
+return 

[11/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index aec4482..c119ebb 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -27,6 +27,8 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CLASS_NAME_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME_INDEX;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_BYTES;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE_BYTES;
@@ -34,6 +36,7 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS_BYT
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL_BYTES;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FAMILY_NAME_INDEX;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES;
@@ -57,6 +60,7 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER_BYTES;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORAGE_SCHEME_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.STORE_NULLS_BYTES;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME_INDEX;
@@ -189,8 +193,11 @@ import org.apache.phoenix.schema.PMetaDataEntity;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.LinkType;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.ViewType;
 import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableType;
@@ -209,10 +216,12 @@ import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.types.PLong;
 import org.apache.phoenix.schema.types.PSmallint;
+import org.apache.phoenix.schema.types.PTinyint;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.KeyValueUtil;
@@ -283,6 +292,8 @@ public class MetaDataEndpointImpl extends MetaDataProtocol 
implements Coprocesso
 TABLE_FAMILY_BYTES, IS_NAMESPACE_MAPPED_BYTES);
 private static final KeyValue AUTO_PARTITION_SEQ_KV = 
createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, 
AUTO_PARTITION_SEQ_BYTES);
 private static final KeyValue APPEND_ONLY_SCHEMA_KV = 
createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, 
APPEND_ONLY_SCHEMA_BYTES);
+private static final KeyValue STORAGE_SCHEME_KV = 
createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, 
STORAGE_SCHEME_BYTES);
+private static final KeyValue ENCODING_SCHEME_KV = 
createFirstOnRow(ByteUtil.EMPTY_BYTE_ARRAY, TABLE_FAMILY_BYTES, 
ENCODING_SCHEME_BYTES);
 
 private static final List TABLE_KV_COLUMNS = 
Arrays.asList(
 EMPTY_KEYVALUE_KV,
@@ -309,7 +320,9 @@ public class 

[09/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
index 5ee1dfb..3b8984a 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/ServerCachingProtos.java
@@ -649,6 +649,4947 @@ public final class ServerCachingProtos {
 // @@protoc_insertion_point(class_scope:ImmutableBytesWritable)
   }
 
+  public interface ColumnReferenceOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// required bytes family = 1;
+/**
+ * required bytes family = 1;
+ */
+boolean hasFamily();
+/**
+ * required bytes family = 1;
+ */
+com.google.protobuf.ByteString getFamily();
+
+// required bytes qualifier = 2;
+/**
+ * required bytes qualifier = 2;
+ */
+boolean hasQualifier();
+/**
+ * required bytes qualifier = 2;
+ */
+com.google.protobuf.ByteString getQualifier();
+  }
+  /**
+   * Protobuf type {@code ColumnReference}
+   */
+  public static final class ColumnReference extends
+  com.google.protobuf.GeneratedMessage
+  implements ColumnReferenceOrBuilder {
+// Use ColumnReference.newBuilder() to construct.
+private ColumnReference(com.google.protobuf.GeneratedMessage.Builder 
builder) {
+  super(builder);
+  this.unknownFields = builder.getUnknownFields();
+}
+private ColumnReference(boolean noInit) { this.unknownFields = 
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final ColumnReference defaultInstance;
+public static ColumnReference getDefaultInstance() {
+  return defaultInstance;
+}
+
+public ColumnReference getDefaultInstanceForType() {
+  return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+  return this.unknownFields;
+}
+private ColumnReference(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+  initFields();
+  int mutable_bitField0_ = 0;
+  com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+  com.google.protobuf.UnknownFieldSet.newBuilder();
+  try {
+boolean done = false;
+while (!done) {
+  int tag = input.readTag();
+  switch (tag) {
+case 0:
+  done = true;
+  break;
+default: {
+  if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+  }
+  break;
+}
+case 10: {
+  bitField0_ |= 0x0001;
+  family_ = input.readBytes();
+  break;
+}
+case 18: {
+  bitField0_ |= 0x0002;
+  qualifier_ = input.readBytes();
+  break;
+}
+  }
+}
+  } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+  } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+  } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+  }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+  return 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.internal_static_ColumnReference_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+  return 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.internal_static_ColumnReference_fieldAccessorTable
+  .ensureFieldAccessorsInitialized(
+  
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ColumnReference.class,
 
org.apache.phoenix.coprocessor.generated.ServerCachingProtos.ColumnReference.Builder.class);
+}
+
+public static com.google.protobuf.Parser PARSER =
+new com.google.protobuf.AbstractParser() {
+  public ColumnReference parsePartialFrom(
+  com.google.protobuf.CodedInputStream input,
+  com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+  throws 

[01/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-0.98 f0e6c4a4f -> dc6a6fc75


http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
index 3396cf8..76479d6 100644
--- 
a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionlessTest.java
@@ -95,8 +95,8 @@ public class ConnectionlessTest {
 "entity_history_id char(12) not null,\n" + 
 "created_by varchar,\n" + 
 "created_date date\n" +
-"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) " +
-(saltBuckets == null ? "" : (PhoenixDatabaseMetaData.SALT_BUCKETS + 
"=" + saltBuckets));
+"CONSTRAINT pk PRIMARY KEY (organization_id, key_prefix, 
entity_history_id) ) COLUMN_ENCODED_BYTES=4 " +
+(saltBuckets == null ? "" : " , " + 
(PhoenixDatabaseMetaData.SALT_BUCKETS + "=" + saltBuckets));
 Properties props = new Properties();
 Connection conn = DriverManager.getConnection(getUrl(), props);
 PreparedStatement statement = conn.prepareStatement(dmlStmt);
@@ -136,38 +136,34 @@ public class ConnectionlessTest {
 conn.rollback(); // to clear the list of mutations for the next
 }
 
-@SuppressWarnings("deprecation")
 private static void assertRow1(Iterator iterator, byte[] 
expectedRowKey1) {
 KeyValue kv;
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
-assertTrue(iterator.hasNext());
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name1, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey1, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
-@SuppressWarnings("deprecation")
 private static void assertRow2(Iterator iterator, byte[] 
expectedRowKey2) {
 KeyValue kv;
-assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
+assertEquals(name2, PVarchar.INSTANCE.toObject(kv.getValue()));
 assertTrue(iterator.hasNext());
 kv = iterator.next();
 assertArrayEquals(expectedRowKey2, kv.getRow());
-assertEquals(QueryConstants.EMPTY_COLUMN_VALUE, 
PVarchar.INSTANCE.toObject(kv.getValue()));
+assertEquals(now, PDate.INSTANCE.toObject(kv.getValue()));
 }
 
 @Test
@@ -206,4 +202,4 @@ public class ConnectionlessTest {
 
 }
 
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
--
diff --git 
a/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
new file mode 100644
index 000..bd70f84
--- /dev/null
+++ 
b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * 

[12/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java 
b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
index 658b4cc..3d178f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/cache/TenantCacheImpl.java
@@ -104,11 +104,11 @@ public class TenantCacheImpl implements TenantCache {
 }
 
 @Override
-public Closeable addServerCache(ImmutableBytesPtr cacheId, 
ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory 
cacheFactory) throws SQLException {
+public Closeable addServerCache(ImmutableBytesPtr cacheId, 
ImmutableBytesWritable cachePtr, byte[] txState, ServerCacheFactory 
cacheFactory, boolean useProtoForIndexMaintainer) throws SQLException {
 MemoryChunk chunk = 
this.getMemoryManager().allocate(cachePtr.getLength() + txState.length);
 boolean success = false;
 try {
-Closeable element = cacheFactory.newCache(cachePtr, txState, 
chunk);
+Closeable element = cacheFactory.newCache(cachePtr, txState, 
chunk, useProtoForIndexMaintainer);
 getServerCaches().put(cacheId, element);
 success = true;
 return element;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
index 07df105..b482998 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/CreateTableCompiler.java
@@ -38,6 +38,7 @@ import org.apache.phoenix.expression.IsNullExpression;
 import org.apache.phoenix.expression.KeyValueColumnExpression;
 import org.apache.phoenix.expression.LiteralExpression;
 import org.apache.phoenix.expression.RowKeyColumnExpression;
+import org.apache.phoenix.expression.SingleCellColumnExpression;
 import 
org.apache.phoenix.expression.visitor.StatelessTraverseNoExpressionVisitor;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixStatement;
@@ -239,7 +240,7 @@ public class CreateTableCompiler {
 }
 }
 
-private static class ViewWhereExpressionVisitor extends 
StatelessTraverseNoExpressionVisitor {
+public static class ViewWhereExpressionVisitor extends 
StatelessTraverseNoExpressionVisitor {
 private boolean isUpdatable = true;
 private final PTable table;
 private int position;
@@ -318,13 +319,18 @@ public class CreateTableCompiler {
 @Override
 public Boolean visit(KeyValueColumnExpression node) {
 try {
-this.position = 
table.getColumnFamily(node.getColumnFamily()).getColumn(node.getColumnName()).getPosition();
+this.position = 
table.getColumnFamily(node.getColumnFamily()).getPColumnForColumnQualifier(node.getColumnQualifier()).getPosition();
 } catch (SQLException e) {
 throw new RuntimeException(e); // Impossible
 }
 return Boolean.TRUE;
 }
 
+@Override
+public Boolean visit(SingleCellColumnExpression node) {
+return visit(node.getKeyValueExpression());
+}
+
 }
 private static class VarbinaryDatum implements PDatum {
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java 
b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 602cd6b..cee545a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -585,7 +585,7 @@ public class DeleteCompiler {
 if (ptr.getLength() > 0) {
 byte[] uuidValue = 
ServerCacheClient.generateId();
 
context.getScan().setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
-
context.getScan().setAttribute(PhoenixIndexCodec.INDEX_MD, ptr.get());
+
context.getScan().setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ptr.get());
 

[06/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
index c67da6e..9ee5ea7 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixTransactionalIndexer.java
@@ -67,7 +67,6 @@ import 
org.apache.phoenix.hbase.index.covered.update.IndexedColumnGroup;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.write.IndexWriter;
 import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
@@ -304,8 +303,16 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 for (ColumnReference ref : mutableColumns) {
 scan.addColumn(ref.getFamily(), ref.getQualifier());
 }
+/*
+ * Indexes inherit the storage scheme of the data table which 
means all the indexes have the same
+ * storage scheme and empty key value qualifier. Note that 
this assumption would be broken if we start
+ * supporting new indexes over existing data tables to have a 
different storage scheme than the data
+ * table.
+ */
+byte[] emptyKeyValueQualifier = 
indexMaintainers.get(0).getEmptyKeyValueQualifier();
+
 // Project empty key value column
-
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
QueryConstants.EMPTY_COLUMN_BYTES);
+
scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), 
emptyKeyValueQualifier);
 ScanRanges scanRanges = 
ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, 
Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, 
KeyRange.EVERYTHING_RANGE, null, true, -1);
 scanRanges.initializeScan(scan);
 TableName tableName = 
env.getRegion().getRegionInfo().getTable();
@@ -356,7 +363,8 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 Map 
mutationsToFindPreviousValue) throws IOException {
 if (scanner != null) {
 Result result;
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0)
+.getDataEmptyKeyValueCF(), 
indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 // Process existing data table rows by removing the old index row 
and adding the new index row
 while ((result = scanner.next()) != null) {
 Mutation m = mutationsToFindPreviousValue.remove(new 
ImmutableBytesPtr(result.getRow()));
@@ -384,7 +392,7 @@ public class PhoenixTransactionalIndexer extends 
BaseRegionObserver {
 // to generate point delete markers for all index rows that were 
added. We don't have Tephra
 // manage index rows in change sets because we don't want to be 
hit with the additional
 // memory hit and do not need to do conflict detection on index 
rows.
-ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 QueryConstants.EMPTY_COLUMN_BYTES);
+ColumnReference emptyColRef = new 
ColumnReference(indexMetaData.getIndexMaintainers().get(0).getDataEmptyKeyValueCF(),
 indexMetaData.getIndexMaintainers().get(0).getEmptyKeyValueQualifier());
 while ((result = scanner.next()) != null) {
 Mutation m = mutations.remove(new 
ImmutableBytesPtr(result.getRow()));
 // Sort by timestamp, type, cf, cq so we can process in time 
batches from oldest to newest

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index c0f9707..33e2f9e 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ 

[18/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

PHOENIX-1598 Encode column names to save space and improve performance(Samarth 
Jain and Thomas D'Silva)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/dc6a6fc7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/dc6a6fc7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/dc6a6fc7

Branch: refs/heads/4.x-HBase-0.98
Commit: dc6a6fc75428a093414a7e801a645819805d6fcb
Parents: f0e6c4a
Author: Samarth 
Authored: Sun Feb 26 19:32:25 2017 -0800
Committer: Samarth 
Committed: Sun Feb 26 19:33:00 2017 -0800

--
 .../phoenix/end2end/AggregateQueryIT.java   |   74 +-
 .../AlterMultiTenantTableWithViewsIT.java   |   25 +-
 .../apache/phoenix/end2end/AlterTableIT.java|  493 +-
 .../phoenix/end2end/AlterTableWithViewsIT.java  |  133 +-
 .../org/apache/phoenix/end2end/ArrayIT.java |   28 +
 .../org/apache/phoenix/end2end/BaseJoinIT.java  |4 +-
 .../org/apache/phoenix/end2end/BaseQueryIT.java |   99 +-
 .../apache/phoenix/end2end/CaseStatementIT.java |   28 +-
 .../apache/phoenix/end2end/CastAndCoerceIT.java |   34 +-
 .../end2end/ClientTimeArithmeticQueryIT.java|   76 +-
 .../end2end/ColumnEncodedBytesPropIT.java   |   95 +
 .../end2end/CountDistinctCompressionIT.java |2 +-
 .../apache/phoenix/end2end/CreateTableIT.java   |   57 +
 .../org/apache/phoenix/end2end/DateTimeIT.java  |2 +-
 .../phoenix/end2end/DefaultColumnValueIT.java   |1 +
 .../apache/phoenix/end2end/DerivedTableIT.java  |2 +-
 .../apache/phoenix/end2end/DistinctCountIT.java |4 +-
 .../apache/phoenix/end2end/DynamicColumnIT.java |   63 +
 .../phoenix/end2end/ExtendedQueryExecIT.java|8 +-
 .../apache/phoenix/end2end/FunkyNamesIT.java|2 +-
 .../org/apache/phoenix/end2end/GroupByIT.java   |  162 +-
 .../phoenix/end2end/ImmutableTablePropIT.java   |  130 -
 .../end2end/ImmutableTablePropertiesIT.java |  189 +
 .../apache/phoenix/end2end/MutableQueryIT.java  |  424 ++
 .../phoenix/end2end/NativeHBaseTypesIT.java |2 +-
 .../org/apache/phoenix/end2end/NotQueryIT.java  |   28 +-
 .../org/apache/phoenix/end2end/OrderByIT.java   |2 -
 .../apache/phoenix/end2end/PercentileIT.java|4 +-
 .../phoenix/end2end/PhoenixRuntimeIT.java   |4 +-
 .../phoenix/end2end/PointInTimeQueryIT.java |   78 +-
 .../phoenix/end2end/ProductMetricsIT.java   |2 +-
 .../end2end/QueryDatabaseMetaDataIT.java|   16 +-
 .../org/apache/phoenix/end2end/QueryIT.java |  112 +-
 .../phoenix/end2end/ReadIsolationLevelIT.java   |2 +-
 .../phoenix/end2end/RowValueConstructorIT.java  |   36 +-
 .../org/apache/phoenix/end2end/ScanQueryIT.java |   93 +-
 .../phoenix/end2end/StatsCollectorIT.java   |  124 +-
 .../apache/phoenix/end2end/StoreNullsIT.java|  310 +-
 .../phoenix/end2end/StoreNullsPropIT.java   |   51 +
 ...SysTableNamespaceMappedStatsCollectorIT.java |4 +-
 .../java/org/apache/phoenix/end2end/TopNIT.java |6 +-
 .../apache/phoenix/end2end/UpsertSelectIT.java  |   10 +-
 .../apache/phoenix/end2end/UpsertValuesIT.java  |   51 +-
 .../phoenix/end2end/UserDefinedFunctionsIT.java |3 +-
 .../phoenix/end2end/VariableLengthPKIT.java |   38 +-
 .../phoenix/end2end/index/DropMetadataIT.java   |   13 +-
 .../phoenix/end2end/index/ImmutableIndexIT.java |   20 +-
 .../end2end/index/IndexExpressionIT.java|   28 +-
 .../apache/phoenix/end2end/index/IndexIT.java   |   58 +-
 .../phoenix/end2end/index/IndexTestUtil.java|   11 +-
 .../end2end/index/MutableIndexFailureIT.java|2 +
 .../phoenix/end2end/index/MutableIndexIT.java   |   21 +-
 .../phoenix/end2end/salted/SaltedTableIT.java   |2 +-
 .../phoenix/tx/ParameterizedTransactionIT.java  |  518 ++
 .../org/apache/phoenix/tx/TransactionIT.java|  589 +-
 .../org/apache/phoenix/tx/TxCheckpointIT.java   |   42 +-
 .../apache/phoenix/cache/ServerCacheClient.java |2 +
 .../org/apache/phoenix/cache/TenantCache.java   |2 +-
 .../apache/phoenix/cache/TenantCacheImpl.java   |4 +-
 .../phoenix/compile/CreateTableCompiler.java|   10 +-
 .../apache/phoenix/compile/DeleteCompiler.java  |2 +-
 .../phoenix/compile/ExpressionCompiler.java |   18 +-
 .../apache/phoenix/compile/FromCompiler.java|   54 +-
 .../apache/phoenix/compile/JoinCompiler.java|   15 +-
 .../phoenix/compile/ListJarsQueryPlan.java  |6 +-
 .../apache/phoenix/compile/PostDDLCompiler.java |   11 +-
 .../compile/PostLocalIndexDDLCompiler.java  |9 +-
 .../phoenix/compile/ProjectionCompiler.java |   35 +-
 .../apache/phoenix/compile/TraceQueryPlan.java  |4 +-
 .../compile/TupleProjectionCompiler.java|   31 +-
 .../apache/phoenix/compile/UnionCompiler.java   |7 +-
 .../apache/phoenix/compile/UpsertCompiler.java  |6 +-
 

[07/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
new file mode 100644
index 000..00e662f
--- /dev/null
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/filter/MultiEncodedCQKeyValueComparisonFilter.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.filter;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.NoSuchElementException;
+import java.util.TreeSet;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.exceptions.DeserializationException;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.KeyValueColumnExpression;
+import org.apache.phoenix.expression.visitor.ExpressionVisitor;
+import 
org.apache.phoenix.expression.visitor.StatelessTraverseAllExpressionVisitor;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import org.apache.phoenix.schema.tuple.BaseTuple;
+
+/**
+ * Filter used for tables that use number based column qualifiers generated by 
one of the encoding schemes in
+ * {@link QualifierEncodingScheme}. Because the qualifiers are number based, 
instead of using a map of cells to track
+ * the columns that have been found, we can use an array of cells where the 
index into the array would be derived by the
+ * number based column qualifier. See {@link EncodedCQIncrementalResultTuple}. 
Using this filter helps us to directly
+ * seek to the next row when the column qualifier that we have encountered is 
greater than the maxQualifier that we
+ * expect. This helps in speeding up the queries filtering on key value 
columns.
+ */
+public class MultiEncodedCQKeyValueComparisonFilter extends 
BooleanExpressionFilter {
+// Smallest qualifier for the columns that are being projected and 
filtered on
+private int minQualifier;
+
+// Largest qualifier for the columns that are being projected and filtered 
on
+private int maxQualifier;
+
+private QualifierEncodingScheme encodingScheme;
+
+// Smallest qualifier for the columns in where expression
+private int whereExpressionMinQualifier;
+
+// Largest qualifier for the columns in where expression
+private int whereExpressionMaxQualifier;
+
+private FilteredKeyValueHolder filteredKeyValues;
+
+// BitSet to track the qualifiers in where expression that we expect to 
find while filtering a row
+private BitSet whereExpressionQualifiers;
+
+// Set to track the column families of the columns in where expression
+private TreeSet cfSet;
+
+// Boolean that tells us whether the result of expression evaluation as 
and when we filter key values in a row
+private Boolean matchedColumn;
+
+// Tuple used to store the relevant key values found while filtering a row
+private EncodedCQIncrementalResultTuple inputTuple = new 
EncodedCQIncrementalResultTuple();
+
+// Member variable to cache the size of whereExpressionQualifiers
+private int expectedCardinality;
+
+private static final byte[] UNITIALIZED_KEY_BUFFER = new byte[0];
+
+public MultiEncodedCQKeyValueComparisonFilter() {}
+
+public MultiEncodedCQKeyValueComparisonFilter(Expression expression, 
QualifierEncodingScheme scheme) {
+super(expression);
+checkArgument(scheme != NON_ENCODED_QUALIFIERS, "Filter can only be 
used for encoded 

[05/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java 
b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 705cde0..5983c85 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -35,6 +35,8 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TABLE_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DATA_TYPE;
@@ -42,9 +44,11 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DECIMAL_DIGITS;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
 import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_TYPE;
@@ -85,9 +89,14 @@ import static 
org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
 import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
 import static 
org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
 import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_DROP_METADATA;
 import static 
org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_UPDATE_STATS_ASYNC;
+import static org.apache.phoenix.schema.PTable.EncodedCQCounter.NULL_COUNTER;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
+import static 
org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
+import static 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
 import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
 import static org.apache.phoenix.schema.PTableType.TABLE;
 import static org.apache.phoenix.schema.PTableType.VIEW;
@@ -187,8 +196,12 @@ import 
org.apache.phoenix.query.ConnectionQueryServices.Feature;
 import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable.EncodedCQCounter;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.LinkType;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
+import 
org.apache.phoenix.schema.PTable.QualifierEncodingScheme.QualifierOutOfRangeException;
 import org.apache.phoenix.schema.PTable.ViewType;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
 import org.apache.phoenix.schema.types.PDataType;
@@ -200,6 +213,7 @@ import org.apache.phoenix.schema.types.PUnsignedLong;
 import org.apache.phoenix.schema.types.PVarbinary;
 import org.apache.phoenix.schema.types.PVarchar;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.MetaDataUtil;
@@ -216,6 +230,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.base.Strings;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.ListMultimap;
 import com.google.common.collect.Lists;
@@ 

[15/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/ImmutableTablePropertiesIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ImmutableTablePropertiesIT.java
 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ImmutableTablePropertiesIT.java
new file mode 100644
index 000..52cfe9c
--- /dev/null
+++ 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ImmutableTablePropertiesIT.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Test;
+
+public class ImmutableTablePropertiesIT extends ParallelStatsDisabledIT {
+
+@Test
+public void testImmutableKeyword() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+String immutableDataTableFullName = SchemaUtil.getTableName("", 
generateUniqueName());
+String mutableDataTableFullName = SchemaUtil.getTableName("", 
generateUniqueName());
+try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+Statement stmt = conn.createStatement();
+// create table with immutable keyword
+String ddl = "CREATE IMMUTABLE TABLE  " + 
immutableDataTableFullName +
+"  (a_string varchar not null, col1 integer" +
+"  CONSTRAINT pk PRIMARY KEY (a_string)) STORE_NULLS=true";
+stmt.execute(ddl);
+
+// create table without immutable keyword
+ddl = "CREATE TABLE  " + mutableDataTableFullName +
+"  (a_string varchar not null, col1 integer" +
+"  CONSTRAINT pk PRIMARY KEY (a_string)) STORE_NULLS=true";
+stmt.execute(ddl);
+
+PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+PTable immutableTable = phxConn.getTable(new PTableKey(null, 
immutableDataTableFullName));
+assertTrue("IMMUTABLE_ROWS should be set to true", 
immutableTable.isImmutableRows());
+PTable mutableTable = phxConn.getTable(new PTableKey(null, 
mutableDataTableFullName));
+assertFalse("IMMUTABLE_ROWS should be set to false", 
mutableTable.isImmutableRows());
+} 
+}
+
+@Test
+public void testImmutableProperty() throws Exception {
+Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+String immutableDataTableFullName = SchemaUtil.getTableName("", 
generateUniqueName());
+String mutableDataTableFullName = SchemaUtil.getTableName("", 
generateUniqueName());
+try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+Statement stmt = conn.createStatement();
+// create table with immutable table property set to true
+String ddl = "CREATE TABLE  " + immutableDataTableFullName +
+"  (a_string varchar not null, col1 integer" +
+"  CONSTRAINT pk PRIMARY KEY (a_string)) 
IMMUTABLE_ROWS=true";
+stmt.execute(ddl);
+
+// create table with immutable table property set to false
+ddl = "CREATE TABLE  " + mutableDataTableFullName +
+"  (a_string varchar not null, col1 integer" +
+"  CONSTRAINT pk PRIMARY KEY (a_string))  
IMMUTABLE_ROWS=false";
+stmt.execute(ddl);
+
+

[13/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
--
diff --git 
a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java 
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
index 410dca5..b76d61d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexIT.java
@@ -61,8 +61,8 @@ import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.parse.NamedTableNode;
 import org.apache.phoenix.parse.TableName;
 import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryConstants;
 import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.util.DateUtil;
@@ -85,27 +85,39 @@ public class IndexIT extends ParallelStatsDisabledIT {
 private final boolean mutable;
 private final String tableDDLOptions;
 
-
-public IndexIT(boolean localIndex, boolean mutable, boolean transactional) 
{
+public IndexIT(boolean localIndex, boolean mutable, boolean transactional, 
boolean columnEncoded) {
 this.localIndex = localIndex;
 this.transactional = transactional;
 this.mutable = mutable;
 StringBuilder optionBuilder = new StringBuilder();
-if (!mutable)
-optionBuilder.append(" IMMUTABLE_ROWS=true ");
+if (!columnEncoded) {
+if (optionBuilder.length()!=0)
+optionBuilder.append(",");
+optionBuilder.append("COLUMN_ENCODED_BYTES=0");
+}
+if (!mutable) {
+if (optionBuilder.length()!=0)
+optionBuilder.append(",");
+optionBuilder.append("IMMUTABLE_ROWS=true");
+if (!columnEncoded) {
+
optionBuilder.append(",IMMUTABLE_STORAGE_SCHEME="+PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN);
+}
+}
 if (transactional) {
-if (!(optionBuilder.length()==0))
+if (optionBuilder.length()!=0)
 optionBuilder.append(",");
 optionBuilder.append(" TRANSACTIONAL=true ");
 }
 this.tableDDLOptions = optionBuilder.toString();
 }
 
-@Parameters(name="IndexIT_localIndex={0},mutable={1},transactional={2}") 
// name is used by failsafe as file name in reports
+
@Parameters(name="IndexIT_localIndex={0},mutable={1},transactional={2},columnEncoded={3}")
 // name is used by failsafe as file name in reports
 public static Collection data() {
 return Arrays.asList(new Boolean[][] {
- { false, false, false }, { false, false, true }, { false, 
true, false }, { false, true, true },
- { true, false, false }, { true, false, true }, { true, true, 
false }, { true, true, true }
+{ false, false, false, false }, { false, false, false, true }, 
{ false, false, true, false }, { false, false, true, true }, 
+{ false, true, false, false }, { false, true, false, true }, { 
false, true, true, false }, { false, true, true, true }, 
+{ true, false, false, false }, { true, false, false, true }, { 
true, false, true, false }, { true, false, true, true }, 
+{ true, true, false, false }, { true, true, false, true }, { 
true, true, true, false }, { true, true, true, true } 
});
 }
 
@@ -780,7 +792,7 @@ public class IndexIT extends ParallelStatsDisabledIT {
 conn.createStatement().execute(
 "CREATE TABLE " + testTable
 + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 
VARCHAR) "
-+ (!tableDDLOptions.isEmpty() ? tableDDLOptions : "") + 
"SPLIT ON ('b')");
++ (!tableDDLOptions.isEmpty() ? tableDDLOptions : "") + " 
SPLIT ON ('b')");
 query = "SELECT * FROM " + testTable;
 rs = conn.createStatement().executeQuery(query);
 assertFalse(rs.next());
@@ -808,23 +820,23 @@ public class IndexIT extends ParallelStatsDisabledIT {
 stmt.execute();
 conn.commit();
 
-// make sure the index is working as expected
-query = "SELECT * FROM " + fullIndexName;
+query = "SELECT /*+ NO_INDEX */ * FROM " + testTable;
 rs = conn.createStatement().executeQuery(query);
 assertTrue(rs.next());
-assertEquals("x", rs.getString(1));
-assertEquals("1", rs.getString(2));
-assertEquals("a", rs.getString(3));
+assertEquals("a", rs.getString(1));
+assertEquals("x", rs.getString(2));
+assertEquals("1", rs.getString(3));
 

[08/18] phoenix git commit: PHOENIX-1598 Encode column names to save space and improve performance(Samarth Jain and Thomas D'Silva)

[samarth]

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index fde403c..8595eda 100644
--- 
a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ 
b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -368,6 +368,10 @@ public enum SQLExceptionCode {
 CANNOT_ALTER_TABLE_PROPERTY_ON_VIEW(1134, "XCL34", "Altering this table 
property on a view is not allowed"),
 
 IMMUTABLE_TABLE_PROPERTY_INVALID(1135, "XCL35", "IMMUTABLE table property 
cannot be used with CREATE IMMUTABLE TABLE statement "),
+
+MAX_COLUMNS_EXCEEDED(1136, "XCL36", "The number of columns exceed the 
maximum supported by the table's qualifier encoding scheme"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES(1137, "XCL37", 
"If IMMUTABLE_STORAGE_SCHEME property is not set to ONE_CELL_PER_COLUMN 
COLUMN_ENCODED_BYTES cannot be 0"),
+INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE(1138, "XCL38", 
"IMMUTABLE_STORAGE_SCHEME property cannot be changed from/to 
ONE_CELL_PER_COLUMN "),
 
 /**
  * Implementation defined class. Phoenix internal error. (errorcode 20, 
sqlstate INT).

http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
--
diff --git 
a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java 
b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
index 352b533..7b0451a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java
@@ -64,11 +64,13 @@ import org.apache.phoenix.schema.PColumn;
 import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTable.IndexType;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.trace.TracingIterator;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.EncodedColumnsUtil;
 import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.LogUtil;
 import org.apache.phoenix.util.SQLCloseable;
@@ -307,10 +309,6 @@ public abstract class BaseQueryPlan implements QueryPlan {
 // project is not present in the index then we need to skip this 
plan.
 if (!dataColumns.isEmpty()) {
 // Set data columns to be join back from data table.
-serializeDataTableColumnsToJoin(scan, dataColumns);
-KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
-// Set key value schema of the data columns.
-serializeSchemaIntoScan(scan, schema);
 PTable parentTable = context.getCurrentTable().getTable();
 String parentSchemaName = 
parentTable.getParentSchemaName().getString();
 String parentTableName = 
parentTable.getParentTableName().getString();
@@ -321,6 +319,12 @@ public abstract class BaseQueryPlan implements QueryPlan {
 FACTORY.namedTable(null, 
TableName.create(parentSchemaName, parentTableName)),
 
context.getConnection()).resolveTable(parentSchemaName, parentTableName);
 PTable dataTable = dataTableRef.getTable();
+// Set data columns to be join back from data table.
+serializeDataTableColumnsToJoin(scan, dataColumns, dataTable);
+KeyValueSchema schema = 
ProjectedColumnExpression.buildSchema(dataColumns);
+// Set key value schema of the data columns.
+serializeSchemaIntoScan(scan, schema);
+
 // Set index maintainer of the local index.
 serializeIndexMaintainerIntoScan(scan, dataTable);
 // Set view constants if exists.
@@ -367,7 +371,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
 }
 ImmutableBytesWritable ptr = new ImmutableBytesWritable();
 IndexMaintainer.serialize(dataTable, ptr, indexes, 
context.getConnection());
-scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
+scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, 
ByteUtil.copyKeyBytesIfNecessary(ptr));
 if (dataTable.isTransactional()) {
 

Jenkins build is back to normal : Phoenix-encode-columns #77

[Apache Jenkins Server]

See 

Apache-Phoenix | EncodeColumns | Build Successful

[Apache Jenkins Server]

encodecolumns2 branch build status Successful

Source repository https://git-wip-us.apache.org/repos/asf?p=phoenix.git;a=shortlog;h=refs/heads/encodecolumns2

Compiled Artifacts https://builds.apache.org/job/Phoenix-encode-columns/lastSuccessfulBuild/artifact/

Test Report https://builds.apache.org/job/Phoenix-encode-columns/lastCompletedBuild/testReport/

Changes


Build times for last couple of runsLatest build time is the right most | Legend blue: normal, red: test failure, gray: timeout

Apache Phoenix - Timeout crawler - Build https://builds.apache.org/job/Phoenix-encode-columns/76/

[Apache Jenkins Server]

[...truncated 21 lines...]
Looking at the log, list of test(s) that timed-out:

Build:
https://builds.apache.org/job/Phoenix-encode-columns/76/


Affected test class(es):
Set(['org.apache.phoenix.end2end.index.LocalIndexIT', 
'org.apache.phoenix.end2end.SysTableNamespaceMappedStatsCollectorIT', 
'org.apache.phoenix.end2end.index.MutableIndexFailureIT', 
'org.apache.phoenix.end2end.index.ImmutableIndexIT', 
'org.apache.phoenix.end2end.StatsCollectorIT'])


Build step 'Execute shell' marked build as failure
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any