http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java index 9b28bad..b0ce8cd 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanQueryIT.java @@ -20,7 +20,6 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.util.TestUtil.A_VALUE; import static org.apache.phoenix.util.TestUtil.B_VALUE; import static org.apache.phoenix.util.TestUtil.C_VALUE; -import static org.apache.phoenix.util.TestUtil.E_VALUE; import static org.apache.phoenix.util.TestUtil.ROW1; import static org.apache.phoenix.util.TestUtil.ROW2; import static org.apache.phoenix.util.TestUtil.ROW3; @@ -39,10 +38,8 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; -import java.util.Arrays; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Properties; import java.util.Set; @@ -53,7 +50,6 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import com.google.common.collect.Lists; import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; @@ -66,13 +62,13 @@ public class ScanQueryIT extends BaseQueryIT { return QueryIT.data(); } - public ScanQueryIT(String indexDDL) { - super(indexDDL); + public ScanQueryIT(String indexDDL, boolean mutable, boolean columnEncoded) { + super(indexDDL, mutable, columnEncoded); } @Test public void testScan() throws Exception { - String query = "SELECT a_string, /* comment ok? */ b_string FROM aTable WHERE ?=organization_id and 5=a_integer"; + String query = "SELECT a_string, /* comment ok? */ b_string FROM " + tableName + " WHERE ?=organization_id and 5=a_integer"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -91,7 +87,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByByteValue() throws Exception { - String query = "SELECT a_string, b_string, a_byte FROM aTable WHERE ?=organization_id and 1=a_byte"; + String query = "SELECT a_string, b_string, a_byte FROM " + tableName + " WHERE ?=organization_id and 1=a_byte"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -111,7 +107,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByShortValue() throws Exception { - String query = "SELECT a_string, b_string, a_short FROM aTable WHERE ?=organization_id and 128=a_short"; + String query = "SELECT a_string, b_string, a_short FROM " + tableName + " WHERE ?=organization_id and 128=a_short"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -131,7 +127,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByFloatValue() throws Exception { - String query = "SELECT a_string, b_string, a_float FROM aTable WHERE ?=organization_id and ?=a_float"; + String query = "SELECT a_string, b_string, a_float FROM " + tableName + " WHERE ?=organization_id and ?=a_float"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -152,7 +148,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByUnsignedFloatValue() throws Exception { - String query = "SELECT a_string, b_string, a_unsigned_float FROM aTable WHERE ?=organization_id and ?=a_unsigned_float"; + String query = "SELECT a_string, b_string, a_unsigned_float FROM " + tableName + " WHERE ?=organization_id and ?=a_unsigned_float"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -173,7 +169,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByDoubleValue() throws Exception { - String query = "SELECT a_string, b_string, a_double FROM aTable WHERE ?=organization_id and ?=a_double"; + String query = "SELECT a_string, b_string, a_double FROM " + tableName + " WHERE ?=organization_id and ?=a_double"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -194,7 +190,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testScanByUnsigned_DoubleValue() throws Exception { - String query = "SELECT a_string, b_string, a_unsigned_double FROM aTable WHERE ?=organization_id and ?=a_unsigned_double"; + String query = "SELECT a_string, b_string, a_unsigned_double FROM " + tableName + " WHERE ?=organization_id and ?=a_unsigned_double"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -215,7 +211,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testAllScan() throws Exception { - String query = "SELECT ALL a_string, b_string FROM aTable WHERE ?=organization_id and 5=a_integer"; + String query = "SELECT ALL a_string, b_string FROM " + tableName + " WHERE ?=organization_id and 5=a_integer"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -234,7 +230,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testDistinctScan() throws Exception { - String query = "SELECT DISTINCT a_string FROM aTable WHERE organization_id=?"; + String query = "SELECT DISTINCT a_string FROM " + tableName + " WHERE organization_id=?"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -256,7 +252,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testDistinctLimitScan() throws Exception { - String query = "SELECT DISTINCT a_string FROM aTable WHERE organization_id=? LIMIT 1"; + String query = "SELECT DISTINCT a_string FROM " + tableName + " WHERE organization_id=? LIMIT 1"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -274,7 +270,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testInListSkipScan() throws Exception { - String query = "SELECT entity_id, b_string FROM aTable WHERE organization_id=? and entity_id IN (?,?)"; + String query = "SELECT entity_id, b_string FROM " + tableName + " WHERE organization_id=? and entity_id IN (?,?)"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -301,7 +297,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testUnboundRangeScan1() throws Exception { - String query = "SELECT entity_id FROM aTable WHERE organization_id <= ?"; + String query = "SELECT entity_id FROM " + tableName + " WHERE organization_id <= ?"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -335,7 +331,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testUnboundRangeScan2() throws Exception { - String query = "SELECT entity_id FROM aTable WHERE organization_id >= ?"; + String query = "SELECT entity_id FROM " + tableName + " WHERE organization_id >= ?"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -369,7 +365,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testUpperLowerBoundRangeScan() throws Exception { - String query = "SELECT entity_id FROM aTable WHERE organization_id=? and substr(entity_id,1,3) > '00A' and substr(entity_id,1,3) < '00C'"; + String query = "SELECT entity_id FROM " + tableName + " WHERE organization_id=? and substr(entity_id,1,3) > '00A' and substr(entity_id,1,3) < '00C'"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -393,7 +389,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testUpperBoundRangeScan() throws Exception { - String query = "SELECT entity_id FROM aTable WHERE organization_id=? and substr(entity_id,1,3) >= '00B' "; + String query = "SELECT entity_id FROM " + tableName + " WHERE organization_id=? and substr(entity_id,1,3) >= '00B' "; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -419,7 +415,7 @@ public class ScanQueryIT extends BaseQueryIT { @Test public void testLowerBoundRangeScan() throws Exception { - String query = "SELECT entity_id FROM aTable WHERE organization_id=? and substr(entity_id,1,3) < '00B' "; + String query = "SELECT entity_id FROM " + tableName + " WHERE organization_id=? and substr(entity_id,1,3) < '00B' "; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -440,57 +436,4 @@ public class ScanQueryIT extends BaseQueryIT { conn.close(); } } - - @SuppressWarnings("unchecked") - @Test - public void testPointInTimeLimitedScan() throws Exception { - // Override value that was set at creation time - String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 1); // Run query at timestamp 5 - Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); - Connection upsertConn = DriverManager.getConnection(url, props); - String upsertStmt = - "upsert into " + - "ATABLE(" + - " ORGANIZATION_ID, " + - " ENTITY_ID, " + - " A_INTEGER) " + - "VALUES (?, ?, ?)"; - upsertConn.setAutoCommit(true); // Test auto commit - // Insert all rows at ts - PreparedStatement stmt = upsertConn.prepareStatement(upsertStmt); - stmt.setString(1, tenantId); - stmt.setString(2, ROW1); - stmt.setInt(3, 6); - stmt.execute(); // should commit too - upsertConn.close(); - - // Override value again, but should be ignored since it's past the SCN - url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 3); - upsertConn = DriverManager.getConnection(url, props); - upsertConn.setAutoCommit(true); // Test auto commit - // Insert all rows at ts - stmt = upsertConn.prepareStatement(upsertStmt); - stmt.setString(1, tenantId); - stmt.setString(2, ROW1); - stmt.setInt(3, 0); - stmt.execute(); // should commit too - upsertConn.close(); - - String query = "SELECT a_integer,b_string FROM atable WHERE organization_id=? and a_integer <= 5 limit 2"; - props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); - Connection conn = DriverManager.getConnection(getUrl(), props); - PreparedStatement statement = conn.prepareStatement(query); - statement.setString(1, tenantId); - ResultSet rs = statement.executeQuery(); - List<List<Object>> expectedResultsA = Lists.newArrayList( - Arrays.<Object>asList(2, C_VALUE), - Arrays.<Object>asList( 3, E_VALUE)); - List<List<Object>> expectedResultsB = Lists.newArrayList( - Arrays.<Object>asList( 5, C_VALUE), - Arrays.<Object>asList(4, B_VALUE)); - // Since we're not ordering and we may be using a descending index, we don't - // know which rows we'll get back. - assertOneOfValuesEqualsResultSet(rs, expectedResultsA,expectedResultsB); - conn.close(); - } }
http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java index 85fc9d7..040c92c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java @@ -40,6 +40,10 @@ import java.util.Random; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; @@ -48,6 +52,7 @@ import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.stats.GuidePostsKey; @@ -58,6 +63,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -68,20 +74,54 @@ import com.google.common.collect.Maps; @RunWith(Parameterized.class) public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { private final String tableDDLOptions; + private final boolean columnEncoded; private String tableName; private String schemaName; private String fullTableName; private String physicalTableName; private final boolean userTableNamespaceMapped; + private final boolean mutable; - public StatsCollectorIT(boolean transactional, boolean userTableNamespaceMapped) { - this.tableDDLOptions= transactional ? " TRANSACTIONAL=true" : ""; + public StatsCollectorIT(boolean mutable, boolean transactional, boolean userTableNamespaceMapped, boolean columnEncoded) { + StringBuilder sb = new StringBuilder(); + if (transactional) { + sb.append("TRANSACTIONAL=true"); + } + if (!columnEncoded) { + if (sb.length()>0) { + sb.append(","); + } + sb.append("COLUMN_ENCODED_BYTES=0"); + } else { + if (sb.length()>0) { + sb.append(","); + } + sb.append("COLUMN_ENCODED_BYTES=4"); + } + if (!mutable) { + if (sb.length()>0) { + sb.append(","); + } + sb.append("IMMUTABLE_ROWS=true"); + if (!columnEncoded) { + sb.append(",IMMUTABLE_STORAGE_SCHEME="+PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN); + } + } + this.tableDDLOptions = sb.toString(); this.userTableNamespaceMapped = userTableNamespaceMapped; + this.columnEncoded = columnEncoded; + this.mutable = mutable; } - @Parameters(name="transactional = {0}, isUserTableNamespaceMapped = {1}") + @Parameters(name="columnEncoded = {0}, mutable = {1}, transactional = {2}, isUserTableNamespaceMapped = {3}") public static Collection<Boolean[]> data() { - return Arrays.asList(new Boolean[][] {{false,true}, {false, false}, {true, false}, {true, true}}); + return Arrays.asList(new Boolean[][] { + { false, false, false, false }, { false, false, false, true }, { false, false, true, false }, { false, false, true, true }, + // no need to test non column encoded mutable case and this is the same as non column encoded immutable + //{ false, true, false, false }, { false, true, false, true }, { false, true, true, false }, { false, true, true, true }, + { true, false, false, false }, { true, false, false, true }, { true, false, true, false }, { true, false, true, true }, + { true, true, false, false }, { true, true, false, true }, { true, true, true, false }, { true, true, true, true } + }); } @BeforeClass @@ -147,25 +187,28 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { "CREATE TABLE " + fullTableName +" ( k VARCHAR PRIMARY KEY, a.v1 VARCHAR, b.v2 VARCHAR ) " + tableDDLOptions + (tableDDLOptions.isEmpty() ? "" : ",") + "SALT_BUCKETS = 3"); conn.createStatement().execute("UPSERT INTO " + fullTableName + "(k,v1) VALUES('a','123456789')"); conn.createStatement().execute("UPDATE STATISTICS " + fullTableName); + ResultSet rs; String explainPlan; rs = conn.createStatement().executeQuery("EXPLAIN SELECT v2 FROM " + fullTableName + " WHERE v2='foo'"); explainPlan = QueryUtil.getExplainPlan(rs); + // if we are using the ONE_CELL_PER_COLUMN_FAMILY storage scheme, we will have the single kv even though there are no values for col family v2 + String stats = columnEncoded && !mutable ? "4-CHUNK 1 ROWS 38 BYTES" : "3-CHUNK 0 ROWS 0 BYTES"; assertEquals( - "CLIENT 3-CHUNK 0 ROWS 0 BYTES PARALLEL 3-WAY FULL SCAN OVER " + physicalTableName + "\n" + + "CLIENT " + stats + " PARALLEL 3-WAY FULL SCAN OVER " + physicalTableName + "\n" + " SERVER FILTER BY B.V2 = 'foo'\n" + "CLIENT MERGE SORT", explainPlan); rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName); explainPlan = QueryUtil.getExplainPlan(rs); assertEquals( - "CLIENT 4-CHUNK 1 ROWS 34 BYTES PARALLEL 3-WAY FULL SCAN OVER " + physicalTableName + "\n" + + "CLIENT 4-CHUNK 1 ROWS " + (columnEncoded ? "28" : "34") + " BYTES PARALLEL 3-WAY FULL SCAN OVER " + physicalTableName + "\n" + "CLIENT MERGE SORT", explainPlan); rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName + " WHERE k = 'a'"); explainPlan = QueryUtil.getExplainPlan(rs); assertEquals( - "CLIENT 1-CHUNK 1 ROWS 202 BYTES PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER " + physicalTableName + "\n" + + "CLIENT 1-CHUNK 1 ROWS " + (columnEncoded ? "204" : "202") + " BYTES PARALLEL 1-WAY POINT LOOKUP ON 1 KEY OVER " + physicalTableName + "\n" + "CLIENT MERGE SORT", explainPlan); @@ -368,11 +411,13 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { } @Test + @Ignore //TODO remove this once https://issues.apache.org/jira/browse/TEPHRA-208 is fixed public void testCompactUpdatesStats() throws Exception { testCompactUpdatesStats(0, fullTableName); } @Test + @Ignore //TODO remove this once https://issues.apache.org/jira/browse/TEPHRA-208 is fixed public void testCompactUpdatesStatsWithMinStatsUpdateFreq() throws Exception { testCompactUpdatesStats(QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS, fullTableName); } @@ -390,6 +435,7 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { Connection conn = getConnection(statsUpdateFreq); PreparedStatement stmt; conn.createStatement().execute("CREATE TABLE " + tableName + "(k CHAR(1) PRIMARY KEY, v INTEGER, w INTEGER) " + + (!tableDDLOptions.isEmpty() ? tableDDLOptions + "," : "") + HColumnDescriptor.KEEP_DELETED_CELLS + "=" + Boolean.FALSE); stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?)"); for (int i = 0; i < nRows; i++) { @@ -399,11 +445,13 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { stmt.executeUpdate(); } conn.commit(); + compactTable(conn, physicalTableName); - if (statsUpdateFreq == null) { + + if (statsUpdateFreq != 0) { invalidateStats(conn, tableName); } else { - // Confirm that when we have a non zero MIN_STATS_UPDATE_FREQ_MS_ATTRIB, after we run + // Confirm that when we have a non zero STATS_UPDATE_FREQ_MS_ATTRIB, after we run // UPDATATE STATISTICS, the new statistics are faulted in as expected. List<KeyRange>keyRanges = getAllSplits(conn, tableName); assertNotEquals(nRows+1, keyRanges.size()); @@ -419,20 +467,40 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { conn.commit(); assertEquals(5, nDeletedRows); + Scan scan = new Scan(); + scan.setRaw(true); + PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class); + try (HTableInterface htable = phxConn.getQueryServices().getTable(Bytes.toBytes(tableName))) { + ResultScanner scanner = htable.getScanner(scan); + Result result; + while ((result = scanner.next())!=null) { + System.out.println(result); + } + } + compactTable(conn, physicalTableName); - if (statsUpdateFreq == null) { - invalidateStats(conn, tableName); + + scan = new Scan(); + scan.setRaw(true); + phxConn = conn.unwrap(PhoenixConnection.class); + try (HTableInterface htable = phxConn.getQueryServices().getTable(Bytes.toBytes(tableName))) { + ResultScanner scanner = htable.getScanner(scan); + Result result; + while ((result = scanner.next())!=null) { + System.out.println(result); + } } - keyRanges = getAllSplits(conn, tableName); - if (statsUpdateFreq != null) { + if (statsUpdateFreq != 0) { + invalidateStats(conn, tableName); + } else { assertEquals(nRows+1, keyRanges.size()); - // If we've set MIN_STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache + // If we've set STATS_UPDATE_FREQ_MS_ATTRIB, an UPDATE STATISTICS will invalidate the cache // and force us to pull over the new stats int rowCount = conn.createStatement().executeUpdate("UPDATE STATISTICS " + tableName); assertEquals(5, rowCount); - keyRanges = getAllSplits(conn, tableName); } + keyRanges = getAllSplits(conn, tableName); assertEquals(nRows/2+1, keyRanges.size()); ResultSet rs = conn.createStatement().executeQuery("SELECT SUM(GUIDE_POSTS_ROW_COUNT) FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " WHERE PHYSICAL_NAME='" + physicalTableName + "'"); @@ -447,7 +515,8 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { PreparedStatement stmt; conn.createStatement().execute( "CREATE TABLE " + fullTableName - + "(k VARCHAR PRIMARY KEY, a.v INTEGER, b.v INTEGER, c.v INTEGER NULL, d.v INTEGER NULL) "); + + "(k VARCHAR PRIMARY KEY, a.v INTEGER, b.v INTEGER, c.v INTEGER NULL, d.v INTEGER NULL) " + + tableDDLOptions ); stmt = conn.prepareStatement("UPSERT INTO " + fullTableName + " VALUES(?,?, ?, ?, ?)"); byte[] val = new byte[250]; for (int i = 0; i < nRows; i++) { @@ -473,7 +542,7 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { List<KeyRange> keyRanges = getAllSplits(conn, fullTableName); assertEquals(26, keyRanges.size()); rs = conn.createStatement().executeQuery("EXPLAIN SELECT * FROM " + fullTableName); - assertEquals("CLIENT 26-CHUNK 25 ROWS 12420 BYTES PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName, + assertEquals("CLIENT 26-CHUNK 25 ROWS " + (columnEncoded ? ( mutable ? "12530" : "13902" ) : "12420") + " BYTES PARALLEL 1-WAY FULL SCAN OVER " + physicalTableName, QueryUtil.getExplainPlan(rs)); ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices(); @@ -485,7 +554,8 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB + "\"=" + Long.toString(1000); conn.createStatement().execute(query); keyRanges = getAllSplits(conn, fullTableName); - assertEquals(12, keyRanges.size()); + boolean oneCellPerColFamliyStorageScheme = !mutable && columnEncoded; + assertEquals(oneCellPerColFamliyStorageScheme ? 13 : 12, keyRanges.size()); rs = conn .createStatement() @@ -496,25 +566,25 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { assertTrue(rs.next()); assertEquals("A", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(12144, rs.getInt(3)); - assertEquals(11, rs.getInt(4)); + assertEquals(columnEncoded ? ( mutable ? 12252 : 13624 ) : 12144, rs.getInt(3)); + assertEquals(oneCellPerColFamliyStorageScheme ? 12 : 11, rs.getInt(4)); assertTrue(rs.next()); assertEquals("B", rs.getString(1)); - assertEquals(20, rs.getInt(2)); - assertEquals(5540, rs.getInt(3)); - assertEquals(5, rs.getInt(4)); + assertEquals(oneCellPerColFamliyStorageScheme ? 24 : 20, rs.getInt(2)); + assertEquals(columnEncoded ? ( mutable ? 5600 : 6972 ) : 5540, rs.getInt(3)); + assertEquals(oneCellPerColFamliyStorageScheme ? 6 : 5, rs.getInt(4)); assertTrue(rs.next()); assertEquals("C", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(6652, rs.getInt(3)); + assertEquals(columnEncoded ? ( mutable ? 6724 : 6988 ) : 6652, rs.getInt(3)); assertEquals(6, rs.getInt(4)); assertTrue(rs.next()); assertEquals("D", rs.getString(1)); assertEquals(24, rs.getInt(2)); - assertEquals(6652, rs.getInt(3)); + assertEquals(columnEncoded ? ( mutable ? 6724 : 6988 ) : 6652, rs.getInt(3)); assertEquals(6, rs.getInt(4)); assertFalse(rs.next()); @@ -539,7 +609,7 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { Connection conn = getConnection(); String ddl = "CREATE TABLE " + fullTableName + " (t_id VARCHAR NOT NULL,\n" + "k1 INTEGER NOT NULL,\n" + "k2 INTEGER NOT NULL,\n" + "C3.k3 INTEGER,\n" + "C2.v1 VARCHAR,\n" - + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2)) split on ('e','j','o')"; + + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2)) " + tableDDLOptions + " split on ('e','j','o')"; conn.createStatement().execute(ddl); String[] strings = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" }; @@ -559,7 +629,7 @@ public class StatsCollectorIT extends BaseUniqueNamesOwnClusterIT { int startIndex = r.nextInt(strings.length); int endIndex = r.nextInt(strings.length - startIndex) + startIndex; long rows = endIndex - startIndex; - long c2Bytes = rows * 35; + long c2Bytes = rows * (columnEncoded ? ( mutable ? 37 : 48 ) : 35); String physicalTableName = SchemaUtil.getPhysicalHBaseTableName(fullTableName, userTableNamespaceMapped, PTableType.TABLE).getString(); rs = conn.createStatement().executeQuery( "SELECT COLUMN_FAMILY,SUM(GUIDE_POSTS_ROW_COUNT),SUM(GUIDE_POSTS_WIDTH) from SYSTEM.STATS where PHYSICAL_NAME = '" http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java index bb13f1b..a37903f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsIT.java @@ -22,29 +22,38 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; -import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.Properties; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.expression.KeyValueColumnExpression; +import org.apache.phoenix.expression.SingleCellColumnExpression; +import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTable.ImmutableStorageScheme; +import org.apache.phoenix.schema.PTableImpl; +import org.apache.phoenix.schema.PTableKey; +import org.apache.phoenix.schema.tuple.ResultTuple; +import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PhoenixRuntime; -import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * Tests to demonstrate and verify the STORE_NULLS option on a table, @@ -52,74 +61,104 @@ import org.junit.Test; * functionality allows having row-level versioning (similar to how KEEP_DELETED_CELLS works), but * also allows permanently deleting a row. */ +@RunWith(Parameterized.class) public class StoreNullsIT extends ParallelStatsDisabledIT { - private static final Log LOG = LogFactory.getLog(StoreNullsIT.class); - private String WITH_NULLS; - private String WITHOUT_NULLS; - private String IMMUTABLE_WITH_NULLS; - private String IMMUTABLE_WITHOUT_NULLS; - private Connection conn; - private Statement stmt; - - @Before - public void setUp() throws SQLException { - WITH_NULLS = generateUniqueName(); - WITHOUT_NULLS = generateUniqueName(); - IMMUTABLE_WITH_NULLS = generateUniqueName(); - IMMUTABLE_WITHOUT_NULLS = generateUniqueName(); - conn = DriverManager.getConnection(getUrl()); - conn.setAutoCommit(true); - - stmt = conn.createStatement(); - stmt.execute("CREATE TABLE " + WITH_NULLS + " (" + - "id SMALLINT NOT NULL PRIMARY KEY, " + - "name VARCHAR) " + - "STORE_NULLS = true, VERSIONS = 1000, KEEP_DELETED_CELLS = false"); - stmt.execute("CREATE TABLE " + WITHOUT_NULLS + " (" + - "id SMALLINT NOT NULL PRIMARY KEY, " + - "name VARCHAR) " + - "VERSIONS = 1000, KEEP_DELETED_CELLS = false"); - stmt.execute("CREATE TABLE " + IMMUTABLE_WITH_NULLS + " (" - + "id SMALLINT NOT NULL PRIMARY KEY, name VARCHAR) " - + "STORE_NULLS = true, VERSIONS = 1, KEEP_DELETED_CELLS = false, IMMUTABLE_ROWS=true"); - stmt.execute("CREATE TABLE " + IMMUTABLE_WITHOUT_NULLS + " (" - + "id SMALLINT NOT NULL PRIMARY KEY, name VARCHAR) " - + "VERSIONS = 1, KEEP_DELETED_CELLS = false, IMMUTABLE_ROWS=true"); + private final boolean mutable; + private final boolean columnEncoded; + private final boolean storeNulls; + private final String ddlFormat; + + private String dataTableName; + + public StoreNullsIT(boolean mutable, boolean columnEncoded, boolean storeNulls) { + this.mutable = mutable; + this.columnEncoded = columnEncoded; + this.storeNulls = storeNulls; + + StringBuilder sb = new StringBuilder("CREATE TABLE %s (id SMALLINT NOT NULL PRIMARY KEY, name VARCHAR) VERSIONS = 1000, KEEP_DELETED_CELLS = false "); + if (!columnEncoded) { + sb.append(",").append("COLUMN_ENCODED_BYTES=0"); + } + if (!mutable) { + sb.append(",").append("IMMUTABLE_ROWS=true"); + if (!columnEncoded) { + sb.append(",IMMUTABLE_STORAGE_SCHEME="+PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN); + } + } + if (storeNulls) { + sb.append(",").append("STORE_NULLS=true"); + } + this.ddlFormat = sb.toString(); } - - @After - public void tearDown() throws SQLException { - stmt.close(); - conn.close(); + + @Parameters(name="StoreNullsIT_mutable={0}, columnEncoded={1}, storeNulls={2}") // name is used by failsafe as file name in reports + public static Collection<Boolean[]> data() { + return Arrays.asList(new Boolean[][] { + { false, false, false }, { false, false, true }, + { false, true, false }, { false, true, true }, + { true, false, false }, { true, false, true }, + { true, true, false }, { true, true, true }}); + } + + + @Before + public void setupTableNames() throws Exception { + dataTableName = generateUniqueName(); } @Test - public void testStoringNulls() throws SQLException, InterruptedException, IOException { - stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITH_NULLS + " VALUES (1, 'v1')"); - stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITHOUT_NULLS + " VALUES (1, 'v1')"); - stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITH_NULLS + " VALUES (2, null)"); - stmt.executeUpdate("UPSERT INTO " + IMMUTABLE_WITHOUT_NULLS + " VALUES (2, null)"); - - ensureNullsNotStored(IMMUTABLE_WITH_NULLS); - ensureNullsNotStored(IMMUTABLE_WITHOUT_NULLS); + public void testStoringNullsForImmutableTables() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + conn.setAutoCommit(true); + stmt.execute(String.format(ddlFormat, dataTableName)); + stmt.executeUpdate("UPSERT INTO " + dataTableName + " VALUES (1, 'v1')"); + stmt.executeUpdate("UPSERT INTO " + dataTableName + " VALUES (2, null)"); + TestUtil.doMajorCompaction(conn, dataTableName); + ensureNullsStoredCorrectly(conn); + } } - private void ensureNullsNotStored(String tableName) throws IOException { - tableName = SchemaUtil.normalizeIdentifier(tableName); - HTable htable = new HTable(getUtility().getConfiguration(), tableName); + private void ensureNullsStoredCorrectly(Connection conn) throws Exception { + ResultSet rs1 = conn.createStatement().executeQuery("SELECT NAME FROM "+dataTableName); + rs1.next(); + assertEquals("v1", rs1.getString(1)); + rs1.next(); + assertNull(rs1.getString(1)); + rs1.next(); + + HTable htable = new HTable(getUtility().getConfiguration(), dataTableName); Scan s = new Scan(); s.setRaw(true); ResultScanner scanner = htable.getScanner(s); // first row has a value for name Result rs = scanner.next(); - assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("NAME"))); - assertTrue(rs.size() == 2); - // 2nd row has not + PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, dataTableName)); + PColumn nameColumn = table.getColumnForColumnName("NAME"); + byte[] qualifier = table.getImmutableStorageScheme()== ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? QueryConstants.SINGLE_KEYVALUE_COLUMN_QUALIFIER_BYTES : nameColumn.getColumnQualifierBytes(); + assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, qualifier)); + assertTrue(rs.size() == 2); // 2 because it also includes the empty key value column + KeyValueColumnExpression colExpression = table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS ? new SingleCellColumnExpression(nameColumn, "NAME", table.getEncodingScheme()) : new KeyValueColumnExpression(nameColumn); + ImmutableBytesPtr ptr = new ImmutableBytesPtr(); + colExpression.evaluate(new ResultTuple(rs), ptr); + assertEquals(new ImmutableBytesPtr(PVarchar.INSTANCE.toBytes("v1")), ptr); rs = scanner.next(); - assertFalse(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, Bytes.toBytes("NAME"))); - // and no delete marker either - assertTrue(rs.size() == 1); + + if ( !mutable && !columnEncoded // we don't issue a put with empty value for immutable tables with cols stored per key value + || (mutable && !storeNulls)) { // for this case we use a delete to represent the null + assertFalse(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, qualifier)); + assertEquals(1, rs.size()); + } + else { + assertTrue(rs.containsColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, qualifier)); + assertEquals(2, rs.size()); + } + // assert null stored correctly + ptr = new ImmutableBytesPtr(); + if (colExpression.evaluate(new ResultTuple(rs), ptr)) { + assertEquals(new ImmutableBytesPtr(ByteUtil.EMPTY_BYTE_ARRAY), ptr); + } assertNull(scanner.next()); scanner.close(); htable.close(); @@ -127,93 +166,80 @@ public class StoreNullsIT extends ParallelStatsDisabledIT { @Test public void testQueryingHistory() throws Exception { - stmt.executeUpdate("UPSERT INTO " + WITH_NULLS + " VALUES (1, 'v1')"); - stmt.executeUpdate("UPSERT INTO " + WITHOUT_NULLS + " VALUES (1, 'v1')"); - - Thread.sleep(10L); - long afterFirstInsert = System.currentTimeMillis(); - Thread.sleep(10L); - - stmt.executeUpdate("UPSERT INTO " + WITH_NULLS + " VALUES (1, null)"); - stmt.executeUpdate("UPSERT INTO " + WITHOUT_NULLS + " VALUES (1, null)"); - Thread.sleep(10L); - - TestUtil.doMajorCompaction(conn, WITH_NULLS); - TestUtil.doMajorCompaction(conn, WITHOUT_NULLS); - - Properties historicalProps = new Properties(); - historicalProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + conn.setAutoCommit(true); + stmt.execute(String.format(ddlFormat, dataTableName)); + stmt.executeUpdate("UPSERT INTO " + dataTableName + " VALUES (1, 'v1')"); + Thread.sleep(10L); + long afterFirstInsert = System.currentTimeMillis(); + Thread.sleep(10L); + + stmt.executeUpdate("UPSERT INTO " + dataTableName + " VALUES (1, null)"); + Thread.sleep(10L); + + TestUtil.doMajorCompaction(conn, dataTableName); + + Properties historicalProps = new Properties(); + historicalProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(afterFirstInsert)); - Connection historicalConn = DriverManager.getConnection(getUrl(), historicalProps); - Statement historicalStmt = historicalConn.createStatement(); - - ResultSet rs = historicalStmt.executeQuery( - "SELECT name FROM " + WITH_NULLS + " WHERE id = 1"); - assertTrue(rs.next()); - assertEquals("v1", rs.getString(1)); - rs.close(); - - // The single null wipes out all history for a field if STORE_NULLS is not enabled - rs = historicalStmt.executeQuery("SELECT name FROM " + WITHOUT_NULLS + " WHERE id = 1"); - assertTrue(rs.next()); - assertNull(rs.getString(1)); - rs.close(); + Connection historicalConn = DriverManager.getConnection(getUrl(), historicalProps); + Statement historicalStmt = historicalConn.createStatement(); + ResultSet rs = historicalStmt.executeQuery( "SELECT name FROM " + dataTableName + " WHERE id = 1"); + + if (storeNulls || !mutable) { // store nulls is set to true if the table is immutable + assertTrue(rs.next()); + assertEquals("v1", rs.getString(1)); + rs.close(); + } + else { + // The single null wipes out all history for a field if STORE_NULLS is not enabled + assertTrue(rs.next()); + assertNull(rs.getString(1)); + } + + rs.close(); + historicalStmt.close(); + historicalConn.close(); + } - historicalStmt.close(); - historicalConn.close(); } // Row deletes should work in the same way regardless of what STORE_NULLS is set to @Test public void testDeletes() throws Exception { - stmt.executeUpdate("UPSERT INTO " + WITH_NULLS + " VALUES (1, 'v1')"); - stmt.executeUpdate("UPSERT INTO " + WITHOUT_NULLS + " VALUES (1, 'v1')"); - - Thread.sleep(10L); - long afterFirstInsert = System.currentTimeMillis(); - Thread.sleep(10L); - - stmt.executeUpdate("DELETE FROM " + WITH_NULLS + " WHERE id = 1"); - stmt.executeUpdate("DELETE FROM " + WITHOUT_NULLS + " WHERE id = 1"); - Thread.sleep(10L); - - TestUtil.doMajorCompaction(conn, WITH_NULLS); - TestUtil.doMajorCompaction(conn, WITHOUT_NULLS); - - Properties historicalProps = new Properties(); - historicalProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, - Long.toString(afterFirstInsert)); - Connection historicalConn = DriverManager.getConnection(getUrl(), historicalProps); - Statement historicalStmt = historicalConn.createStatement(); - - // The row should be completely gone for both tables now - - ResultSet rs = historicalStmt.executeQuery( - "SELECT name FROM " + WITH_NULLS + " WHERE id = 1"); - assertFalse(rs.next()); - rs.close(); - - rs = historicalStmt.executeQuery("SELECT name FROM " + WITHOUT_NULLS + " WHERE id = 1"); - assertFalse(rs.next()); - rs.close(); - } - - @Test - public void testSetStoreNullsDefaultViaConfig() throws SQLException { - Properties props = new Properties(); - props.setProperty(QueryServices.DEFAULT_STORE_NULLS_ATTRIB, "true"); - Connection storeNullsConn = DriverManager.getConnection(getUrl(), props); - - Statement stmt = storeNullsConn.createStatement(); - stmt.execute("CREATE TABLE with_nulls_default (" + - "id smallint primary key," + - "name varchar)"); - - ResultSet rs = stmt.executeQuery("SELECT store_nulls FROM SYSTEM.CATALOG " + - "WHERE table_name = 'WITH_NULLS_DEFAULT' AND store_nulls is not null"); - assertTrue(rs.next()); - assertTrue(rs.getBoolean(1)); + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + conn.setAutoCommit(true); + stmt.execute(String.format(ddlFormat, dataTableName)); + stmt.executeUpdate("UPSERT INTO " + dataTableName + " VALUES (1, 'v1')"); + + Thread.sleep(10L); + long afterFirstInsert = System.currentTimeMillis(); + Thread.sleep(10L); + + stmt.executeUpdate("DELETE FROM " + dataTableName + " WHERE id = 1"); + Thread.sleep(10L); + + TestUtil.doMajorCompaction(conn, dataTableName); + + Properties historicalProps = new Properties(); + historicalProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, + Long.toString(afterFirstInsert)); + Connection historicalConn = DriverManager.getConnection(getUrl(), historicalProps); + Statement historicalStmt = historicalConn.createStatement(); + + // The row should be completely gone for both tables now + + ResultSet rs = historicalStmt.executeQuery( + "SELECT name FROM " + dataTableName + " WHERE id = 1"); + assertFalse(rs.next()); + rs.close(); + + rs = historicalStmt.executeQuery("SELECT name FROM " + dataTableName + " WHERE id = 1"); + assertFalse(rs.next()); + rs.close(); + } } - } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsPropIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsPropIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsPropIT.java new file mode 100644 index 0000000..26ff629 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StoreNullsPropIT.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.junit.Assert.assertTrue; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import org.apache.phoenix.query.QueryServices; +import org.junit.Test; + +public class StoreNullsPropIT extends ParallelStatsDisabledIT { + + @Test + public void testSetStoreNullsDefaultViaConfig() throws SQLException { + Properties props = new Properties(); + props.setProperty(QueryServices.DEFAULT_STORE_NULLS_ATTRIB, "true"); + Connection storeNullsConn = DriverManager.getConnection(getUrl(), props); + + Statement stmt = storeNullsConn.createStatement(); + stmt.execute("CREATE TABLE with_nulls_default (" + + "id smallint primary key," + + "name varchar)"); + + ResultSet rs = stmt.executeQuery("SELECT store_nulls FROM SYSTEM.CATALOG " + + "WHERE table_name = 'WITH_NULLS_DEFAULT' AND store_nulls is not null"); + assertTrue(rs.next()); + assertTrue(rs.getBoolean(1)); + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java index 6b394c1..f9ef0c2 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SysTableNamespaceMappedStatsCollectorIT.java @@ -27,8 +27,8 @@ import com.google.common.collect.Maps; public class SysTableNamespaceMappedStatsCollectorIT extends StatsCollectorIT { - public SysTableNamespaceMappedStatsCollectorIT(boolean transactional, boolean userTableNamespaceMapped) { - super(transactional, userTableNamespaceMapped); + public SysTableNamespaceMappedStatsCollectorIT(boolean mutable, boolean transactional, boolean userTableNamespaceMapped, boolean columnEncoded) { + super(mutable, transactional, userTableNamespaceMapped, columnEncoded); } @BeforeClass http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/TopNIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TopNIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TopNIT.java index ca1cd86..39e8cb6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TopNIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TopNIT.java @@ -50,7 +50,7 @@ public class TopNIT extends BaseClientManagedTimeIT { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl(), null); String query = "SELECT entity_id FROM aTable ORDER BY b_string, entity_id LIMIT 5"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 @@ -80,7 +80,7 @@ public class TopNIT extends BaseClientManagedTimeIT { public void testDescMultiOrderByExpr() throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl(), null); String query = "SELECT entity_id FROM aTable ORDER BY b_string || entity_id desc LIMIT 5"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 @@ -119,7 +119,7 @@ public class TopNIT extends BaseClientManagedTimeIT { private void testTopNDelete(boolean autoCommit) throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts, getUrl(), null); String query = "DELETE FROM aTable ORDER BY b_string, entity_id LIMIT 5"; Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2 http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java index f5905ee..d36e0fe 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java @@ -89,7 +89,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT { private void testUpsertSelect(boolean createIndex) throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl(), null); ensureTableCreated(getUrl(), CUSTOM_ENTITY_DATA_FULL_NAME, CUSTOM_ENTITY_DATA_FULL_NAME, ts-1); String indexName = "IDX1"; @@ -210,7 +210,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT { public void testUpsertSelectEmptyPKColumn() throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl(), null); ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); // Execute at timestamp 1 @@ -386,7 +386,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT { private void testUpsertSelectForAgg(boolean autoCommit) throws Exception { long ts = nextTimestamp(); String tenantId = getOrganizationId(); - initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl()); + initATableValues(ATABLE_NAME, tenantId, getDefaultSplits(tenantId), null, ts-1, getUrl(), null); ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, ts-1); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); // Execute at timestamp 1 @@ -462,7 +462,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT { byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)}; long ts = nextTimestamp(); - ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2); + ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits, ts-2, null); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); Connection conn = DriverManager.getConnection(getUrl(), props); @@ -602,7 +602,7 @@ public class UpsertSelectIT extends BaseClientManagedTimeIT { byte[][] splits = new byte[][] { PInteger.INSTANCE.toBytes(1), PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3), PInteger.INSTANCE.toBytes(4)}; long ts = nextTimestamp(); - ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2); + ensureTableCreated(getUrl(),"IntKeyTest", "IntKeyTest", splits,ts-2, null); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); Connection conn = DriverManager.getConnection(getUrl(), props); http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java index 64935d2..11df167 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertValuesIT.java @@ -35,15 +35,26 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Time; import java.sql.Timestamp; +import java.util.Map.Entry; +import java.util.NavigableMap; import java.util.Properties; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.BaseTest; +import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.util.DateUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -52,7 +63,7 @@ public class UpsertValuesIT extends BaseClientManagedTimeIT { @Test public void testGroupByWithLimitOverRowKey() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),TestUtil.PTSDB_NAME,TestUtil.PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),TestUtil.PTSDB_NAME,TestUtil.PTSDB_NAME, null, ts-2, null); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10)); Connection conn = DriverManager.getConnection(getUrl(), props); @@ -85,7 +96,7 @@ public class UpsertValuesIT extends BaseClientManagedTimeIT { public void testUpsertDateValues() throws Exception { long ts = nextTimestamp(); Date now = new Date(System.currentTimeMillis()); - ensureTableCreated(getUrl(),TestUtil.PTSDB_NAME,TestUtil.PTSDB_NAME,null, ts-2); + ensureTableCreated(getUrl(),TestUtil.PTSDB_NAME,TestUtil.PTSDB_NAME,null, ts-2, null); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); // Execute at timestamp 1 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -114,7 +125,7 @@ public class UpsertValuesIT extends BaseClientManagedTimeIT { @Test public void testUpsertValuesWithExpression() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),"IntKeyTest","IntKeyTest", null, ts-2); + ensureTableCreated(getUrl(),"IntKeyTest","IntKeyTest", null, ts-2, null); Properties props = new Properties(); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 1)); // Execute at timestamp 1 Connection conn = DriverManager.getConnection(getUrl(), props); @@ -847,7 +858,7 @@ public class UpsertValuesIT extends BaseClientManagedTimeIT { assertEquals("KV2", rs.getString(2)); assertFalse(rs.next()); - // Verify now that the data was correctly added to the mutable index too. + // Verify now that the data was correctly added to the immutable index too. stmt = conn.prepareStatement("SELECT KV2 FROM " + tableName + " WHERE PK2 = ? AND KV1 = ?"); stmt.setDate(1, upsertedDate); stmt.setString(2, "KV1"); @@ -960,6 +971,38 @@ public class UpsertValuesIT extends BaseClientManagedTimeIT { } } + public void testColumnQualifierForUpsertedValues() throws Exception { + String schemaName = "A"; + String tableName = "TEST"; + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); + String ddl = "create table " + fullTableName + + " (" + + " K varchar primary key," + + " CF1.V1 varchar, CF2.V2 VARCHAR, CF2.V3 VARCHAR)"; + try (Connection conn = getConnection(nextTimestamp())) { + conn.createStatement().execute(ddl); + } + String dml = "UPSERT INTO " + fullTableName + " VALUES (?, ?, ?, ?)"; + try (Connection conn = getConnection(nextTimestamp())) { + PreparedStatement stmt = conn.prepareStatement(dml); + stmt.setString(1, "KEY1"); + stmt.setString(2, "VALUE1"); + stmt.setString(3, "VALUE2"); + stmt.setString(4, "VALUE3"); + stmt.executeUpdate(); + conn.commit(); + } + // Issue a raw hbase scan and assert that key values have the expected column qualifiers. + try (Connection conn = getConnection(nextTimestamp())) { + HTableInterface table = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(fullTableName)); + ResultScanner scanner = table.getScanner(new Scan()); + Result next = scanner.next(); + assertTrue(next.containsColumn(Bytes.toBytes("CF1"), PInteger.INSTANCE.toBytes(1))); + assertTrue(next.containsColumn(Bytes.toBytes("CF2"), PInteger.INSTANCE.toBytes(2))); + assertTrue(next.containsColumn(Bytes.toBytes("CF2"), PInteger.INSTANCE.toBytes(3))); + } + } + private static Connection getConnection(long ts) throws SQLException { Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java index 8b5a591..0b54e73 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UserDefinedFunctionsIT.java @@ -141,7 +141,7 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT { .append(" throw new ParseException(\"Index cannot be negative :\" + index);\n") .append(" }\n") .append(" Expression arrayExpr = children.get(0);\n") - .append(" return PArrayDataType.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(),getMaxLength());\n") + .append(" return PArrayDataTypeDecoder.positionAtArrayElement(tuple, ptr, index, arrayExpr, getDataType(),getMaxLength());\n") .append(" }\n").toString(); private static String GETY_EVALUATE_METHOD = @@ -217,6 +217,7 @@ public class UserDefinedFunctionsIT extends BaseOwnClusterIT { .append("import org.apache.phoenix.schema.types.PVarchar;\n") .append("import org.apache.phoenix.util.StringUtil;\n") .append("import org.apache.phoenix.schema.types.PArrayDataType;\n") + .append("import org.apache.phoenix.schema.types.PArrayDataTypeDecoder;\n") .append("import org.apache.phoenix.parse.ParseException;\n") .append("public class "+className+" extends ScalarFunction{\n") .append(" public static final String NAME = \""+className+"\";\n") http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java index 6a62673..753f2c8 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/VariableLengthPKIT.java @@ -58,7 +58,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { } protected static void initGroupByRowKeyColumns(long ts) throws Exception { - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; @@ -85,7 +85,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { } protected static void initTableValues(byte[][] splits, long ts) throws Exception { - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, splits, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, splits, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; @@ -106,7 +106,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { stmt.setBigDecimal(4, new BigDecimal(.5)); stmt.execute(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, splits, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, splits, ts-2, null); conn.setAutoCommit(false); // Insert all rows at ts @@ -431,7 +431,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testNullValueEqualityScan() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; @@ -459,7 +459,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testVarLengthPKColScan() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -489,7 +489,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testEscapedQuoteScan() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(), PTSDB_NAME, PTSDB_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -527,7 +527,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { } private static void initPtsdbTableValues(long ts) throws Exception { - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -560,7 +560,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { } private static void initPtsdbTableValues2(long ts, Date d) throws Exception { - ensureTableCreated(getUrl(),PTSDB2_NAME, PTSDB2_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB2_NAME, PTSDB2_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -696,7 +696,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testBatchUpsert() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB2_NAME, PTSDB2_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB2_NAME, PTSDB2_NAME, null, ts-2, null); Date d = new Date(ts); Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts)); @@ -874,7 +874,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testMissingPKColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -894,7 +894,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testNoKVColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -914,7 +914,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { // Broken, since we don't know if insert vs update. @Test public void testMissingKVColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -942,7 +942,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testTooShortKVColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -978,7 +978,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testTooShortPKColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -1014,7 +1014,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testTooLongPKColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -1051,7 +1051,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testTooLongKVColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2); + ensureTableCreated(getUrl(),BTABLE_NAME, BTABLE_NAME, null, ts-2, null); String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; // Insert at timestamp 0 Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); @@ -1481,7 +1481,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testLikeOnColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; @@ -1598,7 +1598,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testILikeOnColumn() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; @@ -1730,7 +1730,7 @@ public class VariableLengthPKIT extends BaseClientManagedTimeIT { @Test public void testIsNullInPK() throws Exception { long ts = nextTimestamp(); - ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2); + ensureTableCreated(getUrl(),PTSDB_NAME, PTSDB_NAME, null, ts-2, null); // Insert all rows at ts String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts; http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java index 4e7d06a..3d0ba8a 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java @@ -43,11 +43,13 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.ColumnNotFoundException; +import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.TableNotFoundException; +import org.apache.phoenix.util.EncodedColumnsUtil; import org.apache.phoenix.util.IndexUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; @@ -175,6 +177,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { // verify that the local index physical table was *not* dropped conn.unwrap(PhoenixConnection.class).getQueryServices().getTableDescriptor(localIndexTablePhysicalName.getBytes()); + PTable localIndex2 = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, localIndexTableName2)); // there should be a single row belonging to localIndexTableName2 Scan scan = new Scan(); @@ -184,7 +187,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { Result result = results.next(); assertNotNull(result); assertNotNull("localIndexTableName2 row is missing", result.getValue(QueryConstants.DEFAULT_LOCAL_INDEX_COLUMN_FAMILY_BYTES, - IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V1").getBytes())); + localIndex2.getColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V1")).getColumnQualifierBytes())); assertNull(results.next()); } } @@ -295,9 +298,11 @@ public class DropMetadataIT extends ParallelStatsDisabledIT { ResultScanner results = table.getScanner(scan); Result result = results.next(); assertNotNull(result); - // there should be a single row belonging to " + viewIndex2 + " - assertNotNull( viewIndex2 + " row is missing", result.getValue(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, - IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V4").getBytes())); + PTable viewIndexPTable = pconn.getTable(new PTableKey(pconn.getTenantId(), viewIndex2)); + PColumn column = viewIndexPTable.getColumnForColumnName(IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, "V4")); + byte[] cq = column.getColumnQualifierBytes(); + // there should be a single row belonging to VIEWINDEX2 + assertNotNull(viewIndex2 + " row is missing", result.getValue(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, cq)); assertNull(results.next()); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java index bc301fa..06802b6 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ImmutableIndexIT.java @@ -52,6 +52,7 @@ import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.schema.PTableImpl; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; @@ -70,6 +71,7 @@ import com.google.common.collect.Maps; public class ImmutableIndexIT extends BaseUniqueNamesOwnClusterIT { private final boolean localIndex; + private final boolean columnEncoded; private final String tableDDLOptions; private volatile boolean stopThreads = false; @@ -78,9 +80,15 @@ public class ImmutableIndexIT extends BaseUniqueNamesOwnClusterIT { private static String INDEX_DDL; public static final AtomicInteger NUM_ROWS = new AtomicInteger(0); - public ImmutableIndexIT(boolean localIndex, boolean transactional) { - this.localIndex = localIndex; + public ImmutableIndexIT(boolean localIndex, boolean transactional, boolean columnEncoded) { StringBuilder optionBuilder = new StringBuilder("IMMUTABLE_ROWS=true"); + this.localIndex = localIndex; + this.columnEncoded = columnEncoded; + if (!columnEncoded) { + if (optionBuilder.length()!=0) + optionBuilder.append(","); + optionBuilder.append("COLUMN_ENCODED_BYTES=0,IMMUTABLE_STORAGE_SCHEME="+PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN); + } if (transactional) { optionBuilder.append(", TRANSACTIONAL=true"); } @@ -98,11 +106,13 @@ public class ImmutableIndexIT extends BaseUniqueNamesOwnClusterIT { setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); } - @Parameters(name="ImmutableIndexIT_localIndex={0},transactional={1}") // name is used by failsafe as file name in reports + @Parameters(name="ImmutableIndexIT_localIndex={0},transactional={1},columnEncoded={2}") // name is used by failsafe as file name in reports public static Collection<Boolean[]> data() { return Arrays.asList(new Boolean[][] { - { false, false }, { false, true }, - { true, false }, { true, true } }); + { false, false, false }, { false, false, true }, + { false, true, false }, { false, true, true }, + { true, false, false }, { true, false, true }, + { true, true, false }, { true, true, true } }); } @Test http://git-wip-us.apache.org/repos/asf/phoenix/blob/dc6a6fc7/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java index 383452f..2395b02 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexExpressionIT.java @@ -410,7 +410,12 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexTableName); assertTrue(rs.next()); assertEquals(2, rs.getInt(1)); - + + String sql = "SELECT LONG_COL1 from " + fullDataTableName + " WHERE LONG_COL2 = 2"; + rs = conn.createStatement().executeQuery(sql); + assertTrue(rs.next()); + assertFalse(rs.next()); + String dml = "DELETE from " + fullDataTableName + " WHERE long_col2 = 2"; assertEquals(1, conn.createStatement().executeUpdate(dml)); conn.commit(); @@ -861,8 +866,10 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { conn.setAutoCommit(false); // make sure that the tables are empty, but reachable - conn.createStatement().execute( - "CREATE TABLE " + dataTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"); + conn.createStatement().execute( + "CREATE TABLE " + dataTableName + + " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + + (!mutable ? " IMMUTABLE_ROWS=true" : "")); query = "SELECT * FROM " + dataTableName ; rs = conn.createStatement().executeQuery(query); assertFalse(rs.next()); @@ -1235,7 +1242,16 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { } @Test - public void testViewUsesTableIndex() throws Exception { + public void testViewUsesMutableTableIndex() throws Exception { + helpTestViewUsesTableIndex(false); + } + + @Test + public void testViewUsesImmutableTableIndex() throws Exception { + helpTestViewUsesTableIndex(true); + } + + private void helpTestViewUsesTableIndex(boolean immutable) throws Exception { Connection conn = DriverManager.getConnection(getUrl()); try { @@ -1244,7 +1260,7 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { String viewName = generateUniqueName(); String indexName2 = generateUniqueName(); ResultSet rs; - String ddl = "CREATE TABLE " + dataTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, s1 VARCHAR, s2 VARCHAR, s3 VARCHAR, s4 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2))"; + String ddl = "CREATE TABLE " + dataTableName + " (k1 INTEGER NOT NULL, k2 INTEGER NOT NULL, s1 VARCHAR, s2 VARCHAR, s3 VARCHAR, s4 VARCHAR CONSTRAINT pk PRIMARY KEY (k1, k2)) " + (immutable ? "IMMUTABLE_ROWS = true" : ""); conn.createStatement().execute(ddl); conn.createStatement().execute("CREATE INDEX " + indexName1 + " ON " + dataTableName + "(k2, s2, s3, s1)"); conn.createStatement().execute("CREATE INDEX " + indexName2 + " ON " + dataTableName + "(k2, s2||'_'||s3, s1, s4)"); @@ -1341,7 +1357,7 @@ public class IndexExpressionIT extends ParallelStatsDisabledIT { try { conn.createStatement().execute( "CREATE TABLE " + dataTableName + " (k VARCHAR NOT NULL PRIMARY KEY, v VARCHAR) " - + (mutable ? "IMMUTABLE_ROWS=true" : "")); + + (!mutable ? "IMMUTABLE_ROWS=true" : "")); String query = "SELECT * FROM " + dataTableName; ResultSet rs = conn.createStatement().executeQuery(query); assertFalse(rs.next());