Merge branch '4.x-HBase-1.3' into omid2
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/24178207 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/24178207 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/24178207 Branch: refs/heads/omid2 Commit: 24178207a94ba1f0900d1962896764c361a1efd6 Parents: 119212c e5379cd Author: James Taylor <jamestay...@apache.org> Authored: Mon Sep 24 08:27:43 2018 -0700 Committer: James Taylor <jamestay...@apache.org> Committed: Mon Sep 24 08:27:43 2018 -0700 ---------------------------------------------------------------------- bin/phoenix_utils.py | 7 + bin/queryserver.py | 4 +- bin/traceserver.py | 6 +- dev/test-patch.properties | 2 +- phoenix-client/pom.xml | 1 + .../StatisticsCollectionRunTrackerIT.java | 2 +- .../AlterMultiTenantTableWithViewsIT.java | 284 +- .../apache/phoenix/end2end/AlterTableIT.java | 45 +- .../phoenix/end2end/AlterTableWithViewsIT.java | 544 ++-- .../phoenix/end2end/AppendOnlySchemaIT.java | 4 +- .../phoenix/end2end/BasePermissionsIT.java | 66 +- .../end2end/BaseTenantSpecificViewIndexIT.java | 42 +- .../org/apache/phoenix/end2end/BaseViewIT.java | 8 +- .../phoenix/end2end/ChangePermissionsIT.java | 4 +- .../phoenix/end2end/ClientHashAggregateIT.java | 208 ++ .../org/apache/phoenix/end2end/DateTimeIT.java | 77 + .../end2end/ExplainPlanWithStatsEnabledIT.java | 69 +- .../org/apache/phoenix/end2end/InListIT.java | 55 + .../apache/phoenix/end2end/InstrFunctionIT.java | 35 + .../MigrateSystemTablesToSystemNamespaceIT.java | 51 +- .../apache/phoenix/end2end/PhoenixDriverIT.java | 37 +- .../ProjectArrayElemAfterHashJoinIT.java | 177 ++ .../end2end/QueryDatabaseMetaDataIT.java | 13 +- .../org/apache/phoenix/end2end/QueryMoreIT.java | 19 +- .../apache/phoenix/end2end/RowTimestampIT.java | 26 +- .../apache/phoenix/end2end/SaltedViewIT.java | 45 - .../phoenix/end2end/SplitSystemCatalogIT.java | 80 + .../end2end/SplitSystemCatalogTests.java | 28 + .../StatsEnabledSplitSystemCatalogIT.java | 261 ++ .../SystemCatalogCreationOnConnectionIT.java | 34 +- .../apache/phoenix/end2end/SystemCatalogIT.java | 31 +- .../phoenix/end2end/TableDDLPermissionsIT.java | 9 +- .../end2end/TenantSpecificTablesDDLIT.java | 15 +- .../end2end/TenantSpecificViewIndexIT.java | 70 +- .../org/apache/phoenix/end2end/UpgradeIT.java | 355 +-- .../end2end/UpsertSelectAutoCommitIT.java | 31 +- .../java/org/apache/phoenix/end2end/ViewIT.java | 1144 ++++++-- .../phoenix/end2end/index/BaseIndexIT.java | 43 +- .../index/ChildViewsUseParentViewIndexIT.java | 11 +- .../phoenix/end2end/index/DropColumnIT.java | 117 - .../index/GlobalIndexOptimizationIT.java | 2 +- .../phoenix/end2end/index/IndexMetadataIT.java | 4 +- .../phoenix/end2end/index/IndexUsageIT.java | 4 +- .../phoenix/end2end/index/LocalIndexIT.java | 2 +- .../end2end/index/MutableIndexFailureIT.java | 124 +- .../MutableIndexFailureWithNamespaceIT.java | 82 + .../phoenix/end2end/index/MutableIndexIT.java | 838 +++--- .../phoenix/end2end/index/ViewIndexIT.java | 74 +- .../end2end/join/HashJoinPersistentCacheIT.java | 167 ++ .../apache/phoenix/execute/PartialCommitIT.java | 2 +- .../monitoring/BasePhoenixMetricsIT.java | 128 + .../monitoring/PhoenixLoggingMetricsIT.java | 290 ++ .../phoenix/monitoring/PhoenixMetricsIT.java | 171 +- .../phoenix/query/ConnectionCachingIT.java | 104 + .../SystemCatalogWALEntryFilterIT.java | 85 +- .../org/apache/phoenix/rpc/UpdateCacheIT.java | 9 +- .../org/apache/phoenix/tx/TransactionIT.java | 6 +- .../regionserver/IndexHalfStoreFileReader.java | 48 +- .../IndexHalfStoreFileReaderGenerator.java | 153 +- .../org/apache/phoenix/cache/GlobalCache.java | 22 +- .../apache/phoenix/cache/ServerCacheClient.java | 118 +- .../org/apache/phoenix/cache/TenantCache.java | 2 +- .../apache/phoenix/cache/TenantCacheImpl.java | 209 +- .../ColumnNameTrackingExpressionCompiler.java | 46 + .../phoenix/compile/CreateTableCompiler.java | 7 +- .../apache/phoenix/compile/DeleteCompiler.java | 4 +- .../apache/phoenix/compile/FromCompiler.java | 22 +- .../apache/phoenix/compile/JoinCompiler.java | 2 +- .../org/apache/phoenix/compile/KeyPart.java | 2 - .../phoenix/compile/ListJarsQueryPlan.java | 2 +- .../apache/phoenix/compile/QueryCompiler.java | 9 +- .../org/apache/phoenix/compile/ScanRanges.java | 77 +- .../phoenix/compile/StatementContext.java | 32 +- .../apache/phoenix/compile/TraceQueryPlan.java | 2 +- .../compile/TupleProjectionCompiler.java | 4 +- .../apache/phoenix/compile/UnionCompiler.java | 4 +- .../apache/phoenix/compile/UpsertCompiler.java | 16 +- .../apache/phoenix/compile/WhereOptimizer.java | 1306 ++++++--- .../GroupedAggregateRegionObserver.java | 8 - .../coprocessor/HashJoinRegionScanner.java | 73 +- .../coprocessor/MetaDataEndpointImpl.java | 2704 +++++++++--------- .../phoenix/coprocessor/MetaDataProtocol.java | 56 +- .../PhoenixMetaDataCoprocessorHost.java | 2 +- .../coprocessor/ServerCachingEndpointImpl.java | 2 +- .../apache/phoenix/coprocessor/TableInfo.java | 79 + .../coprocessor/TableViewFinderResult.java | 48 + .../apache/phoenix/coprocessor/ViewFinder.java | 144 + .../coprocessor/WhereConstantParser.java | 106 + .../coprocessor/generated/MetaDataProtos.java | 969 ++++++- .../coprocessor/generated/PTableProtos.java | 463 ++- .../generated/ServerCachingProtos.java | 221 +- .../phoenix/execute/ClientAggregatePlan.java | 40 +- .../apache/phoenix/execute/HashJoinPlan.java | 104 +- .../apache/phoenix/execute/MutationState.java | 6 +- .../PhoenixTxIndexMutationGenerator.java | 2 +- .../phoenix/expression/InListExpression.java | 5 +- .../phoenix/expression/LikeExpression.java | 2 +- .../expression/function/FunctionExpression.java | 10 +- .../expression/function/InstrFunction.java | 78 +- .../expression/function/InvertFunction.java | 19 +- .../expression/function/PrefixFunction.java | 6 +- .../expression/function/RTrimFunction.java | 6 +- .../function/RoundDateExpression.java | 22 +- .../function/RoundDecimalExpression.java | 7 +- .../expression/function/SubstrFunction.java | 20 + .../apache/phoenix/index/IndexMaintainer.java | 14 +- .../index/PhoenixIndexFailurePolicy.java | 75 +- .../phoenix/iterate/BaseResultIterators.java | 12 +- .../ClientHashAggregatingResultIterator.java | 210 ++ .../apache/phoenix/iterate/ExplainTable.java | 23 +- .../NonAggregateRegionScannerFactory.java | 5 +- .../phoenix/iterate/RegionScannerFactory.java | 7 +- .../phoenix/iterate/TableResultIterator.java | 6 +- .../iterate/TableSnapshotResultIterator.java | 4 +- .../phoenix/jdbc/LoggingPhoenixConnection.java | 50 +- .../jdbc/LoggingPhoenixPreparedStatement.java | 37 +- .../phoenix/jdbc/LoggingPhoenixResultSet.java | 24 +- .../phoenix/jdbc/LoggingPhoenixStatement.java | 49 +- .../apache/phoenix/jdbc/PhoenixConnection.java | 16 +- .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 540 ++-- .../apache/phoenix/jdbc/PhoenixMetricsLog.java | 13 +- .../apache/phoenix/jdbc/PhoenixStatement.java | 8 +- .../apache/phoenix/join/HashCacheClient.java | 24 +- .../apache/phoenix/join/HashCacheFactory.java | 11 + .../java/org/apache/phoenix/log/LogWriter.java | 2 +- .../phoenix/mapreduce/CsvBulkImportUtil.java | 8 +- .../mapreduce/index/IndexScrutinyTool.java | 2 +- .../util/PhoenixConfigurationUtil.java | 7 +- .../phoenix/parse/DropTableStatement.java | 8 +- .../java/org/apache/phoenix/parse/HintNode.java | 9 + .../apache/phoenix/parse/ParseNodeFactory.java | 2 +- .../phoenix/query/ConnectionQueryServices.java | 33 +- .../query/ConnectionQueryServicesImpl.java | 232 +- .../query/ConnectionlessQueryServicesImpl.java | 35 +- .../query/DelegateConnectionQueryServices.java | 19 +- .../java/org/apache/phoenix/query/KeyRange.java | 28 +- .../apache/phoenix/query/QueryConstants.java | 29 +- .../org/apache/phoenix/query/QueryServices.java | 5 + .../phoenix/query/QueryServicesOptions.java | 5 + .../SystemCatalogWALEntryFilter.java | 45 +- .../apache/phoenix/schema/DelegateColumn.java | 15 + .../apache/phoenix/schema/DelegateTable.java | 8 +- .../phoenix/schema/LocalIndexDataColumnRef.java | 2 +- .../apache/phoenix/schema/MetaDataClient.java | 188 +- .../phoenix/schema/MetaDataSplitPolicy.java | 26 +- .../java/org/apache/phoenix/schema/PColumn.java | 12 + .../org/apache/phoenix/schema/PColumnImpl.java | 113 +- .../apache/phoenix/schema/PMetaDataImpl.java | 3 +- .../java/org/apache/phoenix/schema/PTable.java | 20 +- .../org/apache/phoenix/schema/PTableImpl.java | 321 ++- .../org/apache/phoenix/schema/PTableKey.java | 4 +- .../schema/ParentTableNotFoundException.java | 47 + .../org/apache/phoenix/schema/RowKeySchema.java | 78 + .../org/apache/phoenix/schema/SaltingUtil.java | 4 +- .../apache/phoenix/schema/TableProperty.java | 22 +- .../apache/phoenix/schema/types/PVarbinary.java | 4 +- .../java/org/apache/phoenix/util/DateUtil.java | 22 +- .../java/org/apache/phoenix/util/IndexUtil.java | 16 +- .../org/apache/phoenix/util/MetaDataUtil.java | 175 +- .../org/apache/phoenix/util/PhoenixRuntime.java | 1 - .../org/apache/phoenix/util/SchemaUtil.java | 43 +- .../org/apache/phoenix/util/UpgradeUtil.java | 186 +- .../phoenix/util/csv/CsvUpsertExecutor.java | 4 +- .../phoenix/util/json/JsonUpsertExecutor.java | 4 +- .../apache/phoenix/cache/TenantCacheTest.java | 112 +- .../phoenix/compile/QueryCompilerTest.java | 4 +- .../phoenix/compile/QueryOptimizerTest.java | 5 +- .../TenantSpecificViewIndexCompileTest.java | 14 +- .../phoenix/compile/WhereOptimizerTest.java | 371 ++- .../coprocessor/MetaDataEndpointImplTest.java | 299 ++ .../phoenix/execute/CorrelatePlanTest.java | 5 +- .../execute/LiteralResultIteratorPlanTest.java | 6 +- .../phoenix/execute/UnnestArrayPlanTest.java | 7 +- .../expression/ColumnExpressionTest.java | 9 +- .../RoundFloorCeilExpressionsTest.java | 59 +- .../expression/function/InstrFunctionTest.java | 44 +- .../java/org/apache/phoenix/query/BaseTest.java | 312 +- .../apache/phoenix/query/KeyRangeClipTest.java | 2 +- .../org/apache/phoenix/query/QueryPlanTest.java | 8 +- .../apache/phoenix/schema/RowKeySchemaTest.java | 48 + .../util/AbstractUpsertExecutorTest.java | 12 +- .../apache/phoenix/util/MetaDataUtilTest.java | 22 +- .../util/TenantIdByteConversionTest.java | 30 +- .../java/org/apache/phoenix/util/TestUtil.java | 23 +- phoenix-load-balancer/pom.xml | 1 + ...apache.phoenix.queryserver.register.Registry | 1 + phoenix-protocol/src/main/MetaDataService.proto | 12 +- phoenix-protocol/src/main/PTable.proto | 9 +- .../src/main/ServerCachingService.proto | 2 + phoenix-protocol/src/main/build-proto.sh | 6 + .../phoenix/end2end/QueryServerTestUtil.java | 187 ++ .../phoenix/end2end/ServerCustomizersIT.java | 149 + .../AvaticaServerConfigurationFactory.java | 37 + .../phoenix/queryserver/server/QueryServer.java | 173 +- .../server/ServerCustomizersFactory.java | 52 + .../CustomAvaticaServerConfigurationTest.java | 37 + .../server/QueryServerConfigurationTest.java | 26 +- .../server/ServerCustomizersTest.java | 92 + .../apache/phoenix/tracingwebapp/http/Main.java | 13 +- .../tracingwebapp/http/TraceServlet.java | 87 +- pom.xml | 24 +- 201 files changed, 13225 insertions(+), 5431 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java index 3a368bc,e97a40d..3cb4f6c --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableWithViewsIT.java @@@ -33,26 -35,31 +35,36 @@@ import java.sql.ResultSetMetaData import java.sql.SQLException; import java.util.Arrays; import java.util.Collection; + import java.util.List; +import java.util.Properties; import org.apache.commons.lang.ArrayUtils; - import org.apache.hadoop.hbase.client.HTableInterface; + import org.apache.hadoop.hbase.client.HTable; + import org.apache.hadoop.hbase.client.Result; + import org.apache.hadoop.hbase.client.ResultScanner; + import org.apache.hadoop.hbase.client.Scan; + import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.coprocessor.TephraTransactionalProcessor; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.query.QueryServices; + import org.apache.phoenix.schema.ColumnNotFoundException; + import org.apache.phoenix.schema.PColumn; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PNameFactory; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.PTableKey; import org.apache.phoenix.schema.PTableType; + import org.apache.phoenix.schema.TableNotFoundException; +import org.apache.phoenix.transaction.TransactionFactory; + import org.apache.phoenix.util.IndexUtil; + import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; - import org.apache.phoenix.util.StringUtil; + import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@@ -756,15 -867,11 +872,16 @@@ public class AlterTableWithViewsIT exte @Test public void testMakeBaseTableTransactional() throws Exception { - try (Connection conn = DriverManager.getConnection(getUrl()); + if (!TransactionFactory.Provider.TEPHRA.runTests()) { + return; + } + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.setProperty(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + try (Connection conn = DriverManager.getConnection(getUrl(), props); Connection viewConn = isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL1) : conn ) { - String baseTableName = "NONTXNTBL_" + generateUniqueName() + (isMultiTenant ? "0":"1"); - String viewOfTable = baseTableName + "_VIEW"; + String baseTableName = SchemaUtil.getTableName(SCHEMA1, generateUniqueName()); + String viewOfTable = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String ddlFormat = "CREATE TABLE IF NOT EXISTS " + baseTableName + " (" + " %s ID char(1) NOT NULL," + " COL1 integer NOT NULL," http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java index c1c033b,5e465b4..227089b --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewIT.java @@@ -47,145 -74,140 +74,141 @@@ import org.apache.phoenix.jdbc.PhoenixS import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.ColumnAlreadyExistsException; + import org.apache.phoenix.schema.ConcurrentTableMutationException; + import org.apache.phoenix.schema.PColumn; + import org.apache.phoenix.schema.PTable; + import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.ReadOnlyTableException; import org.apache.phoenix.schema.TableNotFoundException; + import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixRuntime; + import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; + import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; + import org.apache.phoenix.util.TestUtil; + import org.junit.BeforeClass; import org.junit.Test; - - - - public class ViewIT extends BaseViewIT { - - public ViewIT(String txProvider) { - super(txProvider); - } - - @Test - public void testReadOnlyOnReadOnlyView() throws Exception { - Connection earlierCon = DriverManager.getConnection(getUrl()); - Connection conn = DriverManager.getConnection(getUrl()); - String ddl = "CREATE TABLE " + fullTableName + " (k INTEGER NOT NULL PRIMARY KEY, v1 DATE) "+ tableDDLOptions; - conn.createStatement().execute(ddl); - String fullParentViewName = "V_" + generateUniqueName(); - ddl = "CREATE VIEW " + fullParentViewName + " (v2 VARCHAR) AS SELECT * FROM " + fullTableName + " WHERE k > 5"; - conn.createStatement().execute(ddl); - try { - conn.createStatement().execute("UPSERT INTO " + fullParentViewName + " VALUES(1)"); - fail(); - } catch (ReadOnlyTableException e) { - - } - for (int i = 0; i < 10; i++) { - conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(" + i + ")"); - } - conn.commit(); - - analyzeTable(conn, fullParentViewName, txProvider != null); - - List<KeyRange> splits = getAllSplits(conn, fullParentViewName); - assertEquals(4, splits.size()); - - int count = 0; - ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM " + fullTableName); - while (rs.next()) { - assertEquals(count++, rs.getInt(1)); - } - assertEquals(10, count); - - count = 0; - rs = conn.createStatement().executeQuery("SELECT k FROM " + fullParentViewName); - while (rs.next()) { - count++; - assertEquals(count + 5, rs.getInt(1)); - } - assertEquals(4, count); - count = 0; - rs = earlierCon.createStatement().executeQuery("SELECT k FROM " + fullParentViewName); - while (rs.next()) { - count++; - assertEquals(count + 5, rs.getInt(1)); - } - assertEquals(4, count); - String fullViewName = "V_" + generateUniqueName(); - ddl = "CREATE VIEW " + fullViewName + " AS SELECT * FROM " + fullParentViewName + " WHERE k < 9"; - conn.createStatement().execute(ddl); - try { - conn.createStatement().execute("UPSERT INTO " + fullViewName + " VALUES(1)"); - fail(); - } catch (ReadOnlyTableException e) { - - } finally { - conn.close(); + import org.junit.runner.RunWith; + import org.junit.runners.Parameterized; + import org.junit.runners.Parameterized.Parameters; + + import com.google.common.base.Predicate; + import com.google.common.collect.Collections2; + import com.google.common.collect.Maps; + @RunWith(Parameterized.class) + public class ViewIT extends SplitSystemCatalogIT { + + protected String tableDDLOptions; - protected boolean transactional; ++ protected String transactionProvider; + protected boolean columnEncoded; + + private static final String FAILED_VIEWNAME = SchemaUtil.getTableName(SCHEMA2, "FAILED_VIEW"); + private static final String SLOW_VIEWNAME_PREFIX = SchemaUtil.getTableName(SCHEMA2, "SLOW_VIEW"); + + private static volatile CountDownLatch latch1 = null; + private static volatile CountDownLatch latch2 = null; + - public ViewIT(boolean transactional, boolean columnEncoded) { ++ public ViewIT(String transactionProvider, boolean columnEncoded) { + StringBuilder optionBuilder = new StringBuilder(); - this.transactional = transactional; ++ this.transactionProvider = transactionProvider; + this.columnEncoded = columnEncoded; - if (transactional) { - optionBuilder.append(" TRANSACTIONAL=true "); ++ if (transactionProvider != null) { ++ optionBuilder.append(" TRANSACTION_PROVIDER='" + transactionProvider + "'"); } - - conn = DriverManager.getConnection(getUrl()); - count = 0; - rs = conn.createStatement().executeQuery("SELECT k FROM " + fullViewName); - while (rs.next()) { - count++; - assertEquals(count + 5, rs.getInt(1)); + if (!columnEncoded) { + if (optionBuilder.length()!=0) + optionBuilder.append(","); + optionBuilder.append("COLUMN_ENCODED_BYTES=0"); } - assertEquals(3, count); + this.tableDDLOptions = optionBuilder.toString(); } - @Test - public void testNonSaltedUpdatableViewWithIndex() throws Exception { - testUpdatableViewWithIndex(null, false); - @Parameters(name="ViewIT_transactional={0}, columnEncoded={1}") // name is used by failsafe as file name in reports - public static Collection<Boolean[]> data() { - return Arrays.asList(new Boolean[][] { - { true, false }, { true, true }, - { false, false }, { false, true }}); ++ @Parameters(name="ViewIT_transactionProvider={0}, columnEncoded={1}") // name is used by failsafe as file name in reports ++ public static Collection<Object[]> data() { ++ return Arrays.asList(new Object[][] { ++ { "TEPHRA", false }, { "TEPHRA", true }, ++ { "OMID", false }, ++ { null, false }, { null, true }}); } - @Test - public void testNonSaltedUpdatableViewWithLocalIndex() throws Exception { - testUpdatableViewWithIndex(null, true); + @BeforeClass + public static void doSetup() throws Exception { + NUM_SLAVES_BASE = 6; + Map<String, String> props = Collections.emptyMap(); + boolean splitSystemCatalog = (driver == null); + Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(1); + serverProps.put(QueryServices.PHOENIX_ACLS_ENABLED, "true"); + serverProps.put(PhoenixMetaDataCoprocessorHost.PHOENIX_META_DATA_COPROCESSOR_CONF_KEY, + TestMetaDataRegionObserver.class.getName()); + serverProps.put("hbase.coprocessor.abortonerror", "false"); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(props.entrySet().iterator())); + // Split SYSTEM.CATALOG once after the mini-cluster is started + if (splitSystemCatalog) { + splitSystemCatalog(); + } } - @Test - public void testUpdatableOnUpdatableView() throws Exception { - String viewName = testUpdatableView(null); - Connection conn = DriverManager.getConnection(getUrl()); - String fullViewName = "V_" + generateUniqueName(); - String ddl = "CREATE VIEW " + fullViewName + " AS SELECT * FROM " + viewName + " WHERE k3 = 2"; - conn.createStatement().execute(ddl); - ResultSet rs = conn.createStatement().executeQuery("SELECT k1, k2, k3 FROM " + fullViewName); - assertTrue(rs.next()); - assertEquals(1, rs.getInt(1)); - assertEquals(109, rs.getInt(2)); - assertEquals(2, rs.getInt(3)); - assertFalse(rs.next()); - - conn.createStatement().execute("UPSERT INTO " + fullViewName + "(k2) VALUES(122)"); - conn.commit(); - rs = conn.createStatement().executeQuery("SELECT k1, k2, k3 FROM " + fullViewName + " WHERE k2 >= 120"); - assertTrue(rs.next()); - assertEquals(1, rs.getInt(1)); - assertEquals(122, rs.getInt(2)); - assertEquals(2, rs.getInt(3)); - assertFalse(rs.next()); + public static class TestMetaDataRegionObserver extends BaseMetaDataEndpointObserver { - try { - conn.createStatement().execute("UPSERT INTO " + fullViewName + "(k2,k3) VALUES(123,3)"); - fail(); - } catch (SQLException e) { - assertEquals(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN.getErrorCode(), e.getErrorCode()); + @Override + public void preAlterTable(ObserverContext<PhoenixMetaDataControllerEnvironment> ctx, String tenantId, + String tableName, TableName physicalTableName, TableName parentPhysicalTableName, PTableType type) throws IOException{ + processTable(tableName); + } + + @Override + public void preCreateTable(ObserverContext<PhoenixMetaDataControllerEnvironment> ctx, String tenantId, + String tableName, TableName physicalTableName, TableName parentPhysicalTableName, PTableType tableType, + Set<byte[]> familySet, Set<TableName> indexes) throws IOException { + processTable(tableName); } - try { - conn.createStatement().execute("UPSERT INTO " + fullViewName + "(k2,k3) select k2, 3 from " + fullViewName); - fail(); - } catch (SQLException e) { - assertEquals(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN.getErrorCode(), e.getErrorCode()); + @Override + public void preDropTable(ObserverContext<PhoenixMetaDataControllerEnvironment> ctx, String tenantId, + String tableName, TableName physicalTableName, TableName parentPhysicalTableName, PTableType tableType, + List<PTable> indexes) throws IOException { + processTable(tableName); } - } + private void processTable(String tableName) throws DoNotRetryIOException { + if (tableName.equals(FAILED_VIEWNAME)) { + // throwing anything other than instances of IOException result + // in this coprocessor being unloaded + // DoNotRetryIOException tells HBase not to retry this mutation + // multiple times + throw new DoNotRetryIOException(); + } else if (tableName.startsWith(SLOW_VIEWNAME_PREFIX)) { + // simulate a slow write to SYSTEM.CATALOG + if (latch1 != null) { + latch1.countDown(); + } + if (latch2 != null) { + try { + // wait till the second task is complete before completing the first task + boolean result = latch2.await(2, TimeUnit.MINUTES); + if (!result) { + throw new RuntimeException("Second task took took long to complete"); + } + } catch (InterruptedException e) { + } + } + } + } + + } + @Test public void testReadOnlyOnUpdatableView() throws Exception { - String viewName = testUpdatableView(null); + String fullTableName = SchemaUtil.getTableName(SCHEMA1, generateUniqueName()); + String fullViewName1 = SchemaUtil.getTableName(SCHEMA2, generateUniqueName()); + String fullViewName2 = SchemaUtil.getTableName(SCHEMA3, generateUniqueName()); + String ddl = + "CREATE VIEW " + fullViewName2 + " AS SELECT * FROM " + fullViewName1 + + " WHERE k3 > 1 and k3 < 50"; + testUpdatableView(fullTableName, fullViewName1, fullViewName2, ddl, null, tableDDLOptions); Connection conn = DriverManager.getConnection(getUrl()); - String fullViewName = "V_" + generateUniqueName(); - String ddl = "CREATE VIEW " + fullViewName + " AS SELECT * FROM " + viewName + " WHERE k3 > 1 and k3 < 50"; - conn.createStatement().execute(ddl); - ResultSet rs = conn.createStatement().executeQuery("SELECT k1, k2, k3 FROM " + fullViewName); + ResultSet rs = conn.createStatement().executeQuery("SELECT k1, k2, k3 FROM " + fullViewName2); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertEquals(109, rs.getInt(2)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexIT.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java index 7efe17e,0611478..f09901e --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureIT.java @@@ -28,8 -28,8 +28,9 @@@ import java.sql.Connection import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; + import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Properties; @@@ -105,15 -110,15 +111,15 @@@ public class MutableIndexFailureIT exte private final boolean throwIndexWriteFailure; private String schema = generateUniqueName(); private List<CommitException> exceptions = Lists.newArrayList(); - private static RegionCoprocessorEnvironment indexRebuildTaskRegionEnvironment; - private static final int forwardOverlapMs = 1000; - private static final int disableTimestampThresholdMs = 10000; - private static final int numRpcRetries = 2; + protected static RegionCoprocessorEnvironment indexRebuildTaskRegionEnvironment; + protected static final int forwardOverlapMs = 1000; + protected static final int disableTimestampThresholdMs = 10000; + protected static final int numRpcRetries = 2; - public MutableIndexFailureIT(boolean transactional, boolean localIndex, boolean isNamespaceMapped, Boolean disableIndexOnWriteFailure, boolean failRebuildTask, Boolean throwIndexWriteFailure) { - this.transactional = transactional; + public MutableIndexFailureIT(String transactionProvider, boolean localIndex, boolean isNamespaceMapped, Boolean disableIndexOnWriteFailure, boolean failRebuildTask, Boolean throwIndexWriteFailure) { + this.transactional = transactionProvider != null; this.localIndex = localIndex; - this.tableDDLOptions = " SALT_BUCKETS=2, COLUMN_ENCODED_BYTES=NONE" + (transactional ? ", TRANSACTIONAL=true " : "") + this.tableDDLOptions = " SALT_BUCKETS=2, COLUMN_ENCODED_BYTES=NONE" + (transactional ? (",TRANSACTIONAL=true,TRANSACTION_PROVIDER='"+transactionProvider+"'") : "") + (disableIndexOnWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.DISABLE_INDEX_ON_WRITE_FAILURE + "=" + disableIndexOnWriteFailure)) + (throwIndexWriteFailure == null ? "" : (", " + PhoenixIndexFailurePolicy.THROW_INDEX_WRITE_FAILURE + "=" + throwIndexWriteFailure)); this.tableName = FailingRegionObserver.FAIL_TABLE_NAME; @@@ -143,53 -165,29 +166,41 @@@ * because we want to control it's execution ourselves */ serverProps.put(QueryServices.INDEX_REBUILD_TASK_INITIAL_DELAY, Long.toString(Long.MAX_VALUE)); - Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); - clientProps.put(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "2"); - NUM_SLAVES_BASE = 4; - setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); - indexRebuildTaskRegionEnvironment = - (RegionCoprocessorEnvironment) getUtility() - .getRSForFirstRegionInTable( - PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME) - .getOnlineRegions(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME) - .get(0).getCoprocessorHost() - .findCoprocessorEnvironment(MetaDataRegionObserver.class.getName()); - MetaDataRegionObserver.initRebuildIndexConnectionProps( - indexRebuildTaskRegionEnvironment.getConfiguration()); + return serverProps; } - @Parameters(name = "MutableIndexFailureIT_transactional={0},localIndex={1},isNamespaceMapped={2},disableIndexOnWriteFailure={3},failRebuildTask={4},throwIndexWriteFailure={5}") // name is used by failsafe as file name in reports - public static List<Object[]> data() { - return Arrays.asList(new Object[][] { - // note - can't disableIndexOnWriteFailure without throwIndexWriteFailure, PHOENIX-4130 - { false, false, false, false, false, false}, - { false, false, false, true, false, null}, - { true, false, false, true, false, null}, - { false, true, false, null, false, null}, - { true, true, false, true, false, null}, - { false, false, false, false, false, null}, - { false, true, false, false, false, null}, - { false, false, false, false, false, null}, - { false, false, false, true, false, null}, - { false, false, false, true, false, null}, - { false, true, false, true, false, null}, - { false, true, false, true, false, null}, - { false, false, false, true, true, null}, - { false, false, false, false, true, false}, - } - ); + @Parameters(name = "MutableIndexFailureIT_transactionProvider={0},localIndex={1},isNamespaceMapped={2},disableIndexOnWriteFailure={3},failRebuildTask={4},throwIndexWriteFailure={5}") // name is used by failsafe as file name in reports + public static Collection<Object[]> data() { + return TestUtil.filterTxParamData( + Arrays.asList(new Object[][] { + // note - can't disableIndexOnWriteFailure without throwIndexWriteFailure, PHOENIX-4130 + { null, false, false, false, false, false}, + { null, false, true, true, false, null}, + { null, false, true, true, false, true}, + { null, false, false, true, false, null}, + { null, true, true, true, false, null}, + { null, true, false, null, false, null}, + { "TEPHRA", true, false, false, false, null}, + { "TEPHRA", true, true, false, false, null}, + { "TEPHRA", false, false, false, false, null}, + { "TEPHRA", false, true, false, false, null}, + { "OMID", true, false, false, false, null}, + { "OMID", true, true, false, false, null}, + { "OMID", false, false, false, false, null}, + { "OMID", false, true, false, false, null}, + + { null, false, false, false, false, null}, + { null, true, false, false, false, null}, + { null, false, false, false, false, null}, + { null, false, false, true, false, null}, + { null, false, false, true, false, null}, + { null, true, false, true, false, null}, + { null, true, false, true, false, null}, + { null, false, false, true, true, null}, + { null, false, true, true, true, null}, + { null, false, false, false, true, false}, + { null, false, true, false, true, false}, + }), 0); } private void runRebuildTask(Connection conn) throws InterruptedException, SQLException { @@@ -409,12 -412,62 +425,62 @@@ stmt.setString(3, "3"); stmt.execute(); conn.commit(); + } + private void addRowsInTableDuringRetry(final String tableName) + throws SQLException, InterruptedException, ExecutionException { + int threads=10; + boolean wasFailWrite = FailingRegionObserver.FAIL_WRITE; + boolean wasToggleFailWriteForRetry = FailingRegionObserver.TOGGLE_FAIL_WRITE_FOR_RETRY; + try { + Callable callable = new Callable() { + + @Override + public Boolean call() { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, String.valueOf(isNamespaceMapped)); + try (Connection conn = driver.connect(url, props)) { + // In case of disable index on failure policy, INDEX will be in PENDING_DISABLE on first retry + // but will + // become active if retry is successfull + PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?)"); + stmt.setString(1, "b"); + stmt.setString(2, "y"); + stmt.setString(3, "2"); + stmt.execute(); + if (!leaveIndexActiveOnFailure && !transactional) { + FailingRegionObserver.FAIL_WRITE = true; + FailingRegionObserver.TOGGLE_FAIL_WRITE_FOR_RETRY = true; + } + conn.commit(); + } catch (SQLException e) { + return false; + } + return true; + } + }; + ExecutorService executor = Executors.newFixedThreadPool(threads); + List<Future<Boolean>> futures = new ArrayList<Future<Boolean>>(); + for (int i = 0; i < threads; i++) { + futures.add(executor.submit(callable)); + } + for (Future<Boolean> future : futures) { + Boolean isSuccess = future.get(); + // transactions can have conflict so ignoring the check for them + if (!transactional) { + assertTrue(isSuccess); + } + } + executor.shutdown(); + } finally { + FailingRegionObserver.FAIL_WRITE = wasFailWrite; + FailingRegionObserver.TOGGLE_FAIL_WRITE_FOR_RETRY = wasToggleFailWriteForRetry; + } } - private void validateDataWithIndex(Connection conn, String fullTableName, String fullIndexName, boolean localIndex) throws SQLException { + private void validateDataWithIndex(Connection conn, String fullTableName, String fullIndexName, boolean localIndex) throws Exception { String query = "SELECT /*+ INDEX(" + fullTableName + " " + SchemaUtil.getTableNameFromFullName(fullIndexName) + ") */ k,v1 FROM " + fullTableName; - ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + query); + ResultSet rs = conn.createStatement().executeQuery(query); String expectedPlan = " OVER " + (localIndex ? Bytes.toString( http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureWithNamespaceIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureWithNamespaceIT.java index 0000000,5ed9e1f..d5f7bd1 mode 000000,100644..100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureWithNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexFailureWithNamespaceIT.java @@@ -1,0 -1,80 +1,82 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.phoenix.end2end.index; + + import java.util.Arrays; + import java.util.List; + import java.util.Map; + + import org.apache.hadoop.hbase.HConstants; + import org.apache.hadoop.hbase.TableName; + import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; + import org.apache.phoenix.coprocessor.MetaDataRegionObserver; + import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; + import org.apache.phoenix.query.QueryServices; + import org.apache.phoenix.util.ReadOnlyProps; + import org.apache.phoenix.util.SchemaUtil; + import org.junit.BeforeClass; + import org.junit.runners.Parameterized.Parameters; + + import com.google.common.collect.Maps; + + /* + * This class is to ensure gets its own cluster with Namespace Enabled + */ + public class MutableIndexFailureWithNamespaceIT extends MutableIndexFailureIT { + - public MutableIndexFailureWithNamespaceIT(boolean transactional, boolean localIndex, boolean isNamespaceMapped, ++ public MutableIndexFailureWithNamespaceIT(String transactionProvider, boolean localIndex, boolean isNamespaceMapped, + Boolean disableIndexOnWriteFailure, boolean failRebuildTask, Boolean throwIndexWriteFailure) { - super(transactional, localIndex, isNamespaceMapped, disableIndexOnWriteFailure, failRebuildTask, ++ super(transactionProvider, localIndex, isNamespaceMapped, disableIndexOnWriteFailure, failRebuildTask, + throwIndexWriteFailure); + } + + @BeforeClass + public static void doSetup() throws Exception { + Map<String, String> serverProps = getServerProps(); + serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.TRUE.toString()); + Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.TRUE.toString()); + clientProps.put(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "2"); + NUM_SLAVES_BASE = 4; + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), new ReadOnlyProps(clientProps.entrySet().iterator())); + TableName systemTable = SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, + true); + indexRebuildTaskRegionEnvironment = (RegionCoprocessorEnvironment)getUtility() + .getRSForFirstRegionInTable(systemTable).getOnlineRegions(systemTable).get(0).getCoprocessorHost() + .findCoprocessorEnvironment(MetaDataRegionObserver.class.getName()); + MetaDataRegionObserver.initRebuildIndexConnectionProps(indexRebuildTaskRegionEnvironment.getConfiguration()); + } + + @Parameters(name = "MutableIndexFailureIT_transactional={0},localIndex={1},isNamespaceMapped={2},disableIndexOnWriteFailure={3},failRebuildTask={4},throwIndexWriteFailure={5}") // name is used by failsafe as file name in reports + public static List<Object[]> data() { + return Arrays.asList(new Object[][] { + // note - can't disableIndexOnWriteFailure without throwIndexWriteFailure, PHOENIX-4130 - { false, false, true, true, false, null}, - { false, false, true, true, false, true}, - { true, false, true, true, false, null}, - { false, true, true, true, false, null}, - { true, true, true, null, false, null}, - { false, false, true, true, true, null}, - { false, false, true, false, true, false}, ++ { null, false, true, true, false, null}, ++ { null, false, true, true, false, true}, ++ { "TEPHRA", false, true, true, false, null}, ++ { "OMID", false, true, true, false, null}, ++ { null, true, true, true, false, null}, ++ { "TEPHRA", true, true, null, false, null}, ++ { "OMID", true, true, null, false, null}, ++ { null, false, true, true, true, null}, ++ { null, false, true, false, true, false}, + } + ); + } + + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java index 85afd66,4433e12..2a58a6e --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java @@@ -106,16 -106,16 +106,16 @@@ public class MutableIndexIT extends Par return getConnection(props); } - @Parameters(name="MutableIndexIT_localIndex={0},transactionProvider={1},columnEncoded={2}") // name is used by failsafe as file name in reports - @Parameters(name="MutableIndexIT_localIndex={0},transactional={1},columnEncoded={2}") // name is used by failsafe as file name in reports ++ @Parameters(name="MutableIndexIT_localIndex={0},transactionProvider={1},columnEncoded={2}") // name is used by failsafe as file name in reports public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] { + return TestUtil.filterTxParamData(Arrays.asList(new Object[][] { { false, null, false }, { false, null, true }, { false, "TEPHRA", false }, { false, "TEPHRA", true }, - //{ false, "OMID", false }, { false, "OMID", true }, + { false, "OMID", false }, { true, null, false }, { true, null, true }, { true, "TEPHRA", false }, { true, "TEPHRA", true }, - //{ true, "OMID", false }, { true, "OMID", true }, - }); + { true, "OMID", false }, + }),1); } @Test http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java index 6a5c73d,feb0ce4..c6dc312 --- a/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/execute/PartialCommitIT.java @@@ -264,16 -284,16 +264,16 @@@ public class PartialCommitIT extends Ba } private PhoenixConnection getConnectionWithTableOrderPreservingMutationState() throws SQLException { - Connection con = driver.connect(url, new Properties()); - PhoenixConnection phxCon = new PhoenixConnection(con.unwrap(PhoenixConnection.class)); - final Map<TableRef, MultiRowMutationState> mutations = Maps.newTreeMap(new TableRefComparator()); - // passing a null mutation state forces the connection.newMutationState() to be used to create the MutationState - return new PhoenixConnection(phxCon, (MutationState)null) { - @Override - protected MutationState newMutationState(int maxSize, int maxSizeBytes) { - return new MutationState(maxSize, maxSizeBytes, this, mutations, false, null); + try (PhoenixConnection con = DriverManager.getConnection(getUrl()).unwrap(PhoenixConnection.class)) { + final Map<TableRef, MultiRowMutationState> mutations = Maps.newTreeMap(new TableRefComparator()); + // passing a null mutation state forces the connection.newMutationState() to be used to create the MutationState - return new PhoenixConnection(con, null) { ++ return new PhoenixConnection(con, (MutationState)null) { + @Override + protected MutationState newMutationState(int maxSize, int maxSizeBytes) { + return new MutationState(maxSize, maxSizeBytes, this, mutations, false, null); + }; }; - }; + } } public static class FailingRegionObserver extends SimpleRegionObserver { http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java index 009ee06,2211d58..0ddbed3 --- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java @@@ -40,8 -41,8 +40,9 @@@ import org.apache.phoenix.query.Connect import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.schema.MetaDataClient; import org.apache.phoenix.schema.PName; + import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.types.PVarchar; +import org.apache.phoenix.transaction.TransactionFactory; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java index 05cf21a,12c3b7a..1a8cb19 --- a/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/tx/TransactionIT.java @@@ -43,12 -40,8 +44,13 @@@ import org.apache.hadoop.hbase.HBaseIOE import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; + import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver; +import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress; import org.apache.hadoop.hbase.util.Bytes; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.exception.SQLExceptionCode; @@@ -473,8 -376,7 +475,8 @@@ public class TransactionIT extends Par } } + - private static void assertTTL(Admin admin, String tableName, int ttl) throws Exception { + private static void assertTTL(Admin admin, String tableName, int ttl) throws TableNotFoundException, IOException { HTableDescriptor tableDesc = admin.getTableDescriptor(TableName.valueOf(tableName)); for (HColumnDescriptor colDesc : tableDesc.getFamilies()) { assertEquals(ttl,Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java index 70b9d34,c2dfeab..583085e --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java @@@ -975,9 -974,7 +975,9 @@@ public class DeleteCompiler private static boolean isMaintainedOnClient(PTable table) { // Test for not being local (rather than being GLOBAL) so that this doesn't fail // when tested with our projected table. - return table.getIndexType() != IndexType.LOCAL && (table.isImmutableRows() || table.isTransactional()); + return (table.getIndexType() != IndexType.LOCAL && (table.isTransactional() || table.isImmutableRows())) || + (table.getIndexType() == IndexType.LOCAL && (table.isTransactional() && + table.getTransactionProvider().getTransactionProvider().isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER) ) ); } - } + } http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java index 6ef6f3b,877c939..1737911 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/PhoenixTxIndexMutationGenerator.java @@@ -191,9 -177,9 +191,9 @@@ public class PhoenixTxIndexMutationGene // Project empty key value column scan.addColumn(indexMaintainers.get(0).getDataEmptyKeyValueCF(), emptyKeyValueQualifier); - ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, KeyRange.EVERYTHING_RANGE, null, true, -1); + ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN, null, true, -1); scanRanges.initializeScan(scan); - Table txTable = indexMetaData.getTransactionContext().getTransactionalTable(htable, isImmutable); + Table txTable = indexMetaData.getTransactionContext().getTransactionalTable(htable, true); // For rollback, we need to see all versions, including // the last committed version as there may be multiple // checkpointed versions. http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java index eb79fb9,0820232..d5a5199 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java @@@ -155,5 -168,19 +168,19 @@@ public interface ConnectionQueryService public QueryLoggerDisruptor getQueryDisruptor(); - public PhoenixTransactionClient initTransactionClient(TransactionFactory.Provider provider); + public PhoenixTransactionClient initTransactionClient(TransactionFactory.Provider provider) throws SQLException; - } + + /** + * Writes a cell to SYSTEM.MUTEX using checkAndPut to ensure only a single client can execute a + * particular task. The params are used to generate the rowkey. + * @return true if this client was able to successfully acquire the mutex + */ + public boolean writeMutexCell(String tenantId, String schemaName, String tableName, + String columnName, String familyName) throws SQLException; + + /** + * Deletes a cell that was written to SYSTEM.MUTEX. The params are used to generate the rowkey. + */ + public void deleteMutexCell(String tenantId, String schemaName, String tableName, + String columnName, String familyName) throws SQLException; -} ++} http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java ---------------------------------------------------------------------- diff --cc phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java index 262da3c,147e873..4be4af8 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java @@@ -367,7 -367,18 +367,18 @@@ public class DelegateConnectionQuerySer } @Override - public PhoenixTransactionClient initTransactionClient(Provider provider) { + public PhoenixTransactionClient initTransactionClient(Provider provider) throws SQLException { return getDelegate().initTransactionClient(provider); } + + @Override + public boolean writeMutexCell(String tenantId, String schemaName, String tableName, + String columnName, String familyName) throws SQLException { + return true; + } + + @Override + public void deleteMutexCell(String tenantId, String schemaName, String tableName, + String columnName, String familyName) throws SQLException { + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/24178207/pom.xml ---------------------------------------------------------------------- diff --cc pom.xml index 14dd94f,4412c15..8e1f05c --- a/pom.xml +++ b/pom.xml @@@ -98,10 -98,9 +98,10 @@@ <!-- Do not change jodatime.version until HBASE-15199 is fixed --> <jodatime.version>1.6</jodatime.version> <joni.version>2.1.2</joni.version> - <avatica.version>1.10.0</avatica.version> + <avatica.version>1.12.0</avatica.version> <jettyVersion>8.1.7.v20120910</jettyVersion> <tephra.version>0.14.0-incubating</tephra.version> + <omid.version>0.8.2.11-SNAPSHOT</omid.version> <spark.version>2.0.2</spark.version> <scala.version>2.11.8</scala.version> <scala.binary.version>2.11</scala.binary.version> @@@ -142,8 -141,8 +142,8 @@@ <artifactId>maven-compiler-plugin</artifactId> <version>3.0</version> <configuration> -- <source>1.7</source> -- <target>1.7</target> ++ <source>1.8</source> ++ <target>1.8</target> </configuration> </plugin> <!--This plugin's configuration is used to store Eclipse m2e settings