PHOENIX-2616 Indexes over immutable tables not marked as immutable
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c9669359 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c9669359 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c9669359 Branch: refs/heads/calcite Commit: c966935921a85a0e6b5f4ba49d26e167f08349ed Parents: 6911770 Author: James Taylor <jamestay...@apache.org> Authored: Wed Jan 20 18:33:12 2016 -0800 Committer: James Taylor <jamestay...@apache.org> Committed: Wed Jan 20 18:33:47 2016 -0800 ---------------------------------------------------------------------- .../apache/phoenix/execute/MutationState.java | 18 ++++---- .../query/ConnectionQueryServicesImpl.java | 43 ++++++++++++++++---- .../apache/phoenix/schema/MetaDataClient.java | 5 ++- .../phoenix/compile/QueryCompilerTest.java | 12 +++--- 4 files changed, 54 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/c9669359/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java index f1a9c02..0a5b053 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java @@ -92,12 +92,6 @@ import org.apache.phoenix.util.TransactionUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - import co.cask.tephra.Transaction; import co.cask.tephra.Transaction.VisibilityLevel; import co.cask.tephra.TransactionAware; @@ -110,12 +104,16 @@ import co.cask.tephra.hbase11.TransactionAwareHTable; import co.cask.tephra.visibility.FenceWait; import co.cask.tephra.visibility.VisibilityFence; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + /** * * Tracks the uncommitted state * - * - * @since 0.1 */ public class MutationState implements SQLCloseable { private static final Logger logger = LoggerFactory.getLogger(MutationState.class); @@ -1227,7 +1225,9 @@ public class MutationState implements SQLCloseable { List<PTable> oldIndexes; PTableRef ptableRef = cache.getTableRef(dataTable.getKey()); oldIndexes = ptableRef.getTable().getIndexes(); - MetaDataMutationResult result = client.updateCache(dataTable.getTenantId(), dataTable.getSchemaName().getString(), dataTable.getTableName().getString()); + // Always check at server for metadata change, as it's possible that the table is configured to not check for metadata changes + // but in this case, the tx manager is telling us it's likely that there has been a change. + MetaDataMutationResult result = client.updateCache(dataTable.getTenantId(), dataTable.getSchemaName().getString(), dataTable.getTableName().getString(), true); long timestamp = TransactionUtil.getResolvedTime(connection, result); tableRef.setTimeStamp(timestamp); PTable updatedDataTable = result.getTable(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c9669359/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 4522cf8..73999b8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -187,6 +187,12 @@ import org.apache.twill.zookeeper.ZKClients; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import co.cask.tephra.TransactionSystemClient; +import co.cask.tephra.TxConstants; +import co.cask.tephra.distributed.PooledClientProvider; +import co.cask.tephra.distributed.TransactionServiceClient; +import co.cask.tephra.hbase11.coprocessor.TransactionProcessor; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Throwables; @@ -199,12 +205,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import co.cask.tephra.TransactionSystemClient; -import co.cask.tephra.TxConstants; -import co.cask.tephra.distributed.PooledClientProvider; -import co.cask.tephra.distributed.TransactionServiceClient; -import co.cask.tephra.hbase11.coprocessor.TransactionProcessor; - public class ConnectionQueryServicesImpl extends DelegateQueryServices implements ConnectionQueryServices { private static final Logger logger = LoggerFactory.getLogger(ConnectionQueryServicesImpl.class); @@ -2423,6 +2423,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement PhoenixDatabaseMetaData.TRANSACTIONAL + " " + PBoolean.INSTANCE.getSqlTypeName()); metaConnection = addColumnsIfNotExists(metaConnection, PhoenixDatabaseMetaData.SYSTEM_CATALOG, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1, PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " " + PLong.INSTANCE.getSqlTypeName()); + setImmutableTableIndexesImmutable(metaConnection); // Drop old stats table so that new stats table is created metaConnection = dropStatsTable(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0); @@ -2527,7 +2528,35 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } } - private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp) + + /** + * Set IMMUTABLE_ROWS to true for all index tables over immutable tables. + * @param metaConnection connection over which to run the upgrade + * @throws SQLException + */ + private static void setImmutableTableIndexesImmutable(PhoenixConnection metaConnection) throws SQLException { + boolean autoCommit = metaConnection.getAutoCommit(); + try { + metaConnection.setAutoCommit(true); + metaConnection.createStatement().execute( + "UPSERT INTO SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, IMMUTABLE_ROWS)\n" + + "SELECT A.TENANT_ID, A.TABLE_SCHEM,B.COLUMN_FAMILY,null,null,true\n" + + "FROM SYSTEM.CATALOG A JOIN SYSTEM.CATALOG B ON (\n" + + " A.TENANT_ID = B.TENANT_ID AND \n" + + " A.TABLE_SCHEM = B.TABLE_SCHEM AND\n" + + " A.TABLE_NAME = B.TABLE_NAME AND\n" + + " A.COLUMN_NAME = B.COLUMN_NAME AND\n" + + " B.LINK_TYPE = 1\n" + + ")\n" + + "WHERE A.COLUMN_FAMILY IS NULL AND\n" + + " B.COLUMN_FAMILY IS NOT NULL AND\n" + + " A.IMMUTABLE_ROWS = TRUE;"); + } finally { + metaConnection.setAutoCommit(autoCommit); + } + } + + private PhoenixConnection dropStatsTable(PhoenixConnection oldMetaConnection, long timestamp) throws SQLException, IOException { Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp-1)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/c9669359/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 064007f..d559842 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -114,6 +114,8 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import co.cask.tephra.TxConstants; + import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -216,8 +218,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; -import co.cask.tephra.TxConstants; - public class MetaDataClient { private static final Logger logger = LoggerFactory.getLogger(MetaDataClient.class); @@ -1580,6 +1580,7 @@ public class MetaDataClient { if (parent != null && tableType == PTableType.INDEX) { timestamp = TransactionUtil.getTableTimestamp(connection, transactional); storeNulls = parent.getStoreNulls(); + isImmutableRows = parent.isImmutableRows(); // Index on view // TODO: Can we support a multi-tenant index directly on a multi-tenant // table instead of only a view? We don't have anywhere to put the link http://git-wip-us.apache.org/repos/asf/phoenix/blob/c9669359/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index 6d6dcdf..b3baa73 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -1189,7 +1189,7 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { conn.createStatement().execute(ddl); assertImmutableRows(conn, "T", true); conn.createStatement().execute(indexDDL); - assertImmutableRows(conn, "T", true); + assertImmutableRows(conn, "I", true); conn.createStatement().execute("DELETE FROM t WHERE v2 = 'foo'"); fail(); } catch (SQLException e) { @@ -1209,13 +1209,13 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { @Test public void testInvalidNegativeArrayIndex() throws Exception { - String query = "SELECT a_double_array[-20] FROM table_with_array"; - Connection conn = DriverManager.getConnection(getUrl()); + String query = "SELECT a_double_array[-20] FROM table_with_array"; + Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute(query); fail(); } catch (Exception e) { - + } } @Test @@ -1232,8 +1232,8 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { @Test public void testNonArrayColumnWithIndex() throws Exception { - String query = "SELECT a_float[1] FROM table_with_array"; - Connection conn = DriverManager.getConnection(getUrl()); + String query = "SELECT a_float[1] FROM table_with_array"; + Connection conn = DriverManager.getConnection(getUrl()); try { conn.createStatement().execute(query); fail();