PHOENIX-1644 Check for min HBase version before creating local index and provide means of disabling usage
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/54c4ed80 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/54c4ed80 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/54c4ed80 Branch: refs/heads/calcite Commit: 54c4ed80869f6ac0011a65d1247285cfae0a6759 Parents: f3c675b Author: James Taylor <jtay...@salesforce.com> Authored: Fri Feb 6 14:43:04 2015 -0800 Committer: James Taylor <jtay...@salesforce.com> Committed: Fri Feb 6 14:59:01 2015 -0800 ---------------------------------------------------------------------- .../phoenix/end2end/DisableLocalIndexIT.java | 99 ++++++++++++++++++++ .../phoenix/exception/SQLExceptionCode.java | 3 + .../phoenix/jdbc/PhoenixDatabaseMetaData.java | 5 +- .../query/ConnectionQueryServicesImpl.java | 8 ++ .../org/apache/phoenix/query/HTableFactory.java | 3 +- .../org/apache/phoenix/query/QueryServices.java | 1 + .../phoenix/query/QueryServicesOptions.java | 1 + .../apache/phoenix/schema/MetaDataClient.java | 13 ++- 8 files changed, 127 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/it/java/org/apache/phoenix/end2end/DisableLocalIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DisableLocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DisableLocalIndexIT.java new file mode 100644 index 0000000..5f18a1c --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DisableLocalIndexIT.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.exception.SQLExceptionCode; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.util.MetaDataUtil; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.google.common.collect.Maps; + +public class DisableLocalIndexIT extends BaseHBaseManagedTimeIT { + @BeforeClass + @Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class) + public static void doSetup() throws Exception { + Map<String,String> props = Maps.newHashMapWithExpectedSize(1); + // Must update config before starting server + props.put(QueryServices.ALLOW_LOCAL_INDEX_ATTRIB, Boolean.FALSE.toString()); + setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator())); + } + + @Test + public void testDisabledLocalIndexes() throws Exception { + Connection conn = DriverManager.getConnection(getUrl()); + conn.setAutoCommit(true); + String tableName = "DISABLE_LOCAL_INDEX_TEST"; + conn.createStatement().execute("CREATE TABLE " + tableName + " (k1 VARCHAR NOT NULL, k2 VARCHAR, CONSTRAINT PK PRIMARY KEY(K1,K2)) MULTI_TENANT=true"); + conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('t1','x')"); + conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('t2','y')"); + HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); + assertFalse(admin.tableExists(Bytes.toBytes(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + tableName))); + admin.close(); + try { + HTableInterface t = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + tableName)); + t.getTableDescriptor(); // Exception no longer thrown by getTable, but instead need to force an RPC + fail("Local index table should not have been created"); + } catch (org.apache.hadoop.hbase.TableNotFoundException e) { + //expected + } finally { + admin.close(); + } + + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); + props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, "t1"); + Connection tsconn = DriverManager.getConnection(getUrl(), props); + + tsconn.createStatement().execute("CREATE VIEW A.BAR(V1 VARCHAR) AS SELECT * FROM " + tableName); + tsconn.createStatement().execute("CREATE INDEX I1 ON A.BAR(V1)"); + tsconn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(MetaDataUtil.VIEW_INDEX_TABLE_PREFIX + tableName)); + + try { + conn.createStatement().execute("CREATE LOCAL INDEX I2 ON " + tableName + "(k2)"); + fail("Should not allow creation of local index"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.UNALLOWED_LOCAL_INDEXES.getErrorCode(), e.getErrorCode()); + } + try { + tsconn.createStatement().execute("CREATE LOCAL INDEX I2 ON A.BAR(k2, v1)"); + fail("Should not allow creation of local index"); + } catch (SQLException e) { + assertEquals(SQLExceptionCode.UNALLOWED_LOCAL_INDEXES.getErrorCode(), e.getErrorCode()); + } + } + +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index 8a6b8d0..19e7cdf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -236,6 +236,9 @@ public enum SQLExceptionCode { CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified"), CANNOT_SET_TABLE_PROPERTY_ADD_COLUMN(1053, "43A10", "Table level property cannot be set when adding a column"), + NO_LOCAL_INDEXES(1054, "43A11", "Local secondary indexes are only supported for HBase version " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.LOCAL_SI_VERSION_THRESHOLD) + " and above."), + UNALLOWED_LOCAL_INDEXES(1055, "43A12", "Local secondary indexes are configured to not be allowed."), + /** Sequence related */ SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() { @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java index b26f408..7ac2bb6 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java @@ -52,17 +52,17 @@ import org.apache.phoenix.iterate.DelegateResultIterator; import org.apache.phoenix.iterate.MaterializedResultIterator; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.query.QueryConstants; -import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.PDatum; import org.apache.phoenix.schema.PName; import org.apache.phoenix.schema.PTable.LinkType; import org.apache.phoenix.schema.PTableType; -import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.tuple.SingleKeyValueTuple; import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.KeyValueUtil; import org.apache.phoenix.util.SchemaUtil; @@ -274,6 +274,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData, org.apache.pho public static final int ESSENTIAL_FAMILY_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "7"); // Version below which we should disallow usage of mutable secondary indexing. public static final int MUTABLE_SI_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "10"); + public static final int LOCAL_SI_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "98", "9"); /** Version below which we fall back on the generic KeyValueBuilder */ public static final int CLIENT_KEY_VALUE_BUILDER_THRESHOLD = VersionUtil.encodeVersion("0", "94", "14"); http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 97efc43..6d58f57 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -1069,6 +1069,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } private void ensureLocalIndexTableCreated(byte[] physicalTableName, Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits) throws SQLException, TableAlreadyExistsException { + + // If we're not allowing local indexes or the hbase version is too low, + // don't create the local index table + if ( !this.getProps().getBoolean(QueryServices.ALLOW_LOCAL_INDEX_ATTRIB, QueryServicesOptions.DEFAULT_ALLOW_LOCAL_INDEX) + || getLowestClusterHBaseVersion() < PhoenixDatabaseMetaData.LOCAL_SI_VERSION_THRESHOLD) { + return; + } + tableProps.put(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME, TRUE_BYTES_AS_STRING); HTableDescriptor desc = ensureTableCreated(physicalTableName, PTableType.TABLE, tableProps, families, splits, true); if (desc != null) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/query/HTableFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/HTableFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/query/HTableFactory.java index 447267c..7a10683 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/HTableFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/HTableFactory.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.concurrent.ExecutorService; import org.apache.hadoop.hbase.client.HConnection; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.HTableInterface; /** @@ -48,7 +47,7 @@ public interface HTableFactory { static class HTableFactoryImpl implements HTableFactory { @Override public HTableInterface getTable(byte[] tableName, HConnection connection, ExecutorService pool) throws IOException { - return new HTable(tableName, connection, pool); + return connection.getTable(tableName, pool); } } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java index ce9016d..d21695d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -123,6 +123,7 @@ public interface QueryServices extends SQLCloseable { public static final String MIN_INDEX_PRIOIRTY_ATTRIB = "phoenix.regionserver.index.priority.min"; public static final String MAX_INDEX_PRIOIRTY_ATTRIB = "phoenix.regionserver.index.priority.max"; public static final String INDEX_HANDLER_COUNT_ATTRIB = "phoenix.regionserver.index.handler.count"; + public static final String ALLOW_LOCAL_INDEX_ATTRIB = "phoenix.index.allowLocalIndex"; // Config parameters for for configuring tracing public static final String TRACING_FREQ_ATTRIB = "phoenix.trace.frequency"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 5913796..0f9139f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -144,6 +144,7 @@ public class QueryServicesOptions { */ public static final int DEFAULT_INDEX_MIN_PRIORITY = 1000; public static final int DEFAULT_INDEX_HANDLER_COUNT = 30; + public static final boolean DEFAULT_ALLOW_LOCAL_INDEX = true; public static final int DEFAULT_TRACING_PAGE_SIZE = 100; /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/54c4ed80/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 09d2f66..effdb54 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -909,6 +909,16 @@ public class MetaDataClient { boolean retry = true; Short indexId = null; boolean allocateIndexId = false; + boolean isLocalIndex = statement.getIndexType() == IndexType.LOCAL; + int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion(); + if (isLocalIndex) { + if (!connection.getQueryServices().getProps().getBoolean(QueryServices.ALLOW_LOCAL_INDEX_ATTRIB, QueryServicesOptions.DEFAULT_ALLOW_LOCAL_INDEX)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNALLOWED_LOCAL_INDEXES).setTableName(indexTableName.getTableName()).build().buildException(); + } + if (hbaseVersion < PhoenixDatabaseMetaData.LOCAL_SI_VERSION_THRESHOLD) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_LOCAL_INDEXES).setTableName(indexTableName.getTableName()).build().buildException(); + } + } while (true) { try { ColumnResolver resolver = FromCompiler.getResolver(statement, connection); @@ -920,7 +930,6 @@ public class MetaDataClient { throw new SQLFeatureNotSupportedException("An index may only be created for a VIEW through a tenant-specific connection"); } } - int hbaseVersion = connection.getQueryServices().getLowestClusterHBaseVersion(); if (!dataTable.isImmutableRows()) { if (hbaseVersion < PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_MUTABLE_INDEXES).setTableName(indexTableName.getTableName()).build().buildException(); @@ -960,7 +969,7 @@ public class MetaDataClient { * 1) for a local index, as all local indexes will reside in the same HBase table * 2) for a view on an index. */ - if (statement.getIndexType() == IndexType.LOCAL || (dataTable.getType() == PTableType.VIEW && dataTable.getViewType() != ViewType.MAPPED)) { + if (isLocalIndex || (dataTable.getType() == PTableType.VIEW && dataTable.getViewType() != ViewType.MAPPED)) { allocateIndexId = true; // Next add index ID column PDataType dataType = MetaDataUtil.getViewIndexIdDataType();
