Repository: phoenix Updated Branches: refs/heads/4.x-HBase-1.1 a1e9b3277 -> d70494a7c
PHOENIX-4027 Addendum - move testRebuildIndexConnectionProperties to its own class. Fix typo for number of rpc retries Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d70494a7 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d70494a7 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d70494a7 Branch: refs/heads/4.x-HBase-1.1 Commit: d70494a7cb51a0aa4c1c893aa1ad787cc1479be4 Parents: a1e9b32 Author: Samarth Jain <[email protected]> Authored: Mon Jul 17 10:16:16 2017 -0700 Committer: Samarth Jain <[email protected]> Committed: Mon Jul 17 10:16:16 2017 -0700 ---------------------------------------------------------------------- .../phoenix/end2end/PhoenixRuntimeIT.java | 65 --------- .../end2end/RebuildIndexConnectionPropsIT.java | 132 +++++++++++++++++++ .../phoenix/query/QueryServicesOptions.java | 2 +- .../java/org/apache/phoenix/util/QueryUtil.java | 6 +- 4 files changed, 135 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/d70494a7/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRuntimeIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRuntimeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRuntimeIT.java index 95ab906..72ff21e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRuntimeIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PhoenixRuntimeIT.java @@ -18,8 +18,6 @@ package org.apache.phoenix.end2end; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -31,9 +29,6 @@ import java.util.HashSet; import java.util.Properties; import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -45,13 +40,9 @@ import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter; import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.phoenix.coprocessor.MetaDataRegionObserver; import org.apache.phoenix.expression.Expression; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; -import org.apache.phoenix.query.ConnectionQueryServices; -import org.apache.phoenix.query.QueryServices; -import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.schema.tuple.ResultTuple; import org.apache.phoenix.schema.types.PVarchar; @@ -60,7 +51,6 @@ import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.TestUtil; import org.junit.Test; -import org.mockito.internal.util.reflection.Whitebox; import com.google.common.collect.Sets; @@ -157,59 +147,4 @@ public class PhoenixRuntimeIT extends ParallelStatsDisabledIT { HTableInterface htable7 = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(tableName)); assertTenantIds(e7, htable7, new FirstKeyOnlyFilter(), new String[] {t1, t2} ); } - - @Test - public void testRebuildIndexConnectionProperties() throws Exception { - try (PhoenixConnection rebuildIndexConnection = - MetaDataRegionObserver.getRebuildIndexConnection(config)) { - try (PhoenixConnection regularConnection = - DriverManager.getConnection(url).unwrap(PhoenixConnection.class)) { - String rebuildUrl = rebuildIndexConnection.getURL(); - // assert that the url ends with expected string - assertTrue( - rebuildUrl.contains(MetaDataRegionObserver.REBUILD_INDEX_APPEND_TO_URL_STRING)); - // assert that the url for regular connection vs the rebuild connection is different - assertFalse(rebuildUrl.equals(regularConnection.getURL())); - Configuration rebuildQueryServicesConfig = - rebuildIndexConnection.getQueryServices().getConfiguration(); - // assert that the properties are part of the query services config - assertEquals(Long.toString(Long.MAX_VALUE), - rebuildQueryServicesConfig.get(PhoenixRuntime.CURRENT_SCN_ATTRIB)); - assertEquals( - Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT), - rebuildQueryServicesConfig.get(QueryServices.THREAD_TIMEOUT_MS_ATTRIB)); - assertEquals( - Long.toString( - QueryServicesOptions.DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT), - rebuildQueryServicesConfig.get(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); - assertEquals(Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_TIMEOUT), - rebuildQueryServicesConfig.get(HConstants.HBASE_RPC_TIMEOUT_KEY)); - assertEquals( - Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER), - rebuildQueryServicesConfig.get(HConstants.HBASE_CLIENT_RETRIES_NUMBER)); - ConnectionQueryServices rebuildQueryServices = rebuildIndexConnection.getQueryServices(); - HConnection rebuildIndexHConnection = - (HConnection) Whitebox.getInternalState(rebuildQueryServices, - "connection"); - HConnection regularHConnection = - (HConnection) Whitebox.getInternalState( - regularConnection.getQueryServices(), "connection"); - // assert that a new HConnection was spawned - assertFalse( - regularHConnection.toString().equals(rebuildIndexHConnection.toString())); - Configuration rebuildHConnectionConfig = rebuildIndexHConnection.getConfiguration(); - // assert that the HConnection has the desired properties needed for rebuilding - // indices - assertEquals( - Long.toString( - QueryServicesOptions.DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT), - rebuildHConnectionConfig.get(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); - assertEquals(Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_TIMEOUT), - rebuildHConnectionConfig.get(HConstants.HBASE_RPC_TIMEOUT_KEY)); - assertEquals( - Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER), - rebuildHConnectionConfig.get(HConstants.HBASE_CLIENT_RETRIES_NUMBER)); - } - } - } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/d70494a7/phoenix-core/src/it/java/org/apache/phoenix/end2end/RebuildIndexConnectionPropsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RebuildIndexConnectionPropsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RebuildIndexConnectionPropsIT.java new file mode 100644 index 0000000..56ae0f7 --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RebuildIndexConnectionPropsIT.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.end2end; + +import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.phoenix.coprocessor.MetaDataRegionObserver; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver; +import org.apache.phoenix.query.ConnectionQueryServices; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.util.PhoenixRuntime; +import org.apache.phoenix.util.PropertiesUtil; +import org.apache.phoenix.util.ReadOnlyProps; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; + +public class RebuildIndexConnectionPropsIT extends BaseUniqueNamesOwnClusterIT { + private static HBaseTestingUtility hbaseTestUtil; + private static String zkQuorum; + private static String url; + private static int NUM_RPC_RETRIES = 1; + + @BeforeClass + public static void doSetup() throws Exception { + Configuration conf = HBaseConfiguration.create(); + hbaseTestUtil = new HBaseTestingUtility(conf); + Map<String, String> serverProps = new HashMap<>(); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + // need at least one retry otherwise test fails + serverProps.put(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER, Long.toString(NUM_RPC_RETRIES)); + setUpConfigForMiniCluster(conf, new ReadOnlyProps(serverProps.entrySet().iterator())); + hbaseTestUtil.startMiniCluster(); + // establish url and quorum. Need to use PhoenixDriver and not PhoenixTestDriver + zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort(); + url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum; + Properties driverProps = PropertiesUtil.deepCopy(TEST_PROPERTIES); + DriverManager.registerDriver(PhoenixDriver.INSTANCE); + try (PhoenixConnection phxConn = + DriverManager.getConnection(url, driverProps).unwrap(PhoenixConnection.class)) { + } + } + + @Test + public void testRebuildIndexConnectionProperties() throws Exception { + try (PhoenixConnection rebuildIndexConnection = + MetaDataRegionObserver.getRebuildIndexConnection(hbaseTestUtil.getMiniHBaseCluster().getConfiguration())) { + try (PhoenixConnection regularConnection = + DriverManager.getConnection(url).unwrap(PhoenixConnection.class)) { + String rebuildUrl = rebuildIndexConnection.getURL(); + // assert that we are working with non-test urls + assertFalse(PhoenixEmbeddedDriver.isTestUrl(url)); + assertFalse(PhoenixEmbeddedDriver.isTestUrl(rebuildUrl)); + // assert that the url ends with expected string + assertTrue( + rebuildUrl.contains(MetaDataRegionObserver.REBUILD_INDEX_APPEND_TO_URL_STRING)); + // assert that the url for regular connection vs the rebuild connection is different + assertFalse(rebuildUrl.equals(regularConnection.getURL())); + Configuration rebuildQueryServicesConfig = + rebuildIndexConnection.getQueryServices().getConfiguration(); + // assert that the properties are part of the query services config + assertEquals(Long.toString(Long.MAX_VALUE), + rebuildQueryServicesConfig.get(PhoenixRuntime.CURRENT_SCN_ATTRIB)); + assertEquals( + Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT), + rebuildQueryServicesConfig.get(QueryServices.THREAD_TIMEOUT_MS_ATTRIB)); + assertEquals( + Long.toString( + QueryServicesOptions.DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT), + rebuildQueryServicesConfig.get(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); + assertEquals(Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_TIMEOUT), + rebuildQueryServicesConfig.get(HConstants.HBASE_RPC_TIMEOUT_KEY)); + assertEquals( + Long.toString(NUM_RPC_RETRIES), + rebuildQueryServicesConfig.get(HConstants.HBASE_CLIENT_RETRIES_NUMBER)); + ConnectionQueryServices rebuildQueryServices = rebuildIndexConnection.getQueryServices(); + HConnection rebuildIndexHConnection = + (HConnection) Whitebox.getInternalState(rebuildQueryServices, + "connection"); + HConnection regularHConnection = + (HConnection) Whitebox.getInternalState( + regularConnection.getQueryServices(), "connection"); + // assert that a new HConnection was created + assertFalse( + regularHConnection.toString().equals(rebuildIndexHConnection.toString())); + Configuration rebuildHConnectionConfig = rebuildIndexHConnection.getConfiguration(); + // assert that the HConnection has the desired properties needed for rebuilding + // indices + assertEquals( + Long.toString( + QueryServicesOptions.DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT), + rebuildHConnectionConfig.get(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)); + assertEquals(Long.toString(QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_TIMEOUT), + rebuildHConnectionConfig.get(HConstants.HBASE_RPC_TIMEOUT_KEY)); + assertEquals( + Long.toString(NUM_RPC_RETRIES), + rebuildHConnectionConfig.get(HConstants.HBASE_CLIENT_RETRIES_NUMBER)); + } + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/d70494a7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index 21d8300..b974d02 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -185,7 +185,7 @@ public class QueryServicesOptions { public static final long DEFAULT_INDEX_REBUILD_QUERY_TIMEOUT = 30000 * 60; // 30 mins public static final long DEFAULT_INDEX_REBUILD_RPC_TIMEOUT = 30000 * 60; // 30 mins public static final long DEFAULT_INDEX_REBUILD_CLIENT_SCANNER_TIMEOUT = 30000 * 60; // 30 mins - public static final int DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER = 1; // no retries at rpc level + public static final int DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER = 0; // no retries at rpc level public static final int DEFAULT_INDEX_REBUILD_DISABLE_TIMESTAMP_THRESHOLD = 30000 * 60; // 30 mins /** http://git-wip-us.apache.org/repos/asf/phoenix/blob/d70494a7/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java index 7d70138..7961453 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java @@ -350,9 +350,7 @@ public final class QueryUtil { return getConnectionUrl(props, conf, null); } /** - * @return connection url using the various properties set in props and conf. This method is an - * alternative to {@link #getConnectionUrlUsingProps(Properties, String)} when all the - * relevant connection properties are passed in both {@link Properties} and {@link Configuration} + * @return connection url using the various properties set in props and conf. */ public static String getConnectionUrl(Properties props, Configuration conf, String principal) throws ClassNotFoundException, SQLException { @@ -376,7 +374,7 @@ public final class QueryUtil { props.getProperty(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, defaultExtraArgs); if (extraArgs.length() > 0) { url += - PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + extraArgs + PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR + extraArgs + PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR; } else { url += PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
