This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new 12bc597 PHOENIX-6601 Fix IndexTools bugs with namespace mapping 12bc597 is described below commit 12bc597d2a06de88ea39645d09a002db8c853945 Author: Istvan Toth <st...@apache.org> AuthorDate: Mon Nov 22 13:00:59 2021 +0100 PHOENIX-6601 Fix IndexTools bugs with namespace mapping --- .../phoenix/end2end/IndexBuildTimestampIT.java | 19 ++- .../org/apache/phoenix/end2end/IndexToolIT.java | 150 +++++++++++++-------- .../apache/phoenix/mapreduce/index/IndexTool.java | 17 +-- 3 files changed, 122 insertions(+), 64 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java index 6c36838..faa6cb0 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexBuildTimestampIT.java @@ -28,6 +28,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Properties; import org.apache.hadoop.hbase.client.ResultScanner; @@ -37,18 +38,22 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixResultSet; import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.util.EnvironmentEdge; import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; + import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; @Category(NeedsOwnMiniClusterTest.class) @RunWith(Parameterized.class) @@ -73,7 +78,19 @@ public class IndexBuildTimestampIT extends BaseTest { @BeforeClass public static synchronized void setup() throws Exception { - IndexToolIT.setup(); + Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2); + serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); + serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5)); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, + QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8)); + Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); + clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), + new ReadOnlyProps(clientProps.entrySet().iterator())); } @Parameters( diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java index c2de883..b121c3f 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java @@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.HBaseIOException; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; @@ -55,7 +54,6 @@ import org.apache.phoenix.query.BaseTest; import org.apache.phoenix.query.ConnectionQueryServices; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; -import org.apache.phoenix.schema.PIndexState; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature; import org.apache.phoenix.transaction.TransactionFactory; @@ -65,11 +63,11 @@ import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TestUtil; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.BeforeParam; import org.junit.runners.Parameterized.Parameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,6 +110,8 @@ import static org.junit.Assert.fail; public class IndexToolIT extends BaseTest { private static final Logger LOGGER = LoggerFactory.getLogger(IndexToolIT.class); + private static String tmpDir; + private final boolean localIndex; private final boolean mutable; private final boolean transactional; @@ -120,15 +120,17 @@ public class IndexToolIT extends BaseTest { private final String indexDDLOptions; private final boolean useSnapshot; private final boolean useTenantId; + private final boolean namespaceMapped; public IndexToolIT(String transactionProvider, boolean mutable, boolean localIndex, - boolean directApi, boolean useSnapshot, boolean useTenantId) { + boolean directApi, boolean useSnapshot, boolean useTenantId, boolean namespaceMapped) { this.localIndex = localIndex; this.mutable = mutable; this.transactional = transactionProvider != null; this.directApi = directApi; this.useSnapshot = useSnapshot; this.useTenantId = useTenantId; + this.namespaceMapped = namespaceMapped; StringBuilder optionBuilder = new StringBuilder(); if (!mutable) { optionBuilder.append(" IMMUTABLE_ROWS=true "); @@ -152,58 +154,81 @@ public class IndexToolIT extends BaseTest { this.indexDDLOptions = indexOptionBuilder.toString(); } - @BeforeClass - public static synchronized void setup() throws Exception { - Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2); - serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); - serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5)); - serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, - QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); - serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8)); - Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); - clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); - clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); - clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); - clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); - setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), - new ReadOnlyProps(clientProps.entrySet().iterator())); + @BeforeParam + public static synchronized void setup( + String transactionProvider, boolean mutable, boolean localIndex, + boolean directApi, boolean useSnapshot, boolean useTenantId, boolean namespaceMapped) + throws Exception { + if (clusterInitialized && Boolean.valueOf(namespaceMapped).equals(utility.getConfiguration() + .getBoolean(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, true))) { + //Perf optimization: no need to re-initialize the minicluster + return; + } else { + //This has been known to cause flakeyness + //Maybe we're gonna have to refactor the namespace + if(clusterInitialized) { + tearDownMiniCluster(0); + System.setProperty("java.io.tmpdir", tmpDir); + } else { + tmpDir = System.getProperty("java.io.tmpdir"); + } + Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2); + serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20)); + serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5)); + serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, + QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + serverProps.put(QueryServices.INDEX_REBUILD_PAGE_SIZE_IN_ROWS, Long.toString(8)); + serverProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, + Boolean.toString(namespaceMapped)); + Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2); + clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true)); + clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5)); + clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString()); + clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString()); + clientProps.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, + Boolean.toString(namespaceMapped)); + setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()), + new ReadOnlyProps(clientProps.entrySet().iterator())); + } } @Parameters( name = "transactionProvider={0},mutable={1},localIndex={2},directApi={3}," + - "useSnapshot={4},useTenant={5}") + "useSnapshot={4},useTenant={5},namespaceMapped={6}") public static synchronized Collection<Object[]> data() { List<Object[]> list = Lists.newArrayListWithExpectedSize(48); boolean[] Booleans = new boolean[] { false, true }; - for (String transactionProvider : new String[] {"TEPHRA", "OMID", null}) { - if(transactionProvider !=null && - !TransactionFactory.Provider.valueOf(transactionProvider).runTests()) { - continue; - } - for (boolean mutable : Booleans) { - for (boolean localIndex : Booleans) { - if (!localIndex - || transactionProvider == null - || !TransactionFactory.getTransactionProvider( - TransactionFactory.Provider.valueOf(transactionProvider)) - .isUnsupported(Feature.ALLOW_LOCAL_INDEX)) { - if (localIndex) { - for (boolean directApi : Booleans) { + for (boolean namespaceMapped : Booleans) { + for (String transactionProvider : new String[] {"TEPHRA", "OMID", null}) { + if(transactionProvider !=null && + !TransactionFactory.Provider.valueOf(transactionProvider).runTests()) { + continue; + } + for (boolean mutable : Booleans) { + for (boolean localIndex : Booleans) { + if (!localIndex + || transactionProvider == null + || !TransactionFactory.getTransactionProvider( + TransactionFactory.Provider.valueOf(transactionProvider)) + .isUnsupported(Feature.ALLOW_LOCAL_INDEX)) { + if (localIndex) { + for (boolean directApi : Booleans) { + list.add(new Object[]{transactionProvider, mutable, localIndex, + directApi, false, false, namespaceMapped}); + } + } + else { + // Due to PHOENIX-5375 and PHOENIX-5376, the snapshot and bulk load options are ignored for global indexes list.add(new Object[]{transactionProvider, mutable, localIndex, - directApi, false, false}); + true, false, false, namespaceMapped}); } } - else { - // Due to PHOENIX-5375 and PHOENIX-5376, the snapshot and bulk load options are ignored for global indexes - list.add(new Object[]{transactionProvider, mutable, localIndex, - true, false, false}); - } } } } } // Add the usetenantId - list.add(new Object[] { null, false, false, true, false, true}); + list.add(new Object[] { null, false, false, true, false, true, false}); return TestUtil.filterTxParamData(list,0); } @@ -230,6 +255,9 @@ public class IndexToolIT extends BaseTest { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { + if(namespaceMapped) { + conn.createStatement().execute("CREATE SCHEMA " + schemaName); + } String stmString1 = "CREATE TABLE " + dataTableFullName + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) " @@ -289,9 +317,9 @@ public class IndexToolIT extends BaseTest { assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType()); assertEquals(dataTableFullName, - explainPlanAttributes.getTableName()); + explainPlanAttributes.getTableName().replaceAll(":", ".")); assertEquals("SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'", - explainPlanAttributes.getServerWhereFilter()); + explainPlanAttributes.getServerWhereFilter().replaceAll(":", ".")); ResultSet rs = stmt1.executeQuery(selectSql); assertTrue(rs.next()); @@ -321,7 +349,7 @@ public class IndexToolIT extends BaseTest { expectedTableName = indexTableFullName; } assertEquals(expectedTableName, - explainPlanAttributes.getTableName()); + explainPlanAttributes.getTableName().replaceAll(":", ".")); rs = conn.createStatement().executeQuery(selectSql); assertTrue(rs.next()); @@ -549,6 +577,9 @@ public class IndexToolIT extends BaseTest { String indexTableName = generateUniqueName(); try (Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES))) { + if(namespaceMapped) { + conn.createStatement().execute("CREATE SCHEMA " + schemaName); + } String dataDDL = "CREATE TABLE " + dataTableFullName + "(\n" + "ID VARCHAR NOT NULL PRIMARY KEY,\n" @@ -592,13 +623,13 @@ public class IndexToolIT extends BaseTest { String schemaName = generateUniqueName(); String dataTableName = generateUniqueName(); String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName); - final TableName dataTN = TableName.valueOf(dataTableFullName); String indexTableName = generateUniqueName(); - String indexTableFullName = SchemaUtil.getTableName(schemaName, indexTableName); - TableName indexTN = TableName.valueOf(indexTableFullName); try (Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)); Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) { + if(namespaceMapped) { + conn.createStatement().execute("CREATE SCHEMA " + schemaName); + } String dataDDL = "CREATE TABLE " + dataTableFullName + "(\n" + "ID VARCHAR NOT NULL PRIMARY KEY,\n" @@ -618,11 +649,14 @@ public class IndexToolIT extends BaseTest { for (String prefix : carNumPrefixes) { splitPoints[--numSplits] = Bytes.toBytes(prefix); } - HTableDescriptor dataTD = admin.getTableDescriptor(dataTN); - admin.disableTable(dataTN); - admin.deleteTable(dataTN); + TableName hDataName = TableName.valueOf( + SchemaUtil.getPhysicalHBaseTableName(schemaName, dataTableName, namespaceMapped) + .getBytes()); + HTableDescriptor dataTD = admin.getTableDescriptor(hDataName); + admin.disableTable(hDataName); + admin.deleteTable(hDataName); admin.createTable(dataTD, splitPoints); - assertEquals(targetNumRegions, admin.getTableRegions(dataTN).size()); + assertEquals(targetNumRegions, admin.getTableRegions(hDataName).size()); // insert data where index column values start with a, b, c, d int idCounter = 1; @@ -649,10 +683,13 @@ public class IndexToolIT extends BaseTest { // run with 50% sampling rate, split if data table more than 3 regions runIndexTool(directApi, useSnapshot, schemaName, dataTableName, indexTableName, "-sp", "50", "-spa", "3"); - assertEquals(targetNumRegions, admin.getTableRegions(indexTN).size()); + TableName hIndexName = TableName.valueOf( + SchemaUtil.getPhysicalHBaseTableName(schemaName, indexTableName, namespaceMapped) + .getBytes()); + assertEquals(targetNumRegions, admin.getTableRegions(hIndexName).size()); List<Cell> values = new ArrayList<>(); // every index region should have been written to, if the index table was properly split uniformly - for (HRegion region : getUtility().getHBaseCluster().getRegions(indexTN)) { + for (HRegion region : getUtility().getHBaseCluster().getRegions(hIndexName)) { values.clear(); RegionScanner scanner = region.getScanner(new Scan()); scanner.next(values); @@ -684,6 +721,9 @@ public class IndexToolIT extends BaseTest { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); try { + if(namespaceMapped) { + conn.createStatement().execute("CREATE SCHEMA " + sSchemaName); + } String stmString1 = "CREATE TABLE " + qDataTableName + " (ID INTEGER NOT NULL PRIMARY KEY, NAME VARCHAR, ZIP INTEGER) " @@ -739,7 +779,7 @@ public class IndexToolIT extends BaseTest { assertEquals(String.format( "CLIENT PARALLEL 1-WAY FULL SCAN OVER %s\n" + " SERVER FILTER BY (LPAD(UPPER(NAME, 'en_US'), 8, 'x') || '_xyz') = 'xxUNAME2_xyz'", - dataTableNameForExplain), actualExplainPlan); + dataTableNameForExplain), actualExplainPlan.replaceAll(":", ".")); rs = stmt1.executeQuery(selectSql); assertTrue(rs.next()); @@ -789,7 +829,7 @@ public class IndexToolIT extends BaseTest { : indexTableFullName); } assertTrue(actualExplainPlan + "\n expected to contain \n" + expectedExplainPlan, - actualExplainPlan.contains(expectedExplainPlan)); + actualExplainPlan.replaceAll(":", ".").contains(expectedExplainPlan)); } public static CounterGroup getMRJobCounters(IndexTool indexTool) throws IOException { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index 34d8b1c..439485b 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -666,9 +666,9 @@ public class IndexTool extends Configured implements Tool { String snapshotName; try { admin = pConnection.getQueryServices().getAdmin(); - String pdataTableName = pDataTable.getName().getString(); - snapshotName = new StringBuilder(pdataTableName).append("-Snapshot").toString(); - admin.snapshot(snapshotName, TableName.valueOf(pdataTableName)); + TableName hDdataTableName = TableName.valueOf(pDataTable.getPhysicalName().getBytes()); + snapshotName = new StringBuilder(hDdataTableName.toString()).append("-Snapshot").toString(); + admin.snapshot(snapshotName, hDdataTableName); } finally { if (admin != null) { admin.close(); @@ -1046,9 +1046,10 @@ public class IndexTool extends Configured implements Tool { throws SQLException, IOException, IllegalArgumentException { int numRegions; + TableName hDataName = TableName.valueOf(pDataTable.getPhysicalName().getBytes()); try (org.apache.hadoop.hbase.client.Connection tempHConn = getTemporaryHConnection(pConnection); RegionLocator regionLocator = - tempHConn.getRegionLocator(TableName.valueOf(qDataTable))) { + tempHConn.getRegionLocator(hDataName)) { numRegions = regionLocator.getStartKeys().length; if (autosplit && (numRegions <= autosplitNumRegions)) { LOGGER.info(String.format( @@ -1089,10 +1090,10 @@ public class IndexTool extends Configured implements Tool { splitPoints[splitIdx++] = b.getRightBoundExclusive(); } // drop table and recreate with appropriate splits - TableName indexTN = TableName.valueOf(pIndexTable.getPhysicalName().getBytes()); - HTableDescriptor descriptor = admin.getTableDescriptor(indexTN); - admin.disableTable(indexTN); - admin.deleteTable(indexTN); + TableName hIndexName = TableName.valueOf(pIndexTable.getPhysicalName().getBytes()); + HTableDescriptor descriptor = admin.getTableDescriptor(hIndexName); + admin.disableTable(hIndexName); + admin.deleteTable(hIndexName); admin.createTable(descriptor, splitPoints); } }