This is an automated email from the ASF dual-hosted git repository. apurtell pushed a commit to branch PHOENIX-7562-feature in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 79df024ab8bbe0b173bda5b6ed9b5975833beb22 Author: Jacob Isaac <[email protected]> AuthorDate: Wed May 14 15:44:58 2025 -0700 PHOENIX-7107 (ADDENDUM) Fixes RowKeyMatcherIT tests silently failing (#2147) --- .../phoenix/end2end/BaseRowKeyMatcherTestIT.java | 31 +++++----------------- .../LongViewIndexDisabledBaseRowKeyMatcherIT.java | 24 +++++++++++++++-- .../LongViewIndexEnabledBaseRowKeyMatcherIT.java | 26 ++++++++++++++++-- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java index b3e68b8212..dd1c65ee78 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java @@ -26,8 +26,6 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.filter.BinaryComparator; -import org.apache.hadoop.hbase.filter.BinaryComponentComparator; import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.RowFilter; @@ -58,7 +56,6 @@ import org.apache.phoenix.schema.PTable; import org.apache.phoenix.schema.RowKeyValueAccessor; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.tuple.ResultTuple; -import org.apache.phoenix.schema.types.PBinary; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PDataType; import org.apache.phoenix.schema.types.PDate; @@ -73,14 +70,12 @@ import org.apache.phoenix.schema.types.PVarbinaryEncoded; import org.apache.phoenix.schema.types.PVarchar; import org.apache.phoenix.thirdparty.com.google.common.collect.Maps; import org.apache.phoenix.util.ByteUtil; -import org.apache.phoenix.util.LogUtil; import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.PropertiesUtil; -import org.apache.phoenix.util.ScanUtil; import org.apache.phoenix.util.SchemaUtil; -import org.apache.phoenix.util.StringUtil; import org.apache.phoenix.util.ViewUtil; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,8 +104,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_LINK_HBASE_ import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -664,7 +657,8 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { } ; } catch (SQLException | IOException e) { - throw new RuntimeException(e); + LOGGER.info(e.getMessage()); + fail(); } return viewToRowKeyMap; @@ -749,17 +743,18 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { allRows.setFilter(andFilter); ResultScanner scanner = tbl.getScanner(allRows); int numMatchingRows = 0; + //TestUtil.dumpTable(connection, org.apache.hadoop.hbase.TableName.valueOf(hbaseTableName)); for (Result result = scanner.next(); result != null; result = scanner.next()) { rowkey = result.getRow(); numMatchingRows++; } - assertEquals(String.format("Expected rows do match for table = %s, rowId = %s", - Bytes.toString(hbaseTableName), rowId), 1, numMatchingRows); - PrefixFilter matchFilter = new PrefixFilter(prefix); LOGGER.debug(String.format("row-key = %s, tenantId = %s, prefix = %s, matched = %s", Bytes.toStringBinary(rowkey), tenantId, Bytes.toStringBinary(prefix), !matchFilter.filterRowKey(KeyValueUtil.createFirstOnRow(rowkey)))); + assertEquals(String.format("Expected rows do match for table = %s, rowId = %s", + Bytes.toString(hbaseTableName), rid), 1, numMatchingRows); + } } @@ -856,15 +851,6 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { List<PDataType[]> testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); - try (Connection conn = DriverManager.getConnection(getUrl()); - Statement stmt = conn.createStatement()) { - //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); - conn.commit(); - } - String tableName = ""; tableName = createViewHierarchy( testCases, sortOrders, 500, 5000, 3, @@ -879,7 +865,6 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { SchemaUtil.getTableNameFromFullName(tableName)); } catch (Exception e) { - e.printStackTrace(); LOGGER.error(e.getMessage()); } } @@ -928,7 +913,6 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { SchemaUtil.getTableNameFromFullName(tableName)); } catch (Exception e) { - e.printStackTrace(); LOGGER.error(e.getMessage()); } } @@ -965,7 +949,6 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT { SchemaUtil.getTableNameFromFullName(tableName)); } catch (Exception e) { - e.printStackTrace(); LOGGER.error(e.getMessage()); } diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java index a8ae4aa759..1136253a21 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java @@ -28,6 +28,9 @@ import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; @@ -38,6 +41,7 @@ public class LongViewIndexDisabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherT public static synchronized void doSetup() throws Exception { final Configuration conf = HBaseConfiguration.create(); conf.set(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true)); + conf.set(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true)); conf.set(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(false)); conf.set(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, "true"); conf.set(IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY, @@ -60,8 +64,24 @@ public class LongViewIndexDisabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherT } }); - Map<String, String> DEFAULT_PROPERTIES = new HashMap() ; - setUpTestDriver(new ReadOnlyProps(DEFAULT_PROPERTIES.entrySet().iterator())); + // Turn off the Long view index feature + Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{ + put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(false)); + }}; + + setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, + DEFAULT_PROPERTIES.entrySet().iterator())); + + // Create the CATALOG indexes for additional verifications using the catalog indexes + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + } @Override diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java index a7778aea1e..3fa29c41b7 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java @@ -20,6 +20,7 @@ package org.apache.phoenix.end2end; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; import org.apache.phoenix.hbase.index.util.IndexManagementUtil; import org.apache.phoenix.query.ConfigurationFactory; import org.apache.phoenix.query.QueryServices; @@ -28,9 +29,14 @@ import org.apache.phoenix.util.ReadOnlyProps; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; import java.util.HashMap; import java.util.Map; +import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; + @Category(NeedsOwnMiniClusterTest.class) public class LongViewIndexEnabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherTestIT { @@ -38,6 +44,7 @@ public class LongViewIndexEnabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherTe public static synchronized void doSetup() throws Exception { final Configuration conf = HBaseConfiguration.create(); conf.set(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true)); + conf.set(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true)); conf.set(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true)); conf.set(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, "true"); conf.set(IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY, @@ -60,8 +67,23 @@ public class LongViewIndexEnabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherTe } }); - Map<String, String> DEFAULT_PROPERTIES = new HashMap() ; - setUpTestDriver(new ReadOnlyProps(DEFAULT_PROPERTIES.entrySet().iterator())); + // Turn on the Long view index feature + Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{ + put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true)); + }}; + + setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, + DEFAULT_PROPERTIES.entrySet().iterator())); + + // Create the CATALOG indexes for additional verifications using the catalog indexes + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } } @Override
