This is an automated email from the ASF dual-hosted git repository. richardantal pushed a commit to branch 4.16 in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.16 by this push: new ecc8862 PHOENIX-6596 Schema extraction double quotes expressions, resulting in un-executabe create statements ecc8862 is described below commit ecc8862fb70ae7d1fc84b6aa51d2a2022a82a62f Author: Richard Antal <antal97rich...@gmail.com> AuthorDate: Thu Dec 9 14:00:08 2021 +0100 PHOENIX-6596 Schema extraction double quotes expressions, resulting in un-executabe create statements Co-authored-by: Istvan Toth <st...@apache.org> --- .../schema/tool/SchemaToolExtractionIT.java | 63 ++++++++++++++++++++-- .../schema/tool/SchemaExtractionProcessor.java | 28 ++++++++-- .../org/apache/phoenix/schema/tool/SchemaTool.java | 1 - .../java/org/apache/phoenix/util/SchemaUtil.java | 23 ++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java index 6ba411c..0a733ae 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java @@ -93,13 +93,18 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String indexName = generateUniqueName(); String indexName1 = generateUniqueName(); String indexName2 = generateUniqueName(); + String indexName3 = generateUniqueName(); String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true"; String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName); - String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, \"v1\" VARCHAR, v2 VARCHAR)" + properties; - String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC) INCLUDE (v2)"; - String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (v1)"; + //FIXME never verified + String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(\"v1\" DESC) INCLUDE (v2)"; + //FIXME never verified + String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (\"v1\")"; String createIndexStatement2 = "CREATE INDEX "+indexName2 + " ON "+pTableFullName+"(k)"; + String createIndexStatement3 ="CREATE INDEX " + indexName3 + " ON " + pTableFullName + + "('QUOTED' || \"v1\" || V2 DESC, \"v1\" DESC, K) INCLUDE (V2)"; List<String> queries = new ArrayList<String>(){}; queries.add(createTableStatement); queries.add(createIndexStatement); @@ -108,6 +113,12 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String result = runSchemaExtractionTool(schemaName, indexName2, null, queries); Assert.assertEquals(createIndexStatement2.toUpperCase(), result.toUpperCase()); + + List<String> queries3 = new ArrayList<String>(){}; + queries3.add(createIndexStatement3); + + String result3 = runSchemaExtractionTool(schemaName, indexName3, null, queries3); + Assert.assertEquals(createIndexStatement3, result3); } @Test @@ -144,17 +155,45 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { String tableName = generateUniqueName(); String schemaName = generateUniqueName(); String indexName = generateUniqueName(); + String indexName2 = generateUniqueName(); String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true"; String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName); String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)" + properties; String createIndexStatement = "CREATE LOCAL INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC, k) INCLUDE (v2)"; + String createIndexStatement2 = "CREATE LOCAL INDEX "+indexName2 + " ON "+pTableFullName+"( LPAD(v1,10) DESC, k) INCLUDE (v2)"; + List<String> queries = new ArrayList<String>(){}; queries.add(createTableStatement); queries.add(createIndexStatement); String result = runSchemaExtractionTool(schemaName, indexName, null, queries); Assert.assertEquals(createIndexStatement.toUpperCase(), result.toUpperCase()); + + List<String> queries2 = new ArrayList<String>(){}; + queries2.add(createIndexStatement2); + + String result2 = runSchemaExtractionTool(schemaName, indexName2, null, queries2); + Assert.assertEquals(createIndexStatement2.toUpperCase(), result2.toUpperCase()); + } + + @Test + public void testCreateLocalIndexStatementLowerCase() throws Exception { + String tableName = generateUniqueName(); + String schemaName = generateUniqueName(); + String indexName = generateUniqueName(); + String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true"; + String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName); + String createTableStatement = "CREATE TABLE "+pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, \"v1\" VARCHAR, V2 VARCHAR)" + + properties; + String createIndexStatement = "CREATE LOCAL INDEX "+indexName + " ON "+pTableFullName+"( LPAD(\"v1\",10) DESC, K) INCLUDE (V2)"; + + List<String> queries = new ArrayList<String>(){}; + queries.add(createTableStatement); + queries.add(createIndexStatement); + + String result = runSchemaExtractionTool(schemaName, indexName, null, queries); + Assert.assertEquals(createIndexStatement, result); } @Test @@ -176,6 +215,24 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT { } @Test + public void testCreateIndexStatementLowerCaseCombined() throws Exception { + String tableName = "lowercase" + generateUniqueName(); + String schemaName = "lowercase" + generateUniqueName(); + String indexName = "\"lowercaseIND" + generateUniqueName() + "\""; + String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true"; + String pTableFullName = SchemaUtil.getEscapedTableName(schemaName, tableName); + String createTableStatement = "CREATE TABLE " + pTableFullName + "(ID varchar primary key, \"number\" integer, \"currency\" decimal(6,2), lista varchar[])" + + properties; + String createIndexStatement = "CREATE INDEX " + indexName + " ON "+ pTableFullName + "(\"number\" * \"currency\", ID) INCLUDE (LISTA)"; + List<String> queries = new ArrayList<String>(){}; + queries.add(createTableStatement); + queries.add(createIndexStatement); + + String result = runSchemaExtractionTool("\"" + schemaName + "\"", indexName, null, queries); + Assert.assertEquals(createIndexStatement, result); + } + + @Test public void testCreateViewStatement() throws Exception { String tableName = generateUniqueName(); String schemaName = generateUniqueName(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java index 656bd31..8383ca5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java @@ -134,8 +134,8 @@ public class SchemaExtractionProcessor implements SchemaProcessor { StringBuilder indexedColumnsBuilder = new StringBuilder(); for (PColumn indexedColumn : indexPK) { - String indexColumn = extractIndexColumn(indexedColumn.getName().getString(), defaultCF); - if(indexColumn.equalsIgnoreCase(MetaDataUtil.VIEW_INDEX_ID_COLUMN_NAME)) { + String indexColumn = extractIndexColumn(indexedColumn.getExpressionStr(), defaultCF); + if (indexColumn == null) { continue; } indexPKName.add(indexColumn); @@ -183,21 +183,37 @@ public class SchemaExtractionProcessor implements SchemaProcessor { } private String extractIndexColumn(String columnName, String defaultCF) { + if (columnName == null) { + return null; + } String [] columnNameSplit = columnName.split(":"); if(columnNameSplit[0].equals("") || columnNameSplit[0].equalsIgnoreCase(defaultCF) || (defaultCF.startsWith("L#") && columnNameSplit[0].equalsIgnoreCase(defaultCF.substring(2)))) { - return SchemaUtil.formatColumnName(columnNameSplit[1]); + return formatColumnOrExpression(columnNameSplit[1]); } else { if (columnNameSplit.length > 1) { String schema = SchemaUtil.formatSchemaName(columnNameSplit[0]); String name = SchemaUtil.formatColumnName(columnNameSplit[1]); return String.format("%s.%s", schema, name); } else { - return SchemaUtil.formatColumnName(columnNameSplit[0]); + return formatColumnOrExpression(columnNameSplit[0]); } } } + private String formatColumnOrExpression(String columnOrExpression) { + if (columnOrExpression.startsWith("(")) { + //Expressions like (a*b) are always parenthesised + return columnOrExpression.substring(1, columnOrExpression.length()-1); + } else if (columnOrExpression.contains("(")) { + //Expressions like like func(a) are always have a parenthesis + return columnOrExpression; + } else { + //If there are no parentheses, this is a column name + return SchemaUtil.formatIndexColumnName(columnOrExpression); + } + } + private String getCoveredColumnsString(PTable indexPTable, String defaultCF) { StringBuilder coveredColumnsBuilder = new StringBuilder(); List<PColumn> pkColumns = indexPTable.getColumns(); @@ -207,7 +223,9 @@ public class SchemaExtractionProcessor implements SchemaProcessor { } if(cc.getFamilyName()!=null) { String indexColumn = extractIndexColumn(cc.getName().getString(), defaultCF); - coveredColumnsBuilder.append(indexColumn); + if (indexColumn != null) { + coveredColumnsBuilder.append(indexColumn); + } } } return coveredColumnsBuilder.toString(); diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java index 0386c78..fdda6e3 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaTool.java @@ -17,7 +17,6 @@ */ package org.apache.phoenix.schema.tool; -import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec; import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.phoenix.thirdparty.org.apache.commons.cli.CommandLineParser; import org.apache.phoenix.thirdparty.org.apache.commons.cli.DefaultParser; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java index 93c98c8..4e35dec 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java @@ -1315,6 +1315,29 @@ public class SchemaUtil { return name; } + public static String formatIndexColumnName(String name) { + String[] splitName = name.split("\\."); + if(splitName.length < 2) { + if (!name.contains("\"")) { + if (quotesNeededForColumn(name)) { + name = "\"" + name + "\""; + } + return name; + } else if (name.startsWith("\"") && name.endsWith("\"")) { + if (quotesNeededForColumn(name.substring(1, name.length() - 1))) { + return name; + } else { + return name.substring(1, name.length() - 1); + } + } else { + return name; + } + } else { + return String.format("%s.%s", formatIndexColumnName(splitName[0]), + formatIndexColumnName(splitName[1])); + } + } + public static boolean containsLowerCase(String name) { if (Strings.isNullOrEmpty(name)) { return false;