This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new f51f108 HIVE-21255: Remove QueryConditionBuilder in JdbcStorageHandler (Daniel Dai, reviewed by Jesus Camacho Rodriguez) f51f108 is described below commit f51f108b761f0c88647f48f30447dae12b308f31 Author: Daniel Dai <dai...@gmail.com> AuthorDate: Fri Mar 1 11:52:18 2019 -0800 HIVE-21255: Remove QueryConditionBuilder in JdbcStorageHandler (Daniel Dai, reviewed by Jesus Camacho Rodriguez) Signed-off-by: Jesus Camacho Rodriguez <jcama...@apache.org> --- .../hive/storage/jdbc/QueryConditionBuilder.java | 186 --------------------- .../jdbc/conf/JdbcStorageConfigManager.java | 11 -- .../storage/jdbc/TestQueryConditionBuilder.java | 150 ----------------- 3 files changed, 347 deletions(-) diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java deleted file mode 100644 index 194fad8..0000000 --- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/QueryConditionBuilder.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; -import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; - -import java.beans.XMLDecoder; -import java.io.ByteArrayInputStream; -import java.util.HashMap; -import java.util.Map; - -/** - * Translates the hive query condition into a condition that can be run on the underlying database - */ -public class QueryConditionBuilder { - - private static final Logger LOGGER = LoggerFactory.getLogger(QueryConditionBuilder.class); - private static final String EMPTY_STRING = ""; - private static QueryConditionBuilder instance = null; - - - public static QueryConditionBuilder getInstance() { - if (instance == null) { - instance = new QueryConditionBuilder(); - } - - return instance; - } - - - private QueryConditionBuilder() { - - } - - - public String buildCondition(Configuration conf) { - if (conf == null) { - return EMPTY_STRING; - } - - String filterXml = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR); - String hiveColumns = conf.get(serdeConstants.LIST_COLUMNS); - String columnMapping = conf.get(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName()); - - if ((filterXml == null) || ((columnMapping == null) && (hiveColumns == null))) { - return EMPTY_STRING; - } - - if (hiveColumns == null) { - hiveColumns = ""; - } - - Map<String, String> columnMap = buildColumnMapping(columnMapping, hiveColumns); - String condition = createConditionString(filterXml, columnMap); - return condition; - } - - - /* - * Build a Hive-to-X column mapping, - * - */ - private Map<String, String> buildColumnMapping(String columnMapping, String hiveColumns) { - if ((columnMapping == null) || (columnMapping.trim().isEmpty())) { - return createIdentityMap(hiveColumns); - } - - Map<String, String> columnMap = new HashMap<String, String>(); - String[] mappingPairs = columnMapping.toLowerCase().split(","); - for (String mapPair : mappingPairs) { - String[] columns = mapPair.split("="); - columnMap.put(columns[0].trim(), columns[1].trim()); - } - - return columnMap; - } - - - /* - * When no mapping is defined, it is assumed that the hive column names are equivalent to the column names in the - * underlying table - */ - private Map<String, String> createIdentityMap(String hiveColumns) { - Map<String, String> columnMap = new HashMap<String, String>(); - String[] columns = hiveColumns.toLowerCase().split(","); - - for (String col : columns) { - columnMap.put(col.trim(), col.trim()); - } - - return columnMap; - } - - - /* - * Walk to Hive AST and translate the hive column names to their equivalent mappings. This is basically a cheat. - * - */ - private String createConditionString(String filterXml, Map<String, String> columnMap) { - if ((filterXml == null) || (filterXml.trim().isEmpty())) { - return EMPTY_STRING; - } - - try (XMLDecoder decoder = new XMLDecoder(new ByteArrayInputStream(filterXml.getBytes("UTF-8")))) { - Object object = decoder.readObject(); - if (!(object instanceof ExprNodeDesc)) { - LOGGER.error("Deserialized filter expression is not of the expected type"); - throw new RuntimeException("Deserialized filter expression is not of the expected type"); - } - - ExprNodeDesc conditionNode = (ExprNodeDesc) object; - walkTreeAndTranslateColumnNames(conditionNode, columnMap); - return conditionNode.getExprString(); - } - catch (Exception e) { - LOGGER.error("Error during condition build", e); - return EMPTY_STRING; - } - } - - - /* - * Translate column names by walking the AST - */ - private void walkTreeAndTranslateColumnNames(ExprNodeDesc node, Map<String, String> columnMap) { - if (node == null) { - return; - } - - if (node instanceof ExprNodeColumnDesc) { - ExprNodeColumnDesc column = (ExprNodeColumnDesc) node; - String hiveColumnName = column.getColumn().toLowerCase(); - if (columnMap.containsKey(hiveColumnName)) { - String dbColumnName = columnMap.get(hiveColumnName); - String finalName = formatColumnName(dbColumnName); - column.setColumn(finalName); - } - } - else { - if (node.getChildren() != null) { - for (ExprNodeDesc childNode : node.getChildren()) { - walkTreeAndTranslateColumnNames(childNode, columnMap); - } - } - } - } - - - /** - * This is an ugly hack for handling date column types because Hive doesn't have a built-in type for dates - */ - private String formatColumnName(String dbColumnName) { - if (dbColumnName.contains(":")) { - String[] typeSplit = dbColumnName.split(":"); - - if (typeSplit[1].equalsIgnoreCase("date")) { - return "{d " + typeSplit[0] + "}"; - } - - return typeSplit[0]; - } - else { - return dbColumnName; - } - } -} diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java index 5679f1b..2039144 100644 --- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java +++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/conf/JdbcStorageConfigManager.java @@ -24,8 +24,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.conf.Configuration; -import org.apache.hive.storage.jdbc.QueryConditionBuilder; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,10 +128,6 @@ public class JdbcStorageConfigManager { if (tableName != null) { // We generate query as select * query = "select * from " + tableName; - String hiveFilterCondition = QueryConditionBuilder.getInstance().buildCondition(config); - if ((hiveFilterCondition != null) && (!hiveFilterCondition.trim().isEmpty())) { - query = query + " WHERE " + hiveFilterCondition; - } } else { query = config.get(Constants.JDBC_QUERY); } @@ -152,11 +146,6 @@ public class JdbcStorageConfigManager { String tableName = config.get(JdbcStorageConfig.TABLE.getPropertyName()); query = "select * from " + tableName; - String hiveFilterCondition = QueryConditionBuilder.getInstance().buildCondition(config); - if ((hiveFilterCondition != null) && (!hiveFilterCondition.trim().isEmpty())) { - query = query + " WHERE " + hiveFilterCondition; - } - return query; } diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java deleted file mode 100644 index a59645d..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.util.Scanner; - -import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; - -public class TestQueryConditionBuilder { - - private static String condition1; - private static String condition2; - - - @BeforeClass - public static void setup() throws IOException { - condition1 = readFileContents("condition1.xml"); - condition2 = readFileContents("condition2.xml"); - } - - - private static String readFileContents(String name) throws IOException { - try (Scanner s = new Scanner(TestQueryConditionBuilder.class.getClassLoader().getResourceAsStream(name))) { - return s.useDelimiter("\\Z").next(); - } - } - - - @Test - public void testSimpleCondition_noTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("(visitor_id = 'x')"))); - } - - - @Test - public void testSimpleCondition_withTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=vid, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("(vid = 'x')"))); - } - - - @Test - public void testSimpleCondition_withDateType() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=vid:date, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); - } - - - @Test - public void testSimpleCondition_withVariedCaseMappings() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_ID,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=VID:date, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); - } - - - @Test - public void testMultipleConditions_noTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("((visitor_id = 'x') and (sentiment = 'y'))"))); - } - - - @Test - public void testMultipleConditions_withTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("((v = 'x') and (s = 'y'))"))); - } - - - @Test - public void testWithNullConf() { - String condition = QueryConditionBuilder.getInstance().buildCondition(null); - assertThat(condition, is(notNullValue())); - assertThat(condition.trim().isEmpty(), is(true)); - } - - - @Test - public void testWithUndefinedFilterExpr() { - Configuration conf = new Configuration(); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition.trim().isEmpty(), is(true)); - } - -}