HIVE-11898 : support default partition in metastoredirectsql (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/116c3e3b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/116c3e3b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/116c3e3b Branch: refs/heads/llap Commit: 116c3e3b2d2b2b22fef9abed8f092b04155784cd Parents: a953b43 Author: Sergey Shelukhin <ser...@apache.org> Authored: Thu Oct 1 14:11:58 2015 -0700 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Thu Oct 1 14:11:58 2015 -0700 ---------------------------------------------------------------------- .../hive/metastore/MetaStoreDirectSql.java | 34 +++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/116c3e3b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 1f89b7c..95b1ccc 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -109,6 +109,7 @@ class MetaStoreDirectSql { private final DB dbType; private final int batchSize; private final boolean convertMapNullsToEmptyStrings; + private final String defaultPartName; /** * Whether direct SQL can be used with the current datastore backing {@link #pm}. @@ -116,6 +117,7 @@ class MetaStoreDirectSql { private final boolean isCompatibleDatastore; private final boolean isAggregateStatsCacheEnabled; private AggregateStatsCache aggrStatsCache; + public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) { this.pm = pm; this.dbType = determineDbType(); @@ -127,6 +129,7 @@ class MetaStoreDirectSql { convertMapNullsToEmptyStrings = HiveConf.getBoolVar(conf, ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS); + defaultPartName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME); String jdoIdFactory = HiveConf.getVar(conf, ConfVars.METASTORE_IDENTIFIER_FACTORY); if (! ("datanucleus1".equalsIgnoreCase(jdoIdFactory))){ @@ -390,7 +393,7 @@ class MetaStoreDirectSql { // Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround. boolean dbHasJoinCastBug = (dbType == DB.DERBY || dbType == DB.ORACLE); String sqlFilter = PartitionFilterGenerator.generateSqlFilter( - table, tree, params, joins, dbHasJoinCastBug); + table, tree, params, joins, dbHasJoinCastBug, defaultPartName); if (sqlFilter == null) { return null; // Cannot make SQL filter to push down. } @@ -490,8 +493,8 @@ class MetaStoreDirectSql { } List<Object> sqlResult = executeWithArray(query, params, queryText); long queryTime = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, queryText, start, queryTime); if (sqlResult.isEmpty()) { - timingTrace(doTrace, queryText, start, queryTime); return new ArrayList<Partition>(); // no partitions, bail early. } @@ -508,7 +511,6 @@ class MetaStoreDirectSql { result = getPartitionsFromPartitionIds(dbName, tblName, isView, sqlResult); } - timingTrace(doTrace, queryText, start, queryTime); query.closeAll(); return result; } @@ -921,14 +923,16 @@ class MetaStoreDirectSql { private final List<Object> params; private final List<String> joins; private final boolean dbHasJoinCastBug; + private final String defaultPartName; - private PartitionFilterGenerator( - Table table, List<Object> params, List<String> joins, boolean dbHasJoinCastBug) { + private PartitionFilterGenerator(Table table, List<Object> params, List<String> joins, + boolean dbHasJoinCastBug, String defaultPartName) { this.table = table; this.params = params; this.joins = joins; this.dbHasJoinCastBug = dbHasJoinCastBug; this.filterBuffer = new FilterBuilder(false); + this.defaultPartName = defaultPartName; } /** @@ -939,13 +943,14 @@ class MetaStoreDirectSql { * @return the string representation of the expression tree */ private static String generateSqlFilter(Table table, ExpressionTree tree, - List<Object> params, List<String> joins, boolean dbHasJoinCastBug) throws MetaException { + List<Object> params, List<String> joins, boolean dbHasJoinCastBug, String defaultPartName) + throws MetaException { assert table != null; if (tree.getRoot() == null) { return ""; } PartitionFilterGenerator visitor = new PartitionFilterGenerator( - table, params, joins, dbHasJoinCastBug); + table, params, joins, dbHasJoinCastBug, defaultPartName); tree.accept(visitor); if (visitor.filterBuffer.hasError()) { LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage()); @@ -1071,28 +1076,33 @@ class MetaStoreDirectSql { // Build the filter and add parameters linearly; we are traversing leaf nodes LTR. String tableValue = "\"FILTER" + partColIndex + "\".\"PART_KEY_VAL\""; + if (node.isReverseOrder) { params.add(nodeValue); } + String tableColumn = tableValue; if (colType != FilterType.String) { // The underlying database field is varchar, we need to compare numbers. - // Note that this won't work with __HIVE_DEFAULT_PARTITION__. It will fail and fall - // back to JDO. That is by design; we could add an ugly workaround here but didn't. if (colType == FilterType.Integral) { tableValue = "cast(" + tableValue + " as decimal(21,0))"; } else if (colType == FilterType.Date) { tableValue = "cast(" + tableValue + " as date)"; } + // Workaround for HIVE_DEFAULT_PARTITION - ignore it like JDO does, for now. + String tableValue0 = tableValue; + tableValue = "(case when " + tableColumn + " <> ?"; + params.add(defaultPartName); + if (dbHasJoinCastBug) { // This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible. - tableValue = "(case when \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and " + tableValue += (" and \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and " + "\"FILTER" + partColIndex + "\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\" and " - + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex + " then " - + tableValue + " else null end)"; + + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex); params.add(table.getTableName().toLowerCase()); params.add(table.getDbName().toLowerCase()); } + tableValue += " then " + tableValue0 + " else null end)"; } if (!node.isReverseOrder) { params.add(nodeValue);