[hive] branch master updated: HIVE-21296: Dropping varchar partition throw exception (Daniel Dai, reviewed by Anishek Agarwal)
This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 5f30498 HIVE-21296: Dropping varchar partition throw exception (Daniel Dai, reviewed by Anishek Agarwal) 5f30498 is described below commit 5f3049828ca872d88c80602e2e7d46d9be6255f2 Author: Daniel Dai AuthorDate: Thu Feb 21 10:16:04 2019 -0800 HIVE-21296: Dropping varchar partition throw exception (Daniel Dai, reviewed by Anishek Agarwal) Signed-off-by: Anishek Agarwal --- .../java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java | 3 ++- ql/src/test/queries/clientpositive/partition_varchar1.q| 2 ++ ql/src/test/results/clientpositive/partition_varchar1.q.out| 10 ++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java index eb5b111..9febee4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java @@ -102,7 +102,8 @@ public class ExprNodeDescUtils { private static boolean isDefaultPartition(ExprNodeDesc origin, String defaultPartitionName) { if (origin instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)origin).getValue() != null && - ((ExprNodeConstantDesc)origin).getValue().equals(defaultPartitionName)) { +((ExprNodeConstantDesc)origin).getValue() instanceof String && ((ExprNodeConstantDesc)origin).getValue() +.equals(defaultPartitionName)) { return true; } else { return false; diff --git a/ql/src/test/queries/clientpositive/partition_varchar1.q b/ql/src/test/queries/clientpositive/partition_varchar1.q index 216bcf5..ce9ee76 100644 --- a/ql/src/test/queries/clientpositive/partition_varchar1.q +++ b/ql/src/test/queries/clientpositive/partition_varchar1.q @@ -42,4 +42,6 @@ select count(*) from partition_varchar_1 where dt <= '2000-01-01' and region = 1 -- 20 select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region = 1; +alter table partition_varchar_1 drop partition (dt = '2000-01-01'); + drop table partition_varchar_1; diff --git a/ql/src/test/results/clientpositive/partition_varchar1.q.out b/ql/src/test/results/clientpositive/partition_varchar1.q.out index 93c9adf..b5d1890 100644 --- a/ql/src/test/results/clientpositive/partition_varchar1.q.out +++ b/ql/src/test/results/clientpositive/partition_varchar1.q.out @@ -190,6 +190,16 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@partition_varchar_1 A masked pattern was here 20 +PREHOOK: query: alter table partition_varchar_1 drop partition (dt = '2000-01-01') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@partition_varchar_1 +PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1 +PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2 +POSTHOOK: query: alter table partition_varchar_1 drop partition (dt = '2000-01-01') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@partition_varchar_1 +POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1 +POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2 PREHOOK: query: drop table partition_varchar_1 PREHOOK: type: DROPTABLE PREHOOK: Input: default@partition_varchar_1
[hive] branch master updated: HIVE-21295: StorageHandler shall convert date to string using Hive convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new e71b096 HIVE-21295: StorageHandler shall convert date to string using Hive convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez) e71b096 is described below commit e71b09677b610c9b77c924105cc624b46e90695c Author: Daniel Dai AuthorDate: Thu Feb 21 10:23:49 2019 -0800 HIVE-21295: StorageHandler shall convert date to string using Hive convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez) Signed-off-by: Jesus Camacho Rodriguez --- .../org/apache/hive/storage/jdbc/JdbcSerDe.java| 7 +++- .../queries/clientpositive/external_jdbc_table.q | 17 +- .../clientpositive/llap/external_jdbc_table.q.out | 38 +++--- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java index 87ba682..aabfd7c 100644 --- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java +++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java @@ -37,6 +37,7 @@ import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.ObjectWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; +import org.apache.hive.common.util.DateUtils; import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; import org.apache.hive.storage.jdbc.dao.DatabaseAccessorFactory; @@ -197,7 +198,11 @@ public class JdbcSerDe extends AbstractSerDe { case CHAR: case VARCHAR: case STRING: - rowVal = rowVal.toString(); + if (rowVal instanceof java.sql.Date) { +rowVal = DateUtils.getDateFormat().format((java.sql.Date)rowVal); + } else { +rowVal = rowVal.toString(); + } break; case DATE: if (rowVal instanceof java.sql.Date) { diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table.q b/ql/src/test/queries/clientpositive/external_jdbc_table.q index 3e629d2..36ed93a 100644 --- a/ql/src/test/queries/clientpositive/external_jdbc_table.q +++ b/ql/src/test/queries/clientpositive/external_jdbc_table.q @@ -34,19 +34,19 @@ FROM src SELECT dboutput ( 'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', -'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE )' ), +'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL, "dkey" DOUBLE, "datekey" DATE)' ), dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', -'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','20','20','20.0','20.0'), +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey") VALUES (?,?,?,?,?)','20','20','20.0','20.0','1999-02-22'), dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', -'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','-20','8','9.0','11.0'), +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey") VALUES (?,?,?,?,?)','-20','8','9.0','11.0','2000-03-15'), dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', -'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','101','-16','66.0','-75.0'), +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey") VALUES (?,?,?,?,?)','101','-16','66.0','-75.0','2010-04-01'), dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','', -'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES (?,?,?,?)','40','50','-455.4543','330.767') +'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey") VALUES (?,?,?,?,?)','40','50','-455.4543','330.767','2010-04-02') limit 1; @@ -75,7 +75,8 @@ CREATE EXTERNAL TABLE ext_simple_derby_table2 ikey int, bkey bigint, fkey float, - dkey double + dkey double, + datekey string ) STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler' TBLPROPERTIES ( @@ -96,7 +97,7 @@ select dkey,fkey,bkey,ikey from ext_simple_derby_table1; select bkey+ikey,fkey+dkey from ext_simple_derby_table1; select abs(dkey),abs(ikey),abs(fkey),abs(bkey) from ext_simple_derby_table1; - +select datekey from ext_simple_derby_table2; --Test aggregation select count(*) from ext_simple_derby_table1; @@ -184,4 +185,4 @@ SELECT bkey FROM ext_simple_derby_table2; ---select dkey from ext_simple_derby_table1 order by d
[hive] branch master updated: HIVE-21232: LLAP: Add a cache-miss friendly split affinity provider (Gopal V, reviewed by Slim Bouguerra)
This is an automated email from the ASF dual-hosted git repository. gopalv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new c45751f HIVE-21232: LLAP: Add a cache-miss friendly split affinity provider (Gopal V, reviewed by Slim Bouguerra) c45751f is described below commit c45751fb4029009e08f6389c21a68c3ba26ec6de Author: Gopal V AuthorDate: Thu Feb 21 17:04:26 2019 -0800 HIVE-21232: LLAP: Add a cache-miss friendly split affinity provider (Gopal V, reviewed by Slim Bouguerra) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +-- .../tez/HostAffinitySplitLocationProvider.java | 45 ++ .../tez/TestHostAffinitySplitLocationProvider.java | 42 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 11f165a..04166db 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4352,9 +4352,9 @@ public class HiveConf extends Configuration { "Whether LLAP daemon web UI should use SSL.", "llap.daemon.service.ssl"), LLAP_CLIENT_CONSISTENT_SPLITS("hive.llap.client.consistent.splits", true, "Whether to setup split locations to match nodes on which llap daemons are running, " + -"instead of using the locations provided by the split itself. If there is no llap daemon " + -"running, fall back to locations provided by the split. This is effective only if " + -"hive.execution.mode is llap"), +"preferring one of the locations provided by the split itself. If there is no llap daemon " + +"running on any of those locations (or on the cloud), fall back to a cache affinity to" + +" an LLAP node. This is effective only if hive.execution.mode is llap."), LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true, "Whether LLAP should reject permissive ACLs in some cases (e.g. its own management\n" + "protocol or ZK paths), similar to how ssh refuses a key with bad access permissions."), diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java index c5d96e5..5224429 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java @@ -15,11 +15,11 @@ package org.apache.hadoop.hive.ql.exec.tez; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.hash.Hashing; +import java.util.Set; import org.apache.hadoop.hive.serde2.SerDeUtils; import org.apache.hadoop.mapred.FileSplit; @@ -29,6 +29,10 @@ import org.apache.hive.common.util.Murmur3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.hash.Hashing; + /** * This maps a split (path + offset) to an index based on the number of locations provided. * @@ -47,18 +51,20 @@ public class HostAffinitySplitLocationProvider implements SplitLocationProvider private final boolean isDebugEnabled = LOG.isDebugEnabled(); private final List locations; + private final Set locationSet; public HostAffinitySplitLocationProvider(List knownLocations) { Preconditions.checkState(knownLocations != null && !knownLocations.isEmpty(), HostAffinitySplitLocationProvider.class.getName() + " needs at least 1 location to function"); this.locations = knownLocations; +this.locationSet = new HashSet(knownLocations); } @Override public String[] getLocations(InputSplit split) throws IOException { if (!(split instanceof FileSplit)) { - if (LOG.isDebugEnabled()) { + if (isDebugEnabled) { LOG.debug("Split: " + split + " is not a FileSplit. Using default locations"); } return split.getLocations(); @@ -66,14 +72,39 @@ public class HostAffinitySplitLocationProvider implements SplitLocationProvider FileSplit fsplit = (FileSplit) split; String splitDesc = "Split at " + fsplit.getPath() + " with offset= " + fsplit.getStart() + ", length=" + fsplit.getLength(); -String location = locations.get(determineLocation( -locations, fsplit.getPath().toString(), fsplit.getStart(), splitDesc)); +List preferredLocations = preferLocations(fsplit); +String location = +preferredLocations.get(determine