[hive] branch master updated: HIVE-21296: Dropping varchar partition throw exception (Daniel Dai, reviewed by Anishek Agarwal)
This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5f30498 HIVE-21296: Dropping varchar partition throw exception
(Daniel Dai, reviewed by Anishek Agarwal)
5f30498 is described below
commit 5f3049828ca872d88c80602e2e7d46d9be6255f2
Author: Daniel Dai
AuthorDate: Thu Feb 21 10:16:04 2019 -0800
HIVE-21296: Dropping varchar partition throw exception (Daniel Dai,
reviewed by Anishek Agarwal)
Signed-off-by: Anishek Agarwal
---
.../java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java | 3 ++-
ql/src/test/queries/clientpositive/partition_varchar1.q| 2 ++
ql/src/test/results/clientpositive/partition_varchar1.q.out| 10 ++
3 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
index eb5b111..9febee4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
@@ -102,7 +102,8 @@ public class ExprNodeDescUtils {
private static boolean isDefaultPartition(ExprNodeDesc origin, String
defaultPartitionName) {
if (origin instanceof ExprNodeConstantDesc &&
((ExprNodeConstantDesc)origin).getValue() != null &&
-
((ExprNodeConstantDesc)origin).getValue().equals(defaultPartitionName)) {
+((ExprNodeConstantDesc)origin).getValue() instanceof String &&
((ExprNodeConstantDesc)origin).getValue()
+.equals(defaultPartitionName)) {
return true;
} else {
return false;
diff --git a/ql/src/test/queries/clientpositive/partition_varchar1.q
b/ql/src/test/queries/clientpositive/partition_varchar1.q
index 216bcf5..ce9ee76 100644
--- a/ql/src/test/queries/clientpositive/partition_varchar1.q
+++ b/ql/src/test/queries/clientpositive/partition_varchar1.q
@@ -42,4 +42,6 @@ select count(*) from partition_varchar_1 where dt <=
'2000-01-01' and region = 1
-- 20
select count(*) from partition_varchar_1 where dt <> '2000-01-01' and region =
1;
+alter table partition_varchar_1 drop partition (dt = '2000-01-01');
+
drop table partition_varchar_1;
diff --git a/ql/src/test/results/clientpositive/partition_varchar1.q.out
b/ql/src/test/results/clientpositive/partition_varchar1.q.out
index 93c9adf..b5d1890 100644
--- a/ql/src/test/results/clientpositive/partition_varchar1.q.out
+++ b/ql/src/test/results/clientpositive/partition_varchar1.q.out
@@ -190,6 +190,16 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@partition_varchar_1
A masked pattern was here
20
+PREHOOK: query: alter table partition_varchar_1 drop partition (dt =
'2000-01-01')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@partition_varchar_1
+PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1
+PREHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2
+POSTHOOK: query: alter table partition_varchar_1 drop partition (dt =
'2000-01-01')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@partition_varchar_1
+POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=1
+POSTHOOK: Output: default@partition_varchar_1@dt=2000-01-01/region=2
PREHOOK: query: drop table partition_varchar_1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@partition_varchar_1
[hive] branch master updated: HIVE-21295: StorageHandler shall convert date to string using Hive convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
This is an automated email from the ASF dual-hosted git repository.
daijy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e71b096 HIVE-21295: StorageHandler shall convert date to string using
Hive convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
e71b096 is described below
commit e71b09677b610c9b77c924105cc624b46e90695c
Author: Daniel Dai
AuthorDate: Thu Feb 21 10:23:49 2019 -0800
HIVE-21295: StorageHandler shall convert date to string using Hive
convention (Daniel Dai, reviewed by Jesus Camacho Rodriguez)
Signed-off-by: Jesus Camacho Rodriguez
---
.../org/apache/hive/storage/jdbc/JdbcSerDe.java| 7 +++-
.../queries/clientpositive/external_jdbc_table.q | 17 +-
.../clientpositive/llap/external_jdbc_table.q.out | 38 +++---
3 files changed, 41 insertions(+), 21 deletions(-)
diff --git
a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java
b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java
index 87ba682..aabfd7c 100644
--- a/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java
+++ b/jdbc-handler/src/main/java/org/apache/hive/storage/jdbc/JdbcSerDe.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
+import org.apache.hive.common.util.DateUtils;
import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager;
import org.apache.hive.storage.jdbc.dao.DatabaseAccessor;
import org.apache.hive.storage.jdbc.dao.DatabaseAccessorFactory;
@@ -197,7 +198,11 @@ public class JdbcSerDe extends AbstractSerDe {
case CHAR:
case VARCHAR:
case STRING:
- rowVal = rowVal.toString();
+ if (rowVal instanceof java.sql.Date) {
+rowVal = DateUtils.getDateFormat().format((java.sql.Date)rowVal);
+ } else {
+rowVal = rowVal.toString();
+ }
break;
case DATE:
if (rowVal instanceof java.sql.Date) {
diff --git a/ql/src/test/queries/clientpositive/external_jdbc_table.q
b/ql/src/test/queries/clientpositive/external_jdbc_table.q
index 3e629d2..36ed93a 100644
--- a/ql/src/test/queries/clientpositive/external_jdbc_table.q
+++ b/ql/src/test/queries/clientpositive/external_jdbc_table.q
@@ -34,19 +34,19 @@ FROM src
SELECT
dboutput (
'jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','',
-'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL,
"dkey" DOUBLE )' ),
+'CREATE TABLE SIMPLE_DERBY_TABLE2 ("ikey" INTEGER, "bkey" BIGINT, "fkey" REAL,
"dkey" DOUBLE, "datekey" DATE)' ),
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','',
-'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES
(?,?,?,?)','20','20','20.0','20.0'),
+'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey")
VALUES (?,?,?,?,?)','20','20','20.0','20.0','1999-02-22'),
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','',
-'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES
(?,?,?,?)','-20','8','9.0','11.0'),
+'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey")
VALUES (?,?,?,?,?)','-20','8','9.0','11.0','2000-03-15'),
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','',
-'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES
(?,?,?,?)','101','-16','66.0','-75.0'),
+'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey")
VALUES (?,?,?,?,?)','101','-16','66.0','-75.0','2010-04-01'),
dboutput('jdbc:derby:;databaseName=${system:test.tmp.dir}/test_derby_as_external_table_db;create=true','','',
-'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey") VALUES
(?,?,?,?)','40','50','-455.4543','330.767')
+'INSERT INTO SIMPLE_DERBY_TABLE2 ("ikey","bkey","fkey","dkey","datekey")
VALUES (?,?,?,?,?)','40','50','-455.4543','330.767','2010-04-02')
limit 1;
@@ -75,7 +75,8 @@ CREATE EXTERNAL TABLE ext_simple_derby_table2
ikey int,
bkey bigint,
fkey float,
- dkey double
+ dkey double,
+ datekey string
)
STORED BY 'org.apache.hive.storage.jdbc.JdbcStorageHandler'
TBLPROPERTIES (
@@ -96,7 +97,7 @@ select dkey,fkey,bkey,ikey from ext_simple_derby_table1;
select bkey+ikey,fkey+dkey from ext_simple_derby_table1;
select abs(dkey),abs(ikey),abs(fkey),abs(bkey) from ext_simple_derby_table1;
-
+select datekey from ext_simple_derby_table2;
--Test aggregation
select count(*) from ext_simple_derby_table1;
@@ -184,4 +185,4 @@ SELECT bkey FROM ext_simple_derby_table2;
---select dkey from ext_simple_derby_table1 order by d
[hive] branch master updated: HIVE-21232: LLAP: Add a cache-miss friendly split affinity provider (Gopal V, reviewed by Slim Bouguerra)
This is an automated email from the ASF dual-hosted git repository.
gopalv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c45751f HIVE-21232: LLAP: Add a cache-miss friendly split affinity
provider (Gopal V, reviewed by Slim Bouguerra)
c45751f is described below
commit c45751fb4029009e08f6389c21a68c3ba26ec6de
Author: Gopal V
AuthorDate: Thu Feb 21 17:04:26 2019 -0800
HIVE-21232: LLAP: Add a cache-miss friendly split affinity provider (Gopal
V, reviewed by Slim Bouguerra)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 +--
.../tez/HostAffinitySplitLocationProvider.java | 45 ++
.../tez/TestHostAffinitySplitLocationProvider.java | 42
3 files changed, 83 insertions(+), 10 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 11f165a..04166db 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4352,9 +4352,9 @@ public class HiveConf extends Configuration {
"Whether LLAP daemon web UI should use SSL.", "llap.daemon.service.ssl"),
LLAP_CLIENT_CONSISTENT_SPLITS("hive.llap.client.consistent.splits", true,
"Whether to setup split locations to match nodes on which llap daemons
are running, " +
-"instead of using the locations provided by the split itself. If there
is no llap daemon " +
-"running, fall back to locations provided by the split. This is
effective only if " +
-"hive.execution.mode is llap"),
+"preferring one of the locations provided by the split itself. If
there is no llap daemon " +
+"running on any of those locations (or on the cloud), fall back to a
cache affinity to" +
+" an LLAP node. This is effective only if hive.execution.mode is
llap."),
LLAP_VALIDATE_ACLS("hive.llap.validate.acls", true,
"Whether LLAP should reject permissive ACLs in some cases (e.g. its
own management\n" +
"protocol or ZK paths), similar to how ssh refuses a key with bad
access permissions."),
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java
index c5d96e5..5224429 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HostAffinitySplitLocationProvider.java
@@ -15,11 +15,11 @@
package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.hash.Hashing;
+import java.util.Set;
import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.mapred.FileSplit;
@@ -29,6 +29,10 @@ import org.apache.hive.common.util.Murmur3;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.hash.Hashing;
+
/**
* This maps a split (path + offset) to an index based on the number of
locations provided.
*
@@ -47,18 +51,20 @@ public class HostAffinitySplitLocationProvider implements
SplitLocationProvider
private final boolean isDebugEnabled = LOG.isDebugEnabled();
private final List locations;
+ private final Set locationSet;
public HostAffinitySplitLocationProvider(List knownLocations) {
Preconditions.checkState(knownLocations != null &&
!knownLocations.isEmpty(),
HostAffinitySplitLocationProvider.class.getName() +
" needs at least 1 location to function");
this.locations = knownLocations;
+this.locationSet = new HashSet(knownLocations);
}
@Override
public String[] getLocations(InputSplit split) throws IOException {
if (!(split instanceof FileSplit)) {
- if (LOG.isDebugEnabled()) {
+ if (isDebugEnabled) {
LOG.debug("Split: " + split + " is not a FileSplit. Using default
locations");
}
return split.getLocations();
@@ -66,14 +72,39 @@ public class HostAffinitySplitLocationProvider implements
SplitLocationProvider
FileSplit fsplit = (FileSplit) split;
String splitDesc = "Split at " + fsplit.getPath() + " with offset= " +
fsplit.getStart()
+ ", length=" + fsplit.getLength();
-String location = locations.get(determineLocation(
-locations, fsplit.getPath().toString(), fsplit.getStart(), splitDesc));
+List preferredLocations = preferLocations(fsplit);
+String location =
+preferredLocations.get(determine
