[hive] branch branch-3 updated: HIVE-25726: Upgrade velocity to 2.3 due to CVE-2020-13936 (Sourabh Goyal via Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 8f4e0ab0dcd HIVE-25726: Upgrade velocity to 2.3 due to CVE-2020-13936
(Sourabh Goyal via Naveen Gangam)
8f4e0ab0dcd is described below
commit 8f4e0ab0dcddc29b2870fb1f6eb41b24d1e02f03
Author: Diksha628 <43694846+diksha...@users.noreply.github.com>
AuthorDate: Tue May 23 20:02:55 2023 +0530
HIVE-25726: Upgrade velocity to 2.3 due to CVE-2020-13936 (Sourabh Goyal
via Naveen Gangam)
Co-authored-by: Sourabh Goyal
Signed-off-by: Sankar Hariappan
Closes (#4308)
---
pom.xml | 4 ++--
vector-code-gen/pom.xml | 11 ---
2 files changed, 2 insertions(+), 13 deletions(-)
diff --git a/pom.xml b/pom.xml
index 399205c76f9..3b48d9cf93f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -205,7 +205,7 @@
1.1
1.1.4
1.4
-1.5
+2.3
2.9.1
3.4.6
1.1
@@ -462,7 +462,7 @@
org.apache.velocity
-velocity
+velocity-engine-core
${velocity.version}
diff --git a/vector-code-gen/pom.xml b/vector-code-gen/pom.xml
index 0c62d604722..ec551ae1f3e 100644
--- a/vector-code-gen/pom.xml
+++ b/vector-code-gen/pom.xml
@@ -49,17 +49,6 @@
ant
${ant.version}
-
- org.apache.velocity
- velocity
- ${velocity.version}
-
-
-commons-collections
-commons-collections
-
-
-
[hive] branch branch-3 updated: HIVE-25468: Authorization for Create/Drop functions in HMS(Saihemanth Gantasala via Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new f906e0246db HIVE-25468: Authorization for Create/Drop functions in
HMS(Saihemanth Gantasala via Naveen Gangam)
f906e0246db is described below
commit f906e0246db8ae7eb573c801a02412f529fcbd50
Author: Diksha628 <43694846+diksha...@users.noreply.github.com>
AuthorDate: Tue May 23 20:00:41 2023 +0530
HIVE-25468: Authorization for Create/Drop functions in HMS(Saihemanth
Gantasala via Naveen Gangam)
Co-authored-by: saihemanth
Signed-off-by: Sankar Hariappan
Closes (#4342)
---
.../AuthorizationPreEventListener.java | 38
.../hadoop/hive/metastore/HiveMetaStore.java | 15 ++--
.../metastore/events/PreCreateFunctionEvent.java | 42 ++
.../metastore/events/PreDropFunctionEvent.java | 42 ++
.../hive/metastore/events/PreEventContext.java | 4 ++-
.../hive/metastore/client/TestFunctions.java | 3 +-
6 files changed, 140 insertions(+), 4 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
index 2cc057ee6e8..fef9fee1afe 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/AuthorizationPreEventListener.java
@@ -42,8 +42,10 @@ import
org.apache.hadoop.hive.metastore.events.PreAlterDatabaseEvent;
import org.apache.hadoop.hive.metastore.events.PreAlterPartitionEvent;
import org.apache.hadoop.hive.metastore.events.PreAlterTableEvent;
import org.apache.hadoop.hive.metastore.events.PreCreateDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.PreCreateFunctionEvent;
import org.apache.hadoop.hive.metastore.events.PreCreateTableEvent;
import org.apache.hadoop.hive.metastore.events.PreDropDatabaseEvent;
+import org.apache.hadoop.hive.metastore.events.PreDropFunctionEvent;
import org.apache.hadoop.hive.metastore.events.PreDropPartitionEvent;
import org.apache.hadoop.hive.metastore.events.PreDropTableEvent;
import org.apache.hadoop.hive.metastore.events.PreEventContext;
@@ -170,6 +172,12 @@ public class AuthorizationPreEventListener extends
MetaStorePreEventListener {
case DROP_DATABASE:
authorizeDropDatabase((PreDropDatabaseEvent)context);
break;
+case CREATE_FUNCTION:
+ authorizeCreateFunction((PreCreateFunctionEvent)context);
+ break;
+case DROP_FUNCTION:
+ authorizeDropFunction((PreDropFunctionEvent)context);
+ break;
case LOAD_PARTITION_DONE:
// noop for now
break;
@@ -402,6 +410,36 @@ public class AuthorizationPreEventListener extends
MetaStorePreEventListener {
}
}
+ private void authorizeCreateFunction(PreCreateFunctionEvent context)
+ throws InvalidOperationException, MetaException {
+try {
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get())
{
+authorizer.authorize(
+HiveOperation.CREATEFUNCTION.getInputRequiredPrivileges(),
+HiveOperation.CREATEFUNCTION.getOutputRequiredPrivileges());
+ }
+} catch (AuthorizationException e) {
+ throw invalidOperationException(e);
+} catch (HiveException e) {
+ throw metaException(e);
+}
+ }
+
+ private void authorizeDropFunction(PreDropFunctionEvent context)
+ throws InvalidOperationException, MetaException {
+try {
+ for (HiveMetastoreAuthorizationProvider authorizer : tAuthorizers.get())
{
+authorizer.authorize(
+HiveOperation.DROPFUNCTION.getInputRequiredPrivileges(),
+HiveOperation.DROPFUNCTION.getOutputRequiredPrivileges());
+ }
+} catch (AuthorizationException e) {
+ throw invalidOperationException(e);
+} catch (HiveException e) {
+ throw metaException(e);
+}
+ }
+
private void authorizeAlterPartition(PreAlterPartitionEvent context)
throws InvalidOperationException, MetaException {
try {
diff --git
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f5d5c5a41a5..8270d8bf282 100644
---
a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++
b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -133,11 +133,13 @@ import
org.apache.hadoop.hive.metastore.events.PreAlterTableEvent;
import org.apache.hadoop.hive.metastore.events.PreAuthorizationCallEvent;
import
[hive] branch branch-3 updated: HIVE-25600: Compaction job creates redundant base/delta folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan) (#4340)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 157d257a0de HIVE-25600: Compaction job creates redundant base/delta
folder within base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan)
(#4340)
157d257a0de is described below
commit 157d257a0de697da24be3597d60d8a92d4025fb8
Author: Diksha628 <43694846+diksha...@users.noreply.github.com>
AuthorDate: Tue May 23 19:57:07 2023 +0530
HIVE-25600: Compaction job creates redundant base/delta folder within
base/delta folder (Nikhil Gupta, reviewed by Sankar Hariappan) (#4340)
Signed-off-by: Sankar Hariappan
Closes (#2705)
Co-authored-by: guptanikhil007
---
.../org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 11 +++
1 file changed, 11 insertions(+)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 95870ad46f9..474f6c53426 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -1184,6 +1184,16 @@ public class CompactorMR {
Path tmpPath = fileStatus.getPath();
//newPath is the base/delta dir
Path newPath = new Path(finalLocation, tmpPath.getName());
+/* rename(A, B) has "interesting" behavior if A and B are directories.
If B doesn't exist,
+* it does the expected operation and everything that was in A is now
in B. If B exists,
+* it will make A a child of B.
+* This issue can happen if the previous MR job succeeded but HMS was
unable to persist compaction result.
+* We will delete the directory B if it exists to avoid the above issue
+*/
+if (fs.exists(newPath)) {
+ LOG.info(String.format("Final path %s already exists. Deleting the
path to avoid redundant base creation", newPath.toString()));
+ fs.delete(newPath, true);
+}
/* Create the markers in the tmp location and rename everything in the
end to prevent race condition between
* marker creation and split read. */
AcidUtils.OrcAcidVersion.writeVersionFile(tmpPath, fs);
@@ -1192,6 +1202,7 @@ public class CompactorMR {
}
fs.delete(tmpLocation, true);
}
+
private void createCompactorMarker(JobConf conf, Path finalLocation,
FileSystem fs)
throws IOException {
if(conf.getBoolean(IS_MAJOR, false)) {
[hive] branch master updated: HIVE-27365: Fix test acid_bloom_filter_orc_file_dump (Krisztian Kasa, reviewed by Stamatis Zampetakis, Attila Turoczy, Kokila N)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b0b3fde70c3 HIVE-27365: Fix test acid_bloom_filter_orc_file_dump
(Krisztian Kasa, reviewed by Stamatis Zampetakis, Attila Turoczy, Kokila N)
b0b3fde70c3 is described below
commit b0b3fde70c39aa1c33296ac48e21665fb67b12c9
Author: Krisztian Kasa
AuthorDate: Tue May 23 10:58:31 2023 +0200
HIVE-27365: Fix test acid_bloom_filter_orc_file_dump (Krisztian Kasa,
reviewed by Stamatis Zampetakis, Attila Turoczy, Kokila N)
---
.../java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
index 45ce9e30479..b8b5e12ea74 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecOrcFileDump.java
@@ -82,13 +82,13 @@ public class PostExecOrcFileDump implements
ExecuteWithHookContext {
List directories;
if (partitionedTable) {
LOG.info("Printing orc file dump for files from partitioned
directory..");
-directories = fetchWork.getPartDir();
+directories = Lists.newArrayList(fetchWork.getPartDir());
} else {
LOG.info("Printing orc file dump for files from table directory..");
-directories = Lists.newArrayList();
-directories.add(fetchWork.getTblDir());
+directories = Lists.newArrayList(fetchWork.getTblDir());
}
+ Collections.sort(directories);
for (Path dir : directories) {
printFileStatus(console, dir.getFileSystem(conf), dir);
}
[hive] branch master updated: HIVE-27088: Incorrect results when inner and outer joins with post join filters are merged (Ryu Kobayashi, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 21020a75196 HIVE-27088: Incorrect results when inner and outer joins
with post join filters are merged (Ryu Kobayashi, reviewed by Krisztian Kasa)
21020a75196 is described below
commit 21020a75196f13b7276b600d15d626af59d15ac3
Author: Ryu Kobayashi
AuthorDate: Tue May 23 17:28:30 2023 +0900
HIVE-27088: Incorrect results when inner and outer joins with post join
filters are merged (Ryu Kobayashi, reviewed by Krisztian Kasa)
---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 +-
.../clientpositive/tez_nway_join_with_filters.q| 94 +++
.../llap/tez_nway_join_with_filters.q.out | 729 +
3 files changed, 828 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index fb9ae347ed0..3f15bc37c17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10766,9 +10766,10 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
isTargetSemiJoin= !target.getNoSemiJoin();
hasTargetPostJoinFilters = target.getPostJoinFilters().size() !=0;
-if((hasNodePostJoinFilters && (isNodeOuterJoin || isNodeSemiJoin))
-|| (hasTargetPostJoinFilters && (isTargetOuterJoin ||
isTargetSemiJoin))) {
- return false;
+if (hasNodePostJoinFilters || hasTargetPostJoinFilters) {
+ if (isNodeOuterJoin || isNodeSemiJoin || isTargetOuterJoin ||
isTargetSemiJoin) {
+return false;
+ }
}
return true;
}
@@ -10810,7 +10811,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
break;
}
if(!shouldMerge(node, target)) {
- // Outer joins with post-filtering conditions cannot be merged
+ // Outer joins or outer and not outer with post-filtering
conditions cannot be merged
break;
}
Pair mergeDetails = findMergePos(node, target);
diff --git a/ql/src/test/queries/clientpositive/tez_nway_join_with_filters.q
b/ql/src/test/queries/clientpositive/tez_nway_join_with_filters.q
new file mode 100644
index 000..aab2529951a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_nway_join_with_filters.q
@@ -0,0 +1,94 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.auto.convert.join=true;
+
+create temporary table foo (id bigint, code string) stored as orc;
+create temporary table bar (id bigint, code string) stored as orc;
+create temporary table baz (id bigint) stored as orc;
+
+-- SORT_QUERY_RESULTS
+
+INSERT INTO foo values
+ (252073, '01'),
+ (252107, '01'),
+ (252111, '01'),
+ (252112, '01'),
+ (252113, '01'),
+ (252114, '01'),
+ (252071, '01A'),
+ (252072, '01A'),
+ (252116, '01A'),
+ (252117, '01A'),
+ (252118, '01A'),
+ (252119, '01A'),
+ (252120, '01A'),
+ (252076, '06'),
+ (252074, '06A'),
+ (252075, '06A');
+INSERT INTO bar values
+ (252071, '01'),
+ (252072, '01'),
+ (252073, '01'),
+ (252116, '01'),
+ (252117, '01'),
+ (252071, '01A'),
+ (252072, '01A'),
+ (252073, '01A'),
+ (252116, '01AS'),
+ (252117, '01AS'),
+ (252071, '01B'),
+ (252072, '01B'),
+ (252073, '01B'),
+ (252116, '01BS'),
+ (252117, '01BS'),
+ (252071, '01C'),
+ (252072, '01C'),
+ (252073, '01C7'),
+ (252116, '01CS'),
+ (252117, '01CS'),
+ (252071, '01D'),
+ (252072, '01D'),
+ (252073, '01D'),
+ (252116, '01DS'),
+ (252117, '01DS');
+INSERT INTO baz values
+ (252071),
+ (252072),
+ (252073),
+ (252074),
+ (252075),
+ (252076),
+ (252107),
+ (252111),
+ (252112),
+ (252113),
+ (252114),
+ (252116),
+ (252117),
+ (252118),
+ (252119),
+ (252120);
+
+set hive.merge.nway.joins=true;
+explain select a.id, b.code, c.id from foo a left outer join bar b on a.id =
b.id and (a.code = '01AS' or b.code = '01BS') left outer join baz c on a.id =
c.id;
+
+set hive.merge.nway.joins=false;
+explain select a.id, b.code, c.id from foo a left outer join bar b on a.id =
b.id and (a.code = '01AS' or b.code = '01BS') left outer join baz c on a.id =
c.id;
+
+set hive.merge.nway.joins=true;
+select a.id, b.code, c.id from foo a left outer join bar b on a.id = b.id and
(a.code = '01AS'
[hive] branch master updated: HIVE-27347: Addendum: count(*) should be using Iceberg stats only if there are no deletes (Denys Kuzmenko, reviewed by Ayush Saxena)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 36f5a09a997 HIVE-27347: Addendum: count(*) should be using Iceberg
stats only if there are no deletes (Denys Kuzmenko, reviewed by Ayush Saxena)
36f5a09a997 is described below
commit 36f5a09a9975415154320c33cec4cf3ca5757f6b
Author: Denys Kuzmenko
AuthorDate: Tue May 23 10:55:28 2023 +0300
HIVE-27347: Addendum: count(*) should be using Iceberg stats only if there
are no deletes (Denys Kuzmenko, reviewed by Ayush Saxena)
Closes #4343
---
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 25 ++--
.../src/test/queries/positive/iceberg_stats.q | 6 ++
.../src/test/results/positive/iceberg_stats.q.out | 73 +-
.../hive/ql/metadata/HiveStorageHandler.java | 8 +++
.../hadoop/hive/ql/optimizer/StatsOptimizer.java | 2 +-
5 files changed, 107 insertions(+), 7 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index ee7fbfaeb0d..66d336a03c9 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -352,9 +352,8 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
org.apache.hadoop.hive.ql.metadata.Table hmsTable = partish.getTable();
// For write queries where rows got modified, don't fetch from cache as
values could have changed.
Table table = getTable(hmsTable);
-String statsSource = HiveConf.getVar(conf,
HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE).toLowerCase();
Map stats = Maps.newHashMap();
-if (statsSource.equals(ICEBERG)) {
+if (getStatsSource().equals(ICEBERG)) {
if (table.currentSnapshot() != null) {
Map summary = table.currentSnapshot().summary();
if (summary != null) {
@@ -405,7 +404,7 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
@Override
public boolean canSetColStatistics(org.apache.hadoop.hive.ql.metadata.Table
hmsTable) {
Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable());
-return table.currentSnapshot() != null ? getStatsSource().equals(ICEBERG)
: false;
+return table.currentSnapshot() != null && getStatsSource().equals(ICEBERG);
}
@Override
@@ -463,11 +462,29 @@ public class HiveIcebergStorageHandler implements
HiveStoragePredicateHandler, H
ByteBuffers.toByteArray(blobMetadataByteBufferPair.second();
return collect.get(blobMetadata.get(0)).get(0).getStatsObj();
} catch (IOException e) {
- LOG.error("Error when trying to read iceberg col stats from puffin
files: {}", e);
+ LOG.error("Error when trying to read iceberg col stats from puffin
files: ", e);
}
return null;
}
+ @Override
+ public boolean
canComputeQueryUsingStats(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
+if (getStatsSource().equals(ICEBERG)) {
+ Table table = getTable(hmsTable);
+ if (table.currentSnapshot() != null) {
+Map summary = table.currentSnapshot().summary();
+if (summary != null &&
summary.containsKey(SnapshotSummary.TOTAL_EQ_DELETES_PROP) &&
+summary.containsKey(SnapshotSummary.TOTAL_POS_DELETES_PROP)) {
+
+ long totalEqDeletes =
Long.parseLong(summary.get(SnapshotSummary.TOTAL_EQ_DELETES_PROP));
+ long totalPosDeletes =
Long.parseLong(summary.get(SnapshotSummary.TOTAL_POS_DELETES_PROP));
+ return totalEqDeletes + totalPosDeletes == 0;
+}
+ }
+}
+return false;
+ }
+
private String getStatsSource() {
return HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_ICEBERG_STATS_SOURCE,
ICEBERG).toLowerCase();
}
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
index 4238cbba4b9..8276cef8aae 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_stats.q
@@ -1,3 +1,5 @@
+--! qt:replace:/(\s+Statistics\: Num rows\: \d+ Data size\:\s+)\S+(\s+Basic
stats\: \S+ Column stats\: \S+)/$1#Masked#$2/
+
set hive.compute.query.using.stats=true;
set hive.explain.user=false;
@@ -14,4 +16,8 @@ delete from ice01 where id in (2,4);
explain select count(*) from ice01;
select count(*) from ice01;
+-- iow
+insert overwrite table ice01 select * from ice01;
+explain select count(*) from ice01;
+
drop table ice01;
\ No newline at end of file
diff --git
