[hive] branch dependabot/maven/storage-api/org.apache.hadoop-hadoop-common-3.2.3 created (now 9a45356819)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/storage-api/org.apache.hadoop-hadoop-common-3.2.3 in repository https://gitbox.apache.org/repos/asf/hive.git at 9a45356819 Bump hadoop-common from 3.1.0 to 3.2.3 in /storage-api No new revisions were added by this update.
[hive] branch dependabot/maven/upgrade-acid/pre-upgrade/org.apache.hadoop-hadoop-common-3.2.3 created (now dbb74a8ff5)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/upgrade-acid/pre-upgrade/org.apache.hadoop-hadoop-common-3.2.3 in repository https://gitbox.apache.org/repos/asf/hive.git at dbb74a8ff5 Bump hadoop-common from 2.7.2 to 3.2.3 in /upgrade-acid/pre-upgrade No new revisions were added by this update.
[hive] branch dependabot/maven/standalone-metastore/org.apache.hadoop-hadoop-common-3.2.3 created (now 746baef3d8)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/standalone-metastore/org.apache.hadoop-hadoop-common-3.2.3 in repository https://gitbox.apache.org/repos/asf/hive.git at 746baef3d8 Bump hadoop-common from 3.1.0 to 3.2.3 in /standalone-metastore No new revisions were added by this update.
[hive] branch dependabot/maven/org.apache.hadoop-hadoop-common-3.2.3 created (now acdcdb6c33)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/org.apache.hadoop-hadoop-common-3.2.3 in repository https://gitbox.apache.org/repos/asf/hive.git at acdcdb6c33 Bump hadoop-common from 3.1.0 to 3.2.3 No new revisions were added by this update.
[hive] branch master updated: disable flaky mapjoin_memcheck
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new ea935a0dd1 disable flaky mapjoin_memcheck ea935a0dd1 is described below commit ea935a0dd101118af10c6378093e3338a3120ff6 Author: Zoltan Haindrich AuthorDate: Tue Apr 12 17:28:25 2022 + disable flaky mapjoin_memcheck --- ql/src/test/queries/clientpositive/mapjoin_memcheck.q | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ql/src/test/queries/clientpositive/mapjoin_memcheck.q b/ql/src/test/queries/clientpositive/mapjoin_memcheck.q index 11cf2cbfc9..efac989c95 100644 --- a/ql/src/test/queries/clientpositive/mapjoin_memcheck.q +++ b/ql/src/test/queries/clientpositive/mapjoin_memcheck.q @@ -1,5 +1,6 @@ --! qt:dataset:src1 --! qt:dataset:src +--! qt:disabled:HIVE-26138 Fix mapjoin_memcheck set hive.mapred.mode=nonstrict; set hive.auto.convert.join = true; @@ -18,4 +19,4 @@ from src0 src1 inner join src0 src2 on src1.key = src2.key; select src1.key as k1, src1.value as v1, src2.key, src2.value from src0 src1 inner join src0 src2 on src1.key = src2.key; -drop table src0; \ No newline at end of file +drop table src0;
[hive] branch master updated: HIVE-26092: Fix javadoc errors for the 4.0.0 release (Peter Vary reviewed by Zoltan Haindrich) (#3185)
This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 431e7d9e54 HIVE-26092: Fix javadoc errors for the 4.0.0 release (Peter Vary reviewed by Zoltan Haindrich) (#3185) 431e7d9e54 is described below commit 431e7d9e5431a808106d8db81e11aea74f040da5 Author: pvary AuthorDate: Tue Apr 12 13:52:52 2022 +0200 HIVE-26092: Fix javadoc errors for the 4.0.0 release (Peter Vary reviewed by Zoltan Haindrich) (#3185) --- Jenkinsfile| 12 .../format/datetime/HiveSqlDateTimeFormatter.java | 32 +++--- .../org/apache/hadoop/hive/common/type/Date.java | 6 ++-- .../apache/hadoop/hive/common/type/Timestamp.java | 6 ++-- .../apache/hive/common/util/TimestampParser.java | 2 +- .../hive/benchmark/calcite/FieldTrimmerBench.java | 6 ++-- .../apache/hive/benchmark/hash/Murmur3Bench.java | 10 +++ .../hive/benchmark/serde/LazySimpleSerDeBench.java | 6 ++-- .../vectorization/VectorizedArithmeticBench.java | 10 +++ .../vectorization/VectorizedComparisonBench.java | 10 +++ .../vectorization/VectorizedLikeBench.java | 10 +++ .../vectorization/VectorizedLogicBench.java| 10 +++ .../hive/ql/qoption/QTestOptionDispatcher.java | 2 +- .../org/apache/hive/jdbc/HiveBaseResultSet.java| 4 +-- .../apache/hive/jdbc/saml/IJdbcBrowserClient.java | 2 +- .../org/apache/hadoop/hive/llap/io/api/LlapIo.java | 2 +- .../security/DefaultJwtSharedSecretProvider.java | 2 +- .../tezplugins/metrics/LlapMetricsListener.java| 2 +- .../org/apache/hadoop/hive/llap/LlapHiveUtils.java | 3 +- .../ql/ddl/table/info/desc/DescTableAnalyzer.java | 6 ++-- .../hadoop/hive/ql/exec/AddToClassPathAction.java | 4 +-- .../java/org/apache/hadoop/hive/ql/exec/Task.java | 2 +- .../org/apache/hadoop/hive/ql/exec/Utilities.java | 3 +- .../hive/ql/exec/WindowFunctionDescription.java| 26 +- .../hive/ql/exec/repl/OptimisedBootstrapUtils.java | 4 +-- .../hadoop/hive/ql/exec/repl/ReplStatsTracker.java | 2 +- .../apache/hadoop/hive/ql/exec/tez/DagUtils.java | 1 - .../expressions/CastDateToCharWithFormat.java | 2 +- .../expressions/CastDateToStringWithFormat.java| 2 +- .../expressions/CastDateToVarCharWithFormat.java | 2 +- .../expressions/CastStringToDateWithFormat.java| 2 +- .../CastStringToTimestampWithFormat.java | 2 +- .../expressions/CastTimestampToCharWithFormat.java | 2 +- .../CastTimestampToStringWithFormat.java | 2 +- .../CastTimestampToVarCharWithFormat.java | 2 +- .../apache/hadoop/hive/ql/io/AcidInputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/encoded/StreamUtils.java | 1 - .../hive/ql/log/syslog/SyslogInputFormat.java | 2 +- .../hadoop/hive/ql/log/syslog/SyslogParser.java| 11 .../org/apache/hadoop/hive/ql/metadata/Hive.java | 2 +- .../hive/ql/metadata/HiveStorageHandler.java | 2 +- .../hive/ql/optimizer/ParallelEdgeFixer.java | 2 +- .../hive/ql/optimizer/SemiJoinReductionMerge.java | 6 ++-- .../calcite/functions/HiveMergeableAggregate.java | 2 +- .../calcite/rules/HiveAggregateSortLimitRule.java | 2 +- .../ql/optimizer/calcite/rules/HiveDruidRules.java | 2 +- .../calcite/rules/HiveHepExtractRelNodeRule.java | 3 +- .../HiveProjectSortExchangeTransposeRule.java | 2 +- .../rules/HiveRewriteToDataSketchesRules.java | 6 ++-- ...regateInsertDeleteIncrementalRewritingRule.java | 14 +- ...iveAggregateInsertIncrementalRewritingRule.java | 6 ++-- ...AggregatePartitionIncrementalRewritingRule.java | 4 +-- ...veJoinInsertDeleteIncrementalRewritingRule.java | 4 +-- .../calcite/translator/RexNodeConverter.java | 4 +-- .../hive/ql/optimizer/topnkey/CommonKeyPrefix.java | 6 ++-- .../ql/optimizer/topnkey/TopNKeyProcessor.java | 2 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 1 - .../hadoop/hive/ql/parse/UnparseTranslator.java| 4 +-- .../hadoop/hive/ql/parse/type/FunctionHelper.java | 2 +- .../hive/ql/txn/compactor/CompactorThread.java | 2 +- .../hive/ql/udf/generic/GenericUDAFRank.java | 2 +- .../hive/ql/udf/generic/GenericUDFCastFormat.java | 2 +- .../org/apache/hadoop/hive/serde2/JsonSerDe.java | 4 +-- .../hadoop/hive/serde2/json/BinaryEncoding.java| 2 +- .../hadoop/hive/serde2/json/HiveJsonReader.java| 6 ++-- .../hive/service/cli/operation/QueryInfoCache.java | 2 +- .../org/apache/hadoop/hive/shims/HadoopShims.java | 2 -- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 2 +- .../hadoop/hive/metastore/IMetaStoreClient.java| 11 ++-- .../hadoop/hive/metastore/utils/FileUtils.java | 1 - .../hadoop/hive/metastore/ExceptionHandler.java| 4 +--
[hive] branch master updated (73b6b7f2ba -> a85f1650e2)
This is an automated email from the ASF dual-hosted git repository.
mbod pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
from 73b6b7f2ba HIVE-25941: Long compilation time of complex query due to
analysis for materialized view rewrite (Krisztian Kasa, reviewed by Zoltan
Haindrich, Aman Sinha)
add a85f1650e2 HIVE-26102: Implement DELETE statements for Iceberg tables
(#3131) (Marton Bod, reviewed by Peter Vary)
No new revisions were added by this update.
Summary of changes:
.../org/apache/iceberg/mr/InputFormatConfig.java | 1 +
.../org/apache/iceberg/mr/hive/FilesForCommit.java | 64 +
.../iceberg/mr/hive/HiveFileWriterFactory.java | 2 +-
.../iceberg/mr/hive/HiveIcebergDeleteWriter.java | 64 +
.../mr/hive/HiveIcebergOutputCommitter.java| 109 +--
.../iceberg/mr/hive/HiveIcebergOutputFormat.java | 17 ++-
.../iceberg/mr/hive/HiveIcebergRecordWriter.java | 82 ++--
.../apache/iceberg/mr/hive/HiveIcebergSerDe.java | 12 +-
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 69 +-
...ergRecordWriter.java => HiveIcebergWriter.java} | 87 +---
.../apache/iceberg/mr/hive/IcebergAcidUtil.java| 139 +++
.../iceberg/mr/mapreduce/IcebergInputFormat.java | 79 ---
.../hive/HiveIcebergStorageHandlerTestUtils.java | 14 +-
.../mr/hive/TestHiveIcebergOutputCommitter.java| 4 +-
.../iceberg/mr/hive/TestHiveIcebergStatistics.java | 4 +-
.../apache/iceberg/mr/hive/TestHiveIcebergV2.java | 82
.../positive/delete_iceberg_partitioned_avro.q | 30 +
.../positive/delete_iceberg_partitioned_orc.q | 30 +
.../positive/delete_iceberg_partitioned_parquet.q | 30 +
.../delete_iceberg_unpartitioned_parquet.q | 30 +
.../positive/delete_iceberg_partitioned_avro.q.out | 148 +
.../positive/delete_iceberg_partitioned_orc.q.out | 148 +
.../delete_iceberg_partitioned_parquet.q.out | 148 +
.../delete_iceberg_unpartitioned_parquet.q.out | 148 +
itests/qtest-iceberg/pom.xml | 6 +
.../apache/hadoop/hive/ql/exec/MapOperator.java| 27 +++-
.../org/apache/hadoop/hive/ql/io/AcidUtils.java| 10 ++
.../hive/ql/io/HiveContextAwareRecordReader.java | 1 +
.../org/apache/hadoop/hive/ql/io/IOContext.java| 10 +-
.../hadoop/hive/ql/io/PositionDeleteInfo.java | 73 ++
.../hive/ql/metadata/HiveStorageHandler.java | 74 +++
.../hadoop/hive/ql/metadata/VirtualColumn.java | 9 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 12 +-
.../hive/ql/parse/RewriteSemanticAnalyzer.java | 3 -
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 37 +++---
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 32 -
36 files changed, 1596 insertions(+), 239 deletions(-)
create mode 100644
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/FilesForCommit.java
create mode 100644
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergDeleteWriter.java
copy
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/{HiveIcebergRecordWriter.java
=> HiveIcebergWriter.java} (54%)
create mode 100644
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
create mode 100644
iceberg/iceberg-handler/src/test/queries/positive/delete_iceberg_partitioned_avro.q
create mode 100644
iceberg/iceberg-handler/src/test/queries/positive/delete_iceberg_partitioned_orc.q
create mode 100644
iceberg/iceberg-handler/src/test/queries/positive/delete_iceberg_partitioned_parquet.q
create mode 100644
iceberg/iceberg-handler/src/test/queries/positive/delete_iceberg_unpartitioned_parquet.q
create mode 100644
iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_partitioned_avro.q.out
create mode 100644
iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_partitioned_orc.q.out
create mode 100644
iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_partitioned_parquet.q.out
create mode 100644
iceberg/iceberg-handler/src/test/results/positive/delete_iceberg_unpartitioned_parquet.q.out
create mode 100644
ql/src/java/org/apache/hadoop/hive/ql/io/PositionDeleteInfo.java
[hive] branch master updated: HIVE-25941: Long compilation time of complex query due to analysis for materialized view rewrite (Krisztian Kasa, reviewed by Zoltan Haindrich, Aman Sinha)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 73b6b7f2ba HIVE-25941: Long compilation time of complex query due to
analysis for materialized view rewrite (Krisztian Kasa, reviewed by Zoltan
Haindrich, Aman Sinha)
73b6b7f2ba is described below
commit 73b6b7f2ba75ba9d78b326a73288f80dfb5a6c8d
Author: Krisztian Kasa
AuthorDate: Tue Apr 12 10:20:32 2022 +0200
HIVE-25941: Long compilation time of complex query due to analysis for
materialized view rewrite (Krisztian Kasa, reviewed by Zoltan Haindrich, Aman
Sinha)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 8 +-
.../ql/metadata/HiveMaterializedViewsRegistry.java | 7 +-
.../ql/metadata/HiveRelOptMaterialization.java | 17 +-
.../hive/ql/metadata/MaterializedViewsCache.java | 75 +++--
...terializedViewASTSubQueryRewriteRexShuttle.java | 67
...eMaterializedViewASTSubQueryRewriteShuttle.java | 181 +
.../rules/views/HiveMaterializedViewUtils.java | 6 +-
.../org/apache/hadoop/hive/ql/parse/CBOPlan.java | 8 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 122 ++
.../apache/hadoop/hive/ql/parse/ParseUtils.java| 2 +-
.../ql/metadata/TestMaterializedViewsCache.java| 71
.../materialized_view_rewrite_by_text_3.q | 4 -
.../materialized_view_rewrite_by_text_6.q | 2 +
.../materialized_view_rewrite_by_text_8.q | 12 +-
.../llap/materialized_view_rewrite_by_text_3.q.out | 15 --
.../llap/materialized_view_rewrite_by_text_8.q.out | 50 +++---
17 files changed, 468 insertions(+), 183 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 73bb924611..c629e8f698 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1914,10 +1914,10 @@ public class HiveConf extends Configuration {
"Whether to try to rewrite queries using the materialized views
enabled for rewriting"),
HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING_SQL("hive.materializedview.rewriting.sql",
true,
"Whether to try to rewrite queries using the materialized views
enabled for rewriting by comparing the sql " +
-"query text with the materialized views query text"),
+"query syntax tree with the materialized views query syntax
tree"),
HIVE_MATERIALIZED_VIEW_ENABLE_AUTO_REWRITING_SUBQUERY_SQL("hive.materializedview.rewriting.sql.subquery",
true,
"Whether to try to rewrite sub-queries using the materialized views
enabled for rewriting by comparing the sql " +
-"sub-query text with the materialized views query text"),
+"sub-query syntax tree with the materialized views query
syntax tree"),
HIVE_MATERIALIZED_VIEW_REWRITING_SELECTION_STRATEGY("hive.materializedview.rewriting.strategy",
"heuristic",
new StringSet("heuristic", "costbased"),
"The strategy that should be used to cost and select the materialized
view rewriting. \n" +
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index fcb7f037b5..48e859b8c8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -191,7 +191,6 @@ import
org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest;
import org.apache.hadoop.hive.metastore.api.WriteNotificationLogBatchRequest;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.ErrorMsg;
@@ -209,6 +208,7 @@ import org.apache.hadoop.hive.ql.lockmgr.LockException;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import
org.apache.hadoop.hive.ql.optimizer.calcite.rules.views.HiveMaterializedViewUtils;
import
org.apache.hadoop.hive.ql.optimizer.listbucketingpruner.ListBucketingPrunerUtils;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
@@ -2258,11 +2258,11 @@ public class Hive {
* @return List of materialized views has matching query definition with
querySql
* @throws HiveException - an exception is thrown during validation or
unable to pull
