[hive] branch master updated (1e853b9 -> a62af3a)
This is an automated email from the ASF dual-hosted git repository. ychena pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 1e853b9 HIVE-21152: Rewrite if expression to case and recognize simple case as an if (#2791) (Zoltan Haindrich reviewed by Krisztian Kasa) add a62af3a HIVE-25957: Fix password based authentication with SAML enabled (#3028) (Yu-Wen Lai, reviewed by Naveen Gangam) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../org/apache/hive/service/auth/AuthType.java | 112 + .../hive/service/cli/thrift/ThriftHttpServlet.java | 59 +-- .../org/apache/hive/service/auth/TestAuthType.java | 112 + .../service/cli/thrift/ThriftHttpServletTest.java | 2 +- 5 files changed, 253 insertions(+), 34 deletions(-) create mode 100644 service/src/java/org/apache/hive/service/auth/AuthType.java create mode 100644 service/src/test/org/apache/hive/service/auth/TestAuthType.java
[hive] branch master updated: HIVE-25957: [Addendum to HIVE-25875] Fix password authentication with SAML (Yu-Wen Lai via Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e82c355 HIVE-25957: [Addendum to HIVE-25875] Fix password
authentication with SAML (Yu-Wen Lai via Naveen Gangam)
e82c355 is described below
commit e82c355644fdd29b57f8fa2aeca812d0bc951471
Author: Yu-Wen Lai
AuthorDate: Mon Feb 14 14:58:37 2022 -0800
HIVE-25957: [Addendum to HIVE-25875] Fix password authentication with SAML
(Yu-Wen Lai via Naveen Gangam)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../org/apache/hive/service/auth/AuthType.java | 112 +
.../hive/service/cli/thrift/ThriftHttpServlet.java | 59 +--
.../org/apache/hive/service/auth/TestAuthType.java | 112 +
.../service/cli/thrift/ThriftHttpServletTest.java | 2 +-
5 files changed, 253 insertions(+), 34 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index e0fcf3c..d8398b3 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4125,7 +4125,7 @@ public class HiveConf extends Configuration {
" (Use with property
hive.server2.custom.authentication.class)\n" +
" PAM: Pluggable authentication module\n" +
" NOSASL: Raw transport\n" +
-" SAML2: SAML 2.0 compliant authentication. This is only supported in
http transport mode."),
+" SAML: SAML 2.0 compliant authentication. This is only supported in
http transport mode."),
HIVE_SERVER2_TRUSTED_DOMAIN("hive.server2.trusted.domain", "",
"Specifies the host or a domain to trust connections from.
Authentication is skipped " +
"for any connection coming from a host whose hostname ends with the
value of this" +
diff --git a/service/src/java/org/apache/hive/service/auth/AuthType.java
b/service/src/java/org/apache/hive/service/auth/AuthType.java
new file mode 100644
index 000..b0c5711
--- /dev/null
+++ b/service/src/java/org/apache/hive/service/auth/AuthType.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.auth;
+
+import org.apache.commons.lang3.EnumUtils;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * AuthType is used to parse and verify
+ * {@link
org.apache.hadoop.hive.conf.HiveConf.ConfVars#HIVE_SERVER2_AUTHENTICATION}.
+ * Throws an exception if the config value is not allowed.
+ */
+public class AuthType {
+ static final Set PASSWORD_BASED_TYPES = new
HashSet<>(Arrays.asList(
+ HiveAuthConstants.AuthTypes.LDAP, HiveAuthConstants.AuthTypes.CUSTOM,
HiveAuthConstants.AuthTypes.PAM));
+ private final BitSet typeBits;
+
+ public AuthType(String authTypes) throws Exception {
+typeBits = new BitSet();
+parseTypes(authTypes);
+verifyTypes(authTypes);
+ }
+
+ private void parseTypes(String authTypes) throws Exception {
+String[] types = authTypes.split(",");
+for (String type : types) {
+ if (!EnumUtils.isValidEnumIgnoreCase(HiveAuthConstants.AuthTypes.class,
type)) {
+throw new Exception(type + " is not a valid authentication type.");
+ }
+
typeBits.set(EnumUtils.getEnumIgnoreCase(HiveAuthConstants.AuthTypes.class,
type).ordinal());
+}
+ }
+
+ private void verifyTypes(String authTypes) throws Exception {
+if (typeBits.cardinality() == 1) {
+ // single authentication type has no conflicts
+ return;
+}
+if (typeBits.get(HiveAuthConstants.AuthTypes.SAML.ordinal()) &&
+!typeBits.get(HiveAuthConstants.AuthTypes.NOSASL.ordinal()) &&
+!typeBits.get(HiveAuthConstants.AuthTypes.KERBEROS.ordinal()) &&
+!typeBits.get(HiveAuthConstants.AuthTypes.NONE.ordinal()) &&
+(!areAnyEnabled(PASSWORD_BASED_TYPES) ||
isExactlyOneEnabled(PASSWORD_BASED_TYPES))) {
+ // SAML
[hive] branch master updated: HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 362461f HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich
reviewed by Krisztian Kasa)
362461f is described below
commit 362461fab1b79a01b6ab966606b5faa694d38d38
Author: Zoltan Haindrich
AuthorDate: Thu Feb 17 18:06:19 2022 +0100
HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich reviewed by
Krisztian Kasa)
---
Jenkinsfile| 14 +++
beeline/pom.xml| 2 +-
common/pom.xml | 2 +-
dev-support/nightly| 45 ++
hcatalog/core/pom.xml | 4 +-
hcatalog/webhcat/java-client/pom.xml | 4 +-
hcatalog/webhcat/svr/pom.xml | 4 +-
iceberg/pom.xml| 4 +-
itests/pom.xml | 8 ++--
llap-server/pom.xml| 4 +-
metastore/pom.xml | 2 +-
pom.xml| 1 +
ql/pom.xml | 6 +--
service/pom.xml| 4 +-
.../metastore-tools/metastore-benchmarks/pom.xml | 3 +-
standalone-metastore/metastore-tools/pom.xml | 3 +-
standalone-metastore/pom.xml | 1 +
17 files changed, 86 insertions(+), 25 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 029d3a4..b079d51 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -292,6 +292,20 @@ mvn verify -DskipITests=false
-Dit.test=ITest${dbType.capitalize()} -Dtest=nosuc
}
}
}
+ branches['nightly-check'] = {
+ executorNode {
+stage('Prepare') {
+loadWS();
+}
+stage('Build') {
+sh '''#!/bin/bash
+set -e
+dev-support/nightly
+'''
+buildHive("install -Dtest=noMatches -Pdist -pl packaging -am")
+}
+ }
+ }
for (int i = 0; i < splits.size(); i++) {
def num = i
def split = splits[num]
diff --git a/beeline/pom.xml b/beeline/pom.xml
index a8d9159..131a3f6 100644
--- a/beeline/pom.xml
+++ b/beeline/pom.xml
@@ -65,7 +65,7 @@
org.apache.hive
hive-standalone-metastore-server
- ${project.version}
+ ${standalone-metastore.version}
diff --git a/common/pom.xml b/common/pom.xml
index f7c6c8a..912de29 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -46,7 +46,7 @@
org.apache.hive
hive-standalone-metastore-common
- ${project.version}
+ ${standalone-metastore.version}
diff --git a/dev-support/nightly b/dev-support/nightly
new file mode 100755
index 000..7045148
--- /dev/null
+++ b/dev-support/nightly
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+DATE="`date +%Y%m%d_%H%M%S`"
+HASH="`git rev-parse --short HEAD`"
+SUFFIX="nightly-$HASH-$DATE"
+
+V="`xmlstarlet sel -t -m /_:project/_:version -v . pom.xml`"
+NEW_HIVE="${NEW_HIVE:-${V/-*}-$SUFFIX}"
+V="`xmlstarlet sel -t -m /_:project/_:version -v . storage-api/pom.xml`"
+NEW_SA="${NEW_SA:-${V/-*}-$SUFFIX}"
+V="`xmlstarlet sel -t -m /_:project/_:version -v .
standalone-metastore/pom.xml`"
+NEW_MS="${NEW_MS:-${V/-*}-$SUFFIX}"
+
+
+mvn_versions_set="mvn versions:set versions:commit -DgenerateBackupPoms=false"
+
+$mvn_versions_set -B -DnewVersion=$NEW_HIVE
+$mvn_versions_set -B -DnewVersion=$NEW_SA -pl storage-api
+$mvn_versions_set -B -DnewVersion=$NEW_MS -pl standalone-metastore
+
+xmlstarlet edit -L -P --update "/_:project/_:properties/_:hive.version" \
+ --value $NEW_HIVE standalone-metastore/pom.xml
+
+xmlstarlet edit -L -P --update "/_:project/_:properties/_:storage-api.version"
\
+ --value $NEW_SA pom.xml standalone-metastore/pom.xml
+
+xmlstarlet edit -L -P --update
"/_:project/_:properties/_:standalone-metastore.version" \
+ --value $NEW_MS pom.xml
+
diff --git a/hcatalog/core/pom.xml
[hive] branch master updated: HIVE-25951: HiveRelMdPredicates extends RelMdPredicates (Alessandro Solimando, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c835e42 HIVE-25951: HiveRelMdPredicates extends RelMdPredicates
(Alessandro Solimando, reviewed by Krisztian Kasa)
c835e42 is described below
commit c835e4254452662b88376bb9480145500c6a0065
Author: Alessandro Solimando
AuthorDate: Thu Feb 17 13:44:54 2022 +0100
HIVE-25951: HiveRelMdPredicates extends RelMdPredicates (Alessandro
Solimando, reviewed by Krisztian Kasa)
---
.../calcite/stats/HiveRelMdPredicates.java | 97 +-
1 file changed, 4 insertions(+), 93 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
index 66c6888..818fcf4 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java
@@ -35,7 +35,6 @@ import org.apache.calcite.linq4j.function.Predicate1;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.Aggregate;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
@@ -43,7 +42,6 @@ import org.apache.calcite.rel.core.Union;
import org.apache.calcite.rel.metadata.BuiltInMetadata;
import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
import org.apache.calcite.rel.metadata.MetadataDef;
-import org.apache.calcite.rel.metadata.MetadataHandler;
import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider;
import org.apache.calcite.rel.metadata.RelMdPredicates;
import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -75,7 +73,7 @@ import com.google.common.collect.Maps;
//TODO: Move this to calcite
-public class HiveRelMdPredicates implements
MetadataHandler {
+public class HiveRelMdPredicates extends RelMdPredicates {
public static final RelMetadataProvider SOURCE =
ChainedRelMetadataProvider.of(
@@ -116,6 +114,7 @@ public class HiveRelMdPredicates implements
MetadataHandler
*/
+ @Override
public RelOptPredicateList getPredicates(Project project, RelMetadataQuery
mq) {
RelNode child = project.getInput();
@@ -168,6 +167,7 @@ public class HiveRelMdPredicates implements
MetadataHandlerPulls up predicates that only contains references to columns in the
- * GroupSet. For e.g.
- *
- *
- * inputPullUpExprs : { a 7, b + c 10, a + e = 9}
- * groupSet : { a, b}
- * pulledUpExprs: { a 7}
- *
- */
- public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq)
{
-final RelNode input = agg.getInput();
-final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input);
-final List aggPullUpPredicates = new ArrayList<>();
-final RexBuilder rexBuilder = agg.getCluster().getRexBuilder();
-
-ImmutableBitSet groupKeys = agg.getGroupSet();
-Mapping m = Mappings.create(MappingType.PARTIAL_FUNCTION,
-input.getRowType().getFieldCount(), agg.getRowType().getFieldCount());
-
-int i = 0;
-for (int j : groupKeys) {
- m.set(j, i++);
-}
-
-for (RexNode r : inputInfo.pulledUpPredicates) {
- ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r);
- if (!rCols.isEmpty() && groupKeys.contains(rCols)) {
-r = r.accept(new RexPermuteInputsShuttle(m, input));
-aggPullUpPredicates.add(r);
- }
-}
-return RelOptPredicateList.of(rexBuilder, aggPullUpPredicates);
- }
-
- /**
- * Infers predicates for a Union.
- */
- public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) {
-RexBuilder rB = union.getCluster().getRexBuilder();
-
-Map finalPreds = new HashMap<>();
-List finalResidualPreds = new ArrayList<>();
-for (int i = 0; i < union.getInputs().size(); i++) {
- RelNode input = union.getInputs().get(i);
- RelOptPredicateList info = mq.getPulledUpPredicates(input);
- if (info.pulledUpPredicates.isEmpty()) {
-return RelOptPredicateList.EMPTY;
- }
- Map preds = new HashMap<>();
- List residualPreds = new ArrayList<>();
- for (RexNode pred : info.pulledUpPredicates) {
-final String predString = pred.toString();
-if (i == 0) {
- preds.put(predString, pred);
- continue;
-}
-if (finalPreds.containsKey(predString)) {
- preds.put(predString, pred);
-} else {
- residualPreds.add(pred);
-}
- }
- // Add new residual preds
- finalResidualPreds.add(RexUtil.composeConjunction(rB,
[hive] branch master updated: HIVE-25884: Improve rule description for rules defined as subclasses (Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new aafced6 HIVE-25884: Improve rule description for rules defined as
subclasses (Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa)
aafced6 is described below
commit aafced6cd2b1bf31bd74c563680e02fd54cd1e01
Author: Alessandro Solimando
AuthorDate: Thu Feb 17 13:25:36 2022 +0100
HIVE-25884: Improve rule description for rules defined as subclasses
(Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa)
---
.../calcite/rules/HiveFilterJoinRule.java | 8
.../calcite/rules/HiveInBetweenExpandRule.java | 9 +---
.../rules/HivePointLookupOptimizerRule.java| 13 +++-
.../rules/HiveRewriteToDataSketchesRules.java | 24 --
.../rules/jdbc/JDBCExpandExpressionsRule.java | 10 -
5 files changed, 36 insertions(+), 28 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
index 335130d..bf08c98 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java
@@ -71,7 +71,7 @@ public abstract class HiveFilterJoinRule extends
FilterJoinRule {
super(
(Config) RelRule.Config.EMPTY
- .withDescription("FilterJoinRule:" + id)
+ .withDescription("HiveFilterJoinRule(" + id + ")")
.withOperandSupplier(b0 ->
b0.exactly(operand))
.as(FilterJoinRule.Config.class)
@@ -108,7 +108,7 @@ public abstract class HiveFilterJoinRule extends
FilterJoinRule {
public static class HiveFilterJoinMergeRule extends HiveFilterJoinRule {
public HiveFilterJoinMergeRule() {
super(operand(Filter.class, operand(Join.class, any())),
- null, true, HiveRelFactories.HIVE_BUILDER);
+ HiveFilterJoinMergeRule.class.getSimpleName(), true,
HiveRelFactories.HIVE_BUILDER);
}
@Override
@@ -130,8 +130,8 @@ public abstract class HiveFilterJoinRule extends
FilterJoinRule {
public static class HiveFilterJoinTransposeRule extends HiveFilterJoinRule {
public HiveFilterJoinTransposeRule() {
- super(RelOptRule.operand(Join.class, RelOptRule.any()),
"HiveFilterJoinRule:no-filter", true,
- HiveRelFactories.HIVE_BUILDER);
+ super(RelOptRule.operand(Join.class, RelOptRule.any()),
+ HiveFilterJoinTransposeRule.class.getSimpleName(), true,
HiveRelFactories.HIVE_BUILDER);
}
@Override
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java
index 0e1c65f..49b67f1 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java
@@ -49,7 +49,8 @@ public class HiveInBetweenExpandRule {
private static class FilterRule extends RelOptRule {
FilterRule() {
- super(operand(Filter.class, any()), HiveRelFactories.HIVE_BUILDER, null);
+ super(operand(Filter.class, any()), HiveRelFactories.HIVE_BUILDER,
+ "HiveInBetweenExpandRule(FilterRule)");
}
@Override
@@ -74,7 +75,8 @@ public class HiveInBetweenExpandRule {
private static class JoinRule extends RelOptRule {
JoinRule() {
- super(operand(Join.class, any()), HiveRelFactories.HIVE_BUILDER, null);
+ super(operand(Join.class, any()), HiveRelFactories.HIVE_BUILDER,
+ "HiveInBetweenExpandRule(JoinRule)");
}
@Override
@@ -103,7 +105,8 @@ public class HiveInBetweenExpandRule {
private static class ProjectRule extends RelOptRule {
ProjectRule() {
- super(operand(Project.class, any()), HiveRelFactories.HIVE_BUILDER,
null);
+ super(operand(Project.class, any()), HiveRelFactories.HIVE_BUILDER,
+ "HiveInBetweenExpandRule(ProjectRule)");
}
@Override
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
index da6e9e7..a351a3d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java
@@ -81,7 +81,8 @@ public abstract class HivePointLookupOptimizerRule extends
RelOptRule {
/** Rule adapter to apply the transformation to Filter conditions. */
public static class
