[hive] branch master updated (1e853b9 -> a62af3a)
This is an automated email from the ASF dual-hosted git repository. ychena pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 1e853b9 HIVE-21152: Rewrite if expression to case and recognize simple case as an if (#2791) (Zoltan Haindrich reviewed by Krisztian Kasa) add a62af3a HIVE-25957: Fix password based authentication with SAML enabled (#3028) (Yu-Wen Lai, reviewed by Naveen Gangam) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../org/apache/hive/service/auth/AuthType.java | 112 + .../hive/service/cli/thrift/ThriftHttpServlet.java | 59 +-- .../org/apache/hive/service/auth/TestAuthType.java | 112 + .../service/cli/thrift/ThriftHttpServletTest.java | 2 +- 5 files changed, 253 insertions(+), 34 deletions(-) create mode 100644 service/src/java/org/apache/hive/service/auth/AuthType.java create mode 100644 service/src/test/org/apache/hive/service/auth/TestAuthType.java
[hive] branch master updated: HIVE-25957: [Addendum to HIVE-25875] Fix password authentication with SAML (Yu-Wen Lai via Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new e82c355 HIVE-25957: [Addendum to HIVE-25875] Fix password authentication with SAML (Yu-Wen Lai via Naveen Gangam) e82c355 is described below commit e82c355644fdd29b57f8fa2aeca812d0bc951471 Author: Yu-Wen Lai AuthorDate: Mon Feb 14 14:58:37 2022 -0800 HIVE-25957: [Addendum to HIVE-25875] Fix password authentication with SAML (Yu-Wen Lai via Naveen Gangam) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../org/apache/hive/service/auth/AuthType.java | 112 + .../hive/service/cli/thrift/ThriftHttpServlet.java | 59 +-- .../org/apache/hive/service/auth/TestAuthType.java | 112 + .../service/cli/thrift/ThriftHttpServletTest.java | 2 +- 5 files changed, 253 insertions(+), 34 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e0fcf3c..d8398b3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4125,7 +4125,7 @@ public class HiveConf extends Configuration { " (Use with property hive.server2.custom.authentication.class)\n" + " PAM: Pluggable authentication module\n" + " NOSASL: Raw transport\n" + -" SAML2: SAML 2.0 compliant authentication. This is only supported in http transport mode."), +" SAML: SAML 2.0 compliant authentication. This is only supported in http transport mode."), HIVE_SERVER2_TRUSTED_DOMAIN("hive.server2.trusted.domain", "", "Specifies the host or a domain to trust connections from. Authentication is skipped " + "for any connection coming from a host whose hostname ends with the value of this" + diff --git a/service/src/java/org/apache/hive/service/auth/AuthType.java b/service/src/java/org/apache/hive/service/auth/AuthType.java new file mode 100644 index 000..b0c5711 --- /dev/null +++ b/service/src/java/org/apache/hive/service/auth/AuthType.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.auth; + +import org.apache.commons.lang3.EnumUtils; + +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +/** + * AuthType is used to parse and verify + * {@link org.apache.hadoop.hive.conf.HiveConf.ConfVars#HIVE_SERVER2_AUTHENTICATION}. + * Throws an exception if the config value is not allowed. + */ +public class AuthType { + static final Set PASSWORD_BASED_TYPES = new HashSet<>(Arrays.asList( + HiveAuthConstants.AuthTypes.LDAP, HiveAuthConstants.AuthTypes.CUSTOM, HiveAuthConstants.AuthTypes.PAM)); + private final BitSet typeBits; + + public AuthType(String authTypes) throws Exception { +typeBits = new BitSet(); +parseTypes(authTypes); +verifyTypes(authTypes); + } + + private void parseTypes(String authTypes) throws Exception { +String[] types = authTypes.split(","); +for (String type : types) { + if (!EnumUtils.isValidEnumIgnoreCase(HiveAuthConstants.AuthTypes.class, type)) { +throw new Exception(type + " is not a valid authentication type."); + } + typeBits.set(EnumUtils.getEnumIgnoreCase(HiveAuthConstants.AuthTypes.class, type).ordinal()); +} + } + + private void verifyTypes(String authTypes) throws Exception { +if (typeBits.cardinality() == 1) { + // single authentication type has no conflicts + return; +} +if (typeBits.get(HiveAuthConstants.AuthTypes.SAML.ordinal()) && +!typeBits.get(HiveAuthConstants.AuthTypes.NOSASL.ordinal()) && +!typeBits.get(HiveAuthConstants.AuthTypes.KERBEROS.ordinal()) && +!typeBits.get(HiveAuthConstants.AuthTypes.NONE.ordinal()) && +(!areAnyEnabled(PASSWORD_BASED_TYPES) || isExactlyOneEnabled(PASSWORD_BASED_TYPES))) { + // SAML
[hive] branch master updated: HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 362461f HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich reviewed by Krisztian Kasa) 362461f is described below commit 362461fab1b79a01b6ab966606b5faa694d38d38 Author: Zoltan Haindrich AuthorDate: Thu Feb 17 18:06:19 2022 +0100 HIVE-25715: Provide nightly builds (#3013) (Zoltan Haindrich reviewed by Krisztian Kasa) --- Jenkinsfile| 14 +++ beeline/pom.xml| 2 +- common/pom.xml | 2 +- dev-support/nightly| 45 ++ hcatalog/core/pom.xml | 4 +- hcatalog/webhcat/java-client/pom.xml | 4 +- hcatalog/webhcat/svr/pom.xml | 4 +- iceberg/pom.xml| 4 +- itests/pom.xml | 8 ++-- llap-server/pom.xml| 4 +- metastore/pom.xml | 2 +- pom.xml| 1 + ql/pom.xml | 6 +-- service/pom.xml| 4 +- .../metastore-tools/metastore-benchmarks/pom.xml | 3 +- standalone-metastore/metastore-tools/pom.xml | 3 +- standalone-metastore/pom.xml | 1 + 17 files changed, 86 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 029d3a4..b079d51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -292,6 +292,20 @@ mvn verify -DskipITests=false -Dit.test=ITest${dbType.capitalize()} -Dtest=nosuc } } } + branches['nightly-check'] = { + executorNode { +stage('Prepare') { +loadWS(); +} +stage('Build') { +sh '''#!/bin/bash +set -e +dev-support/nightly +''' +buildHive("install -Dtest=noMatches -Pdist -pl packaging -am") +} + } + } for (int i = 0; i < splits.size(); i++) { def num = i def split = splits[num] diff --git a/beeline/pom.xml b/beeline/pom.xml index a8d9159..131a3f6 100644 --- a/beeline/pom.xml +++ b/beeline/pom.xml @@ -65,7 +65,7 @@ org.apache.hive hive-standalone-metastore-server - ${project.version} + ${standalone-metastore.version} diff --git a/common/pom.xml b/common/pom.xml index f7c6c8a..912de29 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,7 +46,7 @@ org.apache.hive hive-standalone-metastore-common - ${project.version} + ${standalone-metastore.version} diff --git a/dev-support/nightly b/dev-support/nightly new file mode 100755 index 000..7045148 --- /dev/null +++ b/dev-support/nightly @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +DATE="`date +%Y%m%d_%H%M%S`" +HASH="`git rev-parse --short HEAD`" +SUFFIX="nightly-$HASH-$DATE" + +V="`xmlstarlet sel -t -m /_:project/_:version -v . pom.xml`" +NEW_HIVE="${NEW_HIVE:-${V/-*}-$SUFFIX}" +V="`xmlstarlet sel -t -m /_:project/_:version -v . storage-api/pom.xml`" +NEW_SA="${NEW_SA:-${V/-*}-$SUFFIX}" +V="`xmlstarlet sel -t -m /_:project/_:version -v . standalone-metastore/pom.xml`" +NEW_MS="${NEW_MS:-${V/-*}-$SUFFIX}" + + +mvn_versions_set="mvn versions:set versions:commit -DgenerateBackupPoms=false" + +$mvn_versions_set -B -DnewVersion=$NEW_HIVE +$mvn_versions_set -B -DnewVersion=$NEW_SA -pl storage-api +$mvn_versions_set -B -DnewVersion=$NEW_MS -pl standalone-metastore + +xmlstarlet edit -L -P --update "/_:project/_:properties/_:hive.version" \ + --value $NEW_HIVE standalone-metastore/pom.xml + +xmlstarlet edit -L -P --update "/_:project/_:properties/_:storage-api.version" \ + --value $NEW_SA pom.xml standalone-metastore/pom.xml + +xmlstarlet edit -L -P --update "/_:project/_:properties/_:standalone-metastore.version" \ + --value $NEW_MS pom.xml + diff --git a/hcatalog/core/pom.xml
[hive] branch master updated: HIVE-25951: HiveRelMdPredicates extends RelMdPredicates (Alessandro Solimando, reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new c835e42 HIVE-25951: HiveRelMdPredicates extends RelMdPredicates (Alessandro Solimando, reviewed by Krisztian Kasa) c835e42 is described below commit c835e4254452662b88376bb9480145500c6a0065 Author: Alessandro Solimando AuthorDate: Thu Feb 17 13:44:54 2022 +0100 HIVE-25951: HiveRelMdPredicates extends RelMdPredicates (Alessandro Solimando, reviewed by Krisztian Kasa) --- .../calcite/stats/HiveRelMdPredicates.java | 97 +- 1 file changed, 4 insertions(+), 93 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java index 66c6888..818fcf4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/HiveRelMdPredicates.java @@ -35,7 +35,6 @@ import org.apache.calcite.linq4j.function.Predicate1; import org.apache.calcite.plan.RelOptPredicateList; import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Project; @@ -43,7 +42,6 @@ import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.metadata.BuiltInMetadata; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.MetadataDef; -import org.apache.calcite.rel.metadata.MetadataHandler; import org.apache.calcite.rel.metadata.ReflectiveRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMdPredicates; import org.apache.calcite.rel.metadata.RelMetadataProvider; @@ -75,7 +73,7 @@ import com.google.common.collect.Maps; //TODO: Move this to calcite -public class HiveRelMdPredicates implements MetadataHandler { +public class HiveRelMdPredicates extends RelMdPredicates { public static final RelMetadataProvider SOURCE = ChainedRelMetadataProvider.of( @@ -116,6 +114,7 @@ public class HiveRelMdPredicates implements MetadataHandler */ + @Override public RelOptPredicateList getPredicates(Project project, RelMetadataQuery mq) { RelNode child = project.getInput(); @@ -168,6 +167,7 @@ public class HiveRelMdPredicates implements MetadataHandlerPulls up predicates that only contains references to columns in the - * GroupSet. For e.g. - * - * - * inputPullUpExprs : { a 7, b + c 10, a + e = 9} - * groupSet : { a, b} - * pulledUpExprs: { a 7} - * - */ - public RelOptPredicateList getPredicates(Aggregate agg, RelMetadataQuery mq) { -final RelNode input = agg.getInput(); -final RelOptPredicateList inputInfo = mq.getPulledUpPredicates(input); -final List aggPullUpPredicates = new ArrayList<>(); -final RexBuilder rexBuilder = agg.getCluster().getRexBuilder(); - -ImmutableBitSet groupKeys = agg.getGroupSet(); -Mapping m = Mappings.create(MappingType.PARTIAL_FUNCTION, -input.getRowType().getFieldCount(), agg.getRowType().getFieldCount()); - -int i = 0; -for (int j : groupKeys) { - m.set(j, i++); -} - -for (RexNode r : inputInfo.pulledUpPredicates) { - ImmutableBitSet rCols = RelOptUtil.InputFinder.bits(r); - if (!rCols.isEmpty() && groupKeys.contains(rCols)) { -r = r.accept(new RexPermuteInputsShuttle(m, input)); -aggPullUpPredicates.add(r); - } -} -return RelOptPredicateList.of(rexBuilder, aggPullUpPredicates); - } - - /** - * Infers predicates for a Union. - */ - public RelOptPredicateList getPredicates(Union union, RelMetadataQuery mq) { -RexBuilder rB = union.getCluster().getRexBuilder(); - -Map finalPreds = new HashMap<>(); -List finalResidualPreds = new ArrayList<>(); -for (int i = 0; i < union.getInputs().size(); i++) { - RelNode input = union.getInputs().get(i); - RelOptPredicateList info = mq.getPulledUpPredicates(input); - if (info.pulledUpPredicates.isEmpty()) { -return RelOptPredicateList.EMPTY; - } - Map preds = new HashMap<>(); - List residualPreds = new ArrayList<>(); - for (RexNode pred : info.pulledUpPredicates) { -final String predString = pred.toString(); -if (i == 0) { - preds.put(predString, pred); - continue; -} -if (finalPreds.containsKey(predString)) { - preds.put(predString, pred); -} else { - residualPreds.add(pred); -} - } - // Add new residual preds - finalResidualPreds.add(RexUtil.composeConjunction(rB,
[hive] branch master updated: HIVE-25884: Improve rule description for rules defined as subclasses (Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new aafced6 HIVE-25884: Improve rule description for rules defined as subclasses (Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa) aafced6 is described below commit aafced6cd2b1bf31bd74c563680e02fd54cd1e01 Author: Alessandro Solimando AuthorDate: Thu Feb 17 13:25:36 2022 +0100 HIVE-25884: Improve rule description for rules defined as subclasses (Alessandro Solimando reviewed by Zoltan Haindrich, Krisztian Kasa) --- .../calcite/rules/HiveFilterJoinRule.java | 8 .../calcite/rules/HiveInBetweenExpandRule.java | 9 +--- .../rules/HivePointLookupOptimizerRule.java| 13 +++- .../rules/HiveRewriteToDataSketchesRules.java | 24 -- .../rules/jdbc/JDBCExpandExpressionsRule.java | 10 - 5 files changed, 36 insertions(+), 28 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java index 335130d..bf08c98 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterJoinRule.java @@ -71,7 +71,7 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { super( (Config) RelRule.Config.EMPTY - .withDescription("FilterJoinRule:" + id) + .withDescription("HiveFilterJoinRule(" + id + ")") .withOperandSupplier(b0 -> b0.exactly(operand)) .as(FilterJoinRule.Config.class) @@ -108,7 +108,7 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { public static class HiveFilterJoinMergeRule extends HiveFilterJoinRule { public HiveFilterJoinMergeRule() { super(operand(Filter.class, operand(Join.class, any())), - null, true, HiveRelFactories.HIVE_BUILDER); + HiveFilterJoinMergeRule.class.getSimpleName(), true, HiveRelFactories.HIVE_BUILDER); } @Override @@ -130,8 +130,8 @@ public abstract class HiveFilterJoinRule extends FilterJoinRule { public static class HiveFilterJoinTransposeRule extends HiveFilterJoinRule { public HiveFilterJoinTransposeRule() { - super(RelOptRule.operand(Join.class, RelOptRule.any()), "HiveFilterJoinRule:no-filter", true, - HiveRelFactories.HIVE_BUILDER); + super(RelOptRule.operand(Join.class, RelOptRule.any()), + HiveFilterJoinTransposeRule.class.getSimpleName(), true, HiveRelFactories.HIVE_BUILDER); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java index 0e1c65f..49b67f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInBetweenExpandRule.java @@ -49,7 +49,8 @@ public class HiveInBetweenExpandRule { private static class FilterRule extends RelOptRule { FilterRule() { - super(operand(Filter.class, any()), HiveRelFactories.HIVE_BUILDER, null); + super(operand(Filter.class, any()), HiveRelFactories.HIVE_BUILDER, + "HiveInBetweenExpandRule(FilterRule)"); } @Override @@ -74,7 +75,8 @@ public class HiveInBetweenExpandRule { private static class JoinRule extends RelOptRule { JoinRule() { - super(operand(Join.class, any()), HiveRelFactories.HIVE_BUILDER, null); + super(operand(Join.class, any()), HiveRelFactories.HIVE_BUILDER, + "HiveInBetweenExpandRule(JoinRule)"); } @Override @@ -103,7 +105,8 @@ public class HiveInBetweenExpandRule { private static class ProjectRule extends RelOptRule { ProjectRule() { - super(operand(Project.class, any()), HiveRelFactories.HIVE_BUILDER, null); + super(operand(Project.class, any()), HiveRelFactories.HIVE_BUILDER, + "HiveInBetweenExpandRule(ProjectRule)"); } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java index da6e9e7..a351a3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HivePointLookupOptimizerRule.java @@ -81,7 +81,8 @@ public abstract class HivePointLookupOptimizerRule extends RelOptRule { /** Rule adapter to apply the transformation to Filter conditions. */ public static class