YARN-6638. [ATSv2 Security] Timeline reader side changes for loading auth filters and principals. Contributed by Varun Saxena
(cherry picked from commit 7f06c46065936b88e9864c526bfcda3a89f3e257) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/AbstractTimelineReaderHBaseTestBase.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50dd1372 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50dd1372 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50dd1372 Branch: refs/heads/YARN-5355_branch2 Commit: 50dd1372bf9fb61050f67da39651c9c3b326637a Parents: 9577900 Author: Jian He <jia...@apache.org> Authored: Fri Jun 9 13:42:38 2017 -0700 Committer: Varun Saxena <varunsax...@apache.org> Committed: Tue Oct 17 23:36:39 2017 +0530 ---------------------------------------------------------------------- ...TimelineAuthenticationFilterInitializer.java | 69 +++++++++++--------- .../reader/TimelineReaderServer.java | 61 ++++++++++------- ...neReaderAuthenticationFilterInitializer.java | 53 +++++++++++++++ .../reader/security/package-info.java | 25 +++++++ 4 files changed, 154 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/50dd1372/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java index 4e7c29a..06f9868e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -51,30 +51,18 @@ import java.util.Map; public class TimelineAuthenticationFilterInitializer extends FilterInitializer { /** - * The configuration prefix of timeline HTTP authentication + * The configuration prefix of timeline HTTP authentication. */ public static final String PREFIX = "yarn.timeline-service.http-authentication."; @VisibleForTesting Map<String, String> filterConfig; - /** - * Initializes {@link TimelineAuthenticationFilter} - * <p> - * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN - * configuration properties prefixed with {@value #PREFIX} - * - * @param container - * The filter container - * @param conf - * Configuration for run-time parameters - */ - @Override - public void initFilter(FilterContainer container, Configuration conf) { + protected void setAuthFilterConfig(Configuration conf) { filterConfig = new HashMap<String, String>(); // setting the cookie path to root '/' so it is used for all resources. - filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); + filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/"); for (Map.Entry<String, String> entry : conf) { String name = entry.getKey(); @@ -95,6 +83,41 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer { } } + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = + filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException("Could not resolve Kerberos principal " + + "name: " + ex.toString(), ex); + } + filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, + principal); + } + } + + protected Map<String, String> getFilterConfig() { + return filterConfig; + } + + /** + * Initializes {@link TimelineAuthenticationFilter} + * <p> + * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN + * configuration properties prefixed with {@value #PREFIX} + * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + setAuthFilterConfig(conf); + String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE); if (authType.equals(PseudoAuthenticationHandler.TYPE)) { filterConfig.put(AuthenticationFilter.AUTH_TYPE, @@ -102,23 +125,7 @@ public class TimelineAuthenticationFilterInitializer extends FilterInitializer { } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { filterConfig.put(AuthenticationFilter.AUTH_TYPE, KerberosDelegationTokenAuthenticationHandler.class.getName()); - - // Resolve _HOST into bind address - String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); - String principal = - filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); - if (principal != null) { - try { - principal = SecurityUtil.getServerPrincipal(principal, bindAddress); - } catch (IOException ex) { - throw new RuntimeException( - "Could not resolve Kerberos principal name: " + ex.toString(), ex); - } - filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, - principal); - } } - filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND, TimelineDelegationTokenIdentifier.KIND_NAME.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/50dd1372/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java index cf0fa50..1d5d6e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java @@ -18,12 +18,11 @@ package org.apache.hadoop.yarn.server.timelineservice.reader; -import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER; -import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER; - +import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; -import java.util.HashMap; -import java.util.Map; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,8 +30,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ReflectionUtils; @@ -42,7 +41,9 @@ import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.timelineservice.reader.security.TimelineReaderAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -70,6 +71,17 @@ public class TimelineReaderServer extends CompositeService { if (!YarnConfiguration.timelineServiceV2Enabled(conf)) { throw new YarnException("timeline service v.2 is not enabled"); } + InetSocketAddress bindAddr = conf.getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + // Login from keytab if security is enabled. + try { + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, bindAddr.getHostName()); + } catch(IOException e) { + throw new YarnRuntimeException("Failed to login from keytab", e); + } TimelineReader timelineReaderStore = createTimelineReaderStore(conf); timelineReaderStore.init(conf); @@ -129,36 +141,39 @@ public class TimelineReaderServer extends CompositeService { super.serviceStop(); } - private void startTimelineReaderWebApp() { - Configuration conf = getConfig(); - String bindAddress = WebAppUtils.getWebAppBindURL(conf, - YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, - WebAppUtils.getTimelineReaderWebAppURL(conf)); - LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); + protected void addFilters(Configuration conf) { boolean enableCorsFilter = conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration.TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT); - // setup CORS + // Setup CORS if (enableCorsFilter) { conf.setBoolean(HttpCrossOriginFilterInitializer.PREFIX + HttpCrossOriginFilterInitializer.ENABLED_SUFFIX, true); } + String initializers = conf.get("hadoop.http.filter.initializers", ""); + Set<String> defaultInitializers = new LinkedHashSet<String>(); + if (!initializers.contains( + TimelineReaderAuthenticationFilterInitializer.class.getName())) { + defaultInitializers.add( + TimelineReaderAuthenticationFilterInitializer.class.getName()); + } + TimelineServerUtils.setTimelineFilters( + conf, initializers, defaultInitializers); + } + + private void startTimelineReaderWebApp() { + Configuration conf = getConfig(); + addFilters(conf); + String bindAddress = WebAppUtils.getWebAppBindURL(conf, + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + WebAppUtils.getTimelineReaderWebAppURL(conf)); + LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress); try { HttpServer2.Builder builder = new HttpServer2.Builder() .setName("timeline") .setConf(conf) .addEndpoint(URI.create("http://" + bindAddress)); readerWebServer = builder.build(); - - Map<String, String> options = new HashMap<>(); - String username = conf.get(HADOOP_HTTP_STATIC_USER, - DEFAULT_HADOOP_HTTP_STATIC_USER); - options.put(HADOOP_HTTP_STATIC_USER, username); - HttpServer2.defineFilter(readerWebServer.getWebAppContext(), - "static_user_filter_timeline", - StaticUserWebFilter.StaticUserFilter.class.getName(), - options, new String[] {"/*"}); - readerWebServer.addJerseyResourcePackage( TimelineReaderWebServices.class.getPackage().getName() + ";" + GenericExceptionHandler.class.getPackage().getName() + ";" http://git-wip-us.apache.org/repos/asf/hadoop/blob/50dd1372/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java new file mode 100644 index 0000000..e0e1f4d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/TimelineReaderAuthenticationFilterInitializer.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.reader.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.security.AuthenticationWithProxyUserFilter; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; + +/** + * Filter initializer to initialize {@link AuthenticationWithProxyUserFilter} + * for ATSv2 timeline reader server with timeline service specific + * configurations. + */ +public class TimelineReaderAuthenticationFilterInitializer extends + TimelineAuthenticationFilterInitializer{ + + /** + * Initializes {@link AuthenticationWithProxyUserFilter} + * <p> + * Propagates to {@link AuthenticationWithProxyUserFilter} configuration all + * YARN configuration properties prefixed with + * {@value TimelineAuthenticationFilterInitializer#PREFIX}. + * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + setAuthFilterConfig(conf); + container.addGlobalFilter("Timeline Reader Authentication Filter", + AuthenticationWithProxyUserFilter.class.getName(), + getFilterConfig()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/50dd1372/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java new file mode 100644 index 0000000..5888c98 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/security/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.timelineservice.reader.security contains + * classes to be used to support SPNEGO authentication for timeline reader. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.timelineservice.reader.security; +import org.apache.hadoop.classification.InterfaceAudience; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org