YARN-5647. [ATSv2 Security] Collector side changes for loading auth filters and principals. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0121d46 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0121d46 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0121d46 Branch: refs/heads/YARN-5355 Commit: a0121d46748c914410b84c5fcda9e3148204de64 Parents: 3832fd9 Author: Jian He <jia...@apache.org> Authored: Wed Jun 7 13:45:34 2017 -0700 Committer: Varun Saxena <varunsax...@apache.org> Committed: Tue Aug 22 19:03:03 2017 +0530 ---------------------------------------------------------------------- .../ApplicationHistoryServer.java | 79 ++--- .../security/TimelineAuthenticationFilter.java | 49 --- ...TimelineAuthenticationFilterInitializer.java | 129 ------- ...lineDelegationTokenSecretManagerService.java | 240 -------------- ...neV1DelegationTokenSecretManagerService.java | 225 +++++++++++++ .../TestTimelineAuthenticationFilter.java | 323 ------------------ .../TestTimelineAuthenticationFilterForV1.java | 332 +++++++++++++++++++ ...TimelineAuthenticationFilterInitializer.java | 76 ----- .../security/TimelineAuthenticationFilter.java | 55 +++ ...TimelineAuthenticationFilterInitializer.java | 129 +++++++ ...elineDelgationTokenSecretManagerService.java | 83 +++++ .../server/timeline/security/package-info.java | 26 ++ .../util/timeline/TimelineServerUtils.java | 92 +++++ .../yarn/server/util/timeline/package-info.java | 25 ++ ...TimelineAuthenticationFilterInitializer.java | 76 +++++ .../collector/NodeTimelineCollectorManager.java | 66 +++- .../PerNodeTimelineCollectorsAuxService.java | 5 +- .../collector/TimelineCollectorManager.java | 6 +- ...neV2DelegationTokenSecretManagerService.java | 78 +++++ .../timelineservice/security/package-info.java | 25 ++ 20 files changed, 1227 insertions(+), 892 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java index 85e5f2d..4e3a1e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java @@ -20,14 +20,14 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; -import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.HttpCrossOriginFilterInitializer; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; @@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; -import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; -import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; -import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.timeline.security.TimelineV1DelegationTokenSecretManagerService; import org.apache.hadoop.yarn.server.timeline.webapp.CrossOriginFilterInitializer; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -75,7 +74,7 @@ public class ApplicationHistoryServer extends CompositeService { private ApplicationACLsManager aclsManager; private ApplicationHistoryManager historyManager; private TimelineStore timelineStore; - private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineV1DelegationTokenSecretManagerService secretManagerService; private TimelineDataManager timelineDataManager; private WebApp webApp; private JvmPauseMonitor pauseMonitor; @@ -223,9 +222,9 @@ public class ApplicationHistoryServer extends CompositeService { TimelineStore.class), conf); } - private TimelineDelegationTokenSecretManagerService + private TimelineV1DelegationTokenSecretManagerService createTimelineDelegationTokenSecretManagerService(Configuration conf) { - return new TimelineDelegationTokenSecretManagerService(); + return new TimelineV1DelegationTokenSecretManagerService(); } private TimelineDataManager createTimelineDataManager(Configuration conf) { @@ -237,63 +236,33 @@ public class ApplicationHistoryServer extends CompositeService { @SuppressWarnings("unchecked") private void startWebApp() { Configuration conf = getConfig(); - TimelineAuthenticationFilter.setTimelineDelegationTokenSecretManager( - secretManagerService.getTimelineDelegationTokenSecretManager()); // Always load pseudo authentication filter to parse "user.name" in an URL // to identify a HTTP request's user in insecure mode. // When Kerberos authentication type is set (i.e., secure mode is turned on), // the customized filter will be loaded by the timeline server to do Kerberos // + DT authentication. - String initializers = conf.get("hadoop.http.filter.initializers"); - boolean modifiedInitializers = false; - - initializers = - initializers == null || initializers.length() == 0 ? "" : initializers; - + String initializers = conf.get("hadoop.http.filter.initializers", ""); + Set<String> defaultInitializers = new LinkedHashSet<String>(); + // Add CORS filter if (!initializers.contains(CrossOriginFilterInitializer.class.getName())) { - if(conf.getBoolean(YarnConfiguration - .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, YarnConfiguration - .TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { - if (initializers.contains(HttpCrossOriginFilterInitializer.class.getName())) { - initializers = - initializers.replaceAll(HttpCrossOriginFilterInitializer.class.getName(), + if(conf.getBoolean(YarnConfiguration. + TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED, + YarnConfiguration. + TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT)) { + if (initializers.contains( + HttpCrossOriginFilterInitializer.class.getName())) { + initializers = initializers.replaceAll( + HttpCrossOriginFilterInitializer.class.getName(), CrossOriginFilterInitializer.class.getName()); + } else { + defaultInitializers.add(CrossOriginFilterInitializer.class.getName()); } - else { - if (initializers.length() != 0) { - initializers += ","; - } - initializers += CrossOriginFilterInitializer.class.getName(); - } - modifiedInitializers = true; - } - } - - if (!initializers.contains(TimelineAuthenticationFilterInitializer.class - .getName())) { - if (initializers.length() != 0) { - initializers += ","; - } - initializers += TimelineAuthenticationFilterInitializer.class.getName(); - modifiedInitializers = true; - } - - String[] parts = initializers.split(","); - ArrayList<String> target = new ArrayList<String>(); - for (String filterInitializer : parts) { - filterInitializer = filterInitializer.trim(); - if (filterInitializer.equals(AuthenticationFilterInitializer.class - .getName())) { - modifiedInitializers = true; - continue; } - target.add(filterInitializer); - } - String actualInitializers = - org.apache.commons.lang.StringUtils.join(target, ","); - if (modifiedInitializers) { - conf.set("hadoop.http.filter.initializers", actualInitializers); } + TimelineServerUtils.addTimelineAuthFilter( + initializers, defaultInitializers, secretManagerService); + TimelineServerUtils.setTimelineFilters( + conf, initializers, defaultInitializers); String bindAddress = WebAppUtils.getWebAppBindURL(conf, YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java deleted file mode 100644 index ad8dc2c..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timeline.security; - -import javax.servlet.FilterConfig; -import javax.servlet.ServletException; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; -import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService.TimelineDelegationTokenSecretManager; - -@Private -@Unstable -public class TimelineAuthenticationFilter - extends DelegationTokenAuthenticationFilter { - - private static TimelineDelegationTokenSecretManager secretManager; - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - filterConfig.getServletContext().setAttribute( - DelegationTokenAuthenticationFilter.DELEGATION_TOKEN_SECRET_MANAGER_ATTR, - secretManager); - super.init(filterConfig); - } - - public static void setTimelineDelegationTokenSecretManager( - TimelineDelegationTokenSecretManager secretManager) { - TimelineAuthenticationFilter.secretManager = secretManager; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java deleted file mode 100644 index 4e7c29a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timeline.security; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.FilterContainer; -import org.apache.hadoop.http.FilterInitializer; -import org.apache.hadoop.http.HttpServer2; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.authentication.server.AuthenticationFilter; -import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; -import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; -import org.apache.hadoop.security.authorize.ProxyUsers; -import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; -import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; -import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Initializes {@link TimelineAuthenticationFilter} which provides support for - * Kerberos HTTP SPNEGO authentication. - * <p> - * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the - * timeline server. - * <p> - * Refer to the {@code core-default.xml} file, after the comment 'HTTP - * Authentication' for details on the configuration options. All related - * configuration properties have {@code hadoop.http.authentication.} as prefix. - */ -public class TimelineAuthenticationFilterInitializer extends FilterInitializer { - - /** - * The configuration prefix of timeline HTTP authentication - */ - public static final String PREFIX = "yarn.timeline-service.http-authentication."; - - @VisibleForTesting - Map<String, String> filterConfig; - - /** - * Initializes {@link TimelineAuthenticationFilter} - * <p> - * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN - * configuration properties prefixed with {@value #PREFIX} - * - * @param container - * The filter container - * @param conf - * Configuration for run-time parameters - */ - @Override - public void initFilter(FilterContainer container, Configuration conf) { - filterConfig = new HashMap<String, String>(); - - // setting the cookie path to root '/' so it is used for all resources. - filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); - - for (Map.Entry<String, String> entry : conf) { - String name = entry.getKey(); - if (name.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) { - String value = conf.get(name); - name = name.substring("hadoop.".length()); - filterConfig.put(name, value); - } - } - for (Map.Entry<String, String> entry : conf) { - String name = entry.getKey(); - if (name.startsWith(PREFIX)) { - // yarn.timeline-service.http-authentication.proxyuser will override - // hadoop.proxyuser - String value = conf.get(name); - name = name.substring(PREFIX.length()); - filterConfig.put(name, value); - } - } - - String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE); - if (authType.equals(PseudoAuthenticationHandler.TYPE)) { - filterConfig.put(AuthenticationFilter.AUTH_TYPE, - PseudoDelegationTokenAuthenticationHandler.class.getName()); - } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { - filterConfig.put(AuthenticationFilter.AUTH_TYPE, - KerberosDelegationTokenAuthenticationHandler.class.getName()); - - // Resolve _HOST into bind address - String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); - String principal = - filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); - if (principal != null) { - try { - principal = SecurityUtil.getServerPrincipal(principal, bindAddress); - } catch (IOException ex) { - throw new RuntimeException( - "Could not resolve Kerberos principal name: " + ex.toString(), ex); - } - filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, - principal); - } - } - - filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND, - TimelineDelegationTokenIdentifier.KIND_NAME.toString()); - - container.addGlobalFilter("Timeline Authentication Filter", - TimelineAuthenticationFilter.class.getName(), - filterConfig); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java deleted file mode 100644 index 0c6892a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelegationTokenSecretManagerService.java +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timeline.security; - -import java.io.IOException; -import java.util.Map.Entry; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; -import org.apache.hadoop.security.token.delegation.DelegationKey; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore; -import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; -import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The service wrapper of {@link TimelineDelegationTokenSecretManager} - */ -@Private -@Unstable -public class TimelineDelegationTokenSecretManagerService extends - AbstractService { - - private TimelineDelegationTokenSecretManager secretManager = null; - private TimelineStateStore stateStore = null; - - public TimelineDelegationTokenSecretManagerService() { - super(TimelineDelegationTokenSecretManagerService.class.getName()); - } - - @Override - protected void serviceInit(Configuration conf) throws Exception { - if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED)) { - stateStore = createStateStore(conf); - stateStore.init(conf); - } - - long secretKeyInterval = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL); - long tokenMaxLifetime = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME); - long tokenRenewInterval = - conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, - YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL); - secretManager = new TimelineDelegationTokenSecretManager(secretKeyInterval, - tokenMaxLifetime, tokenRenewInterval, 3600000, stateStore); - super.init(conf); - } - - @Override - protected void serviceStart() throws Exception { - if (stateStore != null) { - stateStore.start(); - TimelineServiceState state = stateStore.loadState(); - secretManager.recover(state); - } - - secretManager.startThreads(); - super.serviceStart(); - } - - @Override - protected void serviceStop() throws Exception { - if (stateStore != null) { - stateStore.stop(); - } - - secretManager.stopThreads(); - super.stop(); - } - - protected TimelineStateStore createStateStore( - Configuration conf) { - return ReflectionUtils.newInstance( - conf.getClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, - LeveldbTimelineStateStore.class, - TimelineStateStore.class), conf); - } - - /** - * Ge the instance of {link #TimelineDelegationTokenSecretManager} - * - * @return the instance of {link #TimelineDelegationTokenSecretManager} - */ - public TimelineDelegationTokenSecretManager - getTimelineDelegationTokenSecretManager() { - return secretManager; - } - - @Private - @Unstable - public static class TimelineDelegationTokenSecretManager extends - AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> { - - public static final Logger LOG = - LoggerFactory.getLogger(TimelineDelegationTokenSecretManager.class); - - private TimelineStateStore stateStore; - - /** - * Create a timeline secret manager - * @param delegationKeyUpdateInterval the number of milliseconds for rolling - * new secret keys. - * @param delegationTokenMaxLifetime the maximum lifetime of the delegation - * tokens in milliseconds - * @param delegationTokenRenewInterval how often the tokens must be renewed - * in milliseconds - * @param delegationTokenRemoverScanInterval how often the tokens are - * scanned for expired tokens in milliseconds - * @param stateStore timeline service state store - */ - public TimelineDelegationTokenSecretManager( - long delegationKeyUpdateInterval, - long delegationTokenMaxLifetime, - long delegationTokenRenewInterval, - long delegationTokenRemoverScanInterval, - TimelineStateStore stateStore) { - super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, - delegationTokenRenewInterval, delegationTokenRemoverScanInterval); - this.stateStore = stateStore; - } - - @Override - public TimelineDelegationTokenIdentifier createIdentifier() { - return new TimelineDelegationTokenIdentifier(); - } - - @Override - protected void storeNewMasterKey(DelegationKey key) throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Storing master key " + key.getKeyId()); - } - try { - if (stateStore != null) { - stateStore.storeTokenMasterKey(key); - } - } catch (IOException e) { - LOG.error("Unable to store master key " + key.getKeyId(), e); - } - } - - @Override - protected void removeStoredMasterKey(DelegationKey key) { - if (LOG.isDebugEnabled()) { - LOG.debug("Removing master key " + key.getKeyId()); - } - try { - if (stateStore != null) { - stateStore.removeTokenMasterKey(key); - } - } catch (IOException e) { - LOG.error("Unable to remove master key " + key.getKeyId(), e); - } - } - - @Override - protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId, - long renewDate) { - if (LOG.isDebugEnabled()) { - LOG.debug("Storing token " + tokenId.getSequenceNumber()); - } - try { - if (stateStore != null) { - stateStore.storeToken(tokenId, renewDate); - } - } catch (IOException e) { - LOG.error("Unable to store token " + tokenId.getSequenceNumber(), e); - } - } - - @Override - protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Storing token " + tokenId.getSequenceNumber()); - } - try { - if (stateStore != null) { - stateStore.removeToken(tokenId); - } - } catch (IOException e) { - LOG.error("Unable to remove token " + tokenId.getSequenceNumber(), e); - } - } - - @Override - protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId, - long renewDate) { - if (LOG.isDebugEnabled()) { - LOG.debug("Updating token " + tokenId.getSequenceNumber()); - } - try { - if (stateStore != null) { - stateStore.updateToken(tokenId, renewDate); - } - } catch (IOException e) { - LOG.error("Unable to update token " + tokenId.getSequenceNumber(), e); - } - } - - public void recover(TimelineServiceState state) throws IOException { - LOG.info("Recovering " + getClass().getSimpleName()); - for (DelegationKey key : state.getTokenMasterKeyState()) { - addKey(key); - } - this.delegationTokenSequenceNumber = state.getLatestSequenceNumber(); - for (Entry<TimelineDelegationTokenIdentifier, Long> entry : - state.getTokenState().entrySet()) { - addPersistedDelegationToken(entry.getKey(), entry.getValue()); - } - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java new file mode 100644 index 0000000..85d8cca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineV1DelegationTokenSecretManagerService.java @@ -0,0 +1,225 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore; +import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; +import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The service wrapper of {@link TimelineV1DelegationTokenSecretManager}. + */ +@Private +@Unstable +public class TimelineV1DelegationTokenSecretManagerService extends + TimelineDelgationTokenSecretManagerService { + private TimelineStateStore stateStore = null; + + public TimelineV1DelegationTokenSecretManagerService() { + super(TimelineV1DelegationTokenSecretManagerService.class.getName()); + } + + @Override + protected void serviceInit(Configuration conf) throws Exception { + if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED)) { + stateStore = createStateStore(conf); + stateStore.init(conf); + } + super.serviceInit(conf); + } + + @Override + protected void serviceStart() throws Exception { + if (stateStore != null) { + stateStore.start(); + TimelineServiceState state = stateStore.loadState(); + ((TimelineV1DelegationTokenSecretManager) + getTimelineDelegationTokenSecretManager()).recover(state); + } + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + if (stateStore != null) { + stateStore.stop(); + } + super.serviceStop(); + } + + @Override + protected AbstractDelegationTokenSecretManager + <TimelineDelegationTokenIdentifier> + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval) { + return new TimelineV1DelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, tokenRemovalScanInterval, + stateStore); + } + + protected TimelineStateStore createStateStore( + Configuration conf) { + return ReflectionUtils.newInstance( + conf.getClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS, + LeveldbTimelineStateStore.class, + TimelineStateStore.class), conf); + } + + /** + * Delegation token secret manager for ATSv1 and ATSv1.5. + */ + @Private + @Unstable + public static class TimelineV1DelegationTokenSecretManager extends + AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> { + + public static final Logger LOG = + LoggerFactory.getLogger(TimelineV1DelegationTokenSecretManager.class); + + private TimelineStateStore stateStore; + + /** + * Create a timeline v1 secret manager. + * @param delegationKeyUpdateInterval the number of milliseconds for rolling + * new secret keys. + * @param delegationTokenMaxLifetime the maximum lifetime of the delegation + * tokens in milliseconds + * @param delegationTokenRenewInterval how often the tokens must be renewed + * in milliseconds + * @param delegationTokenRemoverScanInterval how often the tokens are + * scanned for expired tokens in milliseconds + * @param stateStore timeline service state store + */ + public TimelineV1DelegationTokenSecretManager( + long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval, + TimelineStateStore stateStore) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + this.stateStore = stateStore; + } + + @Override + public TimelineDelegationTokenIdentifier createIdentifier() { + return new TimelineDelegationTokenIdentifier(); + } + + @Override + protected void storeNewMasterKey(DelegationKey key) throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Storing master key " + key.getKeyId()); + } + try { + if (stateStore != null) { + stateStore.storeTokenMasterKey(key); + } + } catch (IOException e) { + LOG.error("Unable to store master key " + key.getKeyId(), e); + } + } + + @Override + protected void removeStoredMasterKey(DelegationKey key) { + if (LOG.isDebugEnabled()) { + LOG.debug("Removing master key " + key.getKeyId()); + } + try { + if (stateStore != null) { + stateStore.removeTokenMasterKey(key); + } + } catch (IOException e) { + LOG.error("Unable to remove master key " + key.getKeyId(), e); + } + } + + @Override + protected void storeNewToken(TimelineDelegationTokenIdentifier tokenId, + long renewDate) { + if (LOG.isDebugEnabled()) { + LOG.debug("Storing token " + tokenId.getSequenceNumber()); + } + try { + if (stateStore != null) { + stateStore.storeToken(tokenId, renewDate); + } + } catch (IOException e) { + LOG.error("Unable to store token " + tokenId.getSequenceNumber(), e); + } + } + + @Override + protected void removeStoredToken(TimelineDelegationTokenIdentifier tokenId) + throws IOException { + if (LOG.isDebugEnabled()) { + LOG.debug("Storing token " + tokenId.getSequenceNumber()); + } + try { + if (stateStore != null) { + stateStore.removeToken(tokenId); + } + } catch (IOException e) { + LOG.error("Unable to remove token " + tokenId.getSequenceNumber(), e); + } + } + + @Override + protected void updateStoredToken(TimelineDelegationTokenIdentifier tokenId, + long renewDate) { + if (LOG.isDebugEnabled()) { + LOG.debug("Updating token " + tokenId.getSequenceNumber()); + } + try { + if (stateStore != null) { + stateStore.updateToken(tokenId, renewDate); + } + } catch (IOException e) { + LOG.error("Unable to update token " + tokenId.getSequenceNumber(), e); + } + } + + public void recover(TimelineServiceState state) throws IOException { + LOG.info("Recovering " + getClass().getSimpleName()); + for (DelegationKey key : state.getTokenMasterKeyState()) { + addKey(key); + } + this.delegationTokenSequenceNumber = state.getLatestSequenceNumber(); + for (Entry<TimelineDelegationTokenIdentifier, Long> entry : + state.getTokenState().entrySet()) { + addPersistedDelegationToken(entry.getKey(), entry.getValue()); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java deleted file mode 100644 index 063f512..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java +++ /dev/null @@ -1,323 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timeline.security; - -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.security.PrivilegedExceptionAction; -import java.util.Arrays; -import java.util.Collection; -import java.util.concurrent.Callable; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.minikdc.MiniKdc; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.KerberosTestUtils; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; -import org.apache.hadoop.security.authorize.AuthorizationException; -import org.apache.hadoop.security.ssl.KeyStoreTestUtil; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; -import org.apache.hadoop.yarn.client.api.TimelineClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; -import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; -import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; -import org.apache.hadoop.yarn.server.timeline.TimelineStore; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -@RunWith(Parameterized.class) -public class TestTimelineAuthenticationFilter { - - private static final String FOO_USER = "foo"; - private static final String BAR_USER = "bar"; - private static final String HTTP_USER = "HTTP"; - - private static final File testRootDir = new File( - System.getProperty("test.build.dir", "target/test-dir"), - TestTimelineAuthenticationFilter.class.getName() + "-root"); - private static File httpSpnegoKeytabFile = new File( - KerberosTestUtils.getKeytabFile()); - private static String httpSpnegoPrincipal = - KerberosTestUtils.getServerPrincipal(); - private static final String BASEDIR = - System.getProperty("test.build.dir", "target/test-dir") + "/" - + TestTimelineAuthenticationFilter.class.getSimpleName(); - - @Parameterized.Parameters - public static Collection<Object[]> withSsl() { - return Arrays.asList(new Object[][] { { false }, { true } }); - } - - private static MiniKdc testMiniKDC; - private static String keystoresDir; - private static String sslConfDir; - private static ApplicationHistoryServer testTimelineServer; - private static Configuration conf; - private static boolean withSsl; - - public TestTimelineAuthenticationFilter(boolean withSsl) { - TestTimelineAuthenticationFilter.withSsl = withSsl; - } - - @BeforeClass - public static void setup() { - try { - testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); - testMiniKDC.start(); - testMiniKDC.createPrincipal( - httpSpnegoKeytabFile, HTTP_USER + "/localhost"); - } catch (Exception e) { - assertTrue("Couldn't setup MiniKDC", false); - } - - try { - testTimelineServer = new ApplicationHistoryServer(); - conf = new Configuration(false); - conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type", - "kerberos"); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + - KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); - conf.set(TimelineAuthenticationFilterInitializer.PREFIX + - KerberosAuthenticationHandler.KEYTAB, - httpSpnegoKeytabFile.getAbsolutePath()); - conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, - "kerberos"); - conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, - httpSpnegoPrincipal); - conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB, - httpSpnegoKeytabFile.getAbsolutePath()); - conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); - conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, - MemoryTimelineStore.class, TimelineStore.class); - conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, - "localhost:10200"); - conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, - "localhost:8188"); - conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, - "localhost:8190"); - conf.set("hadoop.proxyuser.HTTP.hosts", "*"); - conf.set("hadoop.proxyuser.HTTP.users", FOO_USER); - conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1); - - if (withSsl) { - conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, - HttpConfig.Policy.HTTPS_ONLY.name()); - File base = new File(BASEDIR); - FileUtil.fullyDelete(base); - base.mkdirs(); - keystoresDir = new File(BASEDIR).getAbsolutePath(); - sslConfDir = - KeyStoreTestUtil.getClasspathDir(TestTimelineAuthenticationFilter.class); - KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); - } - - UserGroupInformation.setConfiguration(conf); - testTimelineServer.init(conf); - testTimelineServer.start(); - } catch (Exception e) { - assertTrue("Couldn't setup TimelineServer", false); - } - } - - private TimelineClient createTimelineClientForUGI() { - TimelineClient client = TimelineClient.createTimelineClient(); - client.init(conf); - client.start(); - return client; - } - - @AfterClass - public static void tearDown() throws Exception { - if (testMiniKDC != null) { - testMiniKDC.stop(); - } - - if (testTimelineServer != null) { - testTimelineServer.stop(); - } - - if (withSsl) { - KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); - File base = new File(BASEDIR); - FileUtil.fullyDelete(base); - } - } - - @Test - public void testPutTimelineEntities() throws Exception { - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() { - @Override - public Void call() throws Exception { - TimelineClient client = createTimelineClientForUGI(); - TimelineEntity entityToStore = new TimelineEntity(); - entityToStore.setEntityType( - TestTimelineAuthenticationFilter.class.getName()); - entityToStore.setEntityId("entity1"); - entityToStore.setStartTime(0L); - TimelinePutResponse putResponse = client.putEntities(entityToStore); - Assert.assertEquals(0, putResponse.getErrors().size()); - TimelineEntity entityToRead = - testTimelineServer.getTimelineStore().getEntity( - "entity1", TestTimelineAuthenticationFilter.class.getName(), null); - Assert.assertNotNull(entityToRead); - return null; - } - }); - } - - @Test - public void testPutDomains() throws Exception { - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() { - @Override - public Void call() throws Exception { - TimelineClient client = createTimelineClientForUGI(); - TimelineDomain domainToStore = new TimelineDomain(); - domainToStore.setId(TestTimelineAuthenticationFilter.class.getName()); - domainToStore.setReaders("*"); - domainToStore.setWriters("*"); - client.putDomain(domainToStore); - TimelineDomain domainToRead = - testTimelineServer.getTimelineStore().getDomain( - TestTimelineAuthenticationFilter.class.getName()); - Assert.assertNotNull(domainToRead); - return null; - } - }); - } - - @Test - public void testDelegationTokenOperations() throws Exception { - TimelineClient httpUserClient = - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<TimelineClient>() { - @Override - public TimelineClient call() throws Exception { - return createTimelineClientForUGI(); - } - }); - UserGroupInformation httpUser = - KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<UserGroupInformation>() { - @Override - public UserGroupInformation call() throws Exception { - return UserGroupInformation.getCurrentUser(); - } - }); - // Let HTTP user to get the delegation for itself - Token<TimelineDelegationTokenIdentifier> token = - httpUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.assertNotNull(token); - TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); - Assert.assertNotNull(tDT); - Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner()); - - // Renew token - Assert.assertFalse(token.getService().toString().isEmpty()); - // Renew the token from the token service address - long renewTime1 = httpUserClient.renewDelegationToken(token); - Thread.sleep(100); - token.setService(new Text()); - Assert.assertTrue(token.getService().toString().isEmpty()); - // If the token service address is not avaiable, it still can be renewed - // from the configured address - long renewTime2 = httpUserClient.renewDelegationToken(token); - Assert.assertTrue(renewTime1 < renewTime2); - - // Cancel token - Assert.assertTrue(token.getService().toString().isEmpty()); - // If the token service address is not avaiable, it still can be canceled - // from the configured address - httpUserClient.cancelDelegationToken(token); - // Renew should not be successful because the token is canceled - try { - httpUserClient.renewDelegationToken(token); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains( - "Renewal request for unknown token")); - } - - // Let HTTP user to get the delegation token for FOO user - UserGroupInformation fooUgi = UserGroupInformation.createProxyUser( - FOO_USER, httpUser); - TimelineClient fooUserClient = fooUgi.doAs( - new PrivilegedExceptionAction<TimelineClient>() { - @Override - public TimelineClient run() throws Exception { - return createTimelineClientForUGI(); - } - }); - token = fooUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.assertNotNull(token); - tDT = token.decodeIdentifier(); - Assert.assertNotNull(tDT); - Assert.assertEquals(new Text(FOO_USER), tDT.getOwner()); - Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser()); - - // Renew token as the renewer - final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token; - renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew); - renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew); - Assert.assertTrue(renewTime1 < renewTime2); - - // Cancel token - Assert.assertFalse(tokenToRenew.getService().toString().isEmpty()); - // Cancel the token from the token service address - fooUserClient.cancelDelegationToken(tokenToRenew); - - // Renew should not be successful because the token is canceled - try { - httpUserClient.renewDelegationToken(tokenToRenew); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue( - e.getMessage().contains("Renewal request for unknown token")); - } - - // Let HTTP user to get the delegation token for BAR user - UserGroupInformation barUgi = UserGroupInformation.createProxyUser( - BAR_USER, httpUser); - TimelineClient barUserClient = barUgi.doAs( - new PrivilegedExceptionAction<TimelineClient>() { - @Override - public TimelineClient run() { - return createTimelineClientForUGI(); - } - }); - - try { - barUserClient.getDelegationToken(httpUser.getShortUserName()); - Assert.fail(); - } catch (Exception e) { - Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java new file mode 100644 index 0000000..d918e8d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java @@ -0,0 +1,332 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.Callable; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authorize.AuthorizationException; +import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; +import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.client.api.TimelineClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; +import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Test cases for authentication via TimelineAuthenticationFilter while + * publishing entities for ATSv1. + */ +@RunWith(Parameterized.class) +public class TestTimelineAuthenticationFilterForV1 { + + private static final String FOO_USER = "foo"; + private static final String BAR_USER = "bar"; + private static final String HTTP_USER = "HTTP"; + + private static final File TEST_ROOT_DIR = new File( + System.getProperty("test.build.dir", "target/test-dir"), + TestTimelineAuthenticationFilterForV1.class.getName() + "-root"); + private static File httpSpnegoKeytabFile = new File( + KerberosTestUtils.getKeytabFile()); + private static String httpSpnegoPrincipal = + KerberosTestUtils.getServerPrincipal(); + private static final String BASEDIR = + System.getProperty("test.build.dir", "target/test-dir") + "/" + + TestTimelineAuthenticationFilterForV1.class.getSimpleName(); + + @Parameterized.Parameters + public static Collection<Object[]> withSsl() { + return Arrays.asList(new Object[][] {{false}, {true}}); + } + + private static MiniKdc testMiniKDC; + private static String keystoresDir; + private static String sslConfDir; + private static ApplicationHistoryServer testTimelineServer; + private static Configuration conf; + private static boolean withSsl; + + public TestTimelineAuthenticationFilterForV1(boolean withSsl) { + TestTimelineAuthenticationFilterForV1.withSsl = withSsl; + } + + @BeforeClass + public static void setup() { + try { + testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); + testMiniKDC.start(); + testMiniKDC.createPrincipal( + httpSpnegoKeytabFile, HTTP_USER + "/localhost"); + } catch (Exception e) { + assertTrue("Couldn't setup MiniKDC", false); + } + + try { + testTimelineServer = new ApplicationHistoryServer(); + conf = new Configuration(false); + conf.setStrings(TimelineAuthenticationFilterInitializer.PREFIX + "type", + "kerberos"); + conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); + conf.set(TimelineAuthenticationFilterInitializer.PREFIX + + KerberosAuthenticationHandler.KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + conf.set(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, + httpSpnegoPrincipal); + conf.set(YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, + MemoryTimelineStore.class, TimelineStore.class); + conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + "localhost:10200"); + conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, + "localhost:8188"); + conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, + "localhost:8190"); + conf.set("hadoop.proxyuser.HTTP.hosts", "*"); + conf.set("hadoop.proxyuser.HTTP.users", FOO_USER); + conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1); + + if (withSsl) { + conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, + HttpConfig.Policy.HTTPS_ONLY.name()); + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + base.mkdirs(); + keystoresDir = new File(BASEDIR).getAbsolutePath(); + sslConfDir = KeyStoreTestUtil.getClasspathDir( + TestTimelineAuthenticationFilterForV1.class); + KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false); + } + + UserGroupInformation.setConfiguration(conf); + testTimelineServer.init(conf); + testTimelineServer.start(); + } catch (Exception e) { + e.printStackTrace(); + assertTrue("Couldn't setup TimelineServer", false); + } + } + + private TimelineClient createTimelineClientForUGI() { + TimelineClient client = TimelineClient.createTimelineClient(); + client.init(conf); + client.start(); + return client; + } + + @AfterClass + public static void tearDown() throws Exception { + if (testMiniKDC != null) { + testMiniKDC.stop(); + } + + if (testTimelineServer != null) { + testTimelineServer.stop(); + } + + if (withSsl) { + KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); + File base = new File(BASEDIR); + FileUtil.fullyDelete(base); + } + } + + @Test + public void testPutTimelineEntities() throws Exception { + KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() { + @Override + public Void call() throws Exception { + TimelineClient client = createTimelineClientForUGI(); + TimelineEntity entityToStore = new TimelineEntity(); + entityToStore.setEntityType( + TestTimelineAuthenticationFilterForV1.class.getName()); + entityToStore.setEntityId("entity1"); + entityToStore.setStartTime(0L); + TimelinePutResponse putResponse = client.putEntities(entityToStore); + Assert.assertEquals(0, putResponse.getErrors().size()); + TimelineEntity entityToRead = + testTimelineServer.getTimelineStore().getEntity("entity1", + TestTimelineAuthenticationFilterForV1.class.getName(), null); + Assert.assertNotNull(entityToRead); + return null; + } + }); + } + + @Test + public void testPutDomains() throws Exception { + KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() { + @Override + public Void call() throws Exception { + TimelineClient client = createTimelineClientForUGI(); + TimelineDomain domainToStore = new TimelineDomain(); + domainToStore.setId( + TestTimelineAuthenticationFilterForV1.class.getName()); + domainToStore.setReaders("*"); + domainToStore.setWriters("*"); + client.putDomain(domainToStore); + TimelineDomain domainToRead = + testTimelineServer.getTimelineStore().getDomain( + TestTimelineAuthenticationFilterForV1.class.getName()); + Assert.assertNotNull(domainToRead); + return null; + } + }); + } + + @Test + public void testDelegationTokenOperations() throws Exception { + TimelineClient httpUserClient = + KerberosTestUtils.doAs(HTTP_USER + "/localhost", + new Callable<TimelineClient>() { + @Override + public TimelineClient call() throws Exception { + return createTimelineClientForUGI(); + } + }); + UserGroupInformation httpUser = + KerberosTestUtils.doAs(HTTP_USER + "/localhost", + new Callable<UserGroupInformation>() { + @Override + public UserGroupInformation call() throws Exception { + return UserGroupInformation.getCurrentUser(); + } + }); + // Let HTTP user to get the delegation for itself + Token<TimelineDelegationTokenIdentifier> token = + httpUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.assertNotNull(token); + TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); + Assert.assertNotNull(tDT); + Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner()); + + // Renew token + Assert.assertFalse(token.getService().toString().isEmpty()); + // Renew the token from the token service address + long renewTime1 = httpUserClient.renewDelegationToken(token); + Thread.sleep(100); + token.setService(new Text()); + Assert.assertTrue(token.getService().toString().isEmpty()); + // If the token service address is not avaiable, it still can be renewed + // from the configured address + long renewTime2 = httpUserClient.renewDelegationToken(token); + Assert.assertTrue(renewTime1 < renewTime2); + + // Cancel token + Assert.assertTrue(token.getService().toString().isEmpty()); + // If the token service address is not avaiable, it still can be canceled + // from the configured address + httpUserClient.cancelDelegationToken(token); + // Renew should not be successful because the token is canceled + try { + httpUserClient.renewDelegationToken(token); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains( + "Renewal request for unknown token")); + } + + // Let HTTP user to get the delegation token for FOO user + UserGroupInformation fooUgi = UserGroupInformation.createProxyUser( + FOO_USER, httpUser); + TimelineClient fooUserClient = fooUgi.doAs( + new PrivilegedExceptionAction<TimelineClient>() { + @Override + public TimelineClient run() throws Exception { + return createTimelineClientForUGI(); + } + }); + token = fooUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.assertNotNull(token); + tDT = token.decodeIdentifier(); + Assert.assertNotNull(tDT); + Assert.assertEquals(new Text(FOO_USER), tDT.getOwner()); + Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser()); + + // Renew token as the renewer + final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token; + renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew); + renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew); + Assert.assertTrue(renewTime1 < renewTime2); + + // Cancel token + Assert.assertFalse(tokenToRenew.getService().toString().isEmpty()); + // Cancel the token from the token service address + fooUserClient.cancelDelegationToken(tokenToRenew); + + // Renew should not be successful because the token is canceled + try { + httpUserClient.renewDelegationToken(tokenToRenew); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue( + e.getMessage().contains("Renewal request for unknown token")); + } + + // Let HTTP user to get the delegation token for BAR user + UserGroupInformation barUgi = UserGroupInformation.createProxyUser( + BAR_USER, httpUser); + TimelineClient barUserClient = barUgi.doAs( + new PrivilegedExceptionAction<TimelineClient>() { + @Override + public TimelineClient run() { + return createTimelineClientForUGI(); + } + }); + + try { + barUserClient.getDelegationToken(httpUser.getShortUserName()); + Assert.fail(); + } catch (Exception e) { + Assert.assertTrue(e.getCause() instanceof AuthorizationException || + e.getCause() instanceof AuthenticationException); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java deleted file mode 100644 index 430911e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterInitializer.java +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.timeline.security; - -import org.junit.Assert; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.http.FilterContainer; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX; -import org.junit.Test; -import org.mockito.Mockito; - - -public class TestTimelineAuthenticationFilterInitializer { - - @Test - public void testProxyUserConfiguration() { - FilterContainer container = Mockito.mock(FilterContainer.class); - for (int i = 0; i < 3; ++i) { - Configuration conf = new YarnConfiguration(); - switch (i) { - case 0: - // hadoop.proxyuser prefix - conf.set("hadoop.proxyuser.foo.hosts", "*"); - conf.set("hadoop.proxyuser.foo.users", "*"); - conf.set("hadoop.proxyuser.foo.groups", "*"); - break; - case 1: - // yarn.timeline-service.http-authentication.proxyuser prefix - conf.set(PREFIX + "proxyuser.foo.hosts", "*"); - conf.set(PREFIX + "proxyuser.foo.users", "*"); - conf.set(PREFIX + "proxyuser.foo.groups", "*"); - break; - case 2: - // hadoop.proxyuser prefix has been overwritten by - // yarn.timeline-service.http-authentication.proxyuser prefix - conf.set("hadoop.proxyuser.foo.hosts", "bar"); - conf.set("hadoop.proxyuser.foo.users", "bar"); - conf.set("hadoop.proxyuser.foo.groups", "bar"); - conf.set(PREFIX + "proxyuser.foo.hosts", "*"); - conf.set(PREFIX + "proxyuser.foo.users", "*"); - conf.set(PREFIX + "proxyuser.foo.groups", "*"); - break; - default: - break; - } - - TimelineAuthenticationFilterInitializer initializer = - new TimelineAuthenticationFilterInitializer(); - initializer.initFilter(container, conf); - Assert.assertEquals( - "*", initializer.filterConfig.get("proxyuser.foo.hosts")); - Assert.assertEquals( - "*", initializer.filterConfig.get("proxyuser.foo.users")); - Assert.assertEquals( - "*", initializer.filterConfig.get("proxyuser.foo.groups")); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java new file mode 100644 index 0000000..f6d1863 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * Timeline authentication filter provides delegation token support for ATSv1 + * and ATSv2. + */ +@Private +@Unstable +public class TimelineAuthenticationFilter + extends DelegationTokenAuthenticationFilter { + + private static AbstractDelegationTokenSecretManager + <TimelineDelegationTokenIdentifier> secretManager; + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + filterConfig.getServletContext().setAttribute( + DelegationTokenAuthenticationFilter. + DELEGATION_TOKEN_SECRET_MANAGER_ATTR, secretManager); + super.init(filterConfig); + } + + public static void setTimelineDelegationTokenSecretManager( + AbstractDelegationTokenSecretManager + <TimelineDelegationTokenIdentifier> secretMgr) { + TimelineAuthenticationFilter.secretManager = secretMgr; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java new file mode 100644 index 0000000..4e7c29a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; +import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Initializes {@link TimelineAuthenticationFilter} which provides support for + * Kerberos HTTP SPNEGO authentication. + * <p> + * It enables Kerberos HTTP SPNEGO plus delegation token authentication for the + * timeline server. + * <p> + * Refer to the {@code core-default.xml} file, after the comment 'HTTP + * Authentication' for details on the configuration options. All related + * configuration properties have {@code hadoop.http.authentication.} as prefix. + */ +public class TimelineAuthenticationFilterInitializer extends FilterInitializer { + + /** + * The configuration prefix of timeline HTTP authentication + */ + public static final String PREFIX = "yarn.timeline-service.http-authentication."; + + @VisibleForTesting + Map<String, String> filterConfig; + + /** + * Initializes {@link TimelineAuthenticationFilter} + * <p> + * Propagates to {@link TimelineAuthenticationFilter} configuration all YARN + * configuration properties prefixed with {@value #PREFIX} + * + * @param container + * The filter container + * @param conf + * Configuration for run-time parameters + */ + @Override + public void initFilter(FilterContainer container, Configuration conf) { + filterConfig = new HashMap<String, String>(); + + // setting the cookie path to root '/' so it is used for all resources. + filterConfig.put(TimelineAuthenticationFilter.COOKIE_PATH, "/"); + + for (Map.Entry<String, String> entry : conf) { + String name = entry.getKey(); + if (name.startsWith(ProxyUsers.CONF_HADOOP_PROXYUSER)) { + String value = conf.get(name); + name = name.substring("hadoop.".length()); + filterConfig.put(name, value); + } + } + for (Map.Entry<String, String> entry : conf) { + String name = entry.getKey(); + if (name.startsWith(PREFIX)) { + // yarn.timeline-service.http-authentication.proxyuser will override + // hadoop.proxyuser + String value = conf.get(name); + name = name.substring(PREFIX.length()); + filterConfig.put(name, value); + } + } + + String authType = filterConfig.get(AuthenticationFilter.AUTH_TYPE); + if (authType.equals(PseudoAuthenticationHandler.TYPE)) { + filterConfig.put(AuthenticationFilter.AUTH_TYPE, + PseudoDelegationTokenAuthenticationHandler.class.getName()); + } else if (authType.equals(KerberosAuthenticationHandler.TYPE)) { + filterConfig.put(AuthenticationFilter.AUTH_TYPE, + KerberosDelegationTokenAuthenticationHandler.class.getName()); + + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = + filterConfig.get(KerberosAuthenticationHandler.PRINCIPAL); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException( + "Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, + principal); + } + } + + filterConfig.put(DelegationTokenAuthenticationHandler.TOKEN_KIND, + TimelineDelegationTokenIdentifier.KIND_NAME.toString()); + + container.addGlobalFilter("Timeline Authentication Filter", + TimelineAuthenticationFilter.class.getName(), + filterConfig); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java new file mode 100644 index 0000000..2e95af2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineDelgationTokenSecretManagerService.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timeline.security; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.service.AbstractService; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; + +/** + * Abstract implementation of delegation token manager service for different + * versions of timeline service. + */ +public abstract class TimelineDelgationTokenSecretManagerService extends + AbstractService { + + public TimelineDelgationTokenSecretManagerService(String name) { + super(name); + } + + private static long delegationTokenRemovalScanInterval = 3600000L; + + private AbstractDelegationTokenSecretManager + <TimelineDelegationTokenIdentifier> secretManager = null; + + @Override + protected void serviceInit(Configuration conf) throws Exception { + long secretKeyInterval = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL); + long tokenMaxLifetime = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME); + long tokenRenewInterval = + conf.getLong(YarnConfiguration.TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL, + YarnConfiguration.DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL); + secretManager = createTimelineDelegationTokenSecretManager( + secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, + delegationTokenRemovalScanInterval); + super.init(conf); + } + + protected abstract + AbstractDelegationTokenSecretManager<TimelineDelegationTokenIdentifier> + createTimelineDelegationTokenSecretManager(long secretKeyInterval, + long tokenMaxLifetime, long tokenRenewInterval, + long tokenRemovalScanInterval); + + @Override + protected void serviceStart() throws Exception { + secretManager.startThreads(); + super.serviceStart(); + } + + @Override + protected void serviceStop() throws Exception { + secretManager.stopThreads(); + super.stop(); + } + + public AbstractDelegationTokenSecretManager + <TimelineDelegationTokenIdentifier> + getTimelineDelegationTokenSecretManager() { + return secretManager; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0121d46/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java new file mode 100644 index 0000000..14a52e3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/timeline/security/package-info.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Package org.apache.hadoop.server.timeline.security contains classes related + * to timeline authentication filters and abstract delegation token service for + * ATSv1 and ATSv2. + */ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.timeline.security; +import org.apache.hadoop.classification.InterfaceAudience; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
