AMBARI-20891 - Allow extensions to auto-link with supported stack versions
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aa78a172 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aa78a172 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aa78a172 Branch: refs/heads/trunk Commit: aa78a1724feca81e75b5a0f82cf7b048dd2b938e Parents: a92fe1b Author: Tim Thorpe <ttho...@apache.org> Authored: Tue May 9 07:53:39 2017 -0700 Committer: Tim Thorpe <ttho...@apache.org> Committed: Tue May 9 07:53:39 2017 -0700 ---------------------------------------------------------------------- .../AmbariManagementControllerImpl.java | 67 +--- .../controller/AmbariManagementHelper.java | 175 ++++++++ .../ambari/server/orm/dao/ExtensionLinkDAO.java | 36 +- .../orm/entities/ExtensionLinkEntity.java | 1 + .../ambari/server/stack/ExtensionModule.java | 2 + .../ambari/server/stack/StackManager.java | 79 +++- .../apache/ambari/server/stack/StackModule.java | 4 +- .../ambari/server/state/ExtensionInfo.java | 26 +- .../apache/ambari/server/state/StackInfo.java | 27 +- .../state/stack/ExtensionMetainfoXml.java | 11 + .../stack/StackManagerCommonServicesTest.java | 4 +- .../server/stack/StackManagerExtensionTest.java | 73 ++-- .../server/stack/StackManagerMiscTest.java | 13 +- .../ambari/server/stack/StackManagerMock.java | 5 +- .../ambari/server/stack/StackManagerTest.java | 13 +- .../resources/extensions/EXT/0.1/metainfo.xml | 2 +- .../resources/extensions/EXT/0.2/metainfo.xml | 3 +- .../resources/extensions/EXT/0.3/metainfo.xml | 32 ++ .../EXT/0.3/services/OOZIE2/metainfo.xml | 118 ++++++ .../services/OOZIE2/themes/broken_theme.json | 3 + .../stacks_with_extensions/HDP/0.3/metainfo.xml | 22 ++ .../HDP/0.3/repos/repoinfo.xml | 63 +++ .../HDP/0.3/services/HBASE/metainfo.xml | 26 ++ .../0.3/services/HDFS/configuration/global.xml | 145 +++++++ .../services/HDFS/configuration/hadoop-env.xml | 223 +++++++++++ .../services/HDFS/configuration/hbase-site.xml | 137 +++++++ .../services/HDFS/configuration/hdfs-log4j.xml | 199 ++++++++++ .../services/HDFS/configuration/hdfs-site.xml | 396 +++++++++++++++++++ .../HDP/0.3/services/HDFS/metainfo.xml | 30 ++ .../0.3/services/HDFS/package/dummy-script.py | 20 + .../HDP/0.3/services/HIVE/metainfo.xml | 26 ++ .../HDP/0.3/services/MAPREDUCE/metainfo.xml | 23 ++ .../HDP/0.3/services/ZOOKEEPER/metainfo.xml | 26 ++ 33 files changed, 1917 insertions(+), 113 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 751ce08..f9375aa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -120,14 +120,12 @@ import org.apache.ambari.server.orm.dao.WidgetDAO; import org.apache.ambari.server.orm.dao.WidgetLayoutDAO; import org.apache.ambari.server.orm.entities.ClusterEntity; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; -import org.apache.ambari.server.orm.entities.ExtensionEntity; import org.apache.ambari.server.orm.entities.ExtensionLinkEntity; import org.apache.ambari.server.orm.entities.HostEntity; import org.apache.ambari.server.orm.entities.OperatingSystemEntity; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.SettingEntity; -import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.orm.entities.WidgetEntity; import org.apache.ambari.server.orm.entities.WidgetLayoutEntity; import org.apache.ambari.server.orm.entities.WidgetLayoutUserWidgetEntity; @@ -309,11 +307,13 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle private MaintenanceStateHelper maintenanceStateHelper; - @Inject - private ExtensionLinkDAO linkDAO; + private AmbariManagementHelper helper; + @Inject private ExtensionDAO extensionDAO; @Inject + private ExtensionLinkDAO linkDAO; + @Inject private StackDAO stackDAO; /** @@ -392,6 +392,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle mysqljdbcUrl = null; serverDB = null; } + helper = new AmbariManagementHelper(stackDAO, extensionDAO, linkDAO); } @Override @@ -5453,7 +5454,13 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle */ @Override public void createExtensionLink(ExtensionLinkRequest request) throws AmbariException { - validateCreateExtensionLinkRequest(request); + if (StringUtils.isBlank(request.getStackName()) + || StringUtils.isBlank(request.getStackVersion()) + || StringUtils.isBlank(request.getExtensionName()) + || StringUtils.isBlank(request.getExtensionVersion())) { + + throw new IllegalArgumentException("Stack name, stack version, extension name and extension version should be provided"); + } StackInfo stackInfo = ambariMetaInfo.getStack(request.getStackName(), request.getStackVersion()); @@ -5467,24 +5474,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle throw new StackAccessException("extensionName=" + request.getExtensionName() + ", extensionVersion=" + request.getExtensionVersion()); } - ExtensionHelper.validateCreateLink(stackInfo, extensionInfo); - ExtensionLinkEntity linkEntity = createExtensionLinkEntity(request); - ambariMetaInfo.getStackManager().linkStackToExtension(stackInfo, extensionInfo); - - try { - linkDAO.create(linkEntity); - linkEntity = linkDAO.merge(linkEntity); - } catch (RollbackException e) { - String message = "Unable to create extension link"; - LOG.debug(message, e); - String errorMessage = message - + ", stackName=" + request.getStackName() - + ", stackVersion=" + request.getStackVersion() - + ", extensionName=" + request.getExtensionName() - + ", extensionVersion=" + request.getExtensionVersion(); - LOG.warn(errorMessage); - throw new AmbariException(errorMessage, e); - } + helper.createExtensionLink(ambariMetaInfo.getStackManager(), stackInfo, extensionInfo); } /** @@ -5535,37 +5525,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle ambariMetaInfo.getStackManager().linkStackToExtension(stackInfo, extensionInfo); } - private void validateCreateExtensionLinkRequest(ExtensionLinkRequest request) throws AmbariException { - if (request.getStackName() == null - || request.getStackVersion() == null - || request.getExtensionName() == null - || request.getExtensionVersion() == null) { - - throw new IllegalArgumentException("Stack name, stack version, extension name and extension version should be provided"); - } - - ExtensionLinkEntity entity = linkDAO.findByStackAndExtension(request.getStackName(), request.getStackVersion(), - request.getExtensionName(), request.getExtensionVersion()); - - if (entity != null) { - throw new AmbariException("The stack and extension are already linked" - + ", stackName=" + request.getStackName() - + ", stackVersion=" + request.getStackVersion() - + ", extensionName=" + request.getExtensionName() - + ", extensionVersion=" + request.getExtensionVersion()); - } - } - - private ExtensionLinkEntity createExtensionLinkEntity(ExtensionLinkRequest request) throws AmbariException { - StackEntity stack = stackDAO.find(request.getStackName(), request.getStackVersion()); - ExtensionEntity extension = extensionDAO.find(request.getExtensionName(), request.getExtensionVersion()); - - ExtensionLinkEntity linkEntity = new ExtensionLinkEntity(); - linkEntity.setStack(stack); - linkEntity.setExtension(extension); - return linkEntity; - } - @Override public QuickLinkVisibilityController getQuicklinkVisibilityController() { SettingEntity entity = settingDAO.findByName(QuickLinksProfile.SETTING_NAME_QUICKLINKS_PROFILE); http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementHelper.java new file mode 100644 index 0000000..2dd6f12 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementHelper.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import javax.persistence.RollbackException; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.orm.dao.ExtensionDAO; +import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; +import org.apache.ambari.server.orm.dao.StackDAO; +import org.apache.ambari.server.orm.entities.ExtensionEntity; +import org.apache.ambari.server.orm.entities.ExtensionLinkEntity; +import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.stack.ExtensionHelper; +import org.apache.ambari.server.stack.StackManager; +import org.apache.ambari.server.state.ExtensionInfo; +import org.apache.ambari.server.state.StackInfo; +import org.apache.ambari.server.state.stack.ExtensionMetainfoXml; +import org.apache.ambari.server.utils.VersionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +@Singleton +public class AmbariManagementHelper { + + private final static Logger LOG = + LoggerFactory.getLogger(AmbariManagementHelper.class); + + private ExtensionLinkDAO linkDAO; + private ExtensionDAO extensionDAO; + private StackDAO stackDAO; + + @Inject + public AmbariManagementHelper(StackDAO stackDAO, ExtensionDAO extensionDAO, ExtensionLinkDAO linkDAO) { + this.stackDAO = stackDAO; + this.extensionDAO = extensionDAO; + this.linkDAO = linkDAO; + } + + /** + * This method will create a link between an extension version and a stack version (Extension Link). + * + * An extension version is like a stack version but it contains custom services. Linking an extension + * version to the current stack version allows the cluster to install the custom services contained in + * the extension version. + */ + public void createExtensionLink(StackManager stackManager, StackInfo stackInfo, ExtensionInfo extensionInfo) throws AmbariException { + validateCreateExtensionLinkRequest(stackInfo, extensionInfo); + ExtensionHelper.validateCreateLink(stackInfo, extensionInfo); + ExtensionLinkEntity linkEntity = createExtensionLinkEntity(stackInfo, extensionInfo); + stackManager.linkStackToExtension(stackInfo, extensionInfo); + + try { + linkDAO.create(linkEntity); + linkEntity = linkDAO.merge(linkEntity); + } catch (RollbackException e) { + String message = "Unable to create extension link"; + LOG.debug(message, e); + String errorMessage = message + + ", stackName=" + stackInfo.getName() + + ", stackVersion=" + stackInfo.getVersion() + + ", extensionName=" + extensionInfo.getName() + + ", extensionVersion=" + extensionInfo.getVersion(); + LOG.warn(errorMessage); + throw new AmbariException(errorMessage, e); + } + } + + /** + * This method will create a link between an extension version and a stack version (Extension Link). + * + * An extension version is like a stack version but it contains custom services. Linking an extension + * version to the current stack version allows the cluster to install the custom services contained in + * the extension version. + */ + public void createExtensionLinks(StackManager stackManager, List<ExtensionInfo> extensions) throws AmbariException { + Map<String, List<StackInfo>> stackMap = stackManager.getStacksByName(); + for (List<StackInfo> stacks : stackMap.values()) { + Collections.sort(stacks); + Collections.reverse(stacks); + } + + Collections.sort(extensions); + Collections.reverse(extensions); + for (ExtensionInfo extension : extensions) { + if (extension.isActive() && extension.isAutoLink()) { + LOG.debug("Autolink - looking for matching stack versions for extension:{}/{} ", extension.getName(), extension.getVersion()); + for (ExtensionMetainfoXml.Stack supportedStack : extension.getStacks()) { + List<StackInfo> stacks = stackMap.get(supportedStack.getName()); + for (StackInfo stack : stacks) { + // If the stack version is not currently linked to a version of the extension and it meets the minimum stack version then link them + if (stack.getExtension(extension.getName()) == null && VersionUtils.compareVersions(stack.getVersion(), supportedStack.getVersion()) > -1) { + LOG.debug("Autolink - extension: {}/{} stack: {}/{}", extension.getName(), extension.getVersion(), + stack.getName(), stack.getVersion()); + createExtensionLink(stackManager, stack, extension); + } + else { + LOG.debug("Autolink - not a match extension: {}/{} stack: {}/{}", extension.getName(), extension.getVersion(), + stack.getName(), stack.getVersion()); + } + } + } + } + else { + LOG.debug("Autolink - skipping extension: {}/{}. It is either not active or set to autolink.", extension.getName(), extension.getVersion()); + } + } + } + + /** + * Validates the stackInfo and extensionInfo parameters are valid. + * If they are then it confirms that the stack and extension are not already linked. + */ + private void validateCreateExtensionLinkRequest(StackInfo stackInfo, ExtensionInfo extensionInfo) throws AmbariException { + if (stackInfo == null) { + throw new IllegalArgumentException("Stack should be provided"); + } + if (extensionInfo == null) { + throw new IllegalArgumentException("Extension should be provided"); + } + if (StringUtils.isBlank(stackInfo.getName()) + || StringUtils.isBlank(stackInfo.getVersion()) + || StringUtils.isBlank(extensionInfo.getName()) + || StringUtils.isBlank(extensionInfo.getVersion())) { + + throw new IllegalArgumentException("Stack name, stack version, extension name and extension version should be provided"); + } + + ExtensionLinkEntity entity = linkDAO.findByStackAndExtension(stackInfo.getName(), stackInfo.getVersion(), + extensionInfo.getName(), extensionInfo.getVersion()); + + if (entity != null) { + throw new AmbariException("The stack and extension are already linked" + + ", stackName=" + stackInfo.getName() + + ", stackVersion=" + stackInfo.getVersion() + + ", extensionName=" + extensionInfo.getName() + + ", extensionVersion=" + extensionInfo.getVersion()); + } + } + + private ExtensionLinkEntity createExtensionLinkEntity(StackInfo stackInfo, ExtensionInfo extensionInfo) throws AmbariException { + StackEntity stack = stackDAO.find(stackInfo.getName(), stackInfo.getVersion()); + ExtensionEntity extension = extensionDAO.find(extensionInfo.getName(), extensionInfo.getVersion()); + + ExtensionLinkEntity linkEntity = new ExtensionLinkEntity(); + linkEntity.setStack(stack); + linkEntity.setExtension(extension); + return linkEntity; + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExtensionLinkDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExtensionLinkDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExtensionLinkDAO.java index 3df1cd5..7bfa544 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExtensionLinkDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/ExtensionLinkDAO.java @@ -72,16 +72,19 @@ public class ExtensionLinkDAO { } String stackName = request.getStackName(); - String stackVersion = request.getStackName(); - String extensionName = request.getStackName(); - String extensionVersion = request.getStackName(); + String stackVersion = request.getStackVersion(); + String extensionName = request.getExtensionName(); + String extensionVersion = request.getExtensionVersion(); if (stackName != null && stackVersion != null) { - if (extensionName != null && extensionVersion != null) { - ExtensionLinkEntity entity = findByStackAndExtension(stackName, stackVersion, extensionName, extensionVersion); - List<ExtensionLinkEntity> list = new ArrayList<>(); - list.add(entity); - return list; + if (extensionName != null) { + if (extensionVersion != null) { + ExtensionLinkEntity entity = findByStackAndExtension(stackName, stackVersion, extensionName, extensionVersion); + List<ExtensionLinkEntity> list = new ArrayList<>(); + list.add(entity); + return list; + } + return findByStackAndExtensionName(stackName, stackVersion, extensionName); } return findByStack(stackName, stackVersion); } @@ -151,6 +154,23 @@ public class ExtensionLinkDAO { } /** + * Gets the extension link that match the specified stack name, stack version and extension name. + * + * @return the extension link matching the specified stack name, stack version and extension name if any. + */ + @RequiresSession + public List<ExtensionLinkEntity> findByStackAndExtensionName(String stackName, String stackVersion, String extensionName) { + TypedQuery<ExtensionLinkEntity> query = entityManagerProvider.get().createNamedQuery( + "ExtensionLinkEntity.findByStackAndExtensionName", ExtensionLinkEntity.class); + + query.setParameter("stackName", stackName); + query.setParameter("stackVersion", stackVersion); + query.setParameter("extensionName", extensionName); + + return daoUtils.selectList(query); + } + + /** * Gets the extension link that match the specified stack name, stack version, extension name and extension version. * * @return the extension link matching the specified stack name, stack version, extension name and extension version if any. http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExtensionLinkEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExtensionLinkEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExtensionLinkEntity.java index 12b3ce0..e2b48bf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExtensionLinkEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ExtensionLinkEntity.java @@ -42,6 +42,7 @@ import javax.persistence.UniqueConstraint; @TableGenerator(name = "link_id_generator", table = "ambari_sequences", pkColumnName = "sequence_name", valueColumnName = "sequence_value", pkColumnValue = "link_id_seq", initialValue = 0) @NamedQueries({ @NamedQuery(name = "ExtensionLinkEntity.findAll", query = "SELECT link FROM ExtensionLinkEntity link"), + @NamedQuery(name = "ExtensionLinkEntity.findByStackAndExtensionName", query = "SELECT link FROM ExtensionLinkEntity link WHERE link.stack.stackName = :stackName AND link.stack.stackVersion = :stackVersion AND link.extension.extensionName = :extensionName"), @NamedQuery(name = "ExtensionLinkEntity.findByStackAndExtension", query = "SELECT link FROM ExtensionLinkEntity link WHERE link.stack.stackName = :stackName AND link.stack.stackVersion = :stackVersion AND link.extension.extensionName = :extensionName AND link.extension.extensionVersion = :extensionVersion"), @NamedQuery(name = "ExtensionLinkEntity.findByStack", query = "SELECT link FROM ExtensionLinkEntity link WHERE link.stack.stackName = :stackName AND link.stack.stackVersion = :stackVersion"), @NamedQuery(name = "ExtensionLinkEntity.findByExtension", query = "SELECT link FROM ExtensionLinkEntity link WHERE link.extension.extensionName = :extensionName AND link.extension.extensionVersion = :extensionVersion") }) http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java index ce92ecf..3ad0487 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/ExtensionModule.java @@ -394,6 +394,8 @@ public class ExtensionModule extends BaseModule<ExtensionModule, ExtensionInfo> extensionInfo.setParentExtensionVersion(emx.getExtends()); extensionInfo.setStacks(emx.getStacks()); extensionInfo.setExtensions(emx.getExtensions()); + extensionInfo.setActive(emx.getVersion().isActive()); + extensionInfo.setAutoLink(emx.isAutoLink()); } try { http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java index 20e28fc..d19aeba 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java @@ -19,11 +19,13 @@ package org.apache.ambari.server.stack; import java.io.File; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nullable; import javax.xml.XMLConstants; @@ -35,6 +37,7 @@ import javax.xml.validation.Validator; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -56,7 +59,6 @@ import org.xml.sax.SAXException; import com.google.inject.assistedinject.Assisted; import com.google.inject.assistedinject.AssistedInject; - /** * Manages all stack related behavior including parsing of stacks and providing access to * stack information. @@ -109,6 +111,8 @@ public class StackManager { */ private Map<String, ExtensionInfo> extensionMap = new HashMap<>(); + private AmbariManagementHelper helper; + /** * Constructor. Initialize stack manager. * @@ -130,6 +134,8 @@ public class StackManager { * extension DAO automatically injected * @param linkDao * extension link DAO automatically injected + * @param helper + * Ambari management helper automatically injected * * @throws AmbariException * if an exception occurs while processing the stacks @@ -140,7 +146,7 @@ public class StackManager { @Assisted("extensionRoot") @Nullable File extensionRoot, @Assisted OsFamily osFamily, @Assisted boolean validate, MetainfoDAO metaInfoDAO, ActionMetadata actionMetadata, StackDAO stackDao, - ExtensionDAO extensionDao, ExtensionLinkDAO linkDao) + ExtensionDAO extensionDao, ExtensionLinkDAO linkDao, AmbariManagementHelper helper) throws AmbariException { LOG.info("Initializing the stack manager..."); @@ -154,6 +160,7 @@ public class StackManager { stackMap = new HashMap<>(); stackContext = new StackContext(metaInfoDAO, actionMetadata, osFamily); extensionMap = new HashMap<>(); + this.helper = helper; parseDirectories(stackRoot, commonServicesRoot, extensionRoot); @@ -188,6 +195,7 @@ public class StackManager { LOG.info("About to parse extension directories"); extensionModules = parseExtensionDirectory(extensionRoot); } + private void populateDB(StackDAO stackDao, ExtensionDAO extensionDao) throws AmbariException { // for every stack read in, ensure that we have a database entry for it; // don't put try/catch logic around this since a failure here will @@ -226,6 +234,51 @@ public class StackManager { extensionDao.create(extensionEntity); } } + + createLinks(); + } + + /** + * Attempts to automatically create links between extension versions and stack versions. + * This is limited to 'active' extensions that have the 'autolink' attribute set (in the metainfo.xml). + * Stack versions are selected based on the minimum stack versions that the extension supports. + * The extension and stack versions are processed in order of most recent to oldest. + * In this manner, the newest extension version will be autolinked before older extension versions. + * If a different version of the same extension is already linked to a stack version then that stack version + * will be skipped. + */ + private void createLinks() { + LOG.info("Creating links"); + Collection<ExtensionInfo> extensions = getExtensions(); + Set<String> names = new HashSet<String>(); + for(ExtensionInfo extension : extensions){ + names.add(extension.getName()); + } + for(String name : names) { + createLinksForExtension(name); + } + } + + /** + * Attempts to automatically create links between versions of a particular extension and stack versions they support. + * This is limited to 'active' extensions that have the 'autolink' attribute set (in the metainfo.xml). + * Stack versions are selected based on the minimum stack versions that the extension supports. + * The extension and stack versions are processed in order of most recent to oldest. + * In this manner, the newest extension version will be autolinked before older extension versions. + * If a different version of the same extension is already linked to a stack version then that stack version + * will be skipped. + */ + private void createLinksForExtension(String name) { + Collection<ExtensionInfo> collection = getExtensions(name); + List<ExtensionInfo> extensions = new ArrayList<ExtensionInfo>(collection.size()); + extensions.addAll(collection); + try { + helper.createExtensionLinks(this, extensions); + } + catch (AmbariException e) { + String msg = String.format("Failed to create link for extension: %s with exception: %s", name, e.getMessage()); + LOG.error(msg); + } } /** @@ -258,6 +311,24 @@ public class StackManager { } /** + * Obtain all a map of all stacks by name. + * + * @return A map of all stacks with the name as the key. + */ + public Map<String, List<StackInfo>> getStacksByName() { + Map<String, List<StackInfo>> stacks = new HashMap<String, List<StackInfo>>(); + for (StackInfo stack: stackMap.values()) { + List<StackInfo> list = stacks.get(stack.getName()); + if (list == null) { + list = new ArrayList<StackInfo>(); + stacks.put(stack.getName(), list); + } + list.add(stack); + } + return stacks; + } + + /** * Obtain all stacks. * * @return collection of all stacks @@ -459,8 +530,6 @@ public class StackManager { } } - - /** * Validate that the specified extension root is a valid directory. * @@ -567,9 +636,11 @@ public class StackManager { } public void linkStackToExtension(StackInfo stack, ExtensionInfo extension) throws AmbariException { + stack.addExtension(extension); } public void unlinkStackAndExtension(StackInfo stack, ExtensionInfo extension) throws AmbariException { + stack.removeExtension(extension); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java index d3ad351..672b72f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java @@ -408,9 +408,7 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V private void addExtensionServices() throws AmbariException { for (ExtensionModule extension : extensionModules.values()) { - stackInfo.getExtensions().add(extension.getModuleInfo()); - Collection<ServiceModule> services = extension.getServiceModules().values(); - addServices(services); + stackInfo.addExtension(extension.getModuleInfo()); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/state/ExtensionInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ExtensionInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ExtensionInfo.java index 656a8ce..58643cb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/ExtensionInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ExtensionInfo.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.ambari.server.controller.ExtensionVersionResponse; import org.apache.ambari.server.stack.Validable; import org.apache.ambari.server.state.stack.ExtensionMetainfoXml; +import org.apache.ambari.server.utils.VersionUtils; /** * An extension version is like a stack version but it contains custom services. Linking an extension @@ -44,6 +45,8 @@ public class ExtensionInfo implements Comparable<ExtensionInfo>, Validable{ private List<ExtensionMetainfoXml.Stack> stacks; private List<ExtensionMetainfoXml.Extension> extensions; private boolean valid = true; + private boolean autoLink = false; + private boolean active = false; /** * @@ -184,9 +187,10 @@ public class ExtensionInfo implements Comparable<ExtensionInfo>, Validable{ @Override public int compareTo(ExtensionInfo o) { - String myId = name + "-" + version; - String oId = o.name + "-" + o.version; - return myId.compareTo(oId); + if (name.equals(o.name)) { + return VersionUtils.compareVersions(version, o.version); + } + return name.compareTo(o.name); } public List<ExtensionMetainfoXml.Stack> getStacks() { @@ -204,4 +208,20 @@ public class ExtensionInfo implements Comparable<ExtensionInfo>, Validable{ public void setExtensions(List<ExtensionMetainfoXml.Extension> extensions) { this.extensions = extensions; } + + public boolean isAutoLink() { + return autoLink; + } + + public void setAutoLink(boolean autoLink) { + this.autoLink = autoLink; + } + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java index 353dd86..dde9fa5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/StackInfo.java @@ -36,6 +36,7 @@ import org.apache.ambari.server.state.stack.ConfigUpgradePack; import org.apache.ambari.server.state.stack.RepositoryXml; import org.apache.ambari.server.state.stack.StackRoleCommandOrder; import org.apache.ambari.server.state.stack.UpgradePack; +import org.apache.ambari.server.utils.VersionUtils; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimaps; @@ -202,6 +203,7 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ public ExtensionInfo getExtensionByService(String serviceName) { Collection<ExtensionInfo> extensions = getExtensions(); for (ExtensionInfo extension : extensions) { + Collection<ServiceInfo> services = extension.getServices(); for (ServiceInfo service : services) { if (service.getName().equals(serviceName)) return extension; @@ -211,6 +213,24 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ return null; } + public void addExtension(ExtensionInfo extension) { + Collection<ExtensionInfo> extensions = getExtensions(); + extensions.add(extension); + Collection<ServiceInfo> services = getServices(); + for (ServiceInfo service : extension.getServices()) { + services.add(service); + } + } + + public void removeExtension(ExtensionInfo extension) { + Collection<ExtensionInfo> extensions = getExtensions(); + extensions.remove(extension); + Collection<ServiceInfo> services = getServices(); + for (ServiceInfo service : extension.getServices()) { + services.remove(service); + } + } + public List<PropertyInfo> getProperties() { if (properties == null) properties = new ArrayList<>(); return properties; @@ -459,9 +479,10 @@ public class StackInfo implements Comparable<StackInfo>, Validable{ @Override public int compareTo(StackInfo o) { - String myId = name + "-" + version; - String oId = o.name + "-" + o.version; - return myId.compareTo(oId); + if (name.equals(o.name)) { + return VersionUtils.compareVersions(version, o.version); + } + return name.compareTo(o.name); } //todo: ensure that required properties are never modified... http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ExtensionMetainfoXml.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ExtensionMetainfoXml.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ExtensionMetainfoXml.java index 77a7250..4480ef7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ExtensionMetainfoXml.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/ExtensionMetainfoXml.java @@ -77,6 +77,9 @@ public class ExtensionMetainfoXml implements Validable{ @XmlTransient private boolean valid = true; + @XmlElement(name="auto-link") + private boolean autoLink = false; + /** * * @return valid xml flag @@ -201,4 +204,12 @@ public class ExtensionMetainfoXml implements Validable{ } } + public boolean isAutoLink() { + return autoLink; + } + + public void setAutoLink(boolean autoLink) { + this.autoLink = autoLink; + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java index 4e20c2b..1fd0921 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerCommonServicesTest.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -123,10 +124,11 @@ public class StackManagerCommonServicesTest { osFamily = new OsFamily(config); replay(metaInfoDao, actionMetadata); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(new File(stackRoot), new File( commonServicesRoot), new File(extensionRoot), osFamily, true, metaInfoDao, - actionMetadata, stackDao, extensionDao, linkDao); + actionMetadata, stackDao, extensionDao, linkDao, helper); EasyMock.verify( config, stackDao ); http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java index 9900eb5..33717cb 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerExtensionTest.java @@ -30,6 +30,7 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -59,42 +60,57 @@ public class StackManagerExtensionTest { ExtensionLinkDAO linkDao = createNiceMock(ExtensionLinkDAO.class); ActionMetadata actionMetadata = createNiceMock(ActionMetadata.class); OsFamily osFamily = createNiceMock(OsFamily.class); - StackEntity stackEntity = createNiceMock(StackEntity.class); - ExtensionEntity extensionEntity = createNiceMock(ExtensionEntity.class); - ExtensionLinkEntity linkEntity = createNiceMock(ExtensionLinkEntity.class); + StackEntity stack1 = new StackEntity(); + stack1.setStackName("HDP"); + stack1.setStackVersion("0.1"); + StackEntity stack2 = new StackEntity(); + stack2.setStackName("HDP"); + stack2.setStackVersion("0.2"); + StackEntity stack3 = new StackEntity(); + stack3.setStackName("HDP"); + stack3.setStackVersion("0.3"); + ExtensionEntity extension1 = new ExtensionEntity(); + extension1.setExtensionName("EXT"); + extension1.setExtensionVersion("0.1"); + ExtensionEntity extension2 = new ExtensionEntity(); + extension2.setExtensionName("EXT"); + extension2.setExtensionVersion("0.2"); + ExtensionEntity extension3 = new ExtensionEntity(); + extension3.setExtensionName("EXT"); + extension3.setExtensionVersion("0.3"); List<ExtensionLinkEntity> list = new ArrayList<>(); - list.add(linkEntity); - expect( - stackDao.find(EasyMock.anyObject(String.class), - EasyMock.anyObject(String.class))).andReturn(stackEntity).atLeastOnce(); + expect(stackDao.find("HDP", "0.1")).andReturn(stack1).atLeastOnce(); + expect(stackDao.find("HDP", "0.2")).andReturn(stack2).atLeastOnce(); + expect(stackDao.find("HDP", "0.3")).andReturn(stack3).atLeastOnce(); + expect(extensionDao.find("EXT", "0.1")).andReturn(extension1).atLeastOnce(); + expect(extensionDao.find("EXT", "0.2")).andReturn(extension2).atLeastOnce(); + expect(extensionDao.find("EXT", "0.3")).andReturn(extension3).atLeastOnce(); - expect( - extensionDao.find(EasyMock.anyObject(String.class), - EasyMock.anyObject(String.class))).andReturn(extensionEntity).atLeastOnce(); - - expect( - linkDao.findByStack(EasyMock.anyObject(String.class), + expect(linkDao.findByStack(EasyMock.anyObject(String.class), EasyMock.anyObject(String.class))).andReturn(list).atLeastOnce(); - expect( - linkEntity.getExtension()).andReturn(extensionEntity).atLeastOnce(); - - expect( - extensionEntity.getExtensionName()).andReturn("EXT").atLeastOnce(); - - expect( - extensionEntity.getExtensionVersion()).andReturn("0.2").atLeastOnce(); + expect(linkDao.findByStackAndExtension("HDP", "0.2", "EXT", "0.2")).andReturn(null).atLeastOnce(); - replay(actionMetadata, stackDao, metaInfoDao, osFamily, extensionDao, linkDao, extensionEntity, linkEntity); + replay(actionMetadata, stackDao, metaInfoDao, osFamily, extensionDao, linkDao); //linkEntity String stacks = ClassLoader.getSystemClassLoader().getResource("stacks_with_extensions").getPath(); String common = ClassLoader.getSystemClassLoader().getResource("common-services").getPath(); String extensions = ClassLoader.getSystemClassLoader().getResource("extensions").getPath(); - StackManager stackManager = new StackManager(new File(stacks), + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); + + StackManager stackManager = null; + try { + stackManager = new StackManager(new File(stacks), new File(common), new File(extensions), osFamily, false, - metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); + } + catch (Exception e) { + e.printStackTrace(); + } + + assertNotNull("Failed to create Stack Manager", stackManager); ExtensionInfo extension = stackManager.getExtension("EXT", "0.1"); assertNull("EXT 0.1's parent: " + extension.getParentExtensionVersion(), extension.getParentExtensionVersion()); @@ -116,6 +132,7 @@ public class StackManagerExtensionTest { assertNotNull("EXT 0.2's parent: " + extension.getParentExtensionVersion(), extension.getParentExtensionVersion()); assertEquals("EXT 0.2's parent: " + extension.getParentExtensionVersion(), "0.1", extension.getParentExtensionVersion()); assertNotNull(extension.getService("OOZIE2")); + assertTrue("Extension is not set to auto link", extension.isAutoLink()); oozie = extension.getService("OOZIE2"); assertNotNull("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder()); assertTrue("Package dir is " + oozie.getServicePackageFolder(), oozie.getServicePackageFolder().contains("extensions/EXT/0.1/services/OOZIE2/package")); @@ -137,7 +154,13 @@ public class StackManagerExtensionTest { assertTrue("Extensions found: " + stack.getExtensions().size(), stack.getExtensions().size() == 1); extension = stack.getExtensions().iterator().next(); assertEquals("Extension name: " + extension.getName(), extension.getName(), "EXT"); - assertEquals("Extension version: " + extension.getVersion(), extension.getVersion(), "0.2"); + assertEquals("Extension version: " + extension.getVersion(), extension.getVersion(), "0.3"); + + stack = stackManager.getStack("HDP", "0.3"); + assertTrue("Extensions found: " + stack.getExtensions().size(), stack.getExtensions().size() == 1); + extension = stack.getExtensions().iterator().next(); + assertEquals("Extension name: " + extension.getName(), extension.getName(), "EXT"); + assertEquals("Extension version: " + extension.getVersion(), extension.getVersion(), "0.3"); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java index ca24cd9..6df46c3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMiscTest.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.List; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -69,12 +70,13 @@ public class StackManagerMiscTest { EasyMock.anyObject(String.class))).andReturn(list).atLeastOnce(); replay(actionMetadata, stackDao, extensionDao, linkDao, metaInfoDao, osFamily); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); try { String stacksCycle1 = ClassLoader.getSystemClassLoader().getResource("stacks_with_cycle").getPath(); StackManager stackManager = new StackManager(new File(stacksCycle1), null, null, osFamily, false, - metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic stack"); } catch (AmbariException e) { @@ -86,7 +88,7 @@ public class StackManagerMiscTest { "stacks_with_cycle2").getPath(); StackManager stackManager = new StackManager(new File(stacksCycle2), - null, null, osFamily, true, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + null, null, osFamily, true, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic stack"); } catch (AmbariException e) { @@ -124,10 +126,11 @@ public class StackManagerMiscTest { replay(metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata, osFamily); String singleStack = ClassLoader.getSystemClassLoader().getResource("single_stack").getPath(); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(new File(singleStack.replace( StackManager.PATH_DELIMITER, File.separator)), null, null, osFamily, false, metaInfoDao, - actionMetadata, stackDao, extensionDao, linkDao); + actionMetadata, stackDao, extensionDao, linkDao, helper); Collection<StackInfo> stacks = stackManager.getStacks(); assertEquals(1, stacks.size()); @@ -161,11 +164,13 @@ public class StackManagerMiscTest { replay(metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata, osFamily); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); + try { String upgradeCycle = ClassLoader.getSystemClassLoader().getResource("stacks_with_upgrade_cycle").getPath(); StackManager stackManager = new StackManager(new File(upgradeCycle), null, null, osFamily, false, - metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); fail("Expected exception due to cyclic service upgrade xml"); } catch (AmbariException e) { http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java index 0fe1573..4a26020 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerMock.java @@ -27,6 +27,7 @@ import java.util.concurrent.locks.ReentrantLock; import javax.annotation.Nullable; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -134,8 +135,8 @@ public class StackManagerMock extends StackManager { File commonServicesRoot, @Assisted("extensionRoot") @Nullable File extensionRoot, @Assisted OsFamily osFamily, @Assisted boolean validate, MetainfoDAO metaInfoDAO, ActionMetadata actionMetadata, StackDAO stackDao, ExtensionDAO extensionDao, - ExtensionLinkDAO linkDao) throws AmbariException { - super(stackRoot, commonServicesRoot, extensionRoot, osFamily, validate, metaInfoDAO, actionMetadata, stackDao, extensionDao, linkDao); + ExtensionLinkDAO linkDao, AmbariManagementHelper helper) throws AmbariException { + super(stackRoot, commonServicesRoot, extensionRoot, osFamily, validate, metaInfoDAO, actionMetadata, stackDao, extensionDao, linkDao, helper); currentStackRoot = stackRoot; currentCommonServicesRoot = commonServicesRoot; currentExtensionRoot = extensionRoot; http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java index 0a0404a..5149026 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/StackManagerTest.java @@ -44,6 +44,7 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementHelper; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ExtensionDAO; import org.apache.ambari.server.orm.dao.ExtensionLinkDAO; @@ -119,9 +120,10 @@ public class StackManagerTest { replay(config, metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata); osFamily = new OsFamily(config); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(new File(stackRoot), null, null, osFamily, false, - metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); verify(config, metaInfoDao, stackDao, actionMetadata); @@ -794,9 +796,10 @@ public class StackManagerTest { replay(config, metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata); OsFamily osFamily = new OsFamily(config); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, - osFamily, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + osFamily, false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); for (StackInfo stackInfo : stackManager.getStacks()) { for (ServiceInfo serviceInfo : stackInfo.getServices()) { @@ -859,9 +862,10 @@ public class StackManagerTest { replay(config, metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata); OsFamily osFamily = new OsFamily(config); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, osFamily, - false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); String rangerUserSyncRoleCommand = Role.RANGER_USERSYNC + "-" + RoleCommand.START; String rangerAdminRoleCommand = Role.RANGER_ADMIN + "-" + RoleCommand.START; @@ -988,9 +992,10 @@ public class StackManagerTest { replay(config, metaInfoDao, stackDao, extensionDao, linkDao, actionMetadata); OsFamily osFamily = new OsFamily(config); + AmbariManagementHelper helper = new AmbariManagementHelper(stackDao, extensionDao, linkDao); StackManager stackManager = new StackManager(stackRoot, commonServices, extensions, osFamily, - false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao); + false, metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper); String zookeeperServerRoleCommand = Role.ZOOKEEPER_SERVER + "-" + RoleCommand.START; String logsearchServerRoleCommand = Role.LOGSEARCH_SERVER + "-" + RoleCommand.START; http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/extensions/EXT/0.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/extensions/EXT/0.1/metainfo.xml b/ambari-server/src/test/resources/extensions/EXT/0.1/metainfo.xml index 77a832c..27f5902 100644 --- a/ambari-server/src/test/resources/extensions/EXT/0.1/metainfo.xml +++ b/ambari-server/src/test/resources/extensions/EXT/0.1/metainfo.xml @@ -23,7 +23,7 @@ <min-stack-versions> <stack> <name>HDP</name> - <version>0.1.*</version> + <version>0.1</version> </stack> </min-stack-versions> </prerequisites> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/extensions/EXT/0.2/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/extensions/EXT/0.2/metainfo.xml b/ambari-server/src/test/resources/extensions/EXT/0.2/metainfo.xml index 04f733c..0d37b3e 100644 --- a/ambari-server/src/test/resources/extensions/EXT/0.2/metainfo.xml +++ b/ambari-server/src/test/resources/extensions/EXT/0.2/metainfo.xml @@ -20,11 +20,12 @@ <active>true</active> </versions> <extends>0.1</extends> + <auto-link>true</auto-link> <prerequisites> <min-stack-versions> <stack> <name>HDP</name> - <version>0.2.*</version> + <version>0.2</version> </stack> </min-stack-versions> </prerequisites> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/extensions/EXT/0.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/extensions/EXT/0.3/metainfo.xml b/ambari-server/src/test/resources/extensions/EXT/0.3/metainfo.xml new file mode 100644 index 0000000..d827314 --- /dev/null +++ b/ambari-server/src/test/resources/extensions/EXT/0.3/metainfo.xml @@ -0,0 +1,32 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <active>true</active> + </versions> + <extends>0.2</extends> + <auto-link>true</auto-link> + <prerequisites> + <min-stack-versions> + <stack> + <name>HDP</name> + <version>0.2</version> + </stack> + </min-stack-versions> + </prerequisites> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/metainfo.xml b/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/metainfo.xml new file mode 100644 index 0000000..9176551 --- /dev/null +++ b/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/metainfo.xml @@ -0,0 +1,118 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>OOZIE2</name> + <comment>System for workflow coordination and execution of Apache Hadoop jobs</comment> + <version>4.0.0</version> + + <components> + <component> + <name>OOZIE2_SERVER</name> + <category>MASTER</category> + <cardinality>1</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE/MAPREDUCE_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie2_server.py</script> + <scriptType>PYTHON</scriptType> + <timeout>600</timeout> + </commandScript> + </component> + + <component> + <name>OOZIE2_CLIENT</name> + <category>CLIENT</category> + <cardinality>0+</cardinality> + <dependencies> + <dependency> + <name>HDFS/HDFS_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + <dependency> + <name>MAPREDUCE/MAPREDUCE_CLIENT</name> + <scope>host</scope> + <auto-deploy> + <enabled>true</enabled> + </auto-deploy> + </dependency> + </dependencies> + <commandScript> + <script>scripts/oozie2_client.py</script> + <scriptType>PYTHON</scriptType> + </commandScript> + </component> + </components> + + <osSpecifics> + <osSpecific> + <osFamily>any</osFamily> + <packages> + <package> + <name>oozie2.noarch</name> + </package> + <package> + <name>oozie2-client.noarch</name> + </package> + <package> + <name>extjs-2.2-1</name> + </package> + </packages> + </osSpecific> + </osSpecifics> + + <commandScript> + <script>scripts/service_check.py</script> + <scriptType>PYTHON</scriptType> + <timeout>300</timeout> + </commandScript> + + <configuration-dependencies> + <config-type>global</config-type> + <config-type>oozie2-site</config-type> + </configuration-dependencies> + + <themes> + <theme> + <fileName>broken_theme.json</fileName> + <default>true</default> + </theme> + </themes> + + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/themes/broken_theme.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/themes/broken_theme.json b/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/themes/broken_theme.json new file mode 100644 index 0000000..6e8b5bf --- /dev/null +++ b/ambari-server/src/test/resources/extensions/EXT/0.3/services/OOZIE2/themes/broken_theme.json @@ -0,0 +1,3 @@ +{ + "configuration": { +} http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/metainfo.xml b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/metainfo.xml new file mode 100644 index 0000000..b52857b --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/metainfo.xml @@ -0,0 +1,22 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <versions> + <upgrade>0.2</upgrade> + </versions> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/repos/repoinfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/repos/repoinfo.xml b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/repos/repoinfo.xml new file mode 100644 index 0000000..9b3b1c7 --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/repos/repoinfo.xml @@ -0,0 +1,63 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<reposinfo> + <os family="redhat6"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-1.1.1.16/repos/centos6</baseurl> + <repoid>HDP-1.1.1.16</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.15/repos/centos6</baseurl> + <repoid>HDP-UTILS-1.1.0.15</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + <mirrorslist></mirrorslist> + </repo> + <repo> + <baseurl></baseurl> + <repoid>epel</repoid> + <reponame>epel</reponame> + <unique>true</unique> + <mirrorslist><![CDATA[https://mirrors.fedoraproject.org/metalink?repo=epel-6&arch=$basearch]]></mirrorslist> + </repo> + </os> + <os family="centos5"> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-1.1.1.16/repos/centos5</baseurl> + <repoid>HDP-1.1.1.16</repoid> + <reponame>HDP</reponame> + <unique>true</unique> + </repo> + <repo> + <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.15/repos/centos5</baseurl> + <repoid>HDP-UTILS-1.1.0.15</repoid> + <reponame>HDP-UTILS</reponame> + <unique>false</unique> + <mirrorslist></mirrorslist> + </repo> + <repo> + <baseurl></baseurl> + <repoid>epel</repoid> + <reponame>epel</reponame> + <unique>true</unique> + <mirrorslist><![CDATA[https://mirrors.fedoraproject.org/metalink?repo=epel-5&arch=$basearch]]></mirrorslist> + </repo> + </os> +</reposinfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HBASE/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HBASE/metainfo.xml b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HBASE/metainfo.xml new file mode 100644 index 0000000..48123f0 --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HBASE/metainfo.xml @@ -0,0 +1,26 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<metainfo> + <schemaVersion>2.0</schemaVersion> + <services> + <service> + <name>HBASE</name> + <extends>common-services/HBASE/1.0</extends> + </service> + </services> +</metainfo> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/global.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/global.xml b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/global.xml new file mode 100644 index 0000000..bcab577 --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/global.xml @@ -0,0 +1,145 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>namenode_host</name> + <value></value> + <description>NameNode Host.</description> + </property> + <property> + <name>dfs_name_dir</name> + <value>/hadoop/hdfs/namenode</value> + <description>NameNode Directories.</description> + </property> + <property> + <name>snamenode_host</name> + <value></value> + <description>Secondary NameNode.</description> + </property> + <property> + <name>fs_checkpoint_dir</name> + <value>/hadoop/hdfs/namesecondary</value> + <description>Secondary NameNode checkpoint dir.</description> + </property> + <property> + <name>datanode_hosts</name> + <value></value> + <description>List of Datanode Hosts.</description> + </property> + <property> + <name>dfs_data_dir</name> + <value>/hadoop/hdfs/data</value> + <description>Data directories for Data Nodes.</description> + </property> + <property> + <name>hdfs_log_dir_prefix</name> + <value>/var/log/hadoop</value> + <description>Hadoop Log Dir Prefix</description> + </property> + <property> + <name>hadoop_pid_dir_prefix</name> + <value>/var/run/hadoop</value> + <description>Hadoop PID Dir Prefix</description> + </property> + <property> + <name>dfs_webhdfs_enabled</name> + <value>true</value> + <description>WebHDFS enabled</description> + </property> + <property> + <name>hadoop_heapsize</name> + <value>1024</value> + <description>Hadoop maximum Java heap size</description> + </property> + <property> + <name>namenode_heapsize</name> + <value>1024</value> + <description>NameNode Java heap size</description> + </property> + <property> + <name>namenode_opt_newsize</name> + <value>200</value> + <description>Default size of Java new generation for NameNode (Java option -XX:NewSize) Note: The value of NameNode new generation size (default size of Java new generation for NameNode (Java option -XX:NewSize)) should be 1/8 of maximum heap size (-Xmx). Ensure that the value of the namenode_opt_newsize property is 1/8 the value of maximum heap size (-Xmx).</description> + </property> + <property> + <name>namenode_opt_maxnewsize</name> + <value>640</value> + <description>NameNode maximum new generation size</description> + </property> + <property> + <name>namenode_opt_permsize</name> + <value>128</value> + <description>NameNode permanent generation size</description> + </property> + <property> + <name>namenode_opt_maxpermsize</name> + <value>256</value> + <description>NameNode maximum permanent generation size</description> + </property> + <property> + <name>datanode_du_reserved</name> + <value>1</value> + <description>Reserved space for HDFS</description> + </property> + <property> + <name>dtnode_heapsize</name> + <value>1024</value> + <description>DataNode maximum Java heap size</description> + </property> + <property> + <name>dfs_datanode_failed_volume_tolerated</name> + <value>0</value> + <description>DataNode volumes failure toleration</description> + </property> + <property> + <name>fs_checkpoint_period</name> + <value>21600</value> + <description>HDFS Maximum Checkpoint Delay</description> + </property> + <property> + <name>fs_checkpoint_size</name> + <value>0.5</value> + <description>FS Checkpoint Size.</description> + </property> + <property> + <name>security_enabled</name> + <value>false</value> + <description>Hadoop Security</description> + </property> + <property> + <name>kerberos_domain</name> + <value>EXAMPLE.COM</value> + <description>Kerberos realm.</description> + </property> + <property> + <name>kerberos_domain</name> + <value>EXAMPLE.COM</value> + <description>Kerberos realm.</description> + </property> + <property> + <name>keytab_path</name> + <value>/etc/security/keytabs</value> + <description>KeyTab Directory.</description> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/aa78a172/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/hadoop-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/hadoop-env.xml b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/hadoop-env.xml new file mode 100644 index 0000000..8fb8c7f --- /dev/null +++ b/ambari-server/src/test/resources/stacks_with_extensions/HDP/0.3/services/HDFS/configuration/hadoop-env.xml @@ -0,0 +1,223 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>hdfs_log_dir_prefix</name> + <value>/var/log/hadoop</value> + <description>Hadoop Log Dir Prefix</description> + </property> + <property> + <name>hadoop_pid_dir_prefix</name> + <value>/var/run/hadoop</value> + <description>Hadoop PID Dir Prefix</description> + </property> + <property> + <name>hadoop_heapsize</name> + <value>1024</value> + <description>Hadoop maximum Java heap size</description> + </property> + <property> + <name>namenode_heapsize</name> + <value>1024</value> + <description>NameNode Java heap size</description> + </property> + <property> + <name>namenode_opt_newsize</name> + <value>200</value> + <description>Default size of Java new generation for NameNode (Java option -XX:NewSize) Note: The value of NameNode new generation size (default size of Java new generation for NameNode (Java option -XX:NewSize)) should be 1/8 of maximum heap size (-Xmx). Ensure that the value of the namenode_opt_newsize property is 1/8 the value of maximum heap size (-Xmx).</description> + </property> + <property> + <name>namenode_opt_maxnewsize</name> + <value>200</value> + <description>NameNode maximum new generation size</description> + </property> + <property> + <name>namenode_opt_permsize</name> + <value>128</value> + <description>NameNode permanent generation size</description> + </property> + <property> + <name>namenode_opt_maxpermsize</name> + <value>256</value> + <description>NameNode maximum permanent generation size</description> + </property> + <property> + <name>dtnode_heapsize</name> + <value>1024</value> + <description>DataNode maximum Java heap size</description> + </property> + <property> + <name>proxyuser_group</name> + <value>users</value> + <description>Proxy user group.</description> + </property> + <property> + <name>security_enabled</name> + <value>false</value> + <description>Hadoop Security</description> + </property> + <property> + <name>kerberos_domain</name> + <value>EXAMPLE.COM</value> + <description>Kerberos realm.</description> + </property> + <property> + <name>hdfs_user</name> + <value>hdfs</value> + <description>User and Groups.</description> + </property> + <property> + <name>ignore_groupsusers_create</name> + <value>false</value> + <description>Whether to ignores failures on users and group creation</description> + </property> + <property> + <name>smokeuser</name> + <value>ambari-qa</value> + <description>User executing service checks</description> + </property> + <property> + <name>user_group</name> + <value>hadoop</value> + <description>Proxy user group.</description> + </property> + + <!-- hadoop-env.sh --> + <property> + <name>content</name> + <description>hadoop-env.sh content</description> + <value> +# Set Hadoop-specific environment variables here. + +# The only required environment variable is JAVA_HOME. All others are +# optional. When running a distributed configuration it is best to +# set JAVA_HOME in this file, so that it is correctly defined on +# remote nodes. + +# The java implementation to use. Required. +export JAVA_HOME={{java_home}} +export HADOOP_HOME_WARN_SUPPRESS=1 + +# Hadoop home directory +export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop} + +# Hadoop Configuration Directory +#TODO: if env var set that can cause problems +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}} + +{# this is different for HDP1 #} +# Path to jsvc required by secure HDP 2.0 datanode +export JSVC_HOME={{jsvc_path}} + + +# The maximum amount of heap to use, in MB. Default is 1000. +export HADOOP_HEAPSIZE="{{hadoop_heapsize}}" + +export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}" + +# Extra Java runtime options. Empty by default. +export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}" + +# Command specific options appended to HADOOP_OPTS when specified +export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}" +HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}" + +HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}" +HADOOP_DATANODE_OPTS="-XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:ConcGCThreads=4 -XX:+UseConcMarkSweepGC -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}" +HADOOP_BALANCER_OPTS="-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}" + +export HADOOP_SECONDARYNAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps ${HADOOP_NAMENODE_INIT_HEAPSIZE} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_SECONDARYNAMENODE_OPTS}" + +# The following applies to multiple commands (fs, dfs, fsck, distcp etc) +export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS" +# On secure datanodes, user to run the datanode as after dropping privileges +export HADOOP_SECURE_DN_USER={{hdfs_user}} + +# Extra ssh options. Empty by default. +export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR" + +# Where log files are stored. $HADOOP_HOME/logs by default. +export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER + +# History server logs +export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER + +# Where log files are stored in the secure data environment. +export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER + +# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default. +# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves + +# host:path where hadoop code should be rsync'd from. Unset by default. +# export HADOOP_MASTER=master:/home/$USER/src/hadoop + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HADOOP_SLAVE_SLEEP=0.1 + +# The directory where pid files are stored. /tmp by default. +export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER +export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER + +# History server pid +export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER + +YARN_RESOURCEMANAGER_OPTS="-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY" + +# A string representing this instance of hadoop. $USER by default. +export HADOOP_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. + +# export HADOOP_NICENESS=10 + +# Use libraries from standard classpath +JAVA_JDBC_LIBS="" +#Add libraries required by mysql connector +for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +#Add libraries required by oracle connector +for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null` +do + JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile +done +#Add libraries required by nodemanager +MAPREDUCE_LIBS={{mapreduce_libs_path}} +export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} + +if [ -d "/usr/lib/tez" ]; then + export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/tez/*:/usr/lib/tez/lib/*:/etc/tez/conf +fi + +# Setting path to hdfs command line +export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} + +#Mostly required for hadoop 2.0 +export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop/lib/native/Linux-amd64-64 + </value> + </property> + +</configuration>