Repository: incubator-slider Updated Branches: refs/heads/develop 7559f3f2e -> 3dc5abf96
SLIDER-478 move agent bindings to yarn registry Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a7220ed1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a7220ed1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a7220ed1 Branch: refs/heads/develop Commit: a7220ed136f04cdbdfd7384b8f785d85ce2ad3ea Parents: 7559f3f Author: Steve Loughran <ste...@apache.org> Authored: Fri Oct 10 17:08:04 2014 -0700 Committer: Steve Loughran <ste...@apache.org> Committed: Fri Oct 10 17:08:04 2014 -0700 ---------------------------------------------------------------------- .../apache/slider/common/SliderXmlConfKeys.java | 5 +- .../slider/core/launch/AbstractLauncher.java | 8 ++ .../providers/AbstractProviderService.java | 4 + .../providers/agent/AgentProviderService.java | 25 ++--- .../server/appmaster/SliderAppMaster.java | 19 ++-- .../YarnRegistryViewForProviders.java | 100 +++++++++++++++--- .../model/mock/MockRegistryOperations.groovy | 101 +++++++++++++++++++ .../agent/TestAgentProviderService.java | 79 +++++++++++---- 8 files changed, 285 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index d82bbe8..cc2a03f 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -19,6 +19,7 @@ package org.apache.slider.common; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.conf.YarnConfiguration; /** @@ -122,12 +123,12 @@ public interface SliderXmlConfKeys { String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750"; - String REGISTRY_PATH = "slider.registry.path"; + String REGISTRY_PATH = RegistryConstants.KEY_REGISTRY_ZK_ROOT; /** * Default value for the registry: {@value} */ - String DEFAULT_REGISTRY_PATH = "/registry"; + String DEFAULT_REGISTRY_PATH = RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT; String REGISTRY_ZK_QUORUM = "slider.zookeeper.quorum"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java index a8d61e3..658a03a 100644 --- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java +++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java @@ -103,10 +103,18 @@ public abstract class AbstractLauncher extends Configured { return env; } + /** + * Get the launch commands. + * @return the live list of commands + */ public List<String> getCommands() { return commands; } + /** + * Get the map of local resources. + * @return the live map of local resources. + */ public Map<String, LocalResource> getLocalResources() { return localResources; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java index 04f6d8c..36ee910 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java @@ -114,6 +114,10 @@ public abstract class AbstractProviderService this.yarnRegistry = yarnRegistry; } + public YarnRegistryViewForProviders getYarnRegistry() { + return yarnRegistry; + } + @Override public AgentRestOperations getAgentRestOperations() { return restOps; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index 058a838..330ffa3 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -19,7 +19,7 @@ package org.apache.slider.providers.agent; import com.google.common.annotations.VisibleForTesting; -import org.apache.curator.utils.ZKPaths; +import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -340,7 +340,7 @@ public class AgentProviderService extends AbstractProviderService implements String agentConf = instanceDefinition.getAppConfOperations(). getGlobalOptions().getOption(AgentKeys.AGENT_CONF, ""); - if (org.apache.commons.lang.StringUtils.isNotEmpty(agentConf)) { + if (SliderUtils.isSet(agentConf)) { LocalResource agentConfRes = fileSystem.createAmResource(fileSystem .getFileSystem().resolvePath(new Path(agentConf)), LocalResourceType.FILE); @@ -400,7 +400,7 @@ public class AgentProviderService extends AbstractProviderService implements operation.add(getZkRegistryPath()); String debugCmd = agentLaunchParameter.getNextLaunchParameter(role); - if (debugCmd != null && debugCmd.length() != 0) { + if (SliderUtils.isSet(debugCmd)) { operation.add(ARG_DEBUG); operation.add(debugCmd); } @@ -417,16 +417,17 @@ public class AgentProviderService extends AbstractProviderService implements getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME))); } - // build the zookeeper registry path + /** + * build the zookeeper registry path. + * + * @return the path the service registered at + * @throws NullPointerException if the service has not yet registered + */ private String getZkRegistryPath() { - String zkRegistryRoot = getConfig().get(REGISTRY_PATH, - DEFAULT_REGISTRY_PATH); - String appType = APP_TYPE; - String zkRegistryPath = ZKPaths.makePath(zkRegistryRoot, appType); - String clusterName = getAmState().getInternalsSnapshot().get( - OptionKeys.APPLICATION_NAME); - zkRegistryPath = ZKPaths.makePath(zkRegistryPath, clusterName); - return zkRegistryPath; + Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound"); + String path = yarnRegistry.getAbsoluteSelfRegistrationPath(); + Preconditions.checkNotNull(path, "Service record path not defined"); + return path; } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 3f52fd8..8725976 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -965,7 +965,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService //Give the provider restricted access to the state, registry setupInitialRegistryPaths(); yarnRegistryOperations = new YarnRegistryViewForProviders( - registryOperations, service_user_name, + registryOperations, + service_user_name, SliderKeys.APP_TYPE, instanceName, appAttemptID); @@ -997,19 +998,21 @@ public class SliderAppMaster extends AbstractSliderLaunchedService agentStatusURI, serviceRecord); - // store for clients + // register the service's entry log.info("Service Record \n{}", serviceRecord); - String sliderServicePath = yarnRegistryOperations.putService(service_user_name, - SliderKeys.APP_TYPE, - instanceName, - serviceRecord, true); + yarnRegistryOperations.registerSelf(serviceRecord, true); + log.info("Registered service under {}; absolute path {}", + yarnRegistryOperations.getSelfRegistrationPath(), + yarnRegistryOperations.getAbsoluteSelfRegistrationPath()); + boolean isFirstAttempt = 1 == appAttemptID.getAttemptId(); // delete the children in case there are any and this is an AM startup. // just to make sure everything underneath is purged if (isFirstAttempt) { - yarnRegistryOperations.deleteChildren(sliderServicePath, true); + yarnRegistryOperations.deleteChildren( + yarnRegistryOperations.getSelfRegistrationPath(), + true); } - yarnRegistryOperations.setSelfRegistration(serviceRecord); // and a shorter lived binding to the app String attempt = appAttemptID.toString(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java index 345bf05..254bf27 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/yarnregistry/YarnRegistryViewForProviders.java @@ -18,8 +18,9 @@ package org.apache.slider.server.services.yarnregistry; +import com.google.common.base.Preconditions; import org.apache.hadoop.fs.PathNotFoundException; -import org.apache.hadoop.registry.client.types.RegistryPathStatus; +import org.apache.hadoop.registry.client.api.RegistryConstants; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.registry.client.api.BindFlags; import org.apache.hadoop.registry.client.api.RegistryOperations; @@ -27,33 +28,55 @@ import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.slider.common.tools.SliderUtils; import java.io.IOException; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join; +/** + * Registry view for providers. This tracks where the service + * is registered, offers access to the record and other things. + */ public class YarnRegistryViewForProviders { private final RegistryOperations registryOperations; private final String user; - private final String sliderServiceclass; + private final String sliderServiceClass; private final String instanceName; private final ApplicationAttemptId applicationAttemptId; + /** + * Record used where the service registered itself. + * Null until the service is registered + */ private ServiceRecord selfRegistration; + /** + * Path where record was registered + * Null until the service is registered + */ + private String selfRegistrationPath; + public YarnRegistryViewForProviders(RegistryOperations registryOperations, String user, - String sliderServiceclass, + String sliderServiceClass, String instanceName, ApplicationAttemptId applicationAttemptId) { + Preconditions.checkArgument(registryOperations != null, + "null registry operations"); + Preconditions.checkArgument(user != null, "null user"); + Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass), + "unset service class"); + Preconditions.checkArgument(SliderUtils.isSet(instanceName), + "instanceName"); + Preconditions.checkArgument(applicationAttemptId != null, + "null applicationAttemptId"); this.registryOperations = registryOperations; this.user = user; - this.sliderServiceclass = sliderServiceclass; + this.sliderServiceClass = sliderServiceClass; this.instanceName = instanceName; this.applicationAttemptId = applicationAttemptId; } @@ -66,8 +89,8 @@ public class YarnRegistryViewForProviders { return user; } - public String getSliderServiceclass() { - return sliderServiceclass; + public String getSliderServiceClass() { + return sliderServiceClass; } public String getInstanceName() { @@ -82,11 +105,36 @@ public class YarnRegistryViewForProviders { return selfRegistration; } - public void setSelfRegistration(ServiceRecord selfRegistration) { + private void setSelfRegistration(ServiceRecord selfRegistration) { this.selfRegistration = selfRegistration; } /** + * Get the path to where the service has registered itself. + * Null until the service is registered + * @return the service registration path. + */ + public String getSelfRegistrationPath() { + return selfRegistrationPath; + } + + /** + * Get the absolute path to where the service has registered itself. + * This includes the base registry path + * Null until the service is registered + * @return the service registration path. + */ + public String getAbsoluteSelfRegistrationPath() { + if (selfRegistrationPath == null) { + return null; + } + String root = registryOperations.getConfig().getTrimmed( + RegistryConstants.KEY_REGISTRY_ZK_ROOT, + RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); + return RegistryPathUtils.join(root, selfRegistrationPath); + } + + /** * Add a component under the slider name/entry * @param componentName component name * @param record record to put @@ -95,7 +143,7 @@ public class YarnRegistryViewForProviders { public void putComponent(String componentName, ServiceRecord record) throws IOException { - putComponent(sliderServiceclass, instanceName, + putComponent(sliderServiceClass, instanceName, componentName, record); } @@ -116,7 +164,7 @@ public class YarnRegistryViewForProviders { registryOperations.mknode(RegistryPathUtils.parentOf(path), true); registryOperations.bind(path, record, BindFlags.OVERWRITE); } - + /** * Add a service under a path, optionally purging any history * @param username user @@ -159,6 +207,34 @@ public class YarnRegistryViewForProviders { return putService(user, serviceClass, serviceName, record, deleteTreeFirst); } + + /** + * Add a service under a path for the current user + * @param serviceClass service class to use under ~user + * @param serviceName name of the service + * @param record service record + * @param deleteTreeFirst perform recursive delete of the path first + * @return the path the service was created at + * @throws IOException + */ + public String registerSelf( + ServiceRecord record, + boolean deleteTreeFirst) throws IOException { + selfRegistrationPath = + putService(user, sliderServiceClass, instanceName, record, deleteTreeFirst); + setSelfRegistration(record); + return selfRegistrationPath; + } + + /** + * Update the self record by pushing out the latest version of the service + * registration record. + * @throws IOException any failure. + */ + public void updateSelf() throws IOException { + putService(user, sliderServiceClass, instanceName, selfRegistration, false); + } + /** * Delete a component * @param componentName component name @@ -166,7 +242,7 @@ public class YarnRegistryViewForProviders { */ public void deleteComponent(String componentName) throws IOException { String path = RegistryUtils.componentPath( - user, sliderServiceclass, instanceName, + user, sliderServiceClass, instanceName, componentName); registryOperations.delete(path, false); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy new file mode 100644 index 0000000..d70fca9 --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRegistryOperations.groovy @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.mock + +import org.apache.hadoop.fs.FileAlreadyExistsException +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException +import org.apache.hadoop.fs.PathNotFoundException +import org.apache.hadoop.registry.client.api.RegistryOperations +import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException +import org.apache.hadoop.registry.client.exceptions.InvalidRecordException +import org.apache.hadoop.registry.client.exceptions.NoRecordException +import org.apache.hadoop.registry.client.types.RegistryPathStatus +import org.apache.hadoop.registry.client.types.ServiceRecord +import org.apache.hadoop.service.AbstractService + +/** + * Simple stub registry for when one is needed for its API, but the operations + * are not actually required + */ +class MockRegistryOperations extends AbstractService implements RegistryOperations{ + + MockRegistryOperations() { + super("mock") + } + + @Override + boolean mknode(String path, boolean createParents) + throws PathNotFoundException, InvalidPathnameException, IOException { + return true + } + + @Override + void bind(String path, ServiceRecord record, int flags) throws + PathNotFoundException, + FileAlreadyExistsException, + InvalidPathnameException, + IOException { + + } + + @Override + ServiceRecord resolve(String path) throws + PathNotFoundException, + NoRecordException, + InvalidRecordException, + IOException { + throw new PathNotFoundException(path); + } + + @Override + RegistryPathStatus stat(String path) + throws PathNotFoundException, InvalidPathnameException, IOException { + throw new PathNotFoundException(path); + } + + @Override + boolean exists(String path) throws IOException { + return false + } + + @Override + List<String> list(String path) + throws PathNotFoundException, InvalidPathnameException, IOException { + throw new PathNotFoundException(path); + } + + @Override + void delete(String path, boolean recursive) throws + PathNotFoundException, + PathIsNotEmptyDirectoryException, + InvalidPathnameException, + IOException { + + } + + @Override + boolean addWriteAccessor(String id, String pass) throws IOException { + return true + } + + @Override + void clearWriteAccessors() { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a7220ed1/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 10b20d0..addfc8a 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -18,11 +18,12 @@ package org.apache.slider.providers.agent; -import com.sun.jersey.spi.container.servlet.WebConfig; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FilterFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -34,6 +35,7 @@ import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; +import org.apache.slider.common.SliderKeys; import org.apache.slider.common.SliderXmlConfKeys; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.core.conf.AggregateConf; @@ -55,6 +57,9 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup; import org.apache.slider.providers.agent.application.metadata.Metainfo; import org.apache.slider.providers.agent.application.metadata.MetainfoParser; import org.apache.slider.providers.agent.application.metadata.PropertyInfo; +import org.apache.slider.server.appmaster.model.mock.MockRegistryOperations; +import org.apache.slider.server.appmaster.model.mock.MockApplicationAttemptId; +import org.apache.slider.server.appmaster.model.mock.MockApplicationId; import org.apache.slider.server.appmaster.model.mock.MockContainerId; import org.apache.slider.server.appmaster.model.mock.MockFileSystem; import org.apache.slider.server.appmaster.model.mock.MockNodeId; @@ -69,6 +74,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse; import org.apache.slider.server.appmaster.web.rest.agent.Register; import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse; import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus; +import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders; import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -88,12 +94,12 @@ import java.util.List; import java.util.Map; import java.util.Set; -import static org.easymock.EasyMock.anyLong; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyCollection; import static org.mockito.Matchers.anyMap; @@ -254,7 +260,7 @@ public class TestAgentProviderService { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); AggregateConf instanceDefinition = new AggregateConf(); @@ -385,6 +391,7 @@ public class TestAgentProviderService { Assert.assertEquals(2, hbr.getResponseId()); } + private AggregateConf prepareConfForAgentStateTests() { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); @@ -420,7 +427,7 @@ public class TestAgentProviderService { expect(container.getPriority()).andReturn(Priority.newInstance(1)); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doReturn(access).when(mockAps).getAmState(); @@ -600,7 +607,7 @@ public class TestAgentProviderService { @Test public void testRoleHostMapping() throws Exception { - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); StateAccessForProviders appState = new ProviderAppState("undefined", null) { @Override public ClusterDescription getClusterStatus() { @@ -651,7 +658,7 @@ public class TestAgentProviderService { public void testComponentSpecificPublishes() throws Exception { InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes()); Metainfo metainfo = new MetainfoParser().parse(metainfo_1); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); doReturn(metainfo).when(mockAps).getMetainfo(); @@ -691,7 +698,7 @@ public class TestAgentProviderService { InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes()); Metainfo metainfo = new MetainfoParser().parse(metainfo_1); Assert.assertNotNull(metainfo.getApplication()); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); HeartBeat hb = new HeartBeat(); ComponentStatus status = new ComponentStatus(); status.setClusterName("test"); @@ -848,7 +855,7 @@ public class TestAgentProviderService { } Assert.assertEquals("Two config dependencies must be found.", found, 2); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doReturn(metainfo).when(mockAps).getMetainfo(); CommandScript script = mockAps.getScriptPathFromMetainfo("HBASE_MASTER"); @@ -877,7 +884,7 @@ public class TestAgentProviderService { String role_hm = "HBASE_MASTER"; String role_hrs = "HBASE_REGIONSERVER"; - AgentProviderService aps1 = new AgentProviderService(); + AgentProviderService aps1 = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps1); doReturn(metainfo).when(mockAps).getMetainfo(); @@ -906,7 +913,11 @@ public class TestAgentProviderService { ConfTree tree = new ConfTree(); tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, "."); - AgentProviderService aps = new AgentProviderService(); + Configuration conf = new Configuration(); + AgentProviderService aps = createAgentProviderService(conf); + YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry(); + assertNotNull(registryViewForProviders); + ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class); AggregateConf instanceDefinition = new AggregateConf(); @@ -921,6 +932,7 @@ public class TestAgentProviderService { String role_hrs = "HBASE_REGIONSERVER"; SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); + ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class); Path generatedConfPath = new Path(".", "test"); MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); @@ -942,9 +954,6 @@ public class TestAgentProviderService { doReturn(new HashMap<String, DefaultConfig>()).when(mockAps). initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class)); - Configuration conf = new Configuration(); - conf.set(SliderXmlConfKeys.REGISTRY_PATH, - SliderXmlConfKeys.DEFAULT_REGISTRY_PATH); try { doReturn(true).when(mockAps).isMaster(anyString()); @@ -1000,7 +1009,7 @@ public class TestAgentProviderService { appComponent, containerTmpDirPath); - mockAps.buildContainerLaunchContext(launcher, + mockAps.buildContainerLaunchContext(launcher2, instanceDefinition, container, role_hrs, @@ -1156,9 +1165,35 @@ public class TestAgentProviderService { anyCollection()); } - @Test - public void testNotifyContainerCompleted() { + protected AgentProviderService createAgentProviderService(Configuration conf) throws + IOException { AgentProviderService aps = new AgentProviderService(); + YarnRegistryViewForProviders registryViewForProviders = + createYarnRegistryViewForProviders(conf); + aps.bindToYarnRegistry(registryViewForProviders); + return aps; + } + + protected YarnRegistryViewForProviders createYarnRegistryViewForProviders( + Configuration conf) throws IOException { + conf.set(SliderXmlConfKeys.REGISTRY_PATH, + SliderXmlConfKeys.DEFAULT_REGISTRY_PATH); + + RegistryOperations registryOperations = new MockRegistryOperations(); + registryOperations.init(conf); + YarnRegistryViewForProviders registryViewForProviders = + new YarnRegistryViewForProviders(registryOperations, + "hbase", + SliderKeys.APP_TYPE, + "hbase1", + new MockApplicationAttemptId(new MockApplicationId(1), 1)); + registryViewForProviders.registerSelf(new ServiceRecord(), true); + return registryViewForProviders; + } + + @Test + public void testNotifyContainerCompleted() throws IOException { + AgentProviderService aps = createAgentProviderService(new Configuration()); AgentProviderService mockAps = Mockito.spy(aps); doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); @@ -1203,7 +1238,7 @@ public class TestAgentProviderService { public void testAddInstallCommand() throws Exception { InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes()); Metainfo metainfo = new MetainfoParser().parse(metainfo_1); - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); HeartBeatResponse hbr = new HeartBeatResponse(); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); @@ -1252,7 +1287,7 @@ public class TestAgentProviderService { @Test public void testAddStartCommand() throws Exception { - AgentProviderService aps = new AgentProviderService(); + AgentProviderService aps = createAgentProviderService(new Configuration()); HeartBeatResponse hbr = new HeartBeatResponse(); StateAccessForProviders access = createNiceMock(StateAccessForProviders.class); @@ -1335,8 +1370,8 @@ public class TestAgentProviderService { } @Test - public void testParameterParsing() { - AgentProviderService aps = new AgentProviderService(); + public void testParameterParsing() throws IOException { + AgentProviderService aps = createAgentProviderService(new Configuration()); AggregateConf aggConf = new AggregateConf(); ConfTreeOperations treeOps = aggConf.getAppConfOperations(); treeOps.getGlobalOptions().put(AgentKeys.SYSTEM_CONFIGS, "core-site,yarn-site, core-site "); @@ -1347,8 +1382,8 @@ public class TestAgentProviderService { } @Test - public void testDereferenceAllConfig() { - AgentProviderService aps = new AgentProviderService(); + public void testDereferenceAllConfig() throws IOException { + AgentProviderService aps = createAgentProviderService(new Configuration()); Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); Map<String, String> cfg1 = new HashMap<String, String>(); cfg1.put("a1", "${@//site/cfg-2/A1}");
