SLIDER-663. Make it easy to develop and deploy application packages that are essentially shell commands (part-I)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/704e8136 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/704e8136 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/704e8136 Branch: refs/heads/feature/package_simplification_II Commit: 704e81363a72444f0d27b6fe3ab43d46b7791231 Parents: 38fed99 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue Mar 17 17:49:49 2015 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue Mar 17 17:49:49 2015 -0700 ---------------------------------------------------------------------- .../python/agent/CustomServiceOrchestrator.py | 56 ++++- .../python/scripts/shell_cmd/basic_installer.py | 55 +++++ .../src/main/python/scripts/shell_cmd/params.py | 29 +++ .../agent/TestCustomServiceOrchestrator.py | 104 ++++++++- .../org/apache/slider/api/InternalKeys.java | 8 + .../org/apache/slider/client/SliderClient.java | 22 +- .../org/apache/slider/common/SliderKeys.java | 12 +- .../AbstractClusterBuildingActionArgs.java | 31 ++- .../slider/common/params/AddonArgsDelegate.java | 54 +++++ .../apache/slider/common/params/Arguments.java | 3 + .../slider/common/tools/CoreFileSystem.java | 27 ++- .../apache/slider/common/tools/SliderUtils.java | 56 ++++- .../slider/core/build/InstanceBuilder.java | 6 + .../core/persist/AppDefinitionPersister.java | 174 +++++++++++++++ .../slider/core/persist/InstancePaths.java | 4 + .../providers/agent/AgentClientProvider.java | 57 +++-- .../slider/providers/agent/AgentKeys.java | 2 + .../providers/agent/AgentProviderService.java | 211 ++++++++++++++----- .../slider/providers/agent/AgentUtils.java | 30 ++- .../providers/agent/ComponentCommandOrder.java | 4 +- .../agent/application/metadata/Application.java | 94 +++++++-- .../application/metadata/CommandOrder.java | 9 +- .../application/metadata/CommandScript.java | 11 +- .../agent/application/metadata/Component.java | 83 ++++++-- .../application/metadata/ComponentCommand.java | 85 ++++++++ .../agent/application/metadata/ConfigFile.java | 8 +- .../agent/application/metadata/Export.java | 9 +- .../agent/application/metadata/ExportGroup.java | 11 +- .../agent/application/metadata/Metainfo.java | 24 ++- .../application/metadata/MetainfoParser.java | 62 +++++- .../agent/application/metadata/OSPackage.java | 9 +- .../agent/application/metadata/OSSpecific.java | 11 +- .../agent/application/metadata/Package.java | 60 ++++++ .../agent/application/metadata/Validate.java | 27 +++ .../web/rest/agent/ExecutionCommand.java | 18 +- .../test_min_pkg/nc_ping_cmd/appConfig.json | 8 + .../test_min_pkg/nc_ping_cmd/metainfo.json | 28 +++ .../test_min_pkg/nc_ping_cmd/resources.json | 17 ++ .../test_min_pkg/sleep_cmd/metainfo.json | 16 ++ .../test_min_pkg/sleep_cmd/resources.json | 17 ++ .../agent/TestAgentClientProvider2.java | 12 +- .../agent/TestAgentProviderService.java | 129 +++++++----- .../metadata/TestMetainfoParser.java | 98 ++++++++- .../funtest/framework/CommandTestBase.groovy | 80 +++++++ .../funtest/lifecycle/AgentMinSleepIT.groovy | 110 ++++++++++ .../funtest/lifecycle/AgentPingSocketIT.groovy | 142 +++++++++++++ 46 files changed, 1907 insertions(+), 216 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py index 119c926..87ce621 100644 --- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py +++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py @@ -76,23 +76,37 @@ class CustomServiceOrchestrator(): override_output_files=True, store_command=False): allocated_ports = {} try: + py_file_list = [] + json_path = None + script_type = command['commandParams']['script_type'] - script = command['commandParams']['script'] - timeout = int(command['commandParams']['command_timeout']) task_id = command['taskId'] command_name = command['roleCommand'] - script_path = self.resolve_script_path(self.base_dir, script, script_type) - script_tuple = (script_path, self.base_dir) - tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir, "structured-out-{0}.json".format(task_id))) - if script_type.upper() != self.SCRIPT_TYPE_PYTHON: - # We don't support anything else yet + if script_type.upper() == self.SCRIPT_TYPE_PYTHON: + script = command['commandParams']['script'] + timeout = int(command['commandParams']['command_timeout']) + script_path = self.resolve_script_path(self.base_dir, script, script_type) + script_tuple = (script_path, self.base_dir) + py_file_list = [script_tuple] + + json_path = self.dump_command_to_json(command, allocated_ports, store_command) + elif script_type.upper() == "SHELL": + timeout = int(command['commandParams']['command_timeout']) + + json_path = self.dump_command_to_json(command, allocated_ports, store_command) + script_path = os.path.realpath(posixpath.join(self.config.getWorkRootPath(), + "infra", "agent", "slider-agent", "scripts", + "shell_cmd", "basic_installer.py")) + script_tuple = (script_path, self.base_dir) + py_file_list = [script_tuple] + else: + # We don't support anything else yet message = "Unknown script type {0}".format(script_type) raise AgentException(message) - json_path = self.dump_command_to_json(command, allocated_ports, store_command) - py_file_list = [script_tuple] + # filter None values filtered_py_file_list = [i for i in py_file_list if i] logger_level = logging.getLevelName(logger.level) @@ -239,6 +253,7 @@ class CustomServiceOrchestrator(): os.unlink(file_path) self.finalize_command(command, store_command, allocated_ports) + self.finalize_exec_command(command) with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT, 0644), 'w') as f: @@ -246,6 +261,7 @@ class CustomServiceOrchestrator(): f.write(content) return file_path + """ patch content ${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath() @@ -291,6 +307,28 @@ class CustomServiceOrchestrator(): pass """ + configurations/global/exec_cmd should be resolved based on the rest of the config + {$conf:@//site/global/xmx_val} ==> command['configurations']['global']['xmx_val'] + """ + def finalize_exec_command(self, command): + variable_format = "{{$conf:@//site/{0}/{1}}}" + if 'configurations' in command: + if 'global' in command['configurations'] and 'exec_cmd' in command['configurations']['global']: + exec_cmd = command['configurations']['global']['exec_cmd'] + for key in command['configurations']: + if len(command['configurations'][key]) > 0: + for k, value in command['configurations'][key].items(): + replaced_key = variable_format.format(key, k) + exec_cmd = exec_cmd.replace(replaced_key, value) + pass + pass + pass + command['configurations']['global']['exec_cmd'] = exec_cmd + pass + pass + + + """ All unallocated ports should be set to 0 Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}" or "${component.ALLOCATED_PORT}{DEFAULT_port}" http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py new file mode 100644 index 0000000..561fd6c --- /dev/null +++ b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import sys +from resource_management import * + +class BasicInstaller(Script): + def install(self, env): + self.install_packages(env) + pass + + def configure(self, env): + pass + + def start(self, env): + import params + env.set_params(params) + self.configure(env) + process_cmd = format("{cmd}") + + Execute(process_cmd, + logoutput=False, + wait_for_finish=False, + pid_file=params.pid_file, + poll_after = 5 + ) + + def stop(self, env): + import params + env.set_params(params) + + def status(self, env): + import params + env.set_params(params) + check_process_status(params.pid_file) + +if __name__ == "__main__": + BasicInstaller().execute() http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/params.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/main/python/scripts/shell_cmd/params.py b/slider-agent/src/main/python/scripts/shell_cmd/params.py new file mode 100644 index 0000000..9051f43 --- /dev/null +++ b/slider-agent/src/main/python/scripts/shell_cmd/params.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management import * + +# server configurations +config = Script.get_config() + +app_root = config['configurations']['global']['app_root'] +pid_file = config['configurations']['global']['pid_file'] +exec_cmd = config['configurations']['global']['exec_cmd'] +cmd = format(exec_cmd) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py index 30c8d7a..aaef3f9 100644 --- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py +++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py @@ -23,15 +23,13 @@ import pprint from unittest import TestCase import unittest -import threading import tempfile -import time +import posixpath import logging -from threading import Thread from PythonExecutor import PythonExecutor from CustomServiceOrchestrator import CustomServiceOrchestrator -from mock.mock import MagicMock, patch +from mock.mock import MagicMock, patch, call import StringIO import sys from socket import socket @@ -438,7 +436,6 @@ class TestCustomServiceOrchestrator(TestCase): self.assertEqual.__self__.maxDiff = None self.assertEqual(ret['exitcode'], 0) self.assertTrue(run_file_mock.called) - self.assertEqual(orchestrator.stored_command, command) ret = orchestrator.requestComponentStatus(command_get) self.assertEqual(ret['configurations'], expected) @@ -447,6 +444,103 @@ class TestCustomServiceOrchestrator(TestCase): self.assertEqual(ret['configurations'], expected_specific) pass + @patch("hostname.public_hostname") + @patch("os.path.isfile") + @patch("os.unlink") + @patch.object(PythonExecutor, "run_file") + def test_runCommand_with_shell_config(self, + run_file_mock, + unlink_mock, + isfile_mock, + hostname_mock): + hostname_mock.return_value = "test.hst" + isfile_mock.return_value = True + command = { + 'role': 'MEMCACHED', + 'componentName': 'MEMCACHED', + 'hostLevelParams': { + 'jdk_location': 'some_location' + }, + 'commandParams': { + 'script_type': 'SHELL', + 'command_timeout': '600' + }, + 'configurations': { + "memcached-site": { + "memcached.log": "${AGENT_LOG_ROOT}", + "memcached.number": "10485760"}, + "memcached-log4j": {"a": "b"} + }, + 'taskId': '3', + 'roleCommand': 'INSTALL', + 'commandType': 'EXECUTION_COMMAND', + 'commandId': '1-1' + } + + command_get = { + 'roleCommand': 'GET_CONFIG', + 'commandType': 'STATUS_COMMAND' + } + + command_get_specific = { + 'roleCommand': 'GET_CONFIG', + 'commandType': 'STATUS_COMMAND', + 'commandParams': { + 'config_type': 'memcached-site' + } + } + + tempdir = tempfile.gettempdir() + config = MagicMock() + config.get.return_value = "something" + config.getResolvedPath.return_value = tempdir + config.getWorkRootPath.return_value = tempdir + config.getLogPath.return_value = tempdir + + dummy_controller = MagicMock() + orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger) + # normal run case + run_file_mock.return_value = { + 'stdout': 'sss', + 'stderr': 'eee', + 'exitcode': 0, + } + + expected = { + 'memcached-site': { + 'memcached.log': tempdir, 'memcached.number': '10485760'}, + 'memcached-log4j': {'a': 'b'}} + + expected_specific = { + 'memcached-site': { + 'memcached.log': tempdir, 'memcached.number': '10485760'}, + } + + ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True) + self.assertEqual.__self__.maxDiff = None + self.assertEqual(ret['exitcode'], 0) + self.assertTrue(run_file_mock.called) + + ret = orchestrator.requestComponentStatus(command_get) + self.assertEqual(ret['configurations'], expected) + + ret = orchestrator.requestComponentStatus(command_get_specific) + self.assertEqual(ret['configurations'], expected_specific) + + script_path = os.path.realpath(posixpath.join(tempdir, + "infra", "agent", "slider-agent", "scripts", + "shell_cmd", "basic_installer.py")) + run_file_mock.assert_has_calls([call( + script_path, + ['INSTALL', os.path.realpath(posixpath.join(tempdir, 'command-3.json')), + os.path.realpath(posixpath.join(tempdir, 'package'))], + 'out.txt', 'err.txt', 600, + os.path.realpath(posixpath.join(tempdir, 'structured-out-3.json')), + 'INFO', True, + [('PYTHONPATH', ":".join([os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent', 'jinja2')), + os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent'))]))])]) + pass + @patch.object(CustomServiceOrchestrator, "runCommand") def test_requestComponentStatus(self, runCommand_mock): status_command = { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java index b360fbe..fd17ed1 100644 --- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java +++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java @@ -69,6 +69,14 @@ public interface InternalKeys { */ String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path"; /** + * where the app def is stored + */ + String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path"; + /** + * where addons for the app are stored + */ + String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path"; + /** * Time in milliseconds to wait after forking any in-AM * process before attempting to start up the containers: {@value} * http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 84ed5a4..c23177a 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -20,6 +20,7 @@ package org.apache.slider.client; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.io.Files; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -133,6 +134,7 @@ import org.apache.slider.core.launch.LaunchedApplication; import org.apache.slider.core.launch.RunningApplication; import org.apache.slider.core.launch.SerializedApplicationReport; import org.apache.slider.core.main.RunService; +import org.apache.slider.core.persist.AppDefinitionPersister; import org.apache.slider.core.persist.ApplicationReportSerDeser; import org.apache.slider.core.persist.ConfPersister; import org.apache.slider.core.persist.LockAcquireFailedException; @@ -179,7 +181,6 @@ import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1052,7 +1053,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe JSONObject config = null; if(clientInfo.clientConfig != null) { try { - byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath()); + byte[] encoded = Files.toByteArray(clientInfo.clientConfig); config = new JSONObject(new String(encoded, Charset.defaultCharset())); }catch(JSONException jsonEx) { log.error("Unable to read supplied config", jsonEx); @@ -1304,6 +1305,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } + AppDefinitionPersister appDefinitionPersister = new AppDefinitionPersister(sliderFileSystem); + appDefinitionPersister.processSuppliedDefinitions(clustername, buildInfo, appConf); + //get the command line options ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree(); ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree(); @@ -1395,19 +1399,25 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe // make any substitutions needed at this stage replaceTokens(appConf.getConfTree(), getUsername(), clustername); - // providers to validate what there is - AggregateConf instanceDescription = builder.getInstanceDescription(); - validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); - validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); + // TODO: Refactor the validation code and persistence code try { persistInstanceDefinition(overwrite, appconfdir, builder); + appDefinitionPersister.persistPackages(); + } catch (LockAcquireFailedException e) { log.warn("Failed to get a Lock on {} : {}", builder, e); throw new BadClusterStateException("Failed to save " + clustername + ": " + e); } + + // providers to validate what there is + // TODO: Validation should be done before persistence + AggregateConf instanceDescription = builder.getInstanceDescription(); + validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem); + validateInstanceDefinition(provider, instanceDescription, sliderFileSystem); } + protected void persistInstanceDefinition(boolean overwrite, Path appconfdir, InstanceBuilder builder) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java index 5cf7022..30ff258 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java @@ -168,11 +168,21 @@ public interface SliderKeys extends SliderXmlConfKeys { String TMP_LOGDIR_PREFIX = "/tmp/slider-"; String TMP_DIR_PREFIX = "tmp"; String AM_DIR_PREFIX = "appmaster"; - + + /** + * Store the default app definition, e.g. metainfo file or content of a folder + */ + String APP_DEF_DIR = "appdef"; + /** + * Store additional app defs - co-processors + */ + String ADDONS_DIR = "addons"; + String SLIDER_JAR = "slider.jar"; String JCOMMANDER_JAR = "jcommander.jar"; String GSON_JAR = "gson.jar"; String AGENT_TAR = "slider-agent.tar.gz"; + String DEFAULT_APP_PKG = "appPkg.zip"; String DEFAULT_JVM_HEAP = "256M"; int DEFAULT_YARN_MEMORY = 256; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java index 56e01c3..f2e3c61 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java @@ -74,13 +74,6 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Provider of the specific cluster application") public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE; - /* - - @Parameter(names = {ARG_PACKAGE}, - description = "URI to a slider package", - converter = URIArgumentConverter.class ) - public URI packageURI; - */ @Parameter(names = {ARG_PACKAGE}, description = "URI to a slider package") public String packageURI; @@ -93,6 +86,14 @@ public abstract class AbstractClusterBuildingActionArgs extends description = "Template application configuration") public File template; + @Parameter(names = {ARG_METAINFO}, + description = "Application meta info") + public File appMetaInfo; + + @Parameter(names = {ARG_APPDEF}, + description = "Application def (folder or a zip package)") + public File appDef; + @Parameter(names = {ARG_QUEUE}, description = "Queue to submit the application") public String queue; @@ -100,6 +101,9 @@ public abstract class AbstractClusterBuildingActionArgs extends @ParametersDelegate public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate(); + @ParametersDelegate + public AddonArgsDelegate addonDelegate = new AddonArgsDelegate(); + @ParametersDelegate public AppAndResouceOptionArgsDelegate optionsDelegate = @@ -152,6 +156,19 @@ public abstract class AbstractClusterBuildingActionArgs extends return componentDelegate.getComponentMap(); } + @VisibleForTesting + public List<String> getAddonTuples() { + return addonDelegate.getAddonTuples(); + } + + /** + * Get the list of addons (may be empty, but never null) + */ + public Map<String, String> getAddonMap() throws + BadCommandArgumentsException { + return addonDelegate.getAddonMap(); + } + public Path getConfdir() { return confdir; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java new file mode 100644 index 0000000..65ebc4b --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class AddonArgsDelegate extends AbstractArgsDelegate { + + /** + * This is a listing of addon packages + */ + @Parameter(names = {ARG_ADDON}, + arity = 2, + description = "--addon <name> <folder or package>", + splitter = DontSplitArguments.class) + public List<String> addonTuples = new ArrayList<String>(0); + + + /** + * Get the list of addons (may be empty, but never null) + * + * @return map of named addons + * + * @throws BadCommandArgumentsException parse problem + */ + public Map<String, String> getAddonMap() throws BadCommandArgumentsException { + return convertTupleListToMap("addon", addonTuples); + } + + public List<String> getAddonTuples() { + return addonTuples; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java index 9164edc..ea393d5 100644 --- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java +++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java @@ -113,6 +113,9 @@ public interface Arguments { String ARG_ZKPATH = "--zkpath"; String ARG_ZKPORT = "--zkport"; String ARG_CONFIG = "--config"; + String ARG_METAINFO = "--metainfo"; + String ARG_ADDON = "--addon"; + String ARG_APPDEF = "--appdef"; /** http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java index e835312..3df73df 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -122,6 +122,31 @@ public class CoreFileSystem { } /** + * Build up the path string for app def folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAppDefDirPath(String clustername) { + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.APP_DEF_DIR); + } + + /** + * Build up the path string for addon folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAddonDirPath(String clustername, String addonId) { + Preconditions.checkNotNull(addonId); + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId); + } + + /** * Build up the path string for package install location -no attempt to * create the directory is made * @@ -568,7 +593,7 @@ public class CoreFileSystem { /** * List all application instances persisted for this user, giving the - * patha. The instance name is the last element in the path + * path. The instance name is the last element in the path * @return a possibly empty map of application instance names to paths */ public Map<String, Path> listPersistentInstances() throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index ce52b89..1767d9c 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; @@ -107,6 +108,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; /** * These are slider-specific Util methods @@ -1732,7 +1735,58 @@ public final class SliderUtils { } /** - * This wrapps ApplicationReports and generates a string version + * Given a source folder create zipped file + * + * @param srcFolder + * @param zipFile + * + * @throws IOException + */ + public static void zipFolder(File srcFolder, File zipFile) throws IOException { + log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath()); + List<String> files = new ArrayList<>(); + generateFileList(files, srcFolder, srcFolder, true); + + byte[] buffer = new byte[1024]; + + try (FileOutputStream fos = new FileOutputStream(zipFile)) { + try (ZipOutputStream zos = new ZipOutputStream(fos)) { + + for (String file : files) { + ZipEntry ze = new ZipEntry(file); + zos.putNextEntry(ze); + try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) { + int len; + while ((len = in.read(buffer)) > 0) { + zos.write(buffer, 0, len); + } + } + } + } + } + } + + + private static void generateFileList(List<String> fileList, File node, File rootFolder, Boolean relative) { + if (node.isFile()) { + String fileFullPath = node.toString(); + if (relative) { + fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, fileFullPath.length())); + } else { + fileList.add(fileFullPath); + } + } + + if (node.isDirectory()) { + String[] subNode = node.list(); + for (String filename : subNode) { + generateFileList(fileList, new File(node, filename), rootFolder, relative); + } + } + } + + /** + * This wraps ApplicationReports and generates a string version * iff the toString() operator is invoked */ public static class OnDemandReportStringifier { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java index 6812613..0a8dcdb 100644 --- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java +++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java @@ -49,6 +49,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; +import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; +import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE; import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR; import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR; @@ -142,6 +144,10 @@ public class InstanceBuilder { instancePaths.generatedConfPath.toUri()); internalOps.set(INTERNAL_DATA_DIR_PATH, instancePaths.dataPath.toUri()); + internalOps.set(INTERNAL_APPDEF_DIR_PATH, + instancePaths.appDefPath.toUri()); + internalOps.set(INTERNAL_ADDONS_DIR_PATH, + instancePaths.addonsPath.toUri()); internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java new file mode 100644 index 0000000..8f0f7b0 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.persist; + +import com.google.common.io.Files; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.providers.agent.AgentKeys; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Class to prepare and persist app and add-on definitions. + * + * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit + * inclusion of application package in the config. + * + * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a + * temporary folder - upload to HDFS + * + * This class keeps track of all the required operations and allows them to be invoked by build operation + */ +public class AppDefinitionPersister { + private static final Logger log = + LoggerFactory.getLogger(AppDefinitionPersister.class); + + private final SliderFileSystem sliderFileSystem; + private List<AppDefinition> appDefinitions; + + public AppDefinitionPersister(SliderFileSystem sliderFileSystem) { + this.sliderFileSystem = sliderFileSystem; + appDefinitions = new ArrayList<>(); + } + + + /** + * Process the application package or folder by copying it to the cluster path + * + * @param appDefinition details of application package + * + * @throws BadConfigException + * @throws IOException + */ + private void persistDefinitionPackageOrFolder(AppDefinition appDefinition) + throws BadConfigException, IOException { + if (!appDefinition.appDefPkgOrFolder.canRead()) { + throw new BadConfigException("Pkg/Folder cannot be accessed - " + + appDefinition.appDefPkgOrFolder.getAbsolutePath()); + } + + File src = appDefinition.appDefPkgOrFolder; + String targetName = appDefinition.appDefPkgOrFolder.getName(); + + if (appDefinition.appDefPkgOrFolder.isDirectory()) { + log.info("Processing app package/folder {} for {}", + appDefinition.appDefPkgOrFolder.getAbsolutePath(), + appDefinition.pkgName); + File tmpDir = Files.createTempDir(); + File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName); + SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile); + + src = zipFile; + targetName = appDefinition.pkgName; + } + + sliderFileSystem.getFileSystem().copyFromLocalFile( + false, + false, + new Path(src.toURI()), + new Path(appDefinition.targetFolderInFs, targetName)); + } + + public void persistPackages() throws BadConfigException, IOException { + for (AppDefinition appDefinition : appDefinitions) { + persistDefinitionPackageOrFolder(appDefinition); + } + } + + public void processSuppliedDefinitions(String clustername, + AbstractClusterBuildingActionArgs buildInfo, + ConfTreeOperations appConf) + throws BadConfigException, IOException, BadCommandArgumentsException { + // if metainfo is provided add to the app instance + if (buildInfo.appMetaInfo != null) { + File tempDir = Files.createTempDir(); + File pkgSrcDir = new File(tempDir, "default"); + pkgSrcDir.mkdirs(); + Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json")); + + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); + log.info("Using default app def path {}", appDirPath.toString()); + + appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, SliderKeys.DEFAULT_APP_PKG)); + Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); + log.info("Setting app package to {}.", appDefPath); + } + + if (buildInfo.appDef != null) { + Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername); + appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG)); + Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG); + appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath); + log.info("Setting app package to {}.", appDefPath); + } + + if (buildInfo.addonDelegate.getAddonMap().size() > 0) { + List<String> addons = new ArrayList<String>(); + Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap(); + for (String key : addonMap.keySet()) { + Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key); + String addonPkgName = "addon_" + key + ".zip"; + appDefinitions.add(new AppDefinition(addonPath, buildInfo.appDef, addonPkgName)); + String addOnKey = AgentKeys.ADDON_PREFIX + key; + Path addonPkgPath = new Path(addonPath, addonPkgName); + log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath); + appConf.getGlobalOptions().set(addOnKey, addonPkgPath); + } + + String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS); + if (SliderUtils.isUnset(existingList)) { + existingList = ""; + } + appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ",")); + } + } + + + // Helper class to hold details for the app and addon packages + class AppDefinition { + // The target folder where the package will be stored + Path targetFolderInFs; + // The on disk location of the app def package or folder + File appDefPkgOrFolder; + // Package name + String pkgName; + + public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) { + this.targetFolderInFs = targetFolderInFs; + this.appDefPkgOrFolder = appDefPkgOrFolder; + this.pkgName = pkgName; + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java index df1206d..3505ac3 100644 --- a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java +++ b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java @@ -34,6 +34,8 @@ public class InstancePaths { public final Path dataPath; public final Path tmpPath; public final Path tmpPathAM; + public final Path appDefPath; + public final Path addonsPath; public InstancePaths(Path instanceDir) { this.instanceDir = instanceDir; @@ -45,6 +47,8 @@ public class InstancePaths { dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME); tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX); tmpPathAM = new Path(tmpPath, SliderKeys.AM_DIR_PREFIX); + appDefPath = new Path(tmpPath, SliderKeys.APP_DEF_DIR); + addonsPath = new Path(tmpPath, SliderKeys.ADDONS_DIR); } @Override http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java index 9da269d..2e981c6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java @@ -152,6 +152,7 @@ public class AgentClientProvider extends AbstractClientProvider } catch (BadConfigException bce) { throw new BadConfigException("Application definition must be provided. " + bce.getMessage()); } + String appDef = instanceDefinition.getAppConfOperations(). getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); log.info("Validating app definition {}", appDef); @@ -170,7 +171,7 @@ public class AgentClientProvider extends AbstractClientProvider metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef); } catch (IOException ioe) { // Ignore missing metainfo file for now - log.info("Missing metainfo.xml {}", ioe.getMessage()); + log.info("Missing metainfo {}", ioe.getMessage()); } } @@ -215,19 +216,31 @@ public class AgentClientProvider extends AbstractClientProvider "Component %s is not a member of application.", name); } - MapOperations componentConfig = resources.getMandatoryComponent(name); - int count = - componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); - int definedMinCount = componentDef.getMinInstanceCountInt(); - int definedMaxCount = componentDef.getMaxInstanceCountInt(); - if (count < definedMinCount || count > definedMaxCount) { - throw new BadConfigException("Component %s, %s value %d out of range. " - + "Expected minimum is %d and maximum is %d", - name, - ResourceKeys.COMPONENT_INSTANCES, - count, - definedMinCount, - definedMaxCount); + // ensure that intance count is 0 for client components + if ("CLIENT".equals(componentDef.getCategory())) { + MapOperations componentConfig = resources.getMandatoryComponent(name); + int count = + componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); + if (count > 0) { + throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated." + + " Use \"slider client install ...\" command instead.", + name); + } + } else { + MapOperations componentConfig = resources.getMandatoryComponent(name); + int count = + componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES); + int definedMinCount = componentDef.getMinInstanceCountInt(); + int definedMaxCount = componentDef.getMaxInstanceCountInt(); + if (count < definedMinCount || count > definedMaxCount) { + throw new BadConfigException("Component %s, %s value %d out of range. " + + "Expected minimum is %d and maximum is %d", + name, + ResourceKeys.COMPONENT_INSTANCES, + count, + definedMinCount, + definedMaxCount); + } } } } @@ -327,7 +340,19 @@ public class AgentClientProvider extends AbstractClientProvider while (offset < size) { offset += zipInputStream.read(content, offset, size - offset); } - metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content)); + metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content)); + } + } else if ("metainfo.json".equals(zipEntry.getName())) { + int size = (int) zipEntry.getSize(); + if (size != -1) { + log.info("Reading {} of size {}", zipEntry.getName(), + zipEntry.getSize()); + byte[] content = new byte[size]; + int offset = 0; + while (offset < size) { + offset += zipInputStream.read(content, offset, size - offset); + } + metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content)); } } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) { int size = (int) zipEntry.getSize(); @@ -362,7 +387,7 @@ public class AgentClientProvider extends AbstractClientProvider } if (metaInfo == null) { - throw new SliderException("Not a valid app package. Could not read metainfo.xml."); + throw new SliderException("Not a valid app package. Could not read metainfo."); } expandAgentTar(agentPkgDir); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java index 2323f97..963dadc 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java @@ -67,6 +67,8 @@ public interface AgentKeys { String AGENT_MAIN_SCRIPT = "agent/main.py"; String APP_DEF = "application.def"; + String ADDON_PREFIX = "application.addon."; + String ADDONS = "application.addons"; String AGENT_VERSION = "agent.version"; String AGENT_CONF = "agent.conf"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java index bdf168e..4eadf13 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.security.alias.CredentialProviderFactory; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.Container; @@ -38,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.slider.api.ClusterDescription; -import org.apache.slider.api.ClusterDescriptionKeys; import org.apache.slider.api.ClusterNode; import org.apache.slider.api.InternalKeys; import org.apache.slider.api.OptionKeys; @@ -69,6 +67,7 @@ import org.apache.slider.providers.ProviderUtils; import org.apache.slider.providers.agent.application.metadata.Application; import org.apache.slider.providers.agent.application.metadata.CommandScript; import org.apache.slider.providers.agent.application.metadata.Component; +import org.apache.slider.providers.agent.application.metadata.ComponentCommand; import org.apache.slider.providers.agent.application.metadata.ComponentExport; import org.apache.slider.providers.agent.application.metadata.ConfigFile; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; @@ -77,6 +76,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup; import org.apache.slider.providers.agent.application.metadata.Metainfo; import org.apache.slider.providers.agent.application.metadata.OSPackage; import org.apache.slider.providers.agent.application.metadata.OSSpecific; +import org.apache.slider.providers.agent.application.metadata.Package; import org.apache.slider.providers.agent.application.metadata.PropertyInfo; import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss; import org.apache.slider.server.appmaster.actions.RegisterComponentInstance; @@ -157,7 +157,7 @@ public class AgentProviderService extends AbstractProviderService implements private int heartbeatMonitorInterval = 0; private AgentClientProvider clientProvider; private AtomicInteger taskId = new AtomicInteger(0); - private volatile Metainfo metainfo = null; + private volatile Metainfo metaInfo = null; private Map<String, DefaultConfig> defaultConfigs = null; private ComponentCommandOrder commandOrder = null; private HeartbeatMonitor monitor; @@ -233,7 +233,7 @@ public class AgentProviderService extends AbstractProviderService implements Set<String> names = resources.getComponentNames(); names.remove(SliderKeys.COMPONENT_AM); for (String name : names) { - Component componentDef = getMetainfo().getApplicationComponent(name); + Component componentDef = getMetaInfo().getApplicationComponent(name); if (componentDef == null) { throw new BadConfigException( "Component %s is not a member of application.", name); @@ -262,21 +262,20 @@ public class AgentProviderService extends AbstractProviderService implements String appDef = instanceDefinition.getAppConfOperations() .getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF); - if (metainfo == null) { + if (metaInfo == null) { synchronized (syncLock) { - if (metainfo == null) { + if (metaInfo == null) { readAndSetHeartbeatMonitoringInterval(instanceDefinition); initializeAgentDebugCommands(instanceDefinition); - metainfo = getApplicationMetainfo(fileSystem, appDef); - if (metainfo == null || metainfo.getApplication() == null) { + metaInfo = getApplicationMetainfo(fileSystem, appDef); + if (metaInfo == null || metaInfo.getApplication() == null) { log.error("metainfo.xml is unavailable or malformed at {}.", appDef); throw new SliderException( "metainfo.xml is required in app package."); } - commandOrder = new ComponentCommandOrder(metainfo.getApplication() - .getCommandOrder()); - defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo); + commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders()); + defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo); monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval()); monitor.start(); } @@ -717,15 +716,20 @@ public class AgentProviderService extends AbstractProviderService implements StateAccessForProviders accessor = getAmState(); CommandScript cmdScript = getScriptPathFromMetainfo(roleName); + List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands(); - if (cmdScript == null || cmdScript.getScript() == null) { + if ((cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) { log.error("role.script is unavailable for {}. Commands will not be sent.", roleName); return response; } - String scriptPath = cmdScript.getScript(); - long timeout = cmdScript.getTimeout(); + String scriptPath = null; + long timeout = 600L; + if(cmdScript != null) { + scriptPath = cmdScript.getScript(); + timeout = cmdScript.getTimeout(); + } if (timeout == 0L) { timeout = 600L; @@ -778,15 +782,50 @@ public class AgentProviderService extends AbstractProviderService implements if (Command.NOP != command) { if (command == Command.INSTALL) { log.info("Installing {} on {}.", roleName, containerId); - addInstallCommand(roleName, containerId, response, scriptPath, timeout); + if (scriptPath != null) { + addInstallCommand(roleName, containerId, response, scriptPath, null, timeout); + } else { + // commands + ComponentCommand installCmd = null; + for (ComponentCommand compCmd : commands) { + if (compCmd.getName().equals("INSTALL")) { + installCmd = compCmd; + } + } + addInstallCommand(roleName, containerId, response, null, installCmd, timeout); + } componentStatus.commandIssued(command); } else if (command == Command.START) { // check against dependencies boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values()); if (canExecute) { log.info("Starting {} on {}.", roleName, containerId); - addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName)); - componentStatus.commandIssued(command); + if (scriptPath != null) { + addStartCommand(roleName, + containerId, + response, + scriptPath, + null, + null, + timeout, + isMarkedAutoRestart(roleName)); + componentStatus.commandIssued(command); + } else { + ComponentCommand startCmd = null; + for (ComponentCommand compCmd : commands) { + if (compCmd.getName().equals("START")) { + startCmd = compCmd; + } + } + ComponentCommand stopCmd = null; + for (ComponentCommand compCmd : commands) { + if (compCmd.getName().equals("STOP")) { + stopCmd = compCmd; + } + } + addStartCommand(roleName, containerId, response, null, startCmd, stopCmd, timeout, false); + componentStatus.commandIssued(command); + } } else { log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId); } @@ -858,7 +897,6 @@ public class AgentProviderService extends AbstractProviderService implements } } - // component specific publishes processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName); processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName); @@ -1012,8 +1050,8 @@ public class AgentProviderService extends AbstractProviderService implements } @VisibleForTesting - protected Metainfo getMetainfo() { - return this.metainfo; + protected Metainfo getMetaInfo() { + return this.metaInfo; } @VisibleForTesting @@ -1023,7 +1061,7 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException { + String appDef) throws IOException, BadConfigException { return AgentUtils.getApplicationMetainfo(fileSystem, appDef); } @@ -1045,7 +1083,7 @@ public class AgentProviderService extends AbstractProviderService implements */ protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem, String appDef, Metainfo metainfo) throws IOException { - Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>(); + Map<String, DefaultConfig> defaultConfigMap = new HashMap<>(); if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) { for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) { DefaultConfig config = null; @@ -1226,7 +1264,7 @@ public class AgentProviderService extends AbstractProviderService implements log.info("Status report: {}", status.toString()); if (status.getConfigs() != null) { - Application application = getMetainfo().getApplication(); + Application application = getMetaInfo().getApplication(); if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) { // If no Master can explicitly publish then publish if its a master @@ -1370,10 +1408,10 @@ public class AgentProviderService extends AbstractProviderService implements String hostNamePattern = "${THIS_HOST}"; Map<String, String> toPublish = new HashMap<String, String>(); - Application application = getMetainfo().getApplication(); + Application application = getMetaInfo().getApplication(); for (Component component : application.getComponents()) { if (component.getName().equals(componentName)) { - if (!component.getComponentExports().isEmpty()) { + if (component.getComponentExports().size() > 0) { for (ComponentExport export : component.getComponentExports()) { String templateToExport = export.getValue(); @@ -1399,7 +1437,7 @@ public class AgentProviderService extends AbstractProviderService implements } } - if (!toPublish.isEmpty()) { + if (toPublish.size() > 0) { Map<String, String> perContainerData = null; if (!getComponentInstanceData().containsKey(containerId)) { perContainerData = new ConcurrentHashMap<String, String>(); @@ -1420,8 +1458,8 @@ public class AgentProviderService extends AbstractProviderService implements String portVarFormat = "${site.%s}"; String hostNamePattern = "${" + compName + "_HOST}"; - List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups(); - Component component = getMetainfo().getApplicationComponent(compName); + List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups(); + Component component = getMetaInfo().getApplicationComponent(compName); if (component != null && SliderUtils.isSet(component.getCompExports()) && SliderUtils.isNotEmpty(appExportGroups)) { @@ -1523,7 +1561,7 @@ public class AgentProviderService extends AbstractProviderService implements * @return the component entry or null for no match */ protected Component getApplicationComponent(String roleName) { - return getMetainfo().getApplicationComponent(roleName); + return getMetaInfo().getApplicationComponent(roleName); } /** @@ -1583,7 +1621,7 @@ public class AgentProviderService extends AbstractProviderService implements protected boolean isMarkedAutoRestart(String roleName) { Component component = getApplicationComponent(roleName); if (component != null) { - return component.getRequiresAutoRestart(); + return component.getAutoStartOnFailureBoolean(); } return false; } @@ -1595,7 +1633,7 @@ public class AgentProviderService extends AbstractProviderService implements */ protected boolean canAnyMasterPublishConfig() { if (canAnyMasterPublish == null) { - Application application = getMetainfo().getApplication(); + Application application = getMetaInfo().getApplication(); if (application == null) { log.error("Malformed app definition: Expect application as root element in the metainfo.xml"); } else { @@ -1637,6 +1675,7 @@ public class AgentProviderService extends AbstractProviderService implements String containerId, HeartBeatResponse response, String scriptPath, + ComponentCommand compCmd, long timeout) throws SliderException { assert getAmState().isApplicationLive(); @@ -1651,7 +1690,7 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setComponentName(componentName); cmd.setRole(componentName); Map<String, String> hostLevelParams = new TreeMap<String, String>(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME)); + hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); hostLevelParams.put(PACKAGE_LIST, getPackageList()); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); @@ -1659,9 +1698,20 @@ public class AgentProviderService extends AbstractProviderService implements Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName); cmd.setConfigurations(configurations); - cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); + if(SliderUtils.isSet(scriptPath)) { + cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); + } else { + // assume it to be default shell command + ComponentCommand effectiveCommand = compCmd; + if(effectiveCommand == null) { + effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL"); + } + cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false)); + configurations.get("global").put("exec_cmd", effectiveCommand.getExec()); + } cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME)); + response.addExecutionCommand(cmd); } @@ -1670,12 +1720,19 @@ public class AgentProviderService extends AbstractProviderService implements String pkgListFormatString = "[%s]"; List<String> packages = new ArrayList(); if (application != null) { - List<OSSpecific> osSpecifics = application.getOSSpecifics(); - if (osSpecifics != null && osSpecifics.size() > 0) { - for (OSSpecific osSpecific : osSpecifics) { - if (osSpecific.getOsType().equals("any")) { - for (OSPackage osPackage : osSpecific.getPackages()) { - packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName())); + if (application.getPackages().size() > 0) { + List<Package> appPackages = application.getPackages(); + for (Package appPackage : appPackages) { + packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName())); + } + } else { + List<OSSpecific> osSpecifics = application.getOSSpecifics(); + if (osSpecifics != null && osSpecifics.size() > 0) { + for (OSSpecific osSpecific : osSpecifics) { + if (osSpecific.getOsType().equals("any")) { + for (OSPackage osPackage : osSpecific.getPackages()) { + packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName())); + } } } } @@ -1690,7 +1747,7 @@ public class AgentProviderService extends AbstractProviderService implements } private String getPackageList() { - return getPackageListFromApplication(getMetainfo().getApplication()); + return getPackageListFromApplication(getMetaInfo().getApplication()); } private void prepareExecutionCommand(ExecutionCommand cmd) { @@ -1698,8 +1755,8 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setCommandId(cmd.getTaskId() + "-1"); } - private Map<String, String> setCommandParameters(String scriptPath, long timeout, boolean recordConfig) { - Map<String, String> cmdParams = new TreeMap<String, String>(); + private Map<String, String> commandParametersSet(String scriptPath, long timeout, boolean recordConfig) { + Map<String, String> cmdParams = new TreeMap<>(); cmdParams.put("service_package_folder", "${AGENT_WORK_ROOT}/work/app/definition/package"); cmdParams.put("script", scriptPath); @@ -1710,6 +1767,18 @@ public class AgentProviderService extends AbstractProviderService implements return cmdParams; } + private Map<String, String> commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) { + Map<String, String> cmdParams = new TreeMap<>(); + cmdParams.put("service_package_folder", + "${AGENT_WORK_ROOT}/work/app/definition/package"); + cmdParams.put("command", compCmd.getExec()); + cmdParams.put("schema_version", "2.0"); + cmdParams.put("command_timeout", Long.toString(timeout)); + cmdParams.put("script_type", compCmd.getType()); + cmdParams.put("record_config", Boolean.toString(recordConfig)); + return cmdParams; + } + @VisibleForTesting protected void addStatusCommand(String componentName, String containerId, @@ -1730,11 +1799,11 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setRoleCommand(StatusCommand.STATUS_COMMAND); Map<String, String> hostLevelParams = new TreeMap<String, String>(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME)); + hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); - cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false)); + cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false)); Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName); @@ -1767,7 +1836,9 @@ public class AgentProviderService extends AbstractProviderService implements @VisibleForTesting protected void addStartCommand(String componentName, String containerId, HeartBeatResponse response, - String scriptPath, long timeout, boolean isMarkedAutoRestart) + String scriptPath, ComponentCommand startCommand, + ComponentCommand stopCommand, + long timeout, boolean isMarkedAutoRestart) throws SliderException { assert getAmState().isApplicationLive(); @@ -1784,20 +1855,28 @@ public class AgentProviderService extends AbstractProviderService implements cmd.setServiceName(clusterName); cmd.setComponentName(componentName); cmd.setRole(componentName); - Map<String, String> hostLevelParams = new TreeMap<String, String>(); - hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME)); + Map<String, String> hostLevelParams = new TreeMap<>(); + hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir())); hostLevelParams.put(CONTAINER_ID, containerId); cmd.setHostLevelParams(hostLevelParams); - Map<String, String> roleParams = new TreeMap<String, String>(); + Map<String, String> roleParams = new TreeMap<>(); cmd.setRoleParams(roleParams); cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart)); - cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true)); - Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName); - cmd.setConfigurations(configurations); + + if(SliderUtils.isSet(scriptPath)) { + cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true)); + } else { + if(startCommand == null) { + throw new SliderException("Expected START command not found for component " + componentName); + } + cmd.setCommandParams(commandParametersSet(startCommand, timeout, true)); + configurations.get("global").put("exec_cmd", startCommand.getExec()); + } + response.addExecutionCommand(cmd); // With start command, the corresponding command for graceful stop needs to @@ -1815,16 +1894,25 @@ public class AgentProviderService extends AbstractProviderService implements cmdStop.setRole(componentName); Map<String, String> hostLevelParamsStop = new TreeMap<String, String>(); hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions() - .getMandatoryOption(JAVA_HOME)); + .getOption(JAVA_HOME, "")); hostLevelParamsStop.put(CONTAINER_ID, containerId); cmdStop.setHostLevelParams(hostLevelParamsStop); Map<String, String> roleParamsStop = new TreeMap<String, String>(); cmdStop.setRoleParams(roleParamsStop); cmdStop.getRoleParams().put("auto_restart", - Boolean.toString(isMarkedAutoRestart)); + Boolean.toString(isMarkedAutoRestart)); + + if(SliderUtils.isSet(scriptPath)) { + cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true)); + } else { + if(stopCommand == null) { + stopCommand = ComponentCommand.getDefaultComponentCommand("STOP"); + } + cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true)); + configurations.get("global").put("exec_cmd", startCommand.getExec()); + } - cmdStop.setCommandParams(setCommandParameters(scriptPath, timeout, true)); Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations( appConf, containerId, componentName); @@ -1832,6 +1920,19 @@ public class AgentProviderService extends AbstractProviderService implements response.addExecutionCommand(cmdStop); } + protected static String getJDKDir() { + File javaHome = new File(System.getProperty("java.home")).getParentFile(); + File jdkDirectory = null; + if (javaHome.getName().contains("jdk")) { + jdkDirectory = javaHome; + } + if (jdkDirectory != null) { + return jdkDirectory.getAbsolutePath(); + } else { + return ""; + } + } + protected Map<String, String> getAllocatedPorts() { return getAllocatedPorts(SHARED_PORT_TAG); } @@ -1938,7 +2039,7 @@ public class AgentProviderService extends AbstractProviderService implements List<String> configList = new ArrayList<String>(); configList.add(GLOBAL_CONFIG_TAG); - List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles(); + List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles(); for (ConfigFile configFile : configFiles) { log.info("Expecting config type {}.", configFile.getDictionaryName()); configList.add(configFile.getDictionaryName()); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java index 1d61c15..6fe1161 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java @@ -20,6 +20,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadConfigException; import org.apache.slider.providers.agent.application.metadata.DefaultConfig; import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser; import org.apache.slider.providers.agent.application.metadata.Metainfo; @@ -28,7 +29,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileNotFoundException; -import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -39,20 +39,30 @@ public class AgentUtils { private static final Logger log = LoggerFactory.getLogger(AgentUtils.class); public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem, - String appDef) throws IOException { + String appDef) throws IOException, BadConfigException { log.info("Reading metainfo at {}", appDef); FileSystem fs = fileSystem.getFileSystem(); Path appPath = new Path(appDef); - InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream( - fs, appPath, "metainfo.xml"); - if (metainfoStream == null) { - log.error("metainfo.xml is unavailable at {}.", appDef); - throw new FileNotFoundException("metainfo.xml is required in app package. " + - appPath); - } - Metainfo metainfo = new MetainfoParser().parse(metainfoStream); + Metainfo metainfo = null; + MetainfoParser metainfoParser = new MetainfoParser(); + InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream( + fs, appPath, "metainfo.json"); + if (metainfoJsonStream == null) { + InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream( + fs, appPath, "metainfo.xml"); + if (metainfoXMLStream != null) { + metainfo = metainfoParser.fromXmlStream(metainfoXMLStream); + } + } else { + metainfo = metainfoParser.fromJsonStream(metainfoJsonStream); + } + if (metainfo == null) { + log.error("metainfo is unavailable at {}.", appDef); + throw new FileNotFoundException("metainfo.xml/json is required in app package. " + + appPath); + } return metainfo; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java index f4ace5f..194d6ff 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java @@ -49,13 +49,13 @@ public class ComponentCommandOrder { if (requiredStates.size() > 0) { Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command); if (compDep == null) { - compDep = new HashMap<String, List<ComponentState>>(); + compDep = new HashMap<>(); dependencies.put(componentCmd.command, compDep); } List<ComponentState> requirements = compDep.get(componentCmd.componentName); if (requirements == null) { - requirements = new ArrayList<ComponentState>(); + requirements = new ArrayList<>(); compDep.put(componentCmd.componentName, requirements); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java index bc43d4b..cd5555f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java @@ -16,29 +16,31 @@ */ package org.apache.slider.providers.agent.application.metadata; +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.SliderException; +import org.codehaus.jackson.annotate.JsonProperty; + import java.util.ArrayList; import java.util.List; /** * Application type defined in the metainfo */ -public class Application { +public class Application implements Validate { String name; String comment; String version; String exportedConfigs; - List<Component> components; - List<ExportGroup> exportGroups; - List<OSSpecific> osSpecifics; - List<CommandOrder> commandOrders; - List<ConfigFile> configFiles; + List<Component> components = new ArrayList<>(); + List<ExportGroup> exportGroups = new ArrayList<>(); + List<OSSpecific> osSpecifics = new ArrayList<>(); + List<CommandOrder> commandOrders = new ArrayList<>(); + List<ConfigFile> configFiles = new ArrayList<>(); + List<Package> packages = new ArrayList<>(); public Application() { - exportGroups = new ArrayList<ExportGroup>(); - components = new ArrayList<Component>(); - osSpecifics = new ArrayList<OSSpecific>(); - commandOrders = new ArrayList<CommandOrder>(); - configFiles = new ArrayList<ConfigFile>(); } public String getName() { @@ -73,18 +75,20 @@ public class Application { this.exportedConfigs = exportedConfigs; } - public List<ConfigFile> getConfigFiles() { - return configFiles; - } - public void addConfigFile(ConfigFile configFile) { this.configFiles.add(configFile); } + @JsonProperty("configFiles") + public List<ConfigFile> getConfigFiles() { + return configFiles; + } + public void addComponent(Component component) { components.add(component); } + @JsonProperty("components") public List<Component> getComponents() { return components; } @@ -93,6 +97,7 @@ public class Application { exportGroups.add(exportGroup); } + @JsonProperty("exportGroups") public List<ExportGroup> getExportGroups() { return exportGroups; } @@ -101,6 +106,7 @@ public class Application { osSpecifics.add(osSpecific); } + @JsonIgnore public List<OSSpecific> getOSSpecifics() { return osSpecifics; } @@ -109,10 +115,16 @@ public class Application { commandOrders.add(commandOrder); } - public List<CommandOrder> getCommandOrder() { + @JsonProperty("commandOrders") + public List<CommandOrder> getCommandOrders() { return commandOrders; } + @JsonProperty("packages") + public List<Package> getPackages() { + return packages; + } + @Override public String toString() { final StringBuilder sb = @@ -128,4 +140,54 @@ public class Application { sb.append('}'); return sb.toString(); } + + public void validate(String version) throws SliderException { + if(SliderUtils.isUnset(version)) { + throw new BadCommandArgumentsException("schema version cannot be null"); + } + + Metainfo.checkNonNull(getName(), "name", "application"); + + Metainfo.checkNonNull(getVersion(), "version", "application"); + + if(getComponents().size() == 0) { + throw new SliderException("application must contain at least one component"); + } + + if(version.equals(Metainfo.VERSION_TWO_ZERO)) { + if(getPackages().size() > 0) { + throw new SliderException("packages is not supported in version " + version); + } + } + + if(version.equals(Metainfo.VERSION_TWO_ONE)) { + if(getOSSpecifics().size() > 0) { + throw new SliderException("osSpecifics is not supported in version " + version); + } + } + + for(CommandOrder co : getCommandOrders()) { + co.validate(version); + } + + for(Component comp : getComponents()) { + comp.validate(version); + } + + for(ConfigFile cf : getConfigFiles()) { + cf.validate(version); + } + + for(ExportGroup eg : getExportGroups()) { + eg.validate(version); + } + + for(Package pkg : getPackages()) { + pkg.validate(version); + } + + for(OSSpecific os : getOSSpecifics()) { + os.validate(version); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java index 825a104..40d8cc6 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java @@ -16,10 +16,12 @@ */ package org.apache.slider.providers.agent.application.metadata; +import org.apache.slider.core.exceptions.SliderException; + /** * */ -public class CommandOrder { +public class CommandOrder implements Validate { String command; String requires; @@ -51,4 +53,9 @@ public class CommandOrder { sb.append('}'); return sb.toString(); } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getCommand(), "command", "package"); + Metainfo.checkNonNull(getRequires(), "requires", "package"); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java index 612322f..9915ba1 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java @@ -16,10 +16,12 @@ */ package org.apache.slider.providers.agent.application.metadata; +import org.apache.slider.core.exceptions.SliderException; + /** - * + * CommandScript that implements all component commands */ -public class CommandScript { +public class CommandScript implements Validate { String script; String scriptType; long timeout; @@ -62,4 +64,9 @@ public class CommandScript { sb.append('}'); return sb.toString(); } + + public void validate(String version) throws SliderException { + Metainfo.checkNonNull(getScript(), "script", "commandScript"); + Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript"); + } }