Repository: ambari Updated Branches: refs/heads/trunk 46b4a797e -> efb4b8778
AMBARI-14922 - Atlas Integration: Support Atlas / Falcon integration Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efb4b877 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efb4b877 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efb4b877 Branch: refs/heads/trunk Commit: efb4b877850ca5d56153afe278d48a459b0f07b3 Parents: 46b4a79 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Thu Feb 25 09:50:37 2016 -0500 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Thu Feb 25 09:50:37 2016 -0500 ---------------------------------------------------------------------- .../configuration/application-properties.xml | 35 ++++++++++++++++++++ .../FALCON/0.5.0.2.1/package/scripts/falcon.py | 15 +++++++++ .../0.5.0.2.1/package/scripts/params_linux.py | 29 +++++++++++++++- .../configuration/falcon-startup.properties.xml | 10 +----- 4 files changed, 79 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml index ed9f1bc..a6deda1 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml @@ -176,4 +176,39 @@ <value>21443</value> <description></description> </property> + <property> + <name>atlas.http.authentication.type</name> + <value>simple</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.synchronous</name> + <value>false</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.numRetries</name> + <value>3</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.minThreads</name> + <value>5</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.maxThreads</name> + <value>5</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.keepAliveTime</name> + <value>10</value> + <description></description> + </property> + <property> + <name>atlas.hook.falcon.queueSize</name> + <value>1000</value> + <description></description> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py index 81ea39e..ca967b0 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py @@ -27,6 +27,7 @@ from resource_management.core.resources.service import Service from resource_management.core.resources.service import ServiceConfig from resource_management.core.resources.system import Directory from resource_management.core.resources.system import File +from resource_management.core.resources.system import Link from resource_management.libraries.script import Script from resource_management.libraries.resources import PropertiesFile from resource_management.libraries.functions import format @@ -106,6 +107,11 @@ def falcon(type, action = None, upgrade_type=None): create_parents = True, cd_access = "a") + if params.has_atlas: + Link(params.falcon_conf_dir + "/application.properties", + to = params.atlas_conf_dir + "/application.properties" + ) + if type == 'server': if action == 'config': if params.store_uri[0:4] == "hdfs": @@ -176,6 +182,15 @@ def falcon(type, action = None, upgrade_type=None): path = params.hadoop_bin_dir, environment=environment_dictionary) + if params.has_atlas: + atlas_falcon_hook_dir = params.atlas_home_dir + "/hook/falcon" + src_files = os.listdir(atlas_falcon_hook_dir) + for file_name in src_files: + atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name) + falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name) + if (os.path.isfile(atlas_falcon_hook_file_name)): + Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name) + if action == 'stop': Execute(format('{falcon_home}/bin/falcon-stop'), user = params.falcon_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py index 6b37451..74ac010 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py @@ -25,6 +25,7 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script +import os config = Script.get_config() @@ -59,6 +60,8 @@ else: falcon_webapp_dir = '/var/lib/falcon/webapp' falcon_home = '/usr/lib/falcon' +falcon_webinf_lib = falcon_home + "/server/webapp/falcon/WEB-INF/lib" + hadoop_conf_dir = status_params.hadoop_conf_dir falcon_conf_dir = status_params.falcon_conf_dir oozie_user = config['configurations']['oozie-env']['oozie_user'] @@ -106,7 +109,31 @@ supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr'] local_data_mirroring_dir = "/usr/hdp/current/falcon-server/data-mirroring" dfs_data_mirroring_dir = "/apps/data-mirroring" - +atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', []) +has_atlas = len(atlas_hosts) > 0 + +if has_atlas: + atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf' + atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server' + + application_services = "org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.service.LogCleanupService,\ + org.apache.falcon.metadata.MetadataMappingService,\ + org.apache.falcon.atlas.service.AtlasService" +else: + application_services = "org.apache.falcon.security.AuthenticationInitializationService,\ + org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ + org.apache.falcon.service.ProcessSubscriberService,\ + org.apache.falcon.entity.store.ConfigurationStore,\ + org.apache.falcon.rerun.service.RetryService,\ + org.apache.falcon.rerun.service.LateRunService,\ + org.apache.falcon.service.LogCleanupService,\ + org.apache.falcon.metadata.MetadataMappingService" hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml index 4e4ec67..5d4fa1e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml @@ -27,15 +27,7 @@ </property> <property> <name>*.application.services</name> - <value>org.apache.falcon.security.AuthenticationInitializationService,\ - org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ - org.apache.falcon.service.ProcessSubscriberService,\ - org.apache.falcon.entity.store.ConfigurationStore,\ - org.apache.falcon.rerun.service.RetryService,\ - org.apache.falcon.rerun.service.LateRunService,\ - org.apache.falcon.service.LogCleanupService,\ - org.apache.falcon.metadata.MetadataMappingService - </value> + <value>{{application_services}}</value> <description>Falcon Services</description> </property> <property>