Repository: ambari Updated Branches: refs/heads/trunk 7b539281d -> 6f9ef6a9e
AMBARI-13807. Refactor NN HA step3 controller (onechiporenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6f9ef6a9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6f9ef6a9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6f9ef6a9 Branch: refs/heads/trunk Commit: 6f9ef6a9e9a9382786e8ce4e0be75d41de7c90c5 Parents: 7b53928 Author: Oleg Nechiporenko <onechipore...@apache.org> Authored: Tue Nov 10 13:34:30 2015 +0200 Committer: Oleg Nechiporenko <onechipore...@apache.org> Committed: Tue Nov 10 13:34:30 2015 +0200 ---------------------------------------------------------------------- .../nameNode/step3_controller.js | 143 +++-- ambari-web/app/initialize.js | 6 +- ambari-web/app/utils.js | 27 + .../app/utils/configs/config_initializer.js | 414 ++++++--------- .../utils/configs/config_initializer_class.js | 147 ++++++ .../utils/configs/nn_ha_config_initializer.js | 515 +++++++++++++++++++ .../nameNode/step3_controller_test.js | 185 ++++++- .../utils/configs/config_initializer_test.js | 10 +- 8 files changed, 1091 insertions(+), 356 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/controllers/main/admin/highAvailability/nameNode/step3_controller.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/controllers/main/admin/highAvailability/nameNode/step3_controller.js b/ambari-web/app/controllers/main/admin/highAvailability/nameNode/step3_controller.js index 3cb7be7..63f67c1 100644 --- a/ambari-web/app/controllers/main/admin/highAvailability/nameNode/step3_controller.js +++ b/ambari-web/app/controllers/main/admin/highAvailability/nameNode/step3_controller.js @@ -16,8 +16,20 @@ * limitations under the License. */ +/** + * @typedef {object} nnHaConfigDependencies + * @property {string} namespaceId + * @property {object} serverConfigs + * @property {string|number} nnHttpPort + * @property {string|number} nnHttpsPort + * @property {string|number} nnRpcPort + * @property {string|number} zkClientPort + */ + var App = require('app'); +require('utils/configs/nn_ha_config_initializer'); + App.HighAvailabilityWizardStep3Controller = Em.Controller.extend({ name: "highAvailabilityWizardStep3Controller", selectedService: null, @@ -39,7 +51,7 @@ App.HighAvailabilityWizardStep3Controller = Em.Controller.extend({ clearStep: function () { this.get('stepConfigs').clear(); - this.serverConfigData = {}; + this.set('serverConfigData', {}); }, loadStep: function () { @@ -103,90 +115,58 @@ App.HighAvailabilityWizardStep3Controller = Em.Controller.extend({ this.set('isLoaded', true); }, + /** + * Generate set of data used to correctly initialize config values and names + * + * @returns {nnHaConfigDependencies} + * @private + * @method _prepareDependencies + */ + _prepareDependencies: function () { + var ret = {}; + var configsFromServer = this.get('serverConfigData.items'); + ret.namespaceId = this.get('content.nameServiceId'); + ret.serverConfigs = configsFromServer; + var hdfsConfigs = configsFromServer.findProperty('type','hdfs-site').properties; + var zkConfigs = configsFromServer.findProperty('type','zoo.cfg').properties; - tweakServiceConfigs: function(configs) { - var nameServiceId = this.get('content.nameServiceId'); - var nameServiceConfig = configs.findProperty('name','dfs.nameservices'); - this.setConfigInitialValue(nameServiceConfig,nameServiceId); - var defaultFsConfig = configs.findProperty('name','fs.defaultFS'); - this.setConfigInitialValue(defaultFsConfig, "hdfs://" + nameServiceId); - this.tweakServiceConfigNames(configs,nameServiceId); - this.tweakServiceConfigValues(configs,nameServiceId); + var dfsHttpA = hdfsConfigs['dfs.namenode.http-address']; + ret.nnHttpPort = dfsHttpA ? dfsHttpA.split(':')[1] : 50070; + + var dfsHttpsA = hdfsConfigs['dfs.namenode.https-address']; + ret.nnHttpsPort = dfsHttpsA ? dfsHttpsA.split(':')[1] : 50470; + + var dfsRpcA = hdfsConfigs['dfs.namenode.rpc-address']; + ret.nnRpcPort = dfsRpcA ? dfsRpcA.split(':')[1] : 8020; + + ret.zkClientPort = zkConfigs['clientPort'] ? zkConfigs['clientPort'] : 2181; + + return ret; }, - tweakServiceConfigNames: function(configs,nameServiceId) { - var regex = new RegExp("\\$\\{dfs.nameservices\\}","g"); - configs.forEach(function(config) { - if (config.name.contains("${dfs.nameservices}")) { - config.name = config.name.replace(regex,nameServiceId); - config.displayName = config.displayName.replace(regex,nameServiceId); - } - }, this); + /** + * Generate set of data with information about cluster topology + * Used in the configs' initialization process + * + * @returns {extendedTopologyLocalDB} + * @private + * @method _prepareLocalDB + */ + _prepareLocalDB: function () { + var localDB = this.get('content').getProperties(['masterComponentHosts', 'slaveComponentHosts', 'hosts']); + localDB.installedServices = App.Service.find().mapProperty('serviceName'); + return localDB; }, - tweakServiceConfigValues: function(configs,nameServiceId) { - var - value = "", - currentNameNodeHost = this.get('content.masterComponentHosts').filterProperty('component', 'NAMENODE').findProperty('isInstalled', true).hostName, - newNameNodeHost = this.get('content.masterComponentHosts').filterProperty('component', 'NAMENODE').findProperty('isInstalled', false).hostName, - journalNodeHosts = this.get('content.masterComponentHosts').filterProperty('component', 'JOURNALNODE').mapProperty('hostName'), - zooKeeperHosts = this.get('content.masterComponentHosts').filterProperty('component', 'ZOOKEEPER_SERVER').mapProperty('hostName'); - - var nnHttpPort = 50070; - if (this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.http-address']) - nnHttpPort = this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.http-address'].split(':')[1]; - var nnHttpsPort = 50470; - if (this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.https-address']) - nnHttpsPort = this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.https-address'].split(':')[1]; - var nnRpcPort = 8020; - if (this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.rpc-address']) - nnRpcPort = this.get('serverConfigData').items.findProperty('type','hdfs-site').properties['dfs.namenode.rpc-address'].split(':')[1]; - var zkClientPort = 2181; - if (this.get('serverConfigData').items.findProperty('type','zoo.cfg').properties['clientPort']) - zkClientPort = this.get('serverConfigData').items.findProperty('type','zoo.cfg').properties['clientPort']; + tweakServiceConfigs: function(configs) { + var localDB = this._prepareLocalDB(); + var dependencies = this._prepareDependencies(); - var config = configs.findProperty('name','dfs.namenode.rpc-address.' + nameServiceId + '.nn1'); - this.setConfigInitialValue(config,currentNameNodeHost + ':' + nnRpcPort); - config = configs.findProperty('name','dfs.namenode.rpc-address.' + nameServiceId + '.nn2'); - this.setConfigInitialValue(config,newNameNodeHost + ':8020'); - config = configs.findProperty('name','dfs.namenode.http-address.' + nameServiceId + '.nn1'); - this.setConfigInitialValue(config,currentNameNodeHost + ':' + nnHttpPort); - config = configs.findProperty('name','dfs.namenode.http-address.' + nameServiceId + '.nn2'); - this.setConfigInitialValue(config,newNameNodeHost + ':50070'); - config = configs.findProperty('name','dfs.namenode.https-address.' + nameServiceId + '.nn1'); - this.setConfigInitialValue(config,currentNameNodeHost + ':' + nnHttpsPort); - config = configs.findProperty('name','dfs.namenode.https-address.' + nameServiceId + '.nn2'); - this.setConfigInitialValue(config,newNameNodeHost + ':50470'); - config = configs.findProperty('name','dfs.namenode.shared.edits.dir'); - this.setConfigInitialValue(config,'qjournal://' + journalNodeHosts[0] + ':8485;' + journalNodeHosts[1] + ':8485;' + journalNodeHosts[2] + ':8485/' + nameServiceId); - config = configs.findProperty('name','ha.zookeeper.quorum'); - this.setConfigInitialValue(config,zooKeeperHosts[0] + ':' + zkClientPort + ',' + zooKeeperHosts[1] + ':' + zkClientPort + ',' + zooKeeperHosts[2] + ':'+ zkClientPort ); - if (App.Service.find().someProperty('serviceName', 'HBASE')) { - config = configs.filterProperty('filename', 'hbase-site').findProperty('name','hbase.rootdir'); - value = this.get('serverConfigData.items').findProperty('type', 'hbase-site').properties['hbase.rootdir'].replace(/\/\/[^\/]*/, '//' + nameServiceId); - this.setConfigInitialValue(config,value); - } - if (App.Service.find().someProperty('serviceName', 'AMBARI_METRICS')) { - config = configs.filterProperty('filename', 'ams-hbase-site').findProperty('name','hbase.rootdir'); - value = this.get('serverConfigData.items').findProperty('type', 'ams-hbase-site').properties['hbase.rootdir']; - value = (value == "hdfs://" + currentNameNodeHost) ? "hdfs://" + nameServiceId : value; - config.isVisible = config.value != value ; - this.setConfigInitialValue(config,value); - } - config = configs.findProperty('name','instance.volumes'); - config2 = configs.findProperty('name','instance.volumes.replacements'); - if (App.Service.find().someProperty('serviceName', 'ACCUMULO')) { - var oldValue = this.get('serverConfigData.items').findProperty('type', 'accumulo-site').properties['instance.volumes']; - value = oldValue.replace(/\/\/[^\/]*/, '//' + nameServiceId); - var replacements = oldValue + " " + value; - this.setConfigInitialValue(config,value); - this.setConfigInitialValue(config2,replacements) - } - config = configs.findProperty('name','dfs.journalnode.edits.dir'); - if (App.get('isHadoopWindowsStack') && App.Service.find().someProperty('serviceName', 'HDFS')) { - value = this.get('serverConfigData.items').findProperty('type', 'hdfs-site').properties['dfs.journalnode.edits.dir']; - this.setConfigInitialValue(config, value); - } + configs.forEach(function (config) { + App.NnHaConfigInitializer.initialValue(config, localDB, dependencies); + }); + + return configs; }, /** @@ -207,11 +187,6 @@ App.HighAvailabilityWizardStep3Controller = Em.Controller.extend({ return configs; }, - setConfigInitialValue: function(config,value) { - config.value = value; - config.recommendedValue = value; - }, - renderServiceConfigs: function (_serviceConfig) { var serviceConfig = App.ServiceConfig.create({ serviceName: _serviceConfig.serviceName, http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/initialize.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/initialize.js b/ambari-web/app/initialize.js index afcc42c..a749adf 100644 --- a/ambari-web/app/initialize.js +++ b/ambari-web/app/initialize.js @@ -30,11 +30,7 @@ try { require('messages'); } -require('utils/base64'); -require('utils/db'); -require('utils/helper'); -require('utils/config'); -require('utils/configs/config_initializer'); +require('utils'); require('mixins'); require('models'); require('controllers'); http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/utils.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils.js b/ambari-web/app/utils.js new file mode 100644 index 0000000..812bd14 --- /dev/null +++ b/ambari-web/app/utils.js @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +// load needed utils here + +require('utils/base64'); +require('utils/db'); +require('utils/helper'); +require('utils/config'); +require('utils/configs/config_initializer'); +require('utils/configs/nn_ha_config_initializer'); http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/utils/configs/config_initializer.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/config_initializer.js b/ambari-web/app/utils/configs/config_initializer.js index 34b2a54..05efdab 100644 --- a/ambari-web/app/utils/configs/config_initializer.js +++ b/ambari-web/app/utils/configs/config_initializer.js @@ -17,6 +17,7 @@ */ var App = require('app'); +require('utils/configs/config_initializer_class'); /** * Regexp for host with port ('hostName:1234') @@ -40,48 +41,6 @@ var hostWithPrefix = ":\/\/" + hostWithPort; var winRegex = /^([a-z]):\\?$/; /** - * Map with initializers types - * Doesn't contain unique initializes, only common are included - * Key: id - * Value: object with method-name (prefer to start method-name with '_init' or '_initAs') - * Each method here is called with arguments equal to <code>initialValue</code>-call args - * Initializer-settings are added as last argument - * - * @type {object} - */ -var initializerTypes = { - host_with_component: { - method: '_initAsHostWithComponent' - }, - hosts_with_components: { - method: '_initAsHostsWithComponents' - }, - zookeeper_based: { - method: '_initAsZookeeperServersList' - }, - single_mountpoint: { - method: '_initAsSingleMountPoint' - }, - multiple_mountpoints: { - method: '_initAsMultipleMountPoints' - } -}; - -/** - * Map for methods used as value-modifiers for configProperties with values as mount point(s) - * Used if mount point is win-like (@see winRegex) - * Key: id - * Value: method-name - * - * @type {{default: string, file: string, slashes: string}} - */ -var winReplacersMap = { - default: '_defaultWinReplace', - file: '_winReplaceWithFile', - slashes: '_defaultWinReplaceWithAdditionalSlashes' -}; - -/** * Settings for <code>host_with_component</code>-initializer * Used for configs with value equal to hostName that has <code>component</code> * Value may be modified with if <code>withModifier</code> is true (it is by default) @@ -180,10 +139,9 @@ function getComponentsHostsConfig(components, asArray) { */ function getSingleMountPointConfig(components, winReplacer) { winReplacer = winReplacer || 'default'; - Em.assert('Invalid `winReplacer` selected - `' + winReplacer + '`! See winReplacersMap for available replacers.', !!winReplacersMap[winReplacer]); return { components: Em.makeArray(components), - winReplacer: winReplacersMap[winReplacer], + winReplacer: winReplacer, type: 'single_mountpoint' } } @@ -199,124 +157,19 @@ function getSingleMountPointConfig(components, winReplacer) { */ function getMultipleMountPointsConfig(components, winReplacer) { winReplacer = winReplacer || 'default'; - Em.assert('Invalid `winReplacer` selected - `' + winReplacer + '`! See winReplacersMap for available replacers.', !!winReplacersMap[winReplacer]); return { components: Em.makeArray(components), - winReplacer: winReplacersMap[winReplacer], + winReplacer: winReplacer, type: 'multiple_mountpoints' } } /** - * Map with configurations for config initializers - * It's used only for initializers which are common for some configs (if not - use <code>uniqueInitializers</code>-map) - * Key {string} configProperty-name - * Value {object|object[]} settings for initializer - * - * @type {object} - */ -var initializers = { - 'dfs.namenode.rpc-address': getSimpleComponentConfig('NAMENODE'), - 'dfs.http.address': getSimpleComponentConfig('NAMENODE'), - 'dfs.namenode.http-address': getSimpleComponentConfig('NAMENODE'), - 'dfs.https.address': getSimpleComponentConfig('NAMENODE'), - 'dfs.namenode.https-address': getSimpleComponentConfig('NAMENODE'), - 'dfs.secondary.http.address': getSimpleComponentConfig('SECONDARY_NAMENODE'), - 'dfs.namenode.secondary.http-address': getSimpleComponentConfig('SECONDARY_NAMENODE'), - 'yarn.resourcemanager.hostname': getSimpleComponentConfig('RESOURCEMANAGER', false), - 'yarn.resourcemanager.resource-tracker.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.resourcemanager.webapp.https.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.resourcemanager.webapp.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.resourcemanager.scheduler.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.resourcemanager.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.resourcemanager.admin.address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'yarn.timeline-service.webapp.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), - 'yarn.timeline-service.webapp.https.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), - 'yarn.timeline-service.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), - 'mapred.job.tracker': getSimpleComponentConfig('JOBTRACKER'), - 'mapred.job.tracker.http.address': getSimpleComponentConfig('JOBTRACKER'), - 'mapreduce.history.server.http.address': getSimpleComponentConfig('HISTORYSERVER'), - 'hive_hostname': getSimpleComponentConfig('HIVE_SERVER', false), - 'oozie_hostname': getSimpleComponentConfig('OOZIE_SERVER', false), - 'oozie.base.url': getComponentConfigWithAffixes('OOZIE_SERVER', '://'), - 'hawq_dfs_url': getSimpleComponentConfig('NAMENODE'), - 'hawq_rm_yarn_address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'hawq_rm_yarn_scheduler_address': getSimpleComponentConfig('RESOURCEMANAGER'), - 'fs.default.name': getComponentConfigWithAffixes('NAMENODE', '://'), - 'fs.defaultFS': getComponentConfigWithAffixes('NAMENODE', '://'), - 'hbase.rootdir': getComponentConfigWithAffixes('NAMENODE', '://'), - 'instance.volumes': getComponentConfigWithAffixes('NAMENODE', '://'), - 'yarn.log.server.url': getComponentConfigWithAffixes('HISTORYSERVER', '://'), - 'mapreduce.jobhistory.webapp.address': getSimpleComponentConfig('HISTORYSERVER'), - 'mapreduce.jobhistory.address': getSimpleComponentConfig('HISTORYSERVER'), - 'kafka.ganglia.metrics.host': getSimpleComponentConfig('GANGLIA_SERVER', false), - 'hive_master_hosts': getComponentsHostsConfig(['HIVE_METASTORE', 'HIVE_SERVER']), - 'hadoop_host': getSimpleComponentConfig('NAMENODE', false), - 'nimbus.host': getSimpleComponentConfig('NIMBUS', false), - 'nimbus.seeds': getComponentsHostsConfig('NIMBUS', true), - 'storm.zookeeper.servers': getComponentsHostsConfig('ZOOKEEPER_SERVER', true), - 'hawq_master_address_host': getSimpleComponentConfig('HAWQMASTER', false), - 'hawq_standby_address_host': getSimpleComponentConfig('HAWQSTANDBY', false), - - '*.broker.url': { - type: 'host_with_component', - component: 'FALCON_SERVER', - modifier: { - type: 'regexp', - regex: 'localhost' - } - }, - - 'zookeeper.connect': getZKBasedConfig(), - 'hive.zookeeper.quorum': getZKBasedConfig(), - 'templeton.zookeeper.hosts': getZKBasedConfig(), - 'hadoop.registry.zk.quorum': getZKBasedConfig(), - 'hive.cluster.delegation.token.store.zookeeper.connectString': getZKBasedConfig(), - 'instance.zookeeper.host': getZKBasedConfig(), - - 'dfs.name.dir': getMultipleMountPointsConfig('NAMENODE', 'file'), - 'dfs.namenode.name.dir': getMultipleMountPointsConfig('NAMENODE', 'file'), - 'dfs.data.dir': getMultipleMountPointsConfig('DATANODE', 'file'), - 'dfs.datanode.data.dir': getMultipleMountPointsConfig('DATANODE', 'file'), - 'yarn.nodemanager.local-dirs': getMultipleMountPointsConfig('NODEMANAGER'), - 'yarn.nodemanager.log-dirs': getMultipleMountPointsConfig('NODEMANAGER'), - 'mapred.local.dir': getMultipleMountPointsConfig(['TASKTRACKER', 'NODEMANAGER']), - 'log.dirs': getMultipleMountPointsConfig('KAFKA_BROKER'), - - 'fs.checkpoint.dir': getSingleMountPointConfig('SECONDARY_NAMENODE', 'file'), - 'dfs.namenode.checkpoint.dir': getSingleMountPointConfig('SECONDARY_NAMENODE', 'file'), - 'yarn.timeline-service.leveldb-timeline-store.path': getSingleMountPointConfig('APP_TIMELINE_SERVER'), - 'yarn.timeline-service.leveldb-state-store.path': getSingleMountPointConfig('APP_TIMELINE_SERVER'), - 'dataDir': getSingleMountPointConfig('ZOOKEEPER_SERVER'), - 'oozie_data_dir': getSingleMountPointConfig('OOZIE_SERVER'), - 'storm.local.dir': getSingleMountPointConfig(['NODEMANAGER', 'NIMBUS']), - '*.falcon.graph.storage.directory': getSingleMountPointConfig('FALCON_SERVER'), - '*.falcon.graph.serialize.path': getSingleMountPointConfig('FALCON_SERVER') -}; - -/** - * Map with initializers that are used only for one config (are unique) - * Key: configProperty-name - * Value: method-name - * Every method from this map is called with same arguments as <code>initialValue</code> is (prefer to start method-name with '_init' or '_initAs') - * - * @type {object} - */ -var uniqueInitializers = { - 'hive_database': '_initHiveDatabaseValue', - 'templeton.hive.properties': '_initTempletonHiveProperties', - 'hbase.zookeeper.quorum': '_initHBaseZookeeperQuorum', - 'yarn.resourcemanager.zk-address': '_initYarnRMzkAddress', - 'RANGER_HOST': '_initRangerHost', - 'hive.metastore.uris': '_initHiveMetastoreUris' -}; - -/** * Helper-object used to set initial value for some configs * * Usage: * <pre> - * var configProperty = App.ServiceConfigProperty.create({}); + * var configProperty = Object.create({}); * var localDB = { * hosts: [], * masterComponentHosts: [], @@ -326,46 +179,139 @@ var uniqueInitializers = { * configPropertyHelper.initialValue(configProperty, localDB, dependencies); * </pre> * - * @type {object} + * @type {Em.Object} */ -App.ConfigInitializer = Em.Object.create({ +App.ConfigInitializer = App.ConfigInitializerClass.create({ + + initializers: { + 'dfs.namenode.rpc-address': getSimpleComponentConfig('NAMENODE'), + 'dfs.http.address': getSimpleComponentConfig('NAMENODE'), + 'dfs.namenode.http-address': getSimpleComponentConfig('NAMENODE'), + 'dfs.https.address': getSimpleComponentConfig('NAMENODE'), + 'dfs.namenode.https-address': getSimpleComponentConfig('NAMENODE'), + 'dfs.secondary.http.address': getSimpleComponentConfig('SECONDARY_NAMENODE'), + 'dfs.namenode.secondary.http-address': getSimpleComponentConfig('SECONDARY_NAMENODE'), + 'yarn.resourcemanager.hostname': getSimpleComponentConfig('RESOURCEMANAGER', false), + 'yarn.resourcemanager.resource-tracker.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.resourcemanager.webapp.https.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.resourcemanager.webapp.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.resourcemanager.scheduler.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.resourcemanager.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.resourcemanager.admin.address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'yarn.timeline-service.webapp.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), + 'yarn.timeline-service.webapp.https.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), + 'yarn.timeline-service.address': getSimpleComponentConfig('APP_TIMELINE_SERVER'), + 'mapred.job.tracker': getSimpleComponentConfig('JOBTRACKER'), + 'mapred.job.tracker.http.address': getSimpleComponentConfig('JOBTRACKER'), + 'mapreduce.history.server.http.address': getSimpleComponentConfig('HISTORYSERVER'), + 'hive_hostname': getSimpleComponentConfig('HIVE_SERVER', false), + 'oozie_hostname': getSimpleComponentConfig('OOZIE_SERVER', false), + 'oozie.base.url': getComponentConfigWithAffixes('OOZIE_SERVER', '://'), + 'hawq_dfs_url': getSimpleComponentConfig('NAMENODE'), + 'hawq_rm_yarn_address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'hawq_rm_yarn_scheduler_address': getSimpleComponentConfig('RESOURCEMANAGER'), + 'fs.default.name': getComponentConfigWithAffixes('NAMENODE', '://'), + 'fs.defaultFS': getComponentConfigWithAffixes('NAMENODE', '://'), + 'hbase.rootdir': getComponentConfigWithAffixes('NAMENODE', '://'), + 'instance.volumes': getComponentConfigWithAffixes('NAMENODE', '://'), + 'yarn.log.server.url': getComponentConfigWithAffixes('HISTORYSERVER', '://'), + 'mapreduce.jobhistory.webapp.address': getSimpleComponentConfig('HISTORYSERVER'), + 'mapreduce.jobhistory.address': getSimpleComponentConfig('HISTORYSERVER'), + 'kafka.ganglia.metrics.host': getSimpleComponentConfig('GANGLIA_SERVER', false), + 'hive_master_hosts': getComponentsHostsConfig(['HIVE_METASTORE', 'HIVE_SERVER']), + 'hadoop_host': getSimpleComponentConfig('NAMENODE', false), + 'nimbus.host': getSimpleComponentConfig('NIMBUS', false), + 'nimbus.seeds': getComponentsHostsConfig('NIMBUS', true), + 'storm.zookeeper.servers': getComponentsHostsConfig('ZOOKEEPER_SERVER', true), + 'hawq_master_address_host': getSimpleComponentConfig('HAWQMASTER', false), + 'hawq_standby_address_host': getSimpleComponentConfig('HAWQSTANDBY', false), + + '*.broker.url': { + type: 'host_with_component', + component: 'FALCON_SERVER', + modifier: { + type: 'regexp', + regex: 'localhost' + } + }, + + 'zookeeper.connect': getZKBasedConfig(), + 'hive.zookeeper.quorum': getZKBasedConfig(), + 'templeton.zookeeper.hosts': getZKBasedConfig(), + 'hadoop.registry.zk.quorum': getZKBasedConfig(), + 'hive.cluster.delegation.token.store.zookeeper.connectString': getZKBasedConfig(), + 'instance.zookeeper.host': getZKBasedConfig(), + + 'dfs.name.dir': getMultipleMountPointsConfig('NAMENODE', 'file'), + 'dfs.namenode.name.dir': getMultipleMountPointsConfig('NAMENODE', 'file'), + 'dfs.data.dir': getMultipleMountPointsConfig('DATANODE', 'file'), + 'dfs.datanode.data.dir': getMultipleMountPointsConfig('DATANODE', 'file'), + 'yarn.nodemanager.local-dirs': getMultipleMountPointsConfig('NODEMANAGER'), + 'yarn.nodemanager.log-dirs': getMultipleMountPointsConfig('NODEMANAGER'), + 'mapred.local.dir': getMultipleMountPointsConfig(['TASKTRACKER', 'NODEMANAGER']), + 'log.dirs': getMultipleMountPointsConfig('KAFKA_BROKER'), + + 'fs.checkpoint.dir': getSingleMountPointConfig('SECONDARY_NAMENODE', 'file'), + 'dfs.namenode.checkpoint.dir': getSingleMountPointConfig('SECONDARY_NAMENODE', 'file'), + 'yarn.timeline-service.leveldb-timeline-store.path': getSingleMountPointConfig('APP_TIMELINE_SERVER'), + 'yarn.timeline-service.leveldb-state-store.path': getSingleMountPointConfig('APP_TIMELINE_SERVER'), + 'dataDir': getSingleMountPointConfig('ZOOKEEPER_SERVER'), + 'oozie_data_dir': getSingleMountPointConfig('OOZIE_SERVER'), + 'storm.local.dir': getSingleMountPointConfig(['NODEMANAGER', 'NIMBUS']), + '*.falcon.graph.storage.directory': getSingleMountPointConfig('FALCON_SERVER'), + '*.falcon.graph.serialize.path': getSingleMountPointConfig('FALCON_SERVER') + }, + + uniqueInitializers: { + 'hive_database': '_initHiveDatabaseValue', + 'templeton.hive.properties': '_initTempletonHiveProperties', + 'hbase.zookeeper.quorum': '_initHBaseZookeeperQuorum', + 'yarn.resourcemanager.zk-address': '_initYarnRMzkAddress', + 'RANGER_HOST': '_initRangerHost', + 'hive.metastore.uris': '_initHiveMetastoreUris' + }, + + initializerTypes: { + host_with_component: { + method: '_initAsHostWithComponent' + }, + hosts_with_components: { + method: '_initAsHostsWithComponents' + }, + zookeeper_based: { + method: '_initAsZookeeperServersList' + }, + single_mountpoint: { + method: '_initAsSingleMountPoint' + }, + multiple_mountpoints: { + method: '_initAsMultipleMountPoints' + } + }, /** - * Wrapper for common initializers - * Execute initializer if it is a function or throw an error otherwise + * Map for methods used as value-modifiers for configProperties with values as mount point(s) + * Used if mount point is win-like (@see winRegex) + * Key: id + * Value: method-name * - * @param {App.ServiceConfigProperty} configProperty - * @param {topologyLocalDB} localDB - * @param {object} dependencies - * @returns {App.ServiceConfigProperty} - * @private + * @type {{default: string, file: string, slashes: string}} */ - _defaultInitializer: function (configProperty, localDB, dependencies) { - var args = [].slice.call(arguments); - var self = this; - var initializer = initializers[configProperty.get('name')]; - if (initializer) { - Em.makeArray(initializer).forEach(function (init) { - var type = initializerTypes[init.type]; - // add initializer-settings - args.push(init); - var methodName = type.method; - Em.assert('method-initializer is not a function ' + methodName, 'function' === Em.typeOf(self[methodName])); - configProperty = self[methodName].apply(self, args); - }); - } - return configProperty; + winReplacersMap: { + default: '_defaultWinReplace', + file: '_winReplaceWithFile', + slashes: '_defaultWinReplaceWithAdditionalSlashes' }, /** * Initializer for configs with value equal to hostName with needed component * Value example: 'hostName' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies * @param {object} initializer - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initAsHostWithComponent: function (configProperty, localDB, dependencies, initializer) { @@ -395,11 +341,11 @@ App.ConfigInitializer = Em.Object.create({ * Depends on <code>initializer.asArray</code> (true - array, false - string) * Value example: 'hostName1,hostName2,hostName3' or ['hostName1', 'hostName2', 'hostName3'] * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies * @param {object} initializer - * @return {App.ServiceConfigProperty} + * @return {Object} * @private */ _initAsHostsWithComponents: function (configProperty, localDB, dependencies, initializer) { @@ -419,8 +365,8 @@ App.ConfigInitializer = Em.Object.create({ /** * Unique initializer for <code>hive_database</code>-config * - * @param {App.ServiceConfigProperty} configProperty - * @returns {App.ServiceConfigProperty} + * @param {Object} configProperty + * @returns {Object} * @private */ _initHiveDatabaseValue: function (configProperty) { @@ -440,9 +386,9 @@ App.ConfigInitializer = Em.Object.create({ * Initializer for configs with value equal to hostNames-list where ZOOKEEPER_SERVER is installed * Value example: 'host1:2020,host2:2020,host3:2020' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initAsZookeeperServersList: function (configProperty, localDB) { @@ -465,10 +411,10 @@ App.ConfigInitializer = Em.Object.create({ /** * Unique initializer for <code>templeton.hive.properties</code> * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initTempletonHiveProperties: function (configProperty, localDB, dependencies) { @@ -483,9 +429,9 @@ App.ConfigInitializer = Em.Object.create({ /** * Unique initializer for <code>hbase.zookeeper.quorum</code> * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initHBaseZookeeperQuorum: function (configProperty, localDB) { @@ -501,9 +447,9 @@ App.ConfigInitializer = Em.Object.create({ * If RANGER_ADMIN-component isn't installed, this config becomes unneeded (isVisible - false, isRequired - false) * Value example: 'hostName' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initRangerHost: function (configProperty, localDB) { @@ -529,10 +475,10 @@ App.ConfigInitializer = Em.Object.create({ * Port is taken from <code>dependencies.clientPort</code> * Value example: 'host1:111,host2:111,host3:111' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initYarnRMzkAddress: function (configProperty, localDB, dependencies) { @@ -549,10 +495,10 @@ App.ConfigInitializer = Em.Object.create({ /** * Unique initializer for <code>hive.metastore.uris</code> * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies - * @returns {App.ServiceConfigProperty} + * @returns {Object} * @private */ _initHiveMetastoreUris: function (configProperty, localDB, dependencies) { @@ -564,33 +510,6 @@ App.ConfigInitializer = Em.Object.create({ }, /** - * Entry-point for any config's value initializing - * Before calling it, be sure that <code>initializers</code> or <code>uniqueInitializers</code> - * contains record about needed config - * - * @param {App.ServiceConfigProperty} configProperty - * @param {topologyLocalDB} localDB - * @param {object} dependencies - * @returns {App.ServiceConfigProperty} - */ - initialValue: function (configProperty, localDB, dependencies) { - var configName = configProperty.get('name'); - - var initializer = initializers[configName]; - if (initializer) { - return this._defaultInitializer(configProperty, localDB, dependencies); - } - - var uniqueInitializer = uniqueInitializers[configName]; - if (uniqueInitializer) { - var args = [].slice.call(arguments); - return this[uniqueInitializer].apply(this, args); - } - - return configProperty; - }, - - /** * Get hive.metastore.uris initial value * * @param {object[]} hosts @@ -618,10 +537,10 @@ App.ConfigInitializer = Em.Object.create({ * Set <code>value</code> and <code>recommendedValue</code> for <code>configProperty</code> * basing on <code>recommendedValue</code> with replacing <code>regex</code> for <code>replaceWith</code> * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {string} regex * @param {string} replaceWith - * @return {App.ServiceConfigProperty} + * @return {Object} */ setRecommendedValue: function (configProperty, regex, replaceWith) { var recommendedValue = Em.isNone(configProperty.get('recommendedValue')) ? '' : configProperty.get('recommendedValue'); @@ -638,15 +557,16 @@ App.ConfigInitializer = Em.Object.create({ * Hosts with Windows needs additional processing (@see winReplacersMap) * Value example: '/', '/some/cool/dir' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies * @param {object} initializer - * @return {App.ServiceConfigProperty} + * @return {Object} */ _initAsSingleMountPoint: function (configProperty, localDB, dependencies, initializer) { var hostsInfo = this._updateHostInfo(localDB.hosts); var setOfHostNames = this._getSetOfHostNames(localDB, initializer); + var winReplacersMap = this.get('winReplacersMap'); // In Add Host Wizard, if we did not select this slave component for any host, then we don't process any further. if (!setOfHostNames.length) { return configProperty; @@ -660,7 +580,7 @@ App.ConfigInitializer = Em.Object.create({ else { var mp = mPoint.toLowerCase(); if (winRegex.test(mp)) { - var methodName = initializer.winReplacer; + var methodName = winReplacersMap[initializer.winReplacer]; mPoint = this[methodName].call(this, configProperty, mp); } else { @@ -681,16 +601,17 @@ App.ConfigInitializer = Em.Object.create({ * Hosts with Windows needs additional processing (@see winReplacersMap) * Value example: '/\n/some/cool/dir' (`\n` - is divider) * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {topologyLocalDB} localDB * @param {object} dependencies * @param {object} initializer - * @return {App.ServiceConfigProperty} + * @return {Object} */ _initAsMultipleMountPoints: function (configProperty, localDB, dependencies, initializer) { var hostsInfo = this._updateHostInfo(localDB.hosts); var self = this; var setOfHostNames = this._getSetOfHostNames(localDB, initializer); + var winReplacersMap = this.get('winReplacersMap'); // In Add Host Wizard, if we did not select this slave component for any host, then we don't process any further. if (!setOfHostNames.length) { return configProperty; @@ -706,7 +627,7 @@ App.ConfigInitializer = Em.Object.create({ else { var mp = eachDrive.mountpoint.toLowerCase(); if (winRegex.test(mp)) { - var methodName = initializer.winReplacer; + var methodName = winReplacersMap[initializer.winReplacer]; mPoint += self[methodName].call(this, configProperty, mp); } else { @@ -726,7 +647,7 @@ App.ConfigInitializer = Em.Object.create({ /** * Replace drive-based windows-path with 'file:///' * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {string} mountPoint * @returns {string} * @private @@ -739,7 +660,7 @@ App.ConfigInitializer = Em.Object.create({ /** * Replace drive-based windows-path * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {string} mountPoint * @returns {string} * @private @@ -753,7 +674,7 @@ App.ConfigInitializer = Em.Object.create({ /** * Same to <code>_defaultWinReplace</code>, but with extra-slash in the end * - * @param {App.ServiceConfigProperty} configProperty + * @param {Object} configProperty * @param {string} mountPoint * @returns {string} * @private @@ -874,35 +795,6 @@ App.ConfigInitializer = Em.Object.create({ allMountPoints.push(mountPointAsRoot); } return allMountPoints; - }, - - - __testGetInitializers: function () { - if ($.mocho) { - return initializers; - } - Em.assert('Available only for testing', false); - }, - - __testGetUniqueInitializers: function () { - if ($.mocho) { - return uniqueInitializers; - } - Em.assert('Available only for testing', false); - }, - - __testGetInitializerTypes: function () { - if ($.mocho) { - return initializerTypes; - } - Em.assert('Available only for testing', false); - }, - - __testGetWinReplacersMap: function () { - if ($.mocho) { - return winReplacersMap; - } - Em.assert('Available only for testing', false); } -}); +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/utils/configs/config_initializer_class.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/config_initializer_class.js b/ambari-web/app/utils/configs/config_initializer_class.js new file mode 100644 index 0000000..dbfc7ae --- /dev/null +++ b/ambari-web/app/utils/configs/config_initializer_class.js @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var App = require('app'); + +/** + * Basic class for config-initializers + * Each child should fill <code>initializers</code> or <code>uniqueInitializers</code> and <code>initializerTypes</code> + * Usage: + * <pre> + * var myCoolInitializer = App.ConfigInitializerClass.create({ + * initializers: { + * 'my-cool-config': { + * type: 'some_type' + * } + * }, + * + * initializerTypes: { + * some_type: { + * method: '_initAsCool' + * } + * }, + * + * _initAsCool: function (configProperty, localDB, dependencies, initializer) { + * // some magic + * return configProperty; + * } + * }); + * + * var myConfig = { name: 'my-cool-config' }; + * var localDB = getLocalDB(); + * var dependencies = {}; + * myCoolInitializer.initialValue(myConfig, localDB, dependencies); + * </pre> + * + * @type {ConfigInitializerClass} + */ +App.ConfigInitializerClass = Em.Object.extend({ + + /** + * Map with configurations for config initializers + * It's used only for initializers which are common for some configs (if not - use <code>uniqueInitializers</code>-map) + * Key {string} configProperty-name + * Value {object|object[]} settings for initializer + * + * @type {object} + */ + initializers: {}, + + /** + * Map with initializers types + * Doesn't contain unique initializes, only common are included + * Key: id + * Value: object with method-name (prefer to start method-name with '_init' or '_initAs') + * Each method here is called with arguments equal to <code>initialValue</code>-call args + * Initializer-settings are added as last argument + * + * @type {object} + */ + initializerTypes: {}, + + /** + * Map with initializers that are used only for one config (are unique) + * Key: configProperty-name + * Value: method-name + * Every method from this map is called with same arguments as <code>initialValue</code> is (prefer to start method-name with '_init' or '_initAs') + * + * @type {object} + */ + uniqueInitializers: {}, + + /** + * Wrapper for common initializers + * Execute initializer if it is a function or throw an error otherwise + * + * @param {Object} configProperty + * @param {topologyLocalDB} localDB + * @param {object} dependencies + * @returns {Object} + * @private + */ + _defaultInitializer: function (configProperty, localDB, dependencies) { + var args = [].slice.call(arguments); + var self = this; + var initializers = this.get('initializers'); + var initializerTypes = this.get('initializerTypes'); + var initializer = initializers[Em.get(configProperty, 'name')]; + if (initializer) { + initializer = Em.makeArray(initializer); + initializer.forEach(function (init) { + var _args = [].slice.call(args); + var type = initializerTypes[init.type]; + // add initializer-settings + _args.push(init); + var methodName = type.method; + Em.assert('method-initializer is not a function ' + methodName, 'function' === Em.typeOf(self[methodName])); + configProperty = self[methodName].apply(self, _args); + }); + } + return configProperty; + }, + + /** + * Entry-point for any config's value initializing + * Before calling it, be sure that <code>initializers</code> or <code>uniqueInitializers</code> + * contains record about needed configs + * + * @param {Object} configProperty + * @param {topologyLocalDB} localDB + * @param {object} dependencies + * @returns {Object} + */ + initialValue: function (configProperty, localDB, dependencies) { + var configName = Em.get(configProperty, 'name'); + var initializers = this.get('initializers'); + + var initializer = initializers[configName]; + if (initializer) { + return this._defaultInitializer(configProperty, localDB, dependencies); + } + + var uniqueInitializers = this.get('uniqueInitializers'); + var uniqueInitializer = uniqueInitializers[configName]; + if (uniqueInitializer) { + var args = [].slice.call(arguments); + return this[uniqueInitializer].apply(this, args); + } + + return configProperty; + } + +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/app/utils/configs/nn_ha_config_initializer.js ---------------------------------------------------------------------- diff --git a/ambari-web/app/utils/configs/nn_ha_config_initializer.js b/ambari-web/app/utils/configs/nn_ha_config_initializer.js new file mode 100644 index 0000000..ef9e9c8 --- /dev/null +++ b/ambari-web/app/utils/configs/nn_ha_config_initializer.js @@ -0,0 +1,515 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var App = require('app'); +require('utils/configs/config_initializer_class'); + +/** + * @typedef {topologyLocalDB} extendedTopologyLocalDB + * @property {string[]} installedServices list of installed service names + */ + +/** + * Setting for <code>rename</code>-initializer + * Used for configs which should be renamed + * Replace some part if their names with <code>namespaceId</code> (provided by user on the wizard's 1st step) + * + * @param {string} toReplace + * @returns {{type: string, toReplace: string}} + */ +function getRenameWithNamespaceConfig(toReplace) { + return { + type: 'rename', + toReplace: toReplace + }; +} + +/** + * Settings for <code>host_with_port</code>-initializer + * Used for configs with value equal to hostName where some component exists concatenated with port-value + * Port-value is calculated according to <code>port</code> and <code>portFromDependencies</code> values + * If <code>portFromDependencies</code> is <code>true</code>, <code>port</code>-value is used as key of the <code>dependencies</code> (where real port-value is) + * Otherwise - <code>port</code>-value used as is + * Value also may be customized with prefix and suffix + * + * @param {string} component needed component + * @param {boolean} componentExists component already exists or just going to be installed + * @param {string} prefix='' + * @param {string} suffix='' + * @param {string} port + * @param {boolean} portFromDependencies=false + * @returns {{type: string, component: string, componentExists: boolean, modifier: {prefix: (string), suffix: (string)}}} + */ +function getHostWithPortConfig(component, componentExists, prefix, suffix, port, portFromDependencies) { + if (arguments.length < 6) { + portFromDependencies = false; + } + prefix = prefix || ''; + suffix = suffix || ''; + var ret = { + type: 'host_with_port', + component: component, + componentExists: componentExists, + modifier: { + prefix: prefix, + suffix: suffix + } + }; + if (portFromDependencies) { + ret.portKey = port; + } + else { + ret.port = port; + } + return ret; +} + +/** + * Settings for <code>hosts_with_port</code>-initializer + * Used for configs with value equal to the list of hostNames with port + * Value also may be customized with prefix, suffix and delimiter between host:port elements + * Port-value is calculated according to <code>port</code> and <code>portFromDependencies</code> values + * If <code>portFromDependencies</code> is <code>true</code>, <code>port</code>-value is used as key of the <code>dependencies</code> (where real port-value is) + * Otherwise - <code>port</code>-value used as is + * + * @param {string} component hosts where this component exists are used as config-value + * @param {string} prefix='' substring added before hosts-list + * @param {string} suffix='' substring added after hosts-list + * @param {string} delimiter=',' delimiter between hosts in the value + * @param {string} port if <code>portFromDependencies</code> is <code>false</code> this value is used as port for hosts + * if <code>portFromDependencies</code> is <code>true</code> `port` is used as key in the <code>dependencies</code> to get real port-value + * @param {boolean} portFromDependencies=false true - use <code>port</code> as key for <code>dependencies</code> to get real port-value, + * false - use <code>port</code> as port-value + * @returns {{type: string, component: string, modifier: {prefix: (string), suffix: (string), delimiter: (string)}}} + */ +function getHostsWithPortConfig(component, prefix, suffix, delimiter, port, portFromDependencies) { + if (arguments.length < 6) { + portFromDependencies = false; + } + prefix = prefix || ''; + suffix = suffix || ''; + delimiter = delimiter || ','; + var ret = { + type: 'hosts_with_port', + component: component, + modifier: { + prefix: prefix, + suffix: suffix, + delimiter: delimiter + } + }; + if (portFromDependencies) { + ret.portKey = port; + } + else { + ret.port = port; + } + return ret; +} + +/** + * Settings for <code>namespace</code>-initializer + * Used for configs with value equal to the <code>namespaceId</code> (provided by user on the wizard's 1st step) + * Value may be customized with prefix and suffix + * + * @param {string} [prefix=''] substring added before namespace in the replace + * @param {string} [suffix=''] substring added after namespace in the replace + * @returns {{type: string, modifier: {prefix: (string), suffix: (string)}}} + */ +function getNamespaceConfig (prefix, suffix) { + prefix = prefix || ''; + suffix = suffix || ''; + return { + type: 'namespace', + modifier: { + prefix: prefix, + suffix: suffix + } + } +} + +/** + * Settings for <code>replace_namespace</code> + * Used for configs with values that have to be modified with replacing some part of them + * to the <code>namespaceId</code> (provided by user on the wizard's 1st step) + * + * @param {string} toReplace + * @returns {{type: string, toReplace: *}} + */ +function getReplaceNamespaceConfig(toReplace) { + return { + type: 'replace_namespace', + toReplace: toReplace + }; +} + +/** + * + * @class {NnHaConfigInitializer} + */ +App.NnHaConfigInitializer = App.ConfigInitializerClass.create({ + + initializers: { + 'dfs.ha.namenodes.${dfs.nameservices}': getRenameWithNamespaceConfig('${dfs.nameservices}'), + 'dfs.namenode.rpc-address.${dfs.nameservices}.nn1': [ + getHostWithPortConfig('NAMENODE', true, '', '', 'nnRpcPort', true), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.namenode.rpc-address.${dfs.nameservices}.nn2': [ + getHostWithPortConfig('NAMENODE', false, '', '', '8020', false), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.namenode.http-address.${dfs.nameservices}.nn1': [ + getHostWithPortConfig('NAMENODE', true, '', '', 'nnHttpPort', true), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.namenode.http-address.${dfs.nameservices}.nn2': [ + getHostWithPortConfig('NAMENODE', false, '', '', '50070', false), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.namenode.https-address.${dfs.nameservices}.nn1': [ + getHostWithPortConfig('NAMENODE', true, '', '', 'nnHttpsPort', true), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.namenode.https-address.${dfs.nameservices}.nn2': [ + getHostWithPortConfig('NAMENODE', false, '', '', '50470', false), + getRenameWithNamespaceConfig('${dfs.nameservices}') + ], + 'dfs.client.failover.proxy.provider.${dfs.nameservices}': getRenameWithNamespaceConfig('${dfs.nameservices}'), + 'dfs.nameservices': getNamespaceConfig(), + 'fs.defaultFS': getNamespaceConfig('hdfs://'), + 'dfs.namenode.shared.edits.dir': [ + getHostsWithPortConfig('JOURNALNODE', 'qjournal://', '/${dfs.nameservices}', ';', '8485', false), + getReplaceNamespaceConfig('${dfs.nameservices}') + ], + 'ha.zookeeper.quorum': getHostsWithPortConfig('ZOOKEEPER_SERVER', '', '', ',', 'zkClientPort', true) + }, + + uniqueInitializers: { + 'hbase.rootdir': '_initHbaseRootDir', + 'instance.volumes': '_initInstanceVolumes', + 'instance.volumes.replacements': '_initInstanceVolumesReplacements', + 'dfs.journalnode.edits.dir': '_initDfsJnEditsDir' + }, + + initializerTypes: { + rename: { + method: '_initWithRename' + }, + host_with_port: { + method: '_initAsHostWithPort' + }, + hosts_with_port: { + method: '_initAsHostsWithPort' + }, + namespace: { + method: '_initAsNamespace' + }, + replace_namespace: { + method: '_initWithNamespace' + } + }, + + /** + * Initializer for configs that should be renamed + * Some part of their names should be replaced with <code>namespaceId</code> (user input this value on the wizard's 1st step) + * Affects both - name and displayName + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {object} + * @private + * @method _initWithRename + */ + _initWithRename: function (configProperty, localDB, dependencies, initializer) { + var replaceWith = dependencies.namespaceId; + var toReplace = initializer.toReplace; + Em.assert('`dependencies.namespaceId` should be not empty string', !!replaceWith); + var name = Em.getWithDefault(configProperty, 'name', ''); + var displayName = Em.getWithDefault(configProperty, 'displayName', ''); + name = name.replace(toReplace, replaceWith); + displayName = displayName.replace(toReplace, replaceWith); + Em.setProperties(configProperty, { + name: name, + displayName: displayName + }); + return configProperty; + }, + + /** + * Initializer for configs wih value equal to the <code>namespaceId</code> (user input this value on the wizard's 1st step) + * Value may be customized with prefix and suffix (see <code>initializer.modifier</code>) + * Value-examples: 'SOME_COOL_PREFIXmy_namespaceSOME_COOL_SUFFIX', 'my_namespace' + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {object} + * @private + * @method _initAsNamespace + */ + _initAsNamespace: function (configProperty, localDB, dependencies, initializer) { + var value = dependencies.namespaceId; + Em.assert('`dependencies.namespaceId` should be not empty string', !!value); + value = initializer.modifier.prefix + value + initializer.modifier.suffix; + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + return configProperty; + }, + + /** + * Initializer for configs with value that should be modified with replacing some substring + * to the <code>namespaceId</code> (user input this value on the wizard's 1st step) + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {object} + * @private + * @method _initWithNamespace + */ + _initWithNamespace: function (configProperty, localDB, dependencies, initializer) { + var replaceWith = dependencies.namespaceId; + var toReplace = initializer.toReplace; + Em.assert('`dependencies.namespaceId` should be not empty string', !!replaceWith); + var value = Em.get(configProperty, 'value').replace(toReplace, replaceWith); + var recommendedValue = Em.get(configProperty, 'recommendedValue').replace(toReplace, replaceWith); + Em.setProperties(configProperty, { + value: value, + recommendedValue: recommendedValue + }); + return configProperty; + }, + + /** + * Initializer for configs with value equal to the hostName where some component exists + * Value may be customized with prefix and suffix (see <code>initializer.modifier</code>) + * Port-value is calculated according to <code>initializer.portKey</code> or <code>initializer.port</code> values + * Value-examples: 'SOME_COOL_PREFIXhost1:port1SOME_COOL_SUFFIX', 'host1:port2' + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {object} + * @private + * @method _initAsHostWithPort + */ + _initAsHostWithPort: function (configProperty, localDB, dependencies, initializer) { + var hostName = localDB.masterComponentHosts.filterProperty('component', initializer.component).findProperty('isInstalled', initializer.componentExists).hostName; + var port = this.__getPort(dependencies, initializer); + var value = initializer.modifier.prefix + hostName + ':' + port + initializer.modifier.suffix; + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + return configProperty; + }, + + /** + * Initializer for configs with value equal to the list of hosts where some component exists + * Value may be customized with prefix and suffix (see <code>initializer.modifier</code>) + * Delimiter between hostNames also may be customized in the <code>initializer.modifier</code> + * Port-value is calculated according to <code>initializer.portKey</code> or <code>initializer.port</code> values + * Value examples: 'SOME_COOL_PREFIXhost1:port,host2:port,host2:portSOME_COOL_SUFFIX', 'host1:port|||host2:port|||host2:port' + * + * @param {object} configProperty + * @param {topologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {object} + * @private + * @method _initAsHostsWithPort + */ + _initAsHostsWithPort: function (configProperty, localDB, dependencies, initializer) { + var hostNames = localDB.masterComponentHosts.filterProperty('component', initializer.component).mapProperty('hostName'); + var port = this.__getPort(dependencies, initializer); + var value = initializer.modifier.prefix + hostNames.map(function (hostName) { + return hostName + ':' + port; + }).join(initializer.modifier.delimiter) + initializer.modifier.suffix; + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + return configProperty; + }, + + /** + * Returns port-value from <code>dependencies</code> accorfing to <code>initializer.portKey</code> or <code>initializer.port</code> values + * + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @returns {string|number} + * @private + * @method __getPort + */ + __getPort: function (dependencies, initializer) { + var portKey = initializer.portKey; + if (portKey) { + return dependencies[portKey]; + } + return initializer.port; + }, + + /** + * Unique initializer for <code>hbase.rootdir</code> + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initHbaseRootDir + * @return {object} + * @private + */ + _initHbaseRootDir: function (configProperty, localDB, dependencies, initializer) { + var fileName = Em.get(configProperty, 'filename'); + var args = [].slice.call(arguments); + if ('hbase-site' === fileName) { + return this._initHbaseRootDirForHbase.apply(this, args); + } + if('ams-hbase-site' === fileName) { + return this._initHbaseRootDirForAMS.apply(this, args); + } + return configProperty; + }, + + /** + * Unique initializer for <code>hbase.rootdir</code> (HBASE-service) + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initHbaseRootDirForHbase + * @return {object} + * @private + */ + _initHbaseRootDirForHbase: function (configProperty, localDB, dependencies, initializer) { + if (localDB.installedServices.contains('HBASE')) { + var value = dependencies.serverConfigs.findProperty('type', 'hbase-site').properties['hbase.rootdir'].replace(/\/\/[^\/]*/, '//' + dependencies.namespaceId); + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + } + return configProperty; + }, + + /** + * Unique initializer for <code>hbase.rootdir</code> (Ambari Metrics-service) + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initHbaseRootDirForAMS + * @return {object} + * @private + */ + _initHbaseRootDirForAMS: function (configProperty, localDB, dependencies, initializer) { + if (localDB.installedServices.contains('AMBARI_METRICS')) { + var value = dependencies.serverConfigs.findProperty('type', 'ams-hbase-site').properties['hbase.rootdir']; + var currentNameNodeHost = localDB.masterComponentHosts.filterProperty('component', 'NAMENODE').findProperty('isInstalled', true).hostName; + value = (value == "hdfs://" + currentNameNodeHost) ? "hdfs://" + dependencies.namespaceId : value; + configProperty.isVisible = configProperty.value != value; + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + } + return configProperty; + }, + + /** + * Unique initializer for <code>instance.volumes</code> + * + * @param {object} configProperty + * @param {topologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initInstanceVolumes + * @return {object} + * @private + */ + _initInstanceVolumes: function (configProperty, localDB, dependencies, initializer) { + if (localDB.installedServices.contains('ACCUMULO')) { + var oldValue = dependencies.serverConfigs.findProperty('type', 'accumulo-site').properties['instance.volumes']; + var value = oldValue.replace(/\/\/[^\/]*/, '//' + dependencies.namespaceId); + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + } + return configProperty; + }, + + /** + * Unique initializer for <code>instance.volumes.replacements</code> + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initInstanceVolumesReplacements + * @return {object} + * @private + */ + _initInstanceVolumesReplacements: function (configProperty, localDB, dependencies, initializer) { + if (localDB.installedServices.contains('ACCUMULO')) { + var oldValue = dependencies.serverConfigs.findProperty('type', 'accumulo-site').properties['instance.volumes']; + var value = oldValue.replace(/\/\/[^\/]*/, '//' + dependencies.namespaceId); + var replacements = oldValue + " " + value; + Em.setProperties(configProperty, { + value: replacements, + recommendedValue: replacements + }); + } + return configProperty; + }, + + /** + * Unique initializer for <code>dfs.journalnode.edits.dir</code> + * Used only for Windows Stacks + * + * @param {object} configProperty + * @param {extendedTopologyLocalDB} localDB + * @param {nnHaConfigDependencies} dependencies + * @param {object} initializer + * @method _initDfsJnEditsDir + * @return {object} + * @private + */ + _initDfsJnEditsDir: function (configProperty, localDB, dependencies, initializer) { + if (App.get('isHadoopWindowsStack') && localDB.installedServices.contains('HDFS')) { + var value = dependencies.serverConfigs.findProperty('type', 'hdfs-site').properties['dfs.journalnode.edits.dir']; + Em.setProperties(configProperty, { + value: value, + recommendedValue: value + }); + } + return configProperty; + } + +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/test/controllers/main/admin/highAvailability/nameNode/step3_controller_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/controllers/main/admin/highAvailability/nameNode/step3_controller_test.js b/ambari-web/test/controllers/main/admin/highAvailability/nameNode/step3_controller_test.js index 67cf643..75f4e53 100644 --- a/ambari-web/test/controllers/main/admin/highAvailability/nameNode/step3_controller_test.js +++ b/ambari-web/test/controllers/main/admin/highAvailability/nameNode/step3_controller_test.js @@ -18,8 +18,53 @@ var App = require('app'); +var controller; + describe('App.HighAvailabilityWizardStep3Controller', function() { - + + var serverConfigData = { + items: [ + { + type: 'hdfs-site', + properties: { + 'dfs.namenode.http-address': 'h1:1234', + 'dfs.namenode.https-address': 'h1:4321', + 'dfs.namenode.rpc-address': 'h1:1111', + 'dfs.journalnode.edits.dir': '/hadoop/hdfs/journalnode123' + } + }, + { + type: 'zoo.cfg', + properties: { + clientPort: '4444' + } + }, + { + type: 'hbase-site', + properties: { + 'hbase.rootdir': 'hdfs://h34:8020/apps/hbase/data' + } + }, + { + type: 'ams-hbase-site', + properties: { + 'hbase.rootdir': 'file:///var/lib/ambari-metrics-collector/hbase' + } + }, + { + type: 'accumulo-site', + properties: { + 'instance.volumes': 'hdfs://localhost:8020/apps/accumulo/data' + } + } + ] + }; + + beforeEach(function () { + controller = App.HighAvailabilityWizardStep3Controller.create(); + controller.set('serverConfigData', serverConfigData); + }); + describe('#removeConfigs', function() { var tests = [ @@ -144,5 +189,143 @@ describe('App.HighAvailabilityWizardStep3Controller', function() { }); }); }); + + describe('#tweakServiceConfigs', function () { + + var nameServiceId = 'tdk'; + + var masterComponentHosts = [ + {component: 'NAMENODE', isInstalled: true, hostName: 'h1'}, + {component: 'NAMENODE', isInstalled: false, hostName: 'h2'}, + {component: 'JOURNALNODE', hostName: 'h1'}, + {component: 'JOURNALNODE', hostName: 'h2'}, + {component: 'JOURNALNODE', hostName: 'h3'}, + {component: 'ZOOKEEPER_SERVER', hostName: 'h1'}, + {component: 'ZOOKEEPER_SERVER', hostName: 'h2'}, + {component: 'ZOOKEEPER_SERVER', hostName: 'h3'} + ]; + + beforeEach(function () { + controller.set('content', Em.Object.create({ + masterComponentHosts: masterComponentHosts, + slaveComponentHosts: [], + hosts: {}, + nameServiceId: nameServiceId + })); + var get = sinon.stub(App, 'get'); + get.withArgs('isHadoopWindowsStack').returns(true); + sinon.stub(App.Service, 'find', function () { + return [{serviceName: 'HDFS'}, {serviceName: 'HBASE'}, {serviceName: 'AMBARI_METRICS'}, {serviceName: 'ACCUMULO'}] + }); + }); + + afterEach(function () { + App.Service.find.restore(); + App.get.restore(); + }); + + Em.A([ + { + config: { + name: 'dfs.namenode.rpc-address.${dfs.nameservices}.nn1' + }, + value: 'h1:1111', + name: 'dfs.namenode.rpc-address.' + nameServiceId + '.nn1' + }, + { + config: { + name: 'dfs.namenode.rpc-address.${dfs.nameservices}.nn2' + }, + value: 'h2:8020', + name: 'dfs.namenode.rpc-address.' + nameServiceId + '.nn2' + }, + { + config: { + name: 'dfs.namenode.http-address.${dfs.nameservices}.nn1' + }, + value: 'h1:1234', + name: 'dfs.namenode.http-address.' + nameServiceId + '.nn1' + }, + { + config: { + name: 'dfs.namenode.http-address.${dfs.nameservices}.nn2' + }, + value: 'h2:50070', + name: 'dfs.namenode.http-address.' + nameServiceId + '.nn2' + }, + { + config: { + name: 'dfs.namenode.https-address.${dfs.nameservices}.nn1' + }, + value: 'h1:4321', + name: 'dfs.namenode.https-address.' + nameServiceId + '.nn1' + }, + { + config: { + name: 'dfs.namenode.https-address.${dfs.nameservices}.nn2' + }, + value: 'h2:50470', + name: 'dfs.namenode.https-address.' + nameServiceId + '.nn2' + }, + { + config: { + name: 'dfs.namenode.shared.edits.dir' + }, + value: 'qjournal://h1:8485;h2:8485;h3:8485/' + nameServiceId + }, + { + config: { + name: 'ha.zookeeper.quorum' + }, + value: 'h1:4444,h2:4444,h3:4444' + }, + { + config: { + name: 'hbase.rootdir', + filename: 'hbase-site' + }, + value: 'hdfs://' + nameServiceId + '/apps/hbase/data' + }, + { + config: { + name: 'hbase.rootdir', + filename: 'ams-hbase-site' + }, + value: 'file:///var/lib/ambari-metrics-collector/hbase' + }, + { + config: { + name: 'instance.volumes' + }, + value: 'hdfs://' + nameServiceId + '/apps/accumulo/data' + }, + { + config: { + name: 'instance.volumes.replacements' + }, + value: 'hdfs://localhost:8020/apps/accumulo/data hdfs://' + nameServiceId + '/apps/accumulo/data' + }, + { + config: { + name: 'dfs.journalnode.edits.dir' + }, + value: '/hadoop/hdfs/journalnode123' + } + ]).forEach(function (test) { + it(test.config.name, function () { + test.config.displayName = test.config.name; + var configs = [test.config]; + configs = controller.tweakServiceConfigs(configs); + expect(configs[0].value).to.equal(test.value); + expect(configs[0].recommendedValue).to.equal(test.value); + if(test.name) { + expect(configs[0].name).to.equal(test.name); + expect(configs[0].displayName).to.equal(test.name); + } + }); + }); + + }); + }); http://git-wip-us.apache.org/repos/asf/ambari/blob/6f9ef6a9/ambari-web/test/utils/configs/config_initializer_test.js ---------------------------------------------------------------------- diff --git a/ambari-web/test/utils/configs/config_initializer_test.js b/ambari-web/test/utils/configs/config_initializer_test.js index fe55e69..587892f 100644 --- a/ambari-web/test/utils/configs/config_initializer_test.js +++ b/ambari-web/test/utils/configs/config_initializer_test.js @@ -1204,7 +1204,7 @@ describe('App.ConfigInitializer', function () { }); describe('initializerTypes', function () { - var types = App.ConfigInitializer.__testGetInitializerTypes(); + var types = App.ConfigInitializer.get('initializerTypes'); Em.keys(types).forEach(function(type) { it(type, function() { var methodName = types[type].method; @@ -1217,8 +1217,8 @@ describe('App.ConfigInitializer', function () { describe('initializers', function () { - var initializers = App.ConfigInitializer.__testGetInitializers(); - var types = App.ConfigInitializer.__testGetInitializerTypes(); + var initializers = App.ConfigInitializer.get('initializers'); + var types = App.ConfigInitializer.get('initializerTypes'); var typeNames = Em.keys(types); Em.keys(initializers).forEach(function (configName) { @@ -1232,7 +1232,7 @@ describe('App.ConfigInitializer', function () { describe('uniqueInitializers', function () { - var uniqueInitializers = App.ConfigInitializer.__testGetUniqueInitializers(); + var uniqueInitializers = App.ConfigInitializer.get('uniqueInitializers'); var uniqueInitializersNames = Em.keys(uniqueInitializers).map(function (key) { return uniqueInitializers[key]; }); @@ -1251,7 +1251,7 @@ describe('App.ConfigInitializer', function () { describe('winReplacersMap', function () { - var winReplacersMap = App.ConfigInitializer.__testGetWinReplacersMap(); + var winReplacersMap = App.ConfigInitializer.get('winReplacersMap'); var winReplacerNames = Em.keys(winReplacersMap).map(function (key) { return winReplacersMap[key]; });