Merge remote-tracking branch 'remotes/origin/trunk' into branch-3.0-perf
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/af30ab40 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/af30ab40 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/af30ab40 Branch: refs/heads/branch-3.0-perf Commit: af30ab40972022ce4bff6a32b4ebdd29a33ef95d Parents: 7da02bb b4eddc9 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Thu Oct 19 15:54:09 2017 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Thu Oct 19 15:54:09 2017 +0300 ---------------------------------------------------------------------- .../ui/admin-web/app/scripts/i18n.config.js | 2 +- ambari-agent/pom.xml | 4 +- .../main/python/ambari_agent/AmbariConfig.py | 74 +- .../ambari_agent/CustomServiceOrchestrator.py | 6 +- .../src/main/python/ambari_agent/FileCache.py | 5 +- .../main/python/ambari_agent/ProcessHelper.py | 71 -- .../src/main/python/ambari_agent/StatusCheck.py | 142 ---- .../src/main/python/ambari_agent/main.py | 12 +- .../test/python/ambari_agent/TestFileCache.py | 4 +- .../test/python/ambari_agent/TestLiveStatus.py | 5 +- .../src/test/python/ambari_agent/TestMain.py | 13 +- .../python/ambari_agent/TestProcessHelper.py | 70 -- .../test/python/ambari_agent/TestSecurity.py | 1 - .../test/python/ambari_agent/TestStatusCheck.py | 180 ----- .../resource_management/TestPackageResource.py | 6 +- .../python/resource_management/TestScript.py | 26 +- .../core/providers/package/__init__.py | 4 +- .../core/providers/package/apt.py | 35 +- .../core/providers/package/choco.py | 8 +- .../core/providers/package/yumrpm.py | 10 +- .../core/providers/package/zypper.py | 7 +- .../core/resources/packaging.py | 14 +- .../libraries/functions/component_version.py | 26 +- .../libraries/functions/conf_select.py | 362 +++------ .../libraries/functions/repository_util.py | 8 +- .../libraries/functions/stack_select.py | 69 +- .../libraries/script/script.py | 88 ++- ambari-infra/ambari-infra-assembly/pom.xml | 12 + .../src/main/package/deb/manager/postinst | 5 + .../src/main/package/deb/solr-client/postinst | 13 + .../src/main/package/rpm/manager/postinstall.sh | 20 + .../main/package/rpm/solr-client/postinstall.sh | 28 + .../org/apache/ambari/infra/InfraManager.java | 4 +- .../src/main/resources/infraManager.sh | 10 +- ambari-infra/ambari-infra-solr-client/build.xml | 3 + .../src/main/python/solrDataManager.py | 0 .../src/main/resources/solrIndexHelper.sh | 5 +- .../ambari-logsearch-assembly/pom.xml | 4 +- .../ambari-logsearch-logfeeder/pom.xml | 4 + .../ambari-logsearch-server/pom.xml | 4 + .../ambari-logsearch-web/package.json | 19 +- ambari-logsearch/ambari-logsearch-web/pom.xml | 20 +- .../ambari-logsearch-web/src/app/app.module.ts | 8 + .../src/app/classes/active-service-log-entry.ts | 23 + .../src/app/classes/histogram-options.ts | 36 + .../src/app/classes/list-item.class.ts | 25 - .../src/app/classes/list-item.ts | 26 + .../src/app/classes/models/app-settings.ts | 27 + .../src/app/classes/models/app-state.ts | 43 + .../src/app/classes/models/audit-log-field.ts | 225 ++++++ .../src/app/classes/models/audit-log.ts | 46 ++ .../src/app/classes/models/bar-graph.ts | 24 + .../src/app/classes/models/common-entry.ts | 22 + .../src/app/classes/models/count.ts | 22 + .../src/app/classes/models/filter.ts | 25 + .../src/app/classes/models/graph.ts | 23 + .../src/app/classes/models/log-field.ts | 27 + .../src/app/classes/models/log.ts | 38 + .../src/app/classes/models/node.ts | 30 + .../src/app/classes/models/service-log-field.ts | 107 +++ .../src/app/classes/models/service-log.ts | 27 + .../app/classes/models/solr-collection-state.ts | 23 + .../src/app/classes/models/store.ts | 180 +++++ .../src/app/classes/models/user-config.ts | 26 + .../queries/audit-logs-query-params.class.ts | 46 -- .../classes/queries/audit-logs-query-params.ts | 46 ++ .../app/classes/queries/query-params.class.ts | 23 - .../src/app/classes/queries/query-params.ts | 23 + ...ce-logs-histogram-query-params.class.spec.ts | 203 ----- ...service-logs-histogram-query-params.class.ts | 70 -- .../service-logs-histogram-query-params.spec.ts | 203 +++++ .../service-logs-histogram-query-params.ts | 70 ++ .../queries/service-logs-query-params.class.ts | 30 - .../queries/service-logs-query-params.ts | 30 + .../service-logs-truncated-query-params.ts | 36 + .../app/classes/service-log-context-entry.ts | 26 + .../accordion-panel.component.less | 2 +- .../src/app/components/app.component.less | 2 +- .../collapsible-panel.component.html | 27 + .../collapsible-panel.component.less | 42 + .../collapsible-panel.component.spec.ts | 129 +++ .../collapsible-panel.component.ts | 89 +++ .../dropdown-button.component.html | 3 +- .../dropdown-button.component.less | 2 +- .../dropdown-button.component.spec.ts | 5 +- .../dropdown-button.component.ts | 2 +- .../dropdown-list/dropdown-list.component.html | 2 +- .../dropdown-list/dropdown-list.component.less | 2 +- .../dropdown-list.component.spec.ts | 12 +- .../dropdown-list/dropdown-list.component.ts | 11 +- .../filter-button.component.spec.ts | 5 +- .../filter-button/filter-button.component.ts | 2 +- .../filter-dropdown.component.spec.ts | 5 +- .../filters-panel/filters-panel.component.html | 5 +- .../filters-panel/filters-panel.component.less | 8 +- .../filters-panel.component.spec.ts | 5 +- .../filters-panel/filters-panel.component.ts | 4 +- .../log-context/log-context.component.html | 33 + .../log-context/log-context.component.less | 23 + .../log-context/log-context.component.spec.ts | 108 +++ .../log-context/log-context.component.ts | 91 +++ .../log-file-entry.component.html | 20 + .../log-file-entry.component.less | 31 + .../log-file-entry.component.spec.ts | 56 ++ .../log-file-entry/log-file-entry.component.ts | 51 ++ .../logs-container.component.html | 20 +- .../logs-container.component.spec.ts | 10 +- .../logs-container/logs-container.component.ts | 54 +- .../logs-list/logs-list.component.html | 30 +- .../logs-list/logs-list.component.less | 38 +- .../logs-list/logs-list.component.spec.ts | 3 + .../components/logs-list/logs-list.component.ts | 34 +- .../main-container.component.html | 7 + .../main-container.component.less | 6 +- .../main-container.component.spec.ts | 13 +- .../main-container/main-container.component.ts | 32 +- .../src/app/components/main.less | 20 + .../menu-button/menu-button.component.less | 18 +- .../menu-button/menu-button.component.spec.ts | 5 +- .../menu-button/menu-button.component.ts | 7 +- .../src/app/components/mixins.less | 200 +++++ .../pagination-controls.component.less | 2 +- .../pagination/pagination.component.less | 4 +- .../search-box/search-box.component.less | 2 +- .../search-box/search-box.component.ts | 2 +- .../time-histogram.component.less | 24 +- .../time-histogram/time-histogram.component.ts | 94 ++- .../time-range-picker.component.less | 2 +- .../timezone-picker.component.spec.ts | 5 +- .../components/top-menu/top-menu.component.html | 2 +- .../components/top-menu/top-menu.component.less | 2 +- .../src/app/components/variables.less | 93 +-- .../src/app/models/app-settings.model.ts | 27 - .../src/app/models/app-state.model.ts | 35 - .../src/app/models/audit-log-field.model.ts | 225 ------ .../src/app/models/audit-log.model.ts | 46 -- .../src/app/models/bar-graph.model.ts | 24 - .../src/app/models/common-entry.model.ts | 22 - .../src/app/models/count.model.ts | 22 - .../src/app/models/filter.model.ts | 25 - .../src/app/models/graph.model.ts | 23 - .../src/app/models/log-field.model.ts | 27 - .../src/app/models/log.model.ts | 37 - .../src/app/models/node.model.ts | 29 - .../src/app/models/service-log-field.model.ts | 107 --- .../src/app/models/service-log.model.ts | 27 - .../app/models/solr-collection-state.model.ts | 23 - .../src/app/models/store.model.ts | 169 ---- .../src/app/models/user-config.model.ts | 26 - .../services/component-actions.service.spec.ts | 8 +- .../app/services/component-actions.service.ts | 52 +- .../component-generator.service.spec.ts | 10 +- .../app/services/component-generator.service.ts | 23 +- .../src/app/services/filtering.service.spec.ts | 27 + .../src/app/services/filtering.service.ts | 97 ++- .../src/app/services/http-client.service.ts | 15 +- .../app/services/logs-container.service.spec.ts | 8 +- .../src/app/services/logs-container.service.ts | 70 +- .../services/storage/app-settings.service.ts | 4 +- .../app/services/storage/app-state.service.ts | 4 +- .../storage/audit-logs-fields.service.ts | 2 +- .../app/services/storage/audit-logs.service.ts | 2 +- .../app/services/storage/clusters.service.ts | 2 +- .../app/services/storage/components.service.ts | 2 +- .../src/app/services/storage/filters.service.ts | 2 +- .../src/app/services/storage/graphs.service.ts | 2 +- .../src/app/services/storage/hosts.service.ts | 2 +- .../app/services/storage/reducers.service.ts | 2 + .../storage/service-logs-fields.service.ts | 2 +- .../service-logs-histogram-data.service.ts | 2 +- .../storage/service-logs-truncated.service.ts | 32 + .../services/storage/service-logs.service.ts | 2 +- .../services/storage/user-configs.service.ts | 2 +- .../src/assets/i18n/en.json | 14 +- .../ambari-logsearch-web/webpack.config.js | 21 +- ambari-logsearch/ambari-logsearch-web/yarn.lock | 288 +++++-- ambari-metrics/ambari-metrics-assembly/pom.xml | 2 + ambari-metrics/ambari-metrics-common/pom.xml | 6 + .../flume/FlumeTimelineMetricsSinkTest.java | 27 +- .../ambari-metrics-timelineservice/pom.xml | 10 + ambari-server/pom.xml | 2 + ambari-server/src/main/assemblies/server.xml | 30 +- .../actionmanager/ExecutionCommandWrapper.java | 3 +- .../ambari/server/actionmanager/Stage.java | 3 +- .../ambari/server/agent/HeartbeatMonitor.java | 3 +- .../ambari/server/agent/StatusCommand.java | 16 + .../agent/stomp/dto/HostRepositories.java | 19 + .../server/api/services/AmbariMetaInfo.java | 60 +- .../server/checks/AbstractCheckDescriptor.java | 81 +- .../ambari/server/checks/CheckDescription.java | 8 + .../RequiredServicesInRepositoryCheck.java | 91 +++ .../ambari/server/checks/UpgradeCheck.java | 11 + .../AmbariCustomCommandExecutionHelper.java | 10 + .../controller/AmbariManagementController.java | 6 +- .../AmbariManagementControllerImpl.java | 69 +- .../controller/AmbariManagementHelper.java | 28 +- .../server/controller/ControllerModule.java | 4 + .../server/controller/KerberosHelper.java | 9 +- .../server/controller/KerberosHelperImpl.java | 252 +++--- .../controller/ResourceProviderFactory.java | 10 +- .../ServiceComponentHostResponse.java | 15 + .../server/controller/StackVersionResponse.java | 29 - .../AbstractControllerResourceProvider.java | 4 + .../internal/AlertTargetResourceProvider.java | 3 +- .../BlueprintConfigurationProcessor.java | 87 +- .../internal/ClientConfigResourceProvider.java | 2 - .../ClusterStackVersionResourceProvider.java | 31 +- .../internal/DefaultProviderModule.java | 4 - .../internal/ExtensionLinkResourceProvider.java | 15 + .../internal/HostComponentResourceProvider.java | 4 + .../internal/ServiceResourceProvider.java | 8 - .../internal/StackArtifactResourceProvider.java | 35 +- .../internal/UpgradeResourceProvider.java | 77 +- .../internal/ViewInstanceResourceProvider.java | 147 ++-- .../utilities/RemovableIdentities.java | 2 +- .../upgrade/HostVersionOutOfSyncListener.java | 15 + .../ambari/server/metadata/ActionMetadata.java | 1 + .../ambari/server/orm/DBAccessorImpl.java | 16 +- .../orm/entities/RepositoryVersionEntity.java | 39 +- .../orm/entities/ServiceDesiredStateEntity.java | 6 +- .../server/orm/entities/WidgetLayoutEntity.java | 6 +- .../AbstractPrepareKerberosServerAction.java | 14 + .../kerberos/CreateKeytabFilesServerAction.java | 2 +- .../kerberos/CreatePrincipalsServerAction.java | 2 +- .../kerberos/KerberosServerAction.java | 71 +- .../PrepareDisableKerberosServerAction.java | 3 +- .../PrepareEnableKerberosServerAction.java | 6 +- .../PrepareKerberosIdentitiesServerAction.java | 142 +++- .../upgrades/AbstractUpgradeServerAction.java | 6 +- .../AtlasProxyUserConfigCalculation.java | 62 ++ .../upgrades/AutoSkipFailedSummaryAction.java | 15 +- .../upgrades/ComponentVersionCheckAction.java | 2 +- .../serveraction/upgrades/ConfigureAction.java | 16 +- .../upgrades/FinalizeUpgradeAction.java | 23 +- .../FixCapacitySchedulerOrderingPolicy.java | 12 +- .../serveraction/upgrades/FixLzoCodecPath.java | 10 +- .../upgrades/FixOozieAdminUsers.java | 10 +- .../upgrades/FixYarnWebServiceUrl.java | 11 +- .../upgrades/HBaseConfigCalculation.java | 10 +- .../HBaseEnvMaxDirectMemorySizeAction.java | 11 +- .../upgrades/HiveEnvClasspathAction.java | 11 +- .../upgrades/HiveZKQuorumConfigAction.java | 13 +- .../upgrades/KerberosKeytabsAction.java | 13 +- .../upgrades/ManualStageAction.java | 3 +- .../upgrades/OozieConfigCalculation.java | 11 +- .../upgrades/PreconfigureKerberosAction.java | 15 +- .../upgrades/RangerConfigCalculation.java | 11 +- .../RangerKerberosConfigCalculation.java | 11 +- .../upgrades/RangerKmsProxyConfig.java | 11 +- .../RangerUsersyncConfigCalculation.java | 11 +- .../upgrades/RangerWebAlertConfigAction.java | 9 +- .../upgrades/SparkShufflePropertyConfig.java | 11 +- .../upgrades/UpdateDesiredRepositoryAction.java | 10 +- .../upgrades/UpgradeUserKerberosDescriptor.java | 6 +- .../upgrades/YarnConfigCalculation.java | 11 +- .../ambari/server/stack/ExtensionHelper.java | 88 ++- .../ambari/server/stack/ServiceDirectory.java | 3 +- .../ambari/server/stack/StackDirectory.java | 74 +- .../ambari/server/stack/StackManager.java | 5 + .../apache/ambari/server/stack/StackModule.java | 49 +- .../ambari/server/state/ConfigHelper.java | 164 +++- .../org/apache/ambari/server/state/Host.java | 17 + .../ambari/server/state/PropertyInfo.java | 29 + .../ambari/server/state/RefreshCommand.java | 52 ++ .../state/RefreshCommandConfiguration.java | 71 ++ .../ambari/server/state/RepositoryType.java | 18 + .../apache/ambari/server/state/ServiceInfo.java | 2 +- .../apache/ambari/server/state/StackInfo.java | 49 +- .../ambari/server/state/UpgradeContext.java | 79 +- .../ambari/server/state/host/HostImpl.java | 43 + .../kerberos/AbstractKerberosDescriptor.java | 25 + .../AbstractKerberosDescriptorContainer.java | 16 +- .../state/kerberos/KerberosDescriptor.java | 3 +- .../kerberos/KerberosIdentityDescriptor.java | 160 ++++ .../server/state/repository/StackPackage.java | 69 ++ .../state/repository/VersionDefinitionXml.java | 87 ++ .../ambari/server/state/stack/UpgradePack.java | 36 +- .../state/stack/upgrade/ClusterGrouping.java | 13 + .../state/stack/upgrade/ConfigureTask.java | 16 + .../stack/upgrade/ServerSideActionTask.java | 15 + .../state/stack/upgrade/TaskParameter.java | 41 + .../svccomphost/ServiceComponentHostImpl.java | 19 +- .../ServiceComponentHostSummary.java | 123 --- .../server/topology/AsyncCallableService.java | 110 ++- .../ambari/server/topology/TopologyManager.java | 26 +- .../topology/tasks/ConfigureClusterTask.java | 124 +-- .../server/upgrade/UpgradeCatalog260.java | 25 + .../ambari_server/dbConfiguration_linux.py | 9 +- .../python/ambari_server/resourceFilesKeeper.py | 7 +- .../python/ambari_server/serverConfiguration.py | 2 +- .../main/python/ambari_server/setupMpacks.py | 2 +- .../main/resources/Ambari-DDL-Oracle-CREATE.sql | 8 +- .../1.6.1.2.2.0/configuration/accumulo-env.xml | 6 + .../package/scripts/accumulo_script.py | 1 + .../0.1.0/configuration/infra-solr-env.xml | 6 + .../0.1.0/configuration/ams-env.xml | 6 + .../0.1.0/package/scripts/metrics_collector.py | 4 +- .../0.1.0/package/scripts/metrics_grafana.py | 4 +- .../0.1.0/package/scripts/metrics_monitor.py | 4 +- .../AMBARI_METRICS/0.1.0/service_advisor.py | 787 +++++++++++++++++++ .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 6 + .../ATLAS/0.7.0.3.0/configuration/atlas-env.xml | 6 + .../ATLAS/0.7.0.3.0/service_advisor.py | 5 +- .../DRUID/0.10.1/configuration/druid-broker.xml | 106 +++ .../DRUID/0.10.1/configuration/druid-common.xml | 270 +++++++ .../0.10.1/configuration/druid-coordinator.xml | 43 + .../DRUID/0.10.1/configuration/druid-env.xml | 248 ++++++ .../0.10.1/configuration/druid-historical.xml | 94 +++ .../DRUID/0.10.1/configuration/druid-log4j.xml | 84 ++ .../0.10.1/configuration/druid-logrotate.xml | 68 ++ .../configuration/druid-middlemanager.xml | 122 +++ .../0.10.1/configuration/druid-overlord.xml | 52 ++ .../DRUID/0.10.1/configuration/druid-router.xml | 59 ++ .../common-services/DRUID/0.10.1/metainfo.xml | 223 ++++++ .../DRUID/0.10.1/package/scripts/broker.py | 28 + .../DRUID/0.10.1/package/scripts/coordinator.py | 28 + .../DRUID/0.10.1/package/scripts/druid.py | 307 ++++++++ .../DRUID/0.10.1/package/scripts/druid_node.py | 115 +++ .../DRUID/0.10.1/package/scripts/historical.py | 28 + .../0.10.1/package/scripts/middlemanager.py | 28 + .../DRUID/0.10.1/package/scripts/overlord.py | 28 + .../DRUID/0.10.1/package/scripts/params.py | 200 +++++ .../DRUID/0.10.1/package/scripts/router.py | 28 + .../0.10.1/package/scripts/service_check.py | 44 ++ .../0.10.1/package/scripts/status_params.py | 24 + .../DRUID/0.10.1/quicklinks/quicklinks.json | 37 + .../DRUID/0.10.1/role_command_order.json | 17 + .../DRUID/0.10.1/themes/theme.json | 120 +++ .../DRUID/0.9.2/configuration/druid-broker.xml | 100 --- .../DRUID/0.9.2/configuration/druid-common.xml | 270 ------- .../0.9.2/configuration/druid-coordinator.xml | 43 - .../DRUID/0.9.2/configuration/druid-env.xml | 242 ------ .../0.9.2/configuration/druid-historical.xml | 88 --- .../DRUID/0.9.2/configuration/druid-log4j.xml | 84 -- .../0.9.2/configuration/druid-logrotate.xml | 68 -- .../0.9.2/configuration/druid-middlemanager.xml | 122 --- .../0.9.2/configuration/druid-overlord.xml | 52 -- .../DRUID/0.9.2/configuration/druid-router.xml | 59 -- .../common-services/DRUID/0.9.2/metainfo.xml | 223 ------ .../DRUID/0.9.2/package/scripts/broker.py | 28 - .../DRUID/0.9.2/package/scripts/coordinator.py | 28 - .../DRUID/0.9.2/package/scripts/druid.py | 307 -------- .../DRUID/0.9.2/package/scripts/druid_node.py | 114 --- .../DRUID/0.9.2/package/scripts/historical.py | 28 - .../0.9.2/package/scripts/middlemanager.py | 28 - .../DRUID/0.9.2/package/scripts/overlord.py | 28 - .../DRUID/0.9.2/package/scripts/params.py | 200 ----- .../DRUID/0.9.2/package/scripts/router.py | 28 - .../0.9.2/package/scripts/service_check.py | 44 -- .../0.9.2/package/scripts/status_params.py | 24 - .../DRUID/0.9.2/quicklinks/quicklinks.json | 37 - .../DRUID/0.9.2/role_command_order.json | 17 - .../DRUID/0.9.2/themes/theme.json | 120 --- .../0.5.0.2.1/configuration/falcon-env.xml | 6 + .../FLUME/1.4.0.2.0/configuration/flume-env.xml | 6 + .../0.96.0.2.0/configuration/hbase-env.xml | 6 + .../0.96.0.2.0/configuration/hbase-site.xml | 17 +- .../HBASE/2.0.0.3.0/configuration/hbase-env.xml | 6 + .../HDFS/2.1.0.2.0/configuration/core-site.xml | 12 + .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 10 + .../HDFS/2.1.0.2.0/configuration/hdfs-site.xml | 4 + .../HDFS/2.1.0.2.0/package/scripts/datanode.py | 13 +- .../HDFS/2.1.0.2.0/package/scripts/hdfs.py | 62 +- .../2.1.0.2.0/package/scripts/hdfs_client.py | 5 + .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 21 + .../2.1.0.2.0/package/scripts/install_params.py | 6 - .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 21 +- .../2.1.0.2.0/package/scripts/params_linux.py | 2 - .../HDFS/2.1.0.2.0/package/scripts/snamenode.py | 10 + .../HDFS/3.0.0.3.0/configuration/hadoop-env.xml | 10 + .../HDFS/3.0.0.3.0/configuration/hdfs-site.xml | 7 + .../HDFS/3.0.0.3.0/package/scripts/datanode.py | 13 +- .../HDFS/3.0.0.3.0/package/scripts/hdfs.py | 62 +- .../3.0.0.3.0/package/scripts/hdfs_client.py | 5 + .../3.0.0.3.0/package/scripts/hdfs_namenode.py | 20 + .../3.0.0.3.0/package/scripts/install_params.py | 6 - .../HDFS/3.0.0.3.0/package/scripts/namenode.py | 21 +- .../3.0.0.3.0/package/scripts/params_linux.py | 2 - .../HDFS/3.0.0.3.0/package/scripts/snamenode.py | 10 + .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 12 + .../HIVE/0.12.0.2.0/configuration/hive-site.xml | 228 +----- .../HIVE/0.12.0.2.0/package/scripts/hive.py | 34 +- .../package/scripts/hive_interactive.py | 13 + .../package/scripts/hive_server_upgrade.py | 5 - .../0.12.0.2.0/package/scripts/params_linux.py | 9 +- .../0.12.0.2.0/package/scripts/status_params.py | 8 +- .../HIVE/0.12.0.2.0/package/scripts/webhcat.py | 2 +- .../HIVE/2.1.0.3.0/configuration/hive-env.xml | 12 + .../HIVE/2.1.0.3.0/configuration/hive-site.xml | 48 ++ .../HIVE/2.1.0.3.0/package/scripts/hive.py | 32 + .../package/scripts/hive_interactive.py | 13 + .../2.1.0.3.0/package/scripts/params_linux.py | 4 + .../0.10.0.3.0/configuration/kafka-env.xml | 6 + .../KAFKA/0.8.1/configuration/kafka-env.xml | 6 + .../KNOX/0.5.0.2.2/configuration/knox-env.xml | 10 + .../KNOX/0.5.0.3.0/configuration/knox-env.xml | 10 + .../0.5.0/configuration/logsearch-env.xml | 6 + .../1.0.0.2.3/configuration/mahout-env.xml | 6 + .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 10 + .../OOZIE/4.0.0.2.0/package/scripts/oozie.py | 6 +- .../4.0.0.2.0/package/scripts/params_linux.py | 3 - .../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 10 + .../OOZIE/4.2.0.3.0/package/scripts/oozie.py | 5 +- .../4.2.0.3.0/package/scripts/params_linux.py | 3 - .../RANGER/0.4.0/configuration/ranger-env.xml | 10 + .../0.4.0/package/scripts/ranger_admin.py | 36 +- .../0.4.0/package/scripts/ranger_tagsync.py | 23 +- .../1.0.0.3.0/configuration/ranger-env.xml | 10 + .../1.0.0.3.0/package/scripts/ranger_admin.py | 24 +- .../1.0.0.3.0/package/scripts/ranger_tagsync.py | 23 +- .../0.5.0.2.3/configuration/kms-env.xml | 10 + .../0.5.0.2.3/package/scripts/kms_server.py | 21 +- .../1.0.0.3.0/configuration/kms-env.xml | 10 + .../1.0.0.3.0/package/scripts/kms_server.py | 19 +- .../RANGER_KMS/1.0.0.3.0/service_advisor.py | 3 +- .../SPARK/1.2.1/configuration/spark-env.xml | 10 + .../SPARK/2.2.0/configuration/livy-env.xml | 10 + .../SPARK/2.2.0/configuration/spark-env.xml | 10 + .../SPARK2/2.0.0/configuration/spark2-env.xml | 10 + .../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 6 + .../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 6 + .../STORM/0.9.1/configuration/storm-env.xml | 6 + .../package/scripts/supervisord_service.py | 1 + .../STORM/1.0.1.3.0/configuration/storm-env.xml | 6 + .../package/scripts/supervisord_service.py | 1 + .../0.15.0/configuration/superset-env.xml | 6 + .../TEZ/0.4.0.2.1/configuration/tez-env.xml | 10 + .../TEZ/0.9.0.3.0/configuration/tez-env.xml | 10 + .../configuration-mapred/mapred-env.xml | 6 + .../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 + .../2.1.0.2.0/package/scripts/params_linux.py | 34 +- .../2.1.0.2.0/package/scripts/status_params.py | 52 +- .../configuration-mapred/mapred-env.xml | 6 + .../YARN/3.0.0.3.0/configuration/yarn-env.xml | 6 + .../YARN/3.0.0.3.0/configuration/yarn-site.xml | 2 +- .../3.0.0.3.0/package/scripts/params_linux.py | 53 +- .../3.0.0.3.0/package/scripts/status_params.py | 52 +- .../YARN/3.0.0.3.0/service_advisor.py | 3 +- .../0.6.0/configuration/zeppelin-env.xml | 10 + .../0.7.0/configuration/zeppelin-env.xml | 10 + .../ZEPPELIN/0.7.0/package/scripts/master.py | 167 ++-- .../ZEPPELIN/0.7.0/package/scripts/params.py | 26 +- .../3.4.5/configuration/zookeeper-env.xml | 6 + .../src/main/resources/configuration-schema.xsd | 14 +- .../custom_actions/scripts/install_packages.py | 35 +- ambari-server/src/main/resources/kerberos.json | 79 ++ .../src/main/resources/properties.json | 1 + .../main/resources/scripts/Ambaripreupload.py | 4 +- .../scripts/post-user-creation-hook.sh | 2 +- .../stack-hooks/after-INSTALL/scripts/hook.py | 37 + .../stack-hooks/after-INSTALL/scripts/params.py | 108 +++ .../scripts/shared_initialization.py | 132 ++++ .../before-ANY/files/changeToSecureUid.sh | 64 ++ .../stack-hooks/before-ANY/scripts/hook.py | 36 + .../stack-hooks/before-ANY/scripts/params.py | 254 ++++++ .../before-ANY/scripts/shared_initialization.py | 273 +++++++ .../stack-hooks/before-INSTALL/scripts/hook.py | 37 + .../before-INSTALL/scripts/params.py | 115 +++ .../scripts/repo_initialization.py | 73 ++ .../scripts/shared_initialization.py | 37 + .../stack-hooks/before-RESTART/scripts/hook.py | 29 + .../before-START/files/checkForFormat.sh | 65 ++ .../before-START/files/fast-hdfs-resource.jar | Bin 0 -> 28296600 bytes .../before-START/files/task-log4j.properties | 134 ++++ .../before-START/files/topology_script.py | 66 ++ .../before-START/scripts/custom_extensions.py | 173 ++++ .../stack-hooks/before-START/scripts/hook.py | 43 + .../stack-hooks/before-START/scripts/params.py | 371 +++++++++ .../before-START/scripts/rack_awareness.py | 48 ++ .../scripts/shared_initialization.py | 256 ++++++ .../templates/commons-logging.properties.j2 | 43 + .../templates/exclude_hosts_list.j2 | 21 + .../templates/hadoop-metrics2.properties.j2 | 107 +++ .../before-START/templates/health_check.j2 | 81 ++ .../templates/include_hosts_list.j2 | 21 + .../templates/topology_mappings.data.j2 | 24 + .../HDP/2.0.6/configuration/cluster-env.xml | 10 + .../2.0.6/hooks/after-INSTALL/scripts/hook.py | 37 - .../2.0.6/hooks/after-INSTALL/scripts/params.py | 115 --- .../scripts/shared_initialization.py | 132 ---- .../hooks/before-ANY/files/changeToSecureUid.sh | 64 -- .../HDP/2.0.6/hooks/before-ANY/scripts/hook.py | 36 - .../2.0.6/hooks/before-ANY/scripts/params.py | 277 ------- .../before-ANY/scripts/shared_initialization.py | 281 ------- .../2.0.6/hooks/before-INSTALL/scripts/hook.py | 37 - .../hooks/before-INSTALL/scripts/params.py | 115 --- .../scripts/repo_initialization.py | 73 -- .../scripts/shared_initialization.py | 37 - .../2.0.6/hooks/before-RESTART/scripts/hook.py | 29 - .../hooks/before-START/files/checkForFormat.sh | 65 -- .../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes .../before-START/files/task-log4j.properties | 134 ---- .../hooks/before-START/files/topology_script.py | 66 -- .../before-START/scripts/custom_extensions.py | 173 ---- .../2.0.6/hooks/before-START/scripts/hook.py | 43 - .../2.0.6/hooks/before-START/scripts/params.py | 378 --------- .../before-START/scripts/rack_awareness.py | 48 -- .../scripts/shared_initialization.py | 256 ------ .../templates/commons-logging.properties.j2 | 43 - .../templates/exclude_hosts_list.j2 | 21 - .../templates/hadoop-metrics2.properties.j2 | 107 --- .../before-START/templates/health_check.j2 | 81 -- .../templates/include_hosts_list.j2 | 21 - .../templates/topology_mappings.data.j2 | 24 - .../resources/stacks/HDP/2.0.6/kerberos.json | 79 -- .../HDP/2.0.6/properties/stack_packages.json | 8 +- .../stacks/HDP/2.0.6/role_command_order.json | 2 +- .../stacks/HDP/2.0.6/services/stack_advisor.py | 562 +------------ .../resources/stacks/HDP/2.0.6/widgets.json | 95 --- .../services/HBASE/configuration/hbase-env.xml | 6 + .../stacks/HDP/2.2/services/stack_advisor.py | 1 - .../services/ECS/package/scripts/ecs_client.py | 2 +- .../services/ECS/package/scripts/params.py | 2 +- .../services/HDFS/configuration/hdfs-site.xml | 3 + .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 5 + .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 3 + .../stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 1 + .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 4 + .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 3 + .../stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 1 + .../services/ATLAS/configuration/atlas-env.xml | 6 + .../HIVE/configuration/llap-daemon-log4j.xml | 2 +- .../services/SPARK/configuration/livy-env.xml | 10 + .../stacks/HDP/2.5/services/stack_advisor.py | 6 +- .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 10 + .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 7 + .../stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 4 + .../stacks/HDP/2.6/kerberos_preconfigure.json | 9 + .../configuration/application-properties.xml | 15 + .../stacks/HDP/2.6/services/DRUID/kerberos.json | 71 +- .../stacks/HDP/2.6/services/DRUID/metainfo.xml | 5 +- .../configuration/hive-interactive-site.xml | 225 ++++++ .../HIVE/configuration/tez-interactive-site.xml | 6 + .../services/SPARK/configuration/livy-env.xml | 10 + .../services/STORM/configuration/storm-site.xml | 6 +- .../HDP/2.6/services/SUPERSET/kerberos.json | 10 +- .../services/YARN/configuration/yarn-site.xml | 20 +- .../ZEPPELIN/configuration/zeppelin-env.xml | 10 + .../stacks/HDP/2.6/services/stack_advisor.py | 48 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 32 +- .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 15 +- .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 5 + .../HDP/3.0/configuration/cluster-env.xml | 10 + .../HDP/3.0/hooks/after-INSTALL/scripts/hook.py | 37 - .../3.0/hooks/after-INSTALL/scripts/params.py | 109 --- .../scripts/shared_initialization.py | 140 ---- .../hooks/before-ANY/files/changeToSecureUid.sh | 53 -- .../HDP/3.0/hooks/before-ANY/scripts/hook.py | 36 - .../HDP/3.0/hooks/before-ANY/scripts/params.py | 259 ------ .../before-ANY/scripts/shared_initialization.py | 239 ------ .../3.0/hooks/before-INSTALL/scripts/hook.py | 37 - .../3.0/hooks/before-INSTALL/scripts/params.py | 115 --- .../scripts/repo_initialization.py | 76 -- .../scripts/shared_initialization.py | 37 - .../3.0/hooks/before-RESTART/scripts/hook.py | 29 - .../hooks/before-START/files/checkForFormat.sh | 65 -- .../before-START/files/fast-hdfs-resource.jar | Bin 28296600 -> 0 bytes .../before-START/files/task-log4j.properties | 134 ---- .../hooks/before-START/files/topology_script.py | 66 -- .../HDP/3.0/hooks/before-START/scripts/hook.py | 40 - .../3.0/hooks/before-START/scripts/params.py | 364 --------- .../before-START/scripts/rack_awareness.py | 47 -- .../scripts/shared_initialization.py | 249 ------ .../templates/commons-logging.properties.j2 | 43 - .../templates/exclude_hosts_list.j2 | 21 - .../templates/hadoop-metrics2.properties.j2 | 107 --- .../before-START/templates/health_check.j2 | 81 -- .../templates/include_hosts_list.j2 | 21 - .../templates/topology_mappings.data.j2 | 24 - .../main/resources/stacks/HDP/3.0/kerberos.json | 79 -- .../stacks/HDP/3.0/role_command_order.json | 2 +- .../main/resources/stacks/HDP/3.0/widgets.json | 95 --- .../src/main/resources/upgrade-pack.xsd | 9 + ambari-server/src/main/resources/widgets.json | 95 +++ .../server/api/services/AmbariMetaInfoTest.java | 44 +- .../checks/AbstractCheckDescriptorTest.java | 3 + .../checks/AutoStartDisabledCheckTest.java | 12 +- .../checks/ClientRetryPropertyCheckTest.java | 2 + .../checks/ComponentExistsInRepoCheckTest.java | 2 + .../checks/ComponentsInstallationCheckTest.java | 3 + .../checks/ConfigurationMergeCheckTest.java | 2 + .../checks/DruidHighAvailabilityCheckTest.java | 2 + .../checks/HiveMultipleMetastoreCheckTest.java | 3 + .../checks/HiveNotRollingWarningTest.java | 3 + .../server/checks/HostsHeartbeatCheckTest.java | 2 + .../checks/HostsMasterMaintenanceCheckTest.java | 3 + .../checks/HostsRepositoryVersionCheckTest.java | 2 + .../server/checks/InstallPackagesCheckTest.java | 2 + .../server/checks/KafkaKerberosCheckTest.java | 2 + ...duce2JobHistoryStatePreservingCheckTest.java | 2 + .../server/checks/RangerAuditDbCheckTest.java | 3 + .../server/checks/RangerPasswordCheckTest.java | 3 + .../server/checks/RangerSSLConfigCheckTest.java | 3 + .../RequiredServicesInRepositoryCheckTest.java | 115 +++ .../SecondaryNamenodeDeletedCheckTest.java | 3 + .../ServicesMaintenanceModeCheckTest.java | 2 + ...vicesMapReduceDistributedCacheCheckTest.java | 3 + ...rvicesNamenodeHighAvailabilityCheckTest.java | 3 + .../ServicesNamenodeTruncateCheckTest.java | 2 + .../ServicesTezDistributedCacheCheckTest.java | 3 + .../server/checks/ServicesUpCheckTest.java | 2 + .../ServicesYarnWorkPreservingCheckTest.java | 2 + .../server/checks/StormShutdownWarningTest.java | 3 + .../AmbariManagementControllerImplTest.java | 6 +- .../AmbariManagementControllerTest.java | 6 + .../server/controller/KerberosHelperTest.java | 5 - .../BlueprintConfigurationProcessorTest.java | 60 +- .../internal/UpgradeResourceProviderTest.java | 46 +- .../utilities/KerberosIdentityCleanerTest.java | 8 +- .../HostVersionOutOfSyncListenerTest.java | 14 +- .../upgrade/StackVersionListenerTest.java | 1 - .../apache/ambari/server/orm/OrmTestHelper.java | 2 +- ...AbstractPrepareKerberosServerActionTest.java | 5 + .../AtlasProxyUserConfigCalculationTest.java | 127 +++ .../FixCapacitySchedulerOrderingPolicyTest.java | 2 +- .../upgrades/FixOozieAdminUsersTest.java | 2 +- .../upgrades/FixYarnWebServiceUrlTest.java | 2 +- .../HBaseEnvMaxDirectMemorySizeActionTest.java | 2 +- .../upgrades/HiveEnvClasspathActionTest.java | 2 +- .../upgrades/HiveZKQuorumConfigActionTest.java | 2 +- .../upgrades/KerberosKeytabsActionTest.java | 17 + .../upgrades/RangerConfigCalculationTest.java | 2 +- .../RangerKerberosConfigCalculationTest.java | 2 +- .../upgrades/RangerKmsProxyConfigTest.java | 2 +- .../RangerUsersyncConfigCalculationTest.java | 2 +- .../RangerWebAlertConfigActionTest.java | 6 +- .../SparkShufflePropertyConfigTest.java | 2 +- .../UpgradeUserKerberosDescriptorTest.java | 4 +- .../server/stack/StackManagerExtensionTest.java | 49 +- .../ambari/server/stack/StackManagerTest.java | 8 - .../ambari/server/state/CheckHelperTest.java | 7 + .../ambari/server/state/ConfigHelperTest.java | 76 +- .../ambari/server/state/PropertyInfoTest.java | 20 + .../ambari/server/state/UpgradeContextTest.java | 96 ++- .../state/kerberos/KerberosDescriptorTest.java | 198 ++++- .../KerberosDescriptorUpdateHelperTest.java | 1 + .../kerberos/VariableReplacementHelperTest.java | 35 +- .../state/stack/ConfigUpgradeValidityTest.java | 10 + .../svccomphost/ServiceComponentHostTest.java | 102 ++- .../topology/AsyncCallableServiceTest.java | 89 +-- .../ClusterDeployWithStartOnlyTest.java | 6 + ...InstallWithoutStartOnComponentLevelTest.java | 6 + .../ClusterInstallWithoutStartTest.java | 6 + .../topology/ConfigureClusterTaskTest.java | 64 +- .../server/topology/TopologyManagerTest.java | 14 +- .../src/test/python/TestAmbariServer.py | 4 +- ambari-server/src/test/python/TestMpacks.py | 12 +- .../src/test/python/TestResourceFilesKeeper.py | 1 + .../src/test/python/TestUpgradeSummary.py | 6 +- .../AMBARI_METRICS/test_service_advisor.py | 596 ++++++++++++++ .../HIVE/test_jdbc_driver_config.py | 18 +- .../RANGER/test_db_flavor_config.py | 17 +- .../RANGER_KMS/test_db_flavor_config.py | 17 +- .../SQOOP/test_jdbc_driver_config.py | 16 +- .../configs/ranger_admin_default.json | 55 ++ .../custom_actions/TestInstallPackages.py | 50 +- .../stacks/2.0.6/HBASE/test_hbase_client.py | 1 - .../stacks/2.0.6/HBASE/test_hbase_master.py | 8 +- .../2.0.6/HBASE/test_phoenix_queryserver.py | 7 + .../python/stacks/2.0.6/HDFS/test_datanode.py | 55 +- .../python/stacks/2.0.6/HDFS/test_namenode.py | 38 +- .../stacks/2.0.6/HIVE/test_hive_metastore.py | 42 +- .../stacks/2.0.6/HIVE/test_hive_server.py | 148 ++-- .../2.0.6/HIVE/test_hive_service_check.py | 4 +- .../stacks/2.0.6/HIVE/test_webhcat_server.py | 41 +- .../stacks/2.0.6/OOZIE/test_oozie_server.py | 9 + .../2.0.6/OOZIE/test_oozie_service_check.py | 5 +- .../stacks/2.0.6/YARN/test_historyserver.py | 5 +- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 1 + .../stacks/2.0.6/common/test_stack_advisor.py | 576 -------------- .../python/stacks/2.0.6/configs/default.json | 1 + .../hooks/after-INSTALL/test_after_install.py | 88 +-- .../2.0.6/hooks/before-ANY/test_before_any.py | 24 +- .../hooks/before-INSTALL/test_before_install.py | 14 +- .../hooks/before-START/test_before_start.py | 21 +- .../stacks/2.1/FALCON/test_falcon_server.py | 26 +- .../stacks/2.1/HIVE/test_hive_metastore.py | 54 +- .../stacks/2.2/PIG/test_pig_service_check.py | 13 + .../stacks/2.2/SPARK/test_job_history_server.py | 18 +- .../stacks/2.2/common/test_conf_select.py | 13 +- .../stacks/2.2/common/test_stack_advisor.py | 511 ------------ .../2.2/common/test_stack_advisor_perf.py | 66 +- .../stacks/2.3/MAHOUT/test_mahout_client.py | 2 +- .../2.3/MAHOUT/test_mahout_service_check.py | 28 +- .../2.3/SPARK/test_spark_thrift_server.py | 8 +- .../stacks/2.3/common/test_stack_advisor.py | 2 +- .../stacks/2.5/HIVE/test_hive_server_int.py | 16 + .../stacks/2.5/RANGER_KMS/test_kms_server.py | 24 +- .../python/stacks/2.5/SPARK/test_spark_livy.py | 16 +- .../stacks/2.5/ZEPPELIN/test_zeppelin_060.py | 20 +- .../stacks/2.5/common/test_stack_advisor.py | 5 +- .../stacks/2.5/configs/ranger-kms-secured.json | 6 +- .../test/python/stacks/2.6/DRUID/test_druid.py | 22 +- .../stacks/2.6/SPARK2/test_spark_livy2.py | 16 +- .../stacks/2.6/ZEPPELIN/test_zeppelin_070.py | 218 ++--- .../stacks/2.6/common/test_stack_advisor.py | 214 ++++- .../src/test/python/stacks/utils/RMFTestCase.py | 47 +- .../resources/extensions/EXT/0.2/metainfo.xml | 4 +- .../resources/extensions/EXT/0.3/metainfo.xml | 2 +- .../resources/extensions/EXT/0.4/metainfo.xml | 32 + .../EXT/0.4/services/OOZIE2/metainfo.xml | 118 +++ .../services/OOZIE2/themes/broken_theme.json | 3 + ambari-server/src/test/resources/kerberos.json | 42 + .../services/HDFS/configuration/hdfs-site.xml | 8 + .../resources/stacks/HDP/2.0.8/kerberos.json | 42 - .../resources/stacks/OTHER/1.0/widgets.json | 95 --- .../stacks_with_extensions/HDP/0.4/metainfo.xml | 22 + .../HDP/0.4/repos/repoinfo.xml | 63 ++ .../HDP/0.4/services/HBASE/metainfo.xml | 26 + .../0.4/services/HDFS/configuration/global.xml | 145 ++++ .../services/HDFS/configuration/hadoop-env.xml | 223 ++++++ .../services/HDFS/configuration/hbase-site.xml | 137 ++++ .../services/HDFS/configuration/hdfs-log4j.xml | 199 +++++ .../services/HDFS/configuration/hdfs-site.xml | 396 ++++++++++ .../HDP/0.4/services/HDFS/metainfo.xml | 30 + .../0.4/services/HDFS/package/dummy-script.py | 20 + .../HDP/0.4/services/HIVE/metainfo.xml | 26 + .../HDP/0.4/services/MAPREDUCE/metainfo.xml | 23 + .../HDP/0.4/services/ZOOKEEPER/metainfo.xml | 26 + ambari-server/src/test/resources/widgets.json | 95 +++ ambari-utility/pom.xml | 6 + ambari-web/app/app.js | 24 +- ambari-web/app/assets/test/tests.js | 2 + ambari-web/app/controllers.js | 1 + ambari-web/app/controllers/installer.js | 8 +- .../journalNode/progress_controller.js | 4 +- .../journalNode/step4_controller.js | 6 +- .../nameNode/step5_controller.js | 6 +- .../highAvailability/progress_controller.js | 4 +- .../main/admin/kerberos/step2_controller.js | 7 +- .../main/admin/stack_and_upgrade_controller.js | 8 +- ambari-web/app/controllers/main/host/details.js | 2 - .../app/controllers/main/service/info/metric.js | 468 +++++++++++ .../controllers/main/service/info/summary.js | 450 +---------- .../main/service/reassign/step3_controller.js | 18 +- .../main/service/reassign/step4_controller.js | 2 - .../service/widgets/create/wizard_controller.js | 2 +- .../app/controllers/wizard/step6_controller.js | 15 +- .../app/controllers/wizard/step8_controller.js | 15 +- .../app/mappers/repository_version_mapper.js | 2 +- ambari-web/app/messages.js | 10 +- .../app/mixins/common/configs/configs_saver.js | 26 +- .../main/service/configs/config_overridable.js | 1 - .../mixins/wizard/assign_master_components.js | 2 +- ambari-web/app/models/alerts/alert_group.js | 2 +- .../models/configs/service_config_version.js | 9 +- ambari-web/app/models/host_stack_version.js | 4 +- .../models/stack_version/repository_version.js | 2 +- ambari-web/app/routes/add_kerberos_routes.js | 22 +- ambari-web/app/routes/installer.js | 3 +- ambari-web/app/routes/main.js | 1 + ambari-web/app/styles/alerts.less | 14 +- ambari-web/app/styles/application.less | 15 +- ambari-web/app/styles/bootstrap_overrides.less | 4 + ambari-web/app/styles/common.less | 3 +- ambari-web/app/styles/dashboard.less | 28 +- .../app/styles/enhanced_service_dashboard.less | 26 +- ambari-web/app/styles/modal_popups.less | 2 +- ambari-web/app/styles/stack_versions.less | 13 +- .../app/styles/theme/bootstrap-ambari.css | 78 +- ambari-web/app/styles/top-nav.less | 27 +- ambari-web/app/styles/wizard.less | 33 +- ambari-web/app/templates/application.hbs | 39 +- ambari-web/app/templates/common/breadcrumbs.hbs | 4 +- .../common/configs/config_history_flow.hbs | 8 +- .../common/configs/service_version_box.hbs | 8 +- .../templates/common/host_progress_popup.hbs | 6 +- .../modal_popups/widget_browser_popup.hbs | 6 +- ambari-web/app/templates/experimental.hbs | 6 +- ambari-web/app/templates/installer.hbs | 2 +- .../stack_upgrade/stack_upgrade_wizard.hbs | 2 +- .../main/admin/stack_upgrade/versions.hbs | 6 +- ambari-web/app/templates/main/alerts.hbs | 3 +- .../main/alerts/alert_definitions_actions.hbs | 2 +- ambari-web/app/templates/main/dashboard.hbs | 2 +- .../templates/main/dashboard/config_history.hbs | 2 +- ambari-web/app/templates/main/host.hbs | 6 +- .../templates/main/host/bulk_operation_menu.hbs | 134 ++-- .../main/host/details/host_component.hbs | 176 +++-- .../app/templates/main/host/stack_versions.hbs | 4 +- ambari-web/app/templates/main/host/summary.hbs | 120 +-- .../app/templates/main/service/info/metrics.hbs | 104 +++ .../main/service/info/service_alert_popup.hbs | 6 +- .../app/templates/main/service/info/summary.hbs | 105 +-- .../service/info/summary/master_components.hbs | 4 - ambari-web/app/templates/main/service/item.hbs | 7 +- .../templates/main/service/reassign/step3.hbs | 6 +- .../templates/main/service/services/hdfs.hbs | 38 +- ambari-web/app/templates/wizard/step6.hbs | 2 +- ambari-web/app/templates/wizard/step9.hbs | 12 +- ambari-web/app/utils/ajax/ajax.js | 2 - ambari-web/app/utils/helper.js | 23 + ambari-web/app/views.js | 1 + .../stack_upgrade/upgrade_version_box_view.js | 74 +- .../upgrade_version_column_view.js | 2 +- .../admin/stack_upgrade/upgrade_wizard_view.js | 26 +- .../views/main/host/hosts_table_menu_view.js | 4 +- .../app/views/main/host/stack_versions_view.js | 2 +- ambari-web/app/views/main/service/info/menu.js | 7 + .../app/views/main/service/info/metrics_view.js | 290 +++++++ .../app/views/main/service/info/summary.js | 323 ++------ ambari-web/app/views/main/service/item.js | 8 +- .../app/views/main/service/services/hdfs.js | 2 - ambari-web/test/controllers/installer_test.js | 37 +- .../journalNode/progress_controller_test.js | 2 - .../progress_controller_test.js | 3 - .../admin/kerberos/step2_controller_test.js | 9 +- .../main/service/info/metric_test.js | 110 +++ .../main/service/info/summary_test.js | 76 -- .../mixins/common/configs/configs_saver_test.js | 28 +- .../wizard/assign_master_components_test.js | 2 +- .../test/models/alerts/alert_group_test.js | 5 +- .../stack_version/repository_version_test.js | 12 +- .../upgrade_version_box_view_test.js | 149 ++-- .../upgrade_version_column_view_test.js | 6 + .../stack_upgrade/upgrade_wizard_view_test.js | 38 + .../main/service/info/metrics_view_test.js | 334 ++++++++ .../views/main/service/info/summary_test.js | 281 +------ ambari-web/test/views/main/service/item_test.js | 6 +- .../src/main/assemblies/hdf-ambari-mpack.xml | 1 + .../HIVE/package/scripts/hive_client.py | 2 - .../HIVE/package/scripts/hive_metastore.py | 1 - .../HIVE/package/scripts/hive_server.py | 2 +- .../package/scripts/hive_server_interactive.py | 1 - .../HIVE/package/scripts/webhcat_server.py | 2 - .../scripts/application_timeline_server.py | 1 - .../YARN/package/scripts/historyserver.py | 1 - .../YARN/package/scripts/mapreduce2_client.py | 2 - .../YARN/package/scripts/nodemanager.py | 1 - .../YARN/package/scripts/resourcemanager.py | 1 - .../YARN/package/scripts/yarn_client.py | 1 - .../ui/hive-web/app/adapters/application.js | 18 + .../app/controllers/visualization-ui.js | 10 +- .../ui/hive-web/app/utils/constants.js | 1 + .../view/hive20/resources/browser/DDLProxy.java | 154 ++-- .../savedQueries/SavedQueryResourceManager.java | 17 +- .../savedQueries/SavedQueryService.java | 46 +- .../hive20/resources/uploads/UploadService.java | 12 +- .../uploads/query/InsertFromQueryInput.java | 13 +- .../resources/ui/app/components/job-item.js | 49 +- .../resources/ui/app/components/jobs-browser.js | 3 + .../ui/app/components/query-result-table.js | 2 +- .../databases/database/tables/upload-table.js | 12 + .../src/main/resources/ui/app/routes/jobs.js | 11 + .../main/resources/ui/app/routes/queries/new.js | 2 + .../resources/ui/app/routes/queries/query.js | 237 ++++-- .../src/main/resources/ui/app/services/jobs.js | 31 +- .../resources/ui/app/services/saved-queries.js | 21 + .../src/main/resources/ui/app/styles/app.scss | 8 + .../ui/app/templates/components/job-item.hbs | 2 +- .../app/templates/components/jobs-browser.hbs | 2 +- .../main/resources/ui/app/templates/jobs.hbs | 1 + .../hive20/src/main/resources/ui/yarn.lock | 2 +- .../apache/ambari/view/utils/hdfs/HdfsApi.java | 2 +- docs/pom.xml | 61 +- 855 files changed, 21145 insertions(+), 16624 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-agent/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-agent/src/main/python/ambari_agent/AmbariConfig.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-agent/src/main/python/ambari_agent/FileCache.py ---------------------------------------------------------------------- diff --cc ambari-agent/src/main/python/ambari_agent/FileCache.py index 17235bf,28912d1..6994044 --- a/ambari-agent/src/main/python/ambari_agent/FileCache.py +++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py @@@ -82,11 -83,10 +82,10 @@@ class FileCache() Returns a base directory for hooks """ try: - hooks_subpath = command['clusterLevelParams']['hooks_folder'] - hooks_path = command['commandParams']['hooks_folder'] ++ hooks_path = command['clusterLevelParams']['hooks_folder'] except KeyError: return None - subpath = os.path.join(self.STACKS_CACHE_DIRECTORY, hooks_subpath) - return self.provide_directory(self.cache_dir, subpath, + return self.provide_directory(self.cache_dir, hooks_path, server_url_prefix) http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-agent/src/main/python/ambari_agent/main.py ---------------------------------------------------------------------- diff --cc ambari-agent/src/main/python/ambari_agent/main.py index de23ff6,5fcd051..3948290 --- a/ambari-agent/src/main/python/ambari_agent/main.py +++ b/ambari-agent/src/main/python/ambari_agent/main.py @@@ -86,9 -89,9 +86,8 @@@ import tim import locale import platform import ConfigParser - import ProcessHelper import resource from logging.handlers import SysLogHandler -from Controller import Controller import AmbariConfig from NetUtil import NetUtil from PingPortListener import PingPortListener @@@ -298,10 -290,10 +300,10 @@@ def stop_agent() pid = -1 runner = shellRunner() try: - with open(ProcessHelper.pidfile, 'r') as f: + with open(agent_pidfile, 'r') as f: pid = f.read() pid = int(pid) - + runner.run([AMBARI_SUDO_BINARY, 'kill', '-15', str(pid)]) for i in range(GRACEFUL_STOP_TRIES): result = runner.run([AMBARI_SUDO_BINARY, 'kill', '-0', str(pid)]) http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py ---------------------------------------------------------------------- diff --cc ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py index 26fbaaf,c89e767..8a53502 --- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py @@@ -374,62 -216,82 +216,82 @@@ def convert_conf_directories_to_symlink if not os.path.exists(d): need_dirs.append(d) + # log that we'll actually be creating some directories soon if len(need_dirs) > 0: - create(stack_name, package, version) + Logger.info("Package {0} will have the following new configuration directories created: {1}".format( + package, ", ".join(dry_run_directory))) - # find the matching definition and back it up (not the most efficient way) ONLY if there is more than one directory - if len(dirs) > 1: - for need_dir in need_dirs: - for dir_def in dirs: - if 'prefix' in dir_def and need_dir.startswith(dir_def['prefix']): - old_conf = dir_def['conf_dir'] - versioned_conf = need_dir - Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), - only_if = format("ls -d {old_conf}/*")) - elif 1 == len(dirs) and 1 == len(need_dirs): - old_conf = dirs[0]['conf_dir'] - versioned_conf = need_dirs[0] - Execute(as_sudo(["cp", "-R", "-p", os.path.join(old_conf, "*"), versioned_conf], auto_escape=False), - only_if = format("ls -d {old_conf}/*")) + # Create the versioned /etc/[component]/[version]/0 folder (using create-conf-dir) and then + # set it for the installed component: + # - Creates /etc/<component>/<version>/0 + # - Links <stack-root>/<version>/<component>/conf -> /etc/<component>/<version>/0 + select(stack_name, package, version, ignore_errors = True) + # check every existing link to see if it's a link and if it's pointed to the right spot + for directory_struct in dirs: + try: + # check if conf is a link already + old_conf = directory_struct['conf_dir'] + current_dir = directory_struct['current_dir'] + if os.path.islink(old_conf): + # it's already a link; make sure it's a link to where we want it + if os.readlink(old_conf) != current_dir: + # the link isn't to the right spot; re-link it + Logger.info("Re-linking symlink {0} to {1}".format(old_conf, current_dir)) + Link(old_conf, action = "delete") + Link(old_conf, to = current_dir) + else: + Logger.info("{0} is already linked to {1}".format(old_conf, current_dir)) + elif os.path.isdir(old_conf): + # the /etc/<component>/conf directory is not a link, so turn it into one + Logger.info("{0} is a directory - it must be converted into a symlink".format(old_conf)) - # <stack-root>/current/[component] is already set to to the correct version, e.g., <stack-root>/[version]/[component] + backup_dir = _get_backup_conf_directory(old_conf) + Logger.info("Backing up {0} to {1} if destination doesn't exist already.".format(old_conf, backup_dir)) + Execute(("cp", "-R", "-p", old_conf, backup_dir), + not_if = format("test -e {backup_dir}"), sudo = True) - select(stack_name, package, version, ignore_errors = True) + # delete the old /etc/<component>/conf directory now that it's been backed up + Directory(old_conf, action = "delete") - # Symlink /etc/[component]/conf to /etc/[component]/conf.backup - try: - # No more references to /etc/[component]/conf - for dir_def in dirs: - # E.g., /etc/[component]/conf - new_symlink = dir_def['conf_dir'] - - # Delete the existing directory/link so that linking will work - if not os.path.islink(new_symlink): - Directory(new_symlink, action = "delete") - else: - Link(new_symlink, action = "delete") - - old_conf = dir_def['conf_dir'] - backup_dir = _get_backup_conf_directory(old_conf) - # link /etc/[component]/conf -> /etc/[component]/conf.backup - # or - # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf - if link_to == DIRECTORY_TYPE_BACKUP: - Link(new_symlink, to=backup_dir) + # link /etc/[component]/conf -> <stack-root>/current/[component]-client/conf + Link(old_conf, to = current_dir) else: - Link(new_symlink, to=dir_def['current_dir']) - - #HACK + # missing entirely + # /etc/<component>/conf -> <stack-root>/current/<component>/conf if package in ["atlas", ]: - Logger.info("Seeding the new conf symlink {0} from the old backup directory {1} in case any " - "unmanaged artifacts are needed.".format(new_symlink, backup_dir)) - # If /etc/[component]/conf.backup exists, then copy any artifacts not managed by Ambari to the new symlink target - # Be careful not to clobber any existing files. - Execute(as_sudo(["cp", "-R", "--no-clobber", os.path.join(backup_dir, "*"), new_symlink], auto_escape=False), - only_if=format("test -e {new_symlink}")) - except Exception, e: - Logger.warning("Could not change symlink for package {0} to point to {1} directory. Error: {2}".format(package, link_to, e)) + # HACK for Atlas + ''' + In the case of Atlas, the Hive RPM installs /usr/$stack/$version/atlas with some partial packages that + contain Hive hooks, while the Atlas RPM is responsible for installing the full content. - ++ + If the user does not have Atlas currently installed on their stack, then /usr/$stack/current/atlas-client + will be a broken symlink, and we should not create the + symlink /etc/atlas/conf -> /usr/$stack/current/atlas-client/conf . + If we mistakenly create this symlink, then when the user performs an EU/RU and then adds Atlas service + then the Atlas RPM will not be able to copy its artifacts into /etc/atlas/conf directory and therefore + prevent Ambari from by copying those unmanaged contents into /etc/atlas/$version/0 + ''' + component_list = default("/localComponents", []) + if "ATLAS_SERVER" in component_list or "ATLAS_CLIENT" in component_list: + Logger.info("Atlas is installed on this host.") + parent_dir = os.path.dirname(current_dir) + if os.path.exists(parent_dir): + Link(old_conf, to = current_dir) + else: + Logger.info( + "Will not create symlink from {0} to {1} because the destination's parent dir does not exist.".format( + old_conf, current_dir)) + else: + Logger.info( + "Will not create symlink from {0} to {1} because Atlas is not installed on this host.".format( + old_conf, current_dir)) + else: + # Normal path for other packages + Link(old_conf, to = current_dir) + + except Exception, e: + Logger.warning("Could not change symlink for package {0} to point to current directory. Error: {1}".format(package, e)) def _seed_new_configuration_directories(package, created_directories): http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --cc ambari-common/src/main/python/resource_management/libraries/script/script.py index e507b0a,12e6f98..97e7e50 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@@ -744,6 -771,19 +771,19 @@@ class Script(object) """ self.install_packages(env) + def load_available_packages(self): + if self.available_packages_in_repos: + return self.available_packages_in_repos + + - pkg_provider = get_provider("Package") ++ pkg_provider = get_provider("Package") + try: + self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(self.get_config()['repositoryFile']['repositories']) + except Exception as err: + Logger.exception("Unable to load available packages") + self.available_packages_in_repos = [] + + def install_packages(self, env): """ List of packages that are required< by service is received from the server @@@ -763,15 -803,9 +803,9 @@@ return pass try: - package_list_str = config['hostLevelParams']['package_list'] - agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) - agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count']) + package_list_str = config['commandParams']['package_list'] + agent_stack_retry_on_unavailability = bool(config['ambariLevelParams']['agent_stack_retry_on_unavailability']) + agent_stack_retry_count = int(config['ambariLevelParams']['agent_stack_retry_count']) - pkg_provider = get_provider("Package") - try: - available_packages_in_repos = pkg_provider.get_available_packages_in_repos(config['repositoryFile']['repositories']) - except Exception as err: - Logger.exception("Unable to load available packages") - available_packages_in_repos = [] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-infra/ambari-infra-solr-client/src/main/python/solrDataManager.py ---------------------------------------------------------------------- diff --cc ambari-infra/ambari-infra-solr-client/src/main/python/solrDataManager.py index e0356bb,2675bd9..e0356bb mode 100644,100755..100755 --- a/ambari-infra/ambari-infra-solr-client/src/main/python/solrDataManager.py +++ b/ambari-infra/ambari-infra-solr-client/src/main/python/solrDataManager.py http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java index 79ca5ba,ff13d0b..1bf67c3 --- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java @@@ -20,7 -20,9 +20,8 @@@ package org.apache.ambari.server.action import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; + import static org.apache.ambari.server.stack.StackManager.DEFAULT_HOOKS_FOLDER; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java index 76c2019,9f8a095..f6bfdcf --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java @@@ -38,11 -37,12 +38,14 @@@ public class StatusCommand extends Agen private String clusterName; @SerializedName("serviceName") + @com.fasterxml.jackson.annotation.JsonProperty("serviceName") private String serviceName; + @SerializedName("role") + private String role; + @SerializedName("componentName") + @com.fasterxml.jackson.annotation.JsonProperty("componentName") private String componentName; @SerializedName("configurations") http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/HostRepositories.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/HostRepositories.java index 8d0cfb0,0000000..a076c66 mode 100644,000000..100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/HostRepositories.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/stomp/dto/HostRepositories.java @@@ -1,39 -1,0 +1,58 @@@ ++/** ++ * Licensed to the Apache Software Foundation (ASF) under one ++ * or more contributor license agreements. See the NOTICE file ++ * distributed with this work for additional information ++ * regarding copyright ownership. The ASF licenses this file ++ * to you under the Apache License, Version 2.0 (the ++ * "License"); you may not use this file except in compliance ++ * with the License. You may obtain a copy of the License at ++ * <p/> ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * <p/> ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++ +package org.apache.ambari.server.agent.stomp.dto; + +import java.util.Map; + +import org.apache.ambari.server.agent.CommandRepository; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class HostRepositories { + + @JsonProperty("commandRepos") + private Map<Long, CommandRepository> repositories; + + @JsonProperty("componentRepos") + private Map<String, Long> componentRepos; + + public HostRepositories(Map<Long, CommandRepository> repositories, Map<String, Long> componentRepos) { + this.repositories = repositories; + this.componentRepos = componentRepos; + } + + public Map<Long, CommandRepository> getRepositories() { + return repositories; + } + + public void setRepositories(Map<Long, CommandRepository> repositories) { + this.repositories = repositories; + } + + public Map<String, Long> getComponentRepos() { + return componentRepos; + } + + public void setComponentRepos(Map<String, Long> componentRepos) { + this.componentRepos = componentRepos; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java index 407238d,46ee65a..49679b4 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java @@@ -1663,21 -1448,7 +1675,25 @@@ public class AmbariMetaInfo return null; } + /** + * Class that is used to update base urls. There are two implementations of this - when we no + * longer are sure the deprecated repo info can be removed, so too can this class. + */ + @Experimental(feature= ExperimentalFeature.PATCH_UPGRADES) + abstract static class BaseUrlUpdater<T> implements Function<RepositoryVersionEntity, T> { + private T m_default; + + private BaseUrlUpdater(T defaultValue) { + m_default = defaultValue; + } + + private T getDefault() { + return m_default; + } + + } ++ + public File getCommonWidgetsDescriptorFile() { + return commonWidgetsDescriptorFile; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/checks/CheckDescription.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index 5ffdf26,e12477e..7a98b7c --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@@ -84,6 -89,8 +84,7 @@@ import org.apache.ambari.server.state.H import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.PropertyInfo; import org.apache.ambari.server.state.PropertyInfo.PropertyType; + import org.apache.ambari.server.state.RefreshCommandConfiguration; -import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; import org.apache.ambari.server.state.ServiceComponentHost; http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 6a7f050,1b1f524..4f57881 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@@ -177,6 -148,6 +177,7 @@@ import org.apache.ambari.server.servera import org.apache.ambari.server.serveraction.kerberos.KerberosOperationException; import org.apache.ambari.server.stack.ExtensionHelper; import org.apache.ambari.server.stack.RepoUtil; ++import org.apache.ambari.server.stack.StackManager; import org.apache.ambari.server.stageplanner.RoleGraph; import org.apache.ambari.server.stageplanner.RoleGraphFactory; import org.apache.ambari.server.state.Cluster; @@@ -5803,259 -5682,4 +5802,259 @@@ public class AmbariManagementController return QuickLinkVisibilityControllerFactory.get(quickLinkProfileJson); } + /** + * Collects metadata info about clusters for agent. + * @return metadata info about clusters + * @throws AmbariException + */ + public MetadataUpdateEvent getClustersMetadata() throws AmbariException { + TreeMap<String, MetadataCluster> metadataClusters = new TreeMap<>(); + + for (Cluster cl : clusters.getClusters().values()) { + StackId stackId = cl.getDesiredStackVersion(); + + SecurityType securityType = cl.getSecurityType(); + + TreeMap<String, MetadataServiceInfo> serviceLevelParams = new TreeMap<>(); + Collection<ServiceInfo> servicesInfo = ambariMetaInfo.getServices(stackId.getStackName(), + stackId.getStackVersion()).values(); + for (ServiceInfo serviceInfo : servicesInfo) { + Long statusCommandTimeout = null; + if (serviceInfo.getCommandScript() != null) { + statusCommandTimeout = new Long (ambariCustomCommandExecutionHelper.getStatusCommandTimeout(serviceInfo)); + } + + String servicePackageFolder = serviceInfo.getServicePackageFolder(); + + serviceLevelParams.put(serviceInfo.getName(), + new MetadataServiceInfo(serviceInfo.getVersion(), + serviceInfo.isCredentialStoreEnabled(), + statusCommandTimeout, + servicePackageFolder)); + } + + MetadataCluster metadataCluster = new MetadataCluster(securityType, + serviceLevelParams, + getMetadataClusterLevelParams(cl, stackId)); + metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster); + } + + MetadataUpdateEvent metadataUpdateEvent = new MetadataUpdateEvent(metadataClusters, + getMetadataAmbariLevelParams()); + return metadataUpdateEvent; + } + + public MetadataUpdateEvent getClusterMetadata(Cluster cl) throws AmbariException { + TreeMap<String, MetadataCluster> metadataClusters = new TreeMap<>(); + StackId stackId = cl.getDesiredStackVersion(); + + SecurityType securityType = cl.getSecurityType(); + + TreeMap<String, MetadataServiceInfo> serviceLevelParams = new TreeMap<>(); + Collection<ServiceInfo> servicesInfo = ambariMetaInfo.getServices(stackId.getStackName(), + stackId.getStackVersion()).values(); + for (ServiceInfo serviceInfo : servicesInfo) { + Long statusCommandTimeout = null; + if (serviceInfo.getCommandScript() != null) { + statusCommandTimeout = new Long(ambariCustomCommandExecutionHelper.getStatusCommandTimeout(serviceInfo)); + } + + String servicePackageFolder = serviceInfo.getServicePackageFolder(); + + serviceLevelParams.put(serviceInfo.getName(), + new MetadataServiceInfo(serviceInfo.getVersion(), + serviceInfo.isCredentialStoreEnabled(), + statusCommandTimeout, + servicePackageFolder)); + } + + MetadataCluster metadataCluster = new MetadataCluster(securityType, + serviceLevelParams, + getMetadataClusterLevelParams(cl, stackId)); + metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster); + + MetadataUpdateEvent metadataUpdateEvent = new MetadataUpdateEvent(metadataClusters, + null); + return metadataUpdateEvent; + } + + public MetadataUpdateEvent getClusterMetadataOnConfigsUpdate(Cluster cl) throws AmbariException { + TreeMap<String, MetadataCluster> metadataClusters = new TreeMap<>(); + StackId stackId = cl.getDesiredStackVersion(); + + MetadataCluster metadataCluster = new MetadataCluster(null, + new TreeMap<>(), + getMetadataClusterLevelConfigsParams(cl, stackId)); + metadataClusters.put(Long.toString(cl.getClusterId()), metadataCluster); + + MetadataUpdateEvent metadataUpdateEvent = new MetadataUpdateEvent(metadataClusters, + null); + return metadataUpdateEvent; + } + + private String getClientsToUpdateConfigs(ComponentInfo componentInfo) { + List<String> clientsToUpdateConfigsList = componentInfo.getClientsToUpdateConfigs(); + if (clientsToUpdateConfigsList == null) { + clientsToUpdateConfigsList = new ArrayList<>(); + clientsToUpdateConfigsList.add("*"); + } + return gson.toJson(clientsToUpdateConfigsList); + } + + private Boolean getUnlimitedKeyJCERequirement(ComponentInfo componentInfo, SecurityType clusterSecurityType) { + UnlimitedKeyJCERequirement unlimitedKeyJCERequirement = componentInfo.getUnlimitedKeyJCERequired(); + // Ensure that the unlimited key requirement is set. If null, the default value should be used. + if(unlimitedKeyJCERequirement == null) { + unlimitedKeyJCERequirement = UnlimitedKeyJCERequirement.DEFAULT; + } + + return (UnlimitedKeyJCERequirement.ALWAYS == unlimitedKeyJCERequirement) || + ((UnlimitedKeyJCERequirement.KERBEROS_ENABLED == unlimitedKeyJCERequirement) && + (clusterSecurityType == SecurityType.KERBEROS)); + + } + + //TODO will be a need to change to multi-instance usage + public TreeMap<String, String> getTopologyComponentLevelParams(StackId stackId, String serviceName, String componentName, + SecurityType clusterSecurityType) throws AmbariException { + ComponentInfo componentInfo = ambariMetaInfo.getComponent( + stackId.getStackName(), stackId.getStackVersion(), + serviceName, componentName); + + TreeMap<String, String> statusCommandParams = new TreeMap<>(); + statusCommandParams.put(ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS, + getClientsToUpdateConfigs(componentInfo)); + statusCommandParams.put(ExecutionCommand.KeyNames.UNLIMITED_KEY_JCE_REQUIRED, + Boolean.toString(getUnlimitedKeyJCERequirement(componentInfo, clusterSecurityType))); + return statusCommandParams; + } + + //TODO will be a need to change to multi-instance usage + public TreeMap<String, String> getTopologyCommandParams(StackId stackId, String serviceName, String componentName) throws AmbariException { + ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), + stackId.getStackVersion(), serviceName); + ComponentInfo componentInfo = ambariMetaInfo.getComponent( + stackId.getStackName(), stackId.getStackVersion(), + serviceName, componentName); + + String scriptName = null; + String scriptCommandTimeout = ""; + CommandScriptDefinition script = componentInfo.getCommandScript(); + if (serviceInfo.getSchemaVersion().equals(AmbariMetaInfo.SCHEMA_VERSION_2)) { + if (script != null) { + scriptName = script.getScript(); + if (script.getTimeout() > 0) { + scriptCommandTimeout = String.valueOf(script.getTimeout()); + } + } else { + String message = String.format("Component %s of service %s has not " + + "command script defined", componentName, serviceName); + throw new AmbariException(message); + } + } + String agentDefaultCommandTimeout = configs.getDefaultAgentTaskTimeout(false); + String actualTimeout = (!scriptCommandTimeout.equals("") ? scriptCommandTimeout : agentDefaultCommandTimeout); + + TreeMap<String, String> commandParams = new TreeMap<>(); + commandParams.put(COMMAND_TIMEOUT, actualTimeout); + commandParams.put(SCRIPT, scriptName); + commandParams.put(SCRIPT_TYPE, script.getScriptType().toString()); + return commandParams; + } + + public TreeMap<String, String> getMetadataClusterLevelParams(Cluster cluster, StackId stackId) throws AmbariException { + TreeMap<String, String> clusterLevelParams = new TreeMap<>(); + clusterLevelParams.put(STACK_NAME, stackId.getStackName()); + clusterLevelParams.put(STACK_VERSION, stackId.getStackVersion()); + + Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); + if (MapUtils.isNotEmpty(desiredConfigs)) { + + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); + String userList = gson.toJson(userSet); + clusterLevelParams.put(USER_LIST, userList); + + //Create a user_group mapping and send it as part of the hostLevelParams + Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( + stackId, cluster, desiredConfigs); + String userGroups = gson.toJson(userGroupsMap); + clusterLevelParams.put(USER_GROUPS, userGroups); + + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs); + String groupList = gson.toJson(groupSet); + clusterLevelParams.put(GROUP_LIST, groupList); + } + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, + PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); + clusterLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); + + clusterLevelParams.put(CLUSTER_NAME, cluster.getClusterName()); + + StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); - clusterLevelParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); ++ clusterLevelParams.put(HOOKS_FOLDER, StackManager.DEFAULT_HOOKS_FOLDER); + + return clusterLevelParams; + } + + public TreeMap<String, String> getMetadataClusterLevelConfigsParams(Cluster cluster, StackId stackId) throws AmbariException { + TreeMap<String, String> clusterLevelParams = new TreeMap<>(); + + Map<String, DesiredConfig> desiredConfigs = cluster.getDesiredConfigs(); + if (MapUtils.isNotEmpty(desiredConfigs)) { + + Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredConfigs); + String userList = gson.toJson(userSet); + clusterLevelParams.put(USER_LIST, userList); + + //Create a user_group mapping and send it as part of the hostLevelParams + Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( + stackId, cluster, desiredConfigs); + String userGroups = gson.toJson(userGroupsMap); + clusterLevelParams.put(USER_GROUPS, userGroups); + + Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredConfigs); + String groupList = gson.toJson(groupSet); + clusterLevelParams.put(GROUP_LIST, groupList); + } + Set<String> notManagedHdfsPathSet = configHelper.getPropertyValuesWithPropertyType(stackId, + PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs); + String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); + clusterLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); + + return clusterLevelParams; + } + + public TreeMap<String, String> getMetadataAmbariLevelParams() throws AmbariException { + TreeMap<String, String> clusterLevelParams = new TreeMap<>(); + clusterLevelParams.put(JDK_LOCATION, getJdkResourceUrl()); + clusterLevelParams.put(JAVA_HOME, getJavaHome()); + clusterLevelParams.put(JAVA_VERSION, String.valueOf(configs.getJavaVersion())); + clusterLevelParams.put(JDK_NAME, getJDKName()); + clusterLevelParams.put(JCE_NAME, getJCEName()); + clusterLevelParams.put(DB_NAME, getServerDB()); + clusterLevelParams.put(MYSQL_JDBC_URL, getMysqljdbcUrl()); + clusterLevelParams.put(ORACLE_JDBC_URL, getOjdbcUrl()); + clusterLevelParams.put(DB_DRIVER_FILENAME, configs.getMySQLJarName()); + clusterLevelParams.putAll(getRcaParameters()); + clusterLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); + clusterLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); + clusterLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); + + return clusterLevelParams; + } + + @Override + public HostRepositories retrieveHostRepositories(Cluster cluster, Host host) throws AmbariException { + List<ServiceComponentHost> hostComponents = cluster.getServiceComponentHosts(host.getHostName()); + Map<Long, CommandRepository> hostRepositories = new HashMap<>(); + Map<String, Long> componentsRepos = new HashMap<>(); + for (ServiceComponentHost serviceComponentHost : hostComponents) { + CommandRepository commandRepository = ambariMetaInfo.getCommandRepository(cluster, + serviceComponentHost.getServiceComponent(), host); + hostRepositories.put(commandRepository.getM_repoVersionId(), commandRepository); + componentsRepos.put(serviceComponentHost.getServiceComponentName(), commandRepository.getM_repoVersionId()); + } + return new HostRepositories(hostRepositories, componentsRepos); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ServiceResourceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java ---------------------------------------------------------------------- diff --cc ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java index 2fd5abe,1c0853b9..2637f8e --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/CreatePrincipalsServerAction.java @@@ -128,10 -128,11 +128,10 @@@ public class CreatePrincipalsServerActi seenPrincipals.add(evaluatedPrincipal); boolean processPrincipal; - boolean regenerateKeytabs = "true".equalsIgnoreCase(getCommandParameterValue(getCommandParameters(), REGENERATE_ALL)); + boolean regenerateKeytabs = getOperationType(getCommandParameters()) == OperationType.RECREATE_ALL; if (regenerateKeytabs) { - // do not process cached identities that can be passed as is(headless identities) - processPrincipal = "false".equals(identityRecord.get(KerberosIdentityDataFileReader.ONLY_KEYTAB_WRITE).toLowerCase()); + processPrincipal = true; } else { KerberosPrincipalEntity kerberosPrincipalEntity = kerberosPrincipalDAO.find(evaluatedPrincipal); http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/RangerUsersyncConfigCalculation.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/af30ab40/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java ----------------------------------------------------------------------
