[ambari] branch branch-2.7 updated: [AMBARI-25093] "Spark2 Thrift Server alert does not work with HTTPS/SSL (apappu) (#2883)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 1d90d11 [AMBARI-25093] "Spark2 Thrift Server alert does not work with
HTTPS/SSL (apappu) (#2883)
1d90d11 is described below
commit 1d90d1136f1b231e31d894fb089db56bf526838c
Author: amarnathreddy pappu
AuthorDate: Mon Apr 1 06:16:06 2019 -0700
[AMBARI-25093] "Spark2 Thrift Server alert does not work with HTTPS/SSL
(apappu) (#2883)
* [AMBARI-25093] Spark2 Thrift Server alert does not work with HTTPS
(apappu)
* [AMBARI-25093] Spark2 Thrift Server alert does not work with HTTPS/SSL
(apappu)
---
.../scripts/alerts/alert_spark2_thrift_port.py | 32 +++---
1 file changed, 28 insertions(+), 4 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py
index d3660de..dbbbabf 100644
---
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py
+++
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/alerts/alert_spark2_thrift_port.py
@@ -36,6 +36,8 @@ OK_MESSAGE = "TCP OK - {0:.3f}s response on port {1}"
CRITICAL_MESSAGE = "Connection failed on host {0}:{1} ({2})"
HIVE_SERVER_THRIFT_PORT_KEY =
'{{spark2-hive-site-override/hive.server2.thrift.port}}'
+HIVE_SERVER_THRIFT_HTTP_PORT_KEY =
'{{spark2-hive-site-override/hive.server2.thrift.http.port}}'
+
HIVE_SERVER_TRANSPORT_MODE_KEY =
'{{spark2-hive-site-override/hive.server2.transport.mode}}'
SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
@@ -43,6 +45,10 @@ HIVE_SERVER2_AUTHENTICATION_KEY =
'{{hive-site/hive.server2.authentication}}'
HIVE_SERVER2_KERBEROS_KEYTAB =
'{{hive-site/hive.server2.authentication.kerberos.keytab}}'
HIVE_SERVER2_PRINCIPAL_KEY =
'{{hive-site/hive.server2.authentication.kerberos.principal}}'
+SPARK_SSL_ENABLED = '{{spark2-defaults/spark.ssl.enabled}}'
+SPARK_TRUST_STORE_PATH = '{{spark2-defaults/spark.ssl.trustStore}}'
+SPARK_TRUST_STORE_PASS = '{{spark2-defaults/spark.ssl.trustStorePassword}}'
+
# The configured Kerberos executable search paths, if any
KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY =
'{{kerberos-env/executable_search_paths}}'
@@ -63,7 +69,7 @@ def get_tokens():
to build the dictionary passed into execute
"""
return (HIVE_SERVER_THRIFT_PORT_KEY, HIVE_SERVER_TRANSPORT_MODE_KEY,
SECURITY_ENABLED_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY,
-HIVEUSER_DEFAULT, HIVE_SERVER2_KERBEROS_KEYTAB,
HIVE_SERVER2_PRINCIPAL_KEY)
+HIVEUSER_DEFAULT, HIVE_SERVER2_KERBEROS_KEYTAB,
HIVE_SERVER2_PRINCIPAL_KEY, SPARK_SSL_ENABLED, SPARK_TRUST_STORE_PATH,
SPARK_TRUST_STORE_PASS, HIVE_SERVER_THRIFT_HTTP_PORT_KEY)
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def execute(configurations={}, parameters={}, host_name=None):
@@ -88,7 +94,9 @@ def execute(configurations={}, parameters={}, host_name=None):
port = THRIFT_PORT_DEFAULT
if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in
configurations:
port = int(configurations[HIVE_SERVER_THRIFT_PORT_KEY])
-
+elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY
in configurations:
+ port = int(configurations[HIVE_SERVER_THRIFT_HTTP_PORT_KEY])
+
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper()
== 'TRUE'
@@ -105,6 +113,15 @@ def execute(configurations={}, parameters={},
host_name=None):
hive_principal = configurations[HIVE_SERVER2_PRINCIPAL_KEY]
hive_principal = hive_principal.replace('_HOST',host_name.lower())
+# Get the Trust store and pass
+spark_truststore_path = None
+spark_truststore_pass = None
+spark_ssl_enabled = False
+if SPARK_SSL_ENABLED in configurations:
+ spark_truststore_path = configurations[SPARK_TRUST_STORE_PATH]
+ spark_truststore_pass = configurations[SPARK_TRUST_STORE_PASS]
+ spark_ssl_enabled = str(configurations[SPARK_SSL_ENABLED]).upper() ==
'TRUE'
+
# Get the configured Kerberos executable search paths, if any
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths =
configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
@@ -131,9 +148,16 @@ def execute(configurations={}, parameters={},
host_name=None):
host_
[ambari] branch branch-2.6 updated: [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2872)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/branch-2.6 by this push: new d8a20d2 [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2872) d8a20d2 is described below commit d8a20d2ec9e609d5a077dfd92d315ed2e93c1f96 Author: vbrodetskyi AuthorDate: Mon Mar 18 13:37:32 2019 +0200 [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2872) --- .../main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 8 .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml| 11 +++ .../main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml| 10 ++ .../main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml | 8 .../stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml| 4 .../main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml| 4 6 files changed, 45 insertions(+) diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index d0f83f2..e5fc98e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -189,6 +189,14 @@ + + + +livy2-conf + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index a780bdc..9882701 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -115,6 +115,7 @@ SPARK2_JOBHISTORYSERVER SPARK2_THRIFTSERVER +LIVY2_SERVER @@ -698,6 +699,10 @@ + + + + diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index a3c0c9b..f8373b2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -897,6 +897,10 @@ + + + +
[ambari] branch branch-2.6 updated: [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2860)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/branch-2.6 by this push: new a7e11a9 [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2860) a7e11a9 is described below commit a7e11a923ace156d6e1fd83c1846fd453fde1fc7 Author: vbrodetskyi AuthorDate: Wed Mar 13 16:10:29 2019 +0200 [AMBARI-25189] Enable livy.server.access-control.enabled by default.(vbrodetskyi) (#2860) --- .../stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml | 8 1 file changed, 8 insertions(+) diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml index 428ab44..7c7da88 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml @@ -110,5 +110,13 @@ + +livy.server.access-control.enabled +true + +Property to configure Livy user access. + + +
[ambari] branch trunk updated: [AMBARI-24800] service adviser changes for cluster specific configs.(vbrodetskyi) (#2484)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9124745 [AMBARI-24800] service adviser changes for cluster specific
configs.(vbrodetskyi) (#2484)
9124745 is described below
commit 91247455a30f4de0a5af8bcfba92302360a47bff
Author: vbrodetskyi
AuthorDate: Thu Oct 18 18:22:51 2018 +0300
[AMBARI-24800] service adviser changes for cluster specific
configs.(vbrodetskyi) (#2484)
---
ambari-server/src/main/resources/stacks/stack_advisor.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py
b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 850c21c..7b53648 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -1181,7 +1181,7 @@ class DefaultStackAdvisor(StackAdvisor):
- def getConfigurationClusterSummary(self, servicesList, hosts, components,
services):
+ def getConfigurationClusterSummary(self, servicesList, hosts, components,
services, cpu_only_mode = False):
"""
Copied from HDP 2.0.6 so that it could be used by Service Advisors.
:return: Dictionary of memory and CPU attributes in the cluster
@@ -1304,8 +1304,9 @@ class DefaultStackAdvisor(StackAdvisor):
'''containers = max(3, min (2*cores,min (1.8*DISKS,(Total available RAM) /
MIN_CONTAINER_SIZE'''
+core_multiplier = 2 if not cpu_only_mode else 1
cluster["containers"] = int(round(max(3,
- min(2 * cluster["cpu"],
+ min(core_multiplier * cluster["cpu"],
min(ceil(1.8 * cluster["disk"]),
cluster["totalAvailableRam"]
/ cluster["minContainerSize"])
self.logger.info("Containers per node - cluster[containers]: " +
str(cluster["containers"]))
[ambari] branch branch-2.7 updated: [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 … (#2405)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 0e8dcfd [AMBARI-24718] STS fails after start, after stack upgrade
from 3.0.1 … (#2405)
0e8dcfd is described below
commit 0e8dcfd9be2d1ccd526126cf9bc0073e3e8a1c73
Author: vbrodetskyi
AuthorDate: Mon Oct 1 23:18:41 2018 +0300
[AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 …
(#2405)
* [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 to
3.0.3.(vbrodetskyi)
* [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 to
3.0.3.(vbrodetskyi)
---
.../upgrades/FixSparkYarnIdentity.java | 87 ++
1 file changed, 87 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
new file mode 100644
index 000..3d1b994
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.SecurityType;
+
+/**
+ * Check if spark.yarn.kaytab/principal property exists
+ * if not, then we are creating them and copy pasting
+ * values from spark.history.kerberos.keytab/principal properties.
+ * Works only for kerberized cluster.
+ */
+public class FixSparkYarnIdentity extends AbstractUpgradeServerAction {
+ private static final String SPARK2_THRIFT_SPARKCONF_CONFIG_TYPE =
"spark2-thrift-sparkconf";
+ private static final String SPARK2_DEFAULTS_CONFIG_TYPE = "spark2-defaults";
+
+ private static final String SPARK_YARN_KEYTAB_PROPERTY_NAME =
"spark.yarn.keytab";
+ private static final String SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME =
"spark.yarn.principal";
+
+ private static final String SPARK_HISTORY_KERBEROS_KEYTAB_PROPERTY_NAME =
"spark.history.kerberos.keytab";
+ private static final String SPARK_HISTORY_KERBEROS_PRINCIPAL_PROPERTY_NAME =
"spark.history.kerberos.principal";
+
+
+ @Override
+ public CommandReport execute(ConcurrentMap
requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+String clusterName = getExecutionCommand().getClusterName();
+
+Cluster cluster = getClusters().getCluster(clusterName);
+
+if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+
+ Config spark2ThriftSparkConfConfig =
cluster.getDesiredConfigByType(SPARK2_THRIFT_SPARKCONF_CONFIG_TYPE);
+
+ if (spark2ThriftSparkConfConfig != null) {
+Map spark2ThriftSparkConfProperties =
spark2ThriftSparkConfConfig.getProperties();
+if
(!spark2ThriftSparkConfProperties.containsKey(SPARK_YARN_KEYTAB_PROPERTY_NAME)
&&
+
!spark2ThriftSparkConfProperties.containsKey(SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME))
{
+
+ Config spark2DefaultsConfig =
cluster.getDesiredConfigByType(SPARK2_DEFAULTS_CONFIG_TYPE);
+ if (spark2DefaultsConfig != null) {
+Map spark2DefultsProperties = spark2DefaultsConfig.getProperties();
+
+
spark2ThriftSparkConfProperties.put(SPARK_YARN_KEYTAB_PROPERTY_NAME,
spark2DefultsProperties.get(SPARK_HISTORY_KERBEROS_KEYTAB_PROPERTY_NAME));
+
spark2ThriftSparkConfProperties.put(SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME,
spark2DefultsProperties.get(SPARK_HISTORY_KER
[ambari] branch trunk updated: [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 … (#2406)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6976220 [AMBARI-24718] STS fails after start, after stack upgrade
from 3.0.1 … (#2406)
6976220 is described below
commit 69762201ee278a4be486a1656f3c56a775c24518
Author: vbrodetskyi
AuthorDate: Mon Oct 1 23:18:51 2018 +0300
[AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 …
(#2406)
* [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 to
3.0.3.(vbrodetskyi)
* [AMBARI-24718] STS fails after start, after stack upgrade from 3.0.1 to
3.0.3.(vbrodetskyi)
---
.../upgrades/FixSparkYarnIdentity.java | 87 ++
1 file changed, 87 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
new file mode 100644
index 000..3d1b994
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixSparkYarnIdentity.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Host;
+import org.apache.ambari.server.state.SecurityType;
+
+/**
+ * Check if spark.yarn.kaytab/principal property exists
+ * if not, then we are creating them and copy pasting
+ * values from spark.history.kerberos.keytab/principal properties.
+ * Works only for kerberized cluster.
+ */
+public class FixSparkYarnIdentity extends AbstractUpgradeServerAction {
+ private static final String SPARK2_THRIFT_SPARKCONF_CONFIG_TYPE =
"spark2-thrift-sparkconf";
+ private static final String SPARK2_DEFAULTS_CONFIG_TYPE = "spark2-defaults";
+
+ private static final String SPARK_YARN_KEYTAB_PROPERTY_NAME =
"spark.yarn.keytab";
+ private static final String SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME =
"spark.yarn.principal";
+
+ private static final String SPARK_HISTORY_KERBEROS_KEYTAB_PROPERTY_NAME =
"spark.history.kerberos.keytab";
+ private static final String SPARK_HISTORY_KERBEROS_PRINCIPAL_PROPERTY_NAME =
"spark.history.kerberos.principal";
+
+
+ @Override
+ public CommandReport execute(ConcurrentMap
requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+String clusterName = getExecutionCommand().getClusterName();
+
+Cluster cluster = getClusters().getCluster(clusterName);
+
+if (cluster.getSecurityType() == SecurityType.KERBEROS) {
+
+ Config spark2ThriftSparkConfConfig =
cluster.getDesiredConfigByType(SPARK2_THRIFT_SPARKCONF_CONFIG_TYPE);
+
+ if (spark2ThriftSparkConfConfig != null) {
+Map spark2ThriftSparkConfProperties =
spark2ThriftSparkConfConfig.getProperties();
+if
(!spark2ThriftSparkConfProperties.containsKey(SPARK_YARN_KEYTAB_PROPERTY_NAME)
&&
+
!spark2ThriftSparkConfProperties.containsKey(SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME))
{
+
+ Config spark2DefaultsConfig =
cluster.getDesiredConfigByType(SPARK2_DEFAULTS_CONFIG_TYPE);
+ if (spark2DefaultsConfig != null) {
+Map spark2DefultsProperties = spark2DefaultsConfig.getProperties();
+
+
spark2ThriftSparkConfProperties.put(SPARK_YARN_KEYTAB_PROPERTY_NAME,
spark2DefultsProperties.get(SPARK_HISTORY_KERBEROS_KEYTAB_PROPERTY_NAME));
+
spark2ThriftSparkConfProperties.put(SPARK_YARN_KEYTAB_PRINCIPAL_PROPERTY_NAME,
spark2DefultsProperties.get(SPARK_HISTORY_KER
[ambari] branch trunk updated: [AMBARI-24327] Zeppelin server is shown as started even if it fails during start up.(vbrodetskyi) (#1830)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5fa81ce [AMBARI-24327] Zeppelin server is shown as started even if it
fails during start up.(vbrodetskyi) (#1830)
5fa81ce is described below
commit 5fa81ce899319710e99d4f33309201641f20f6b2
Author: vbrodetskyi
AuthorDate: Mon Jul 23 14:57:23 2018 +0300
[AMBARI-24327] Zeppelin server is shown as started even if it fails during
start up.(vbrodetskyi) (#1830)
---
.../common-services/ZEPPELIN/0.6.0/alerts.json | 2 +-
.../scripts/alerts}/alert_check_zeppelin.py| 0
.../common-services/ZEPPELIN/0.7.0/alerts.json | 2 +-
.../scripts/alerts}/alert_check_zeppelin.py| 0
.../package/scripts/alerts/alert_check_zeppelin.py | 108 +
5 files changed, 110 insertions(+), 2 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
index 4b62236..b854e92 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
@@ -10,7 +10,7 @@
"scope": "ANY",
"source": {
"type": "SCRIPT",
- "path": "ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py"
+ "path":
"ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py"
}
}
]
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py
similarity index 100%
rename from
ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py
rename to
ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
index 53dc4a2..4d7d568 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
@@ -10,7 +10,7 @@
"scope": "ANY",
"source": {
"type": "SCRIPT",
- "path": "ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py"
+ "path":
"ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py"
}
}
]
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py
similarity index 100%
rename from
ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py
rename to
ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py
diff --git
a/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
b/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
new file mode 100644
index 000..442095c
--- /dev/null
+++
b/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
@@ -0,0 +1,108 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import socket
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.core.resources import Execute
+
+Z
[ambari] branch branch-2.7 updated: [AMBARI-24327] Zeppelin server is shown as started even if it fails during start up.(vbrodetskyi)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new cb69172 [AMBARI-24327] Zeppelin server is shown as started even if it
fails during start up.(vbrodetskyi)
cb69172 is described below
commit cb69172a701aa05eb07ed1c1ff9b6ba51d245311
Author: Vitaly Brodetskyi
AuthorDate: Fri Jul 20 22:01:08 2018 +0300
[AMBARI-24327] Zeppelin server is shown as started even if it fails during
start up.(vbrodetskyi)
---
.../common-services/ZEPPELIN/0.6.0/alerts.json | 2 +-
.../scripts/alerts}/alert_check_zeppelin.py| 0
.../common-services/ZEPPELIN/0.7.0/alerts.json | 2 +-
.../scripts/alerts}/alert_check_zeppelin.py| 0
.../package/scripts/alerts/alert_check_zeppelin.py | 108 +
5 files changed, 110 insertions(+), 2 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
index 4b62236..b854e92 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/alerts.json
@@ -10,7 +10,7 @@
"scope": "ANY",
"source": {
"type": "SCRIPT",
- "path": "ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py"
+ "path":
"ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py"
}
}
]
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py
similarity index 100%
rename from
ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py
rename to
ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alerts/alert_check_zeppelin.py
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
index 53dc4a2..4d7d568 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/alerts.json
@@ -10,7 +10,7 @@
"scope": "ANY",
"source": {
"type": "SCRIPT",
- "path": "ZEPPELIN/0.7.0/package/scripts/alert_check_zeppelin.py"
+ "path":
"ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py"
}
}
]
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py
similarity index 100%
rename from
ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/package/scripts/alert_check_zeppelin.py
rename to
ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/alerts/alert_check_zeppelin.py
diff --git
a/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
b/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
new file mode 100644
index 000..442095c
--- /dev/null
+++
b/src/main/resources/stacks/HDP/3.0/services/ZEPPELIN/package/scripts/alerts/alert_check_zeppelin.py
@@ -0,0 +1,108 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import socket
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.core.resources import Execute
+
+Z
[ambari] branch branch-2.7 updated: [AMBARI-24271] Spark thrift server is not starting on Upgraded cluster.(vbrodetskyi) (#1730)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 2be530f [AMBARI-24271] Spark thrift server is not starting on
Upgraded cluster.(vbrodetskyi) (#1730)
2be530f is described below
commit 2be530f7a95542bd154b38e8917cfc9310d8d050
Author: vbrodetskyi
AuthorDate: Tue Jul 10 02:21:56 2018 +0300
[AMBARI-24271] Spark thrift server is not starting on Upgraded
cluster.(vbrodetskyi) (#1730)
---
.../upgrades/AddSparkUserToYarnACLAdminQueue.java | 79 ++
1 file changed, 79 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
new file mode 100644
index 000..0fdc197
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Host;
+
+
+public class AddSparkUserToYarnACLAdminQueue extends
AbstractUpgradeServerAction {
+ private static final String CAPACITY_SCHEDULER_CONFIG_TYPE =
"capacity-scheduler";
+ private static final String SPARK_ENV_CONFIG_TYPE = "spark2-env";
+
+ private static final String YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME =
"yarn.scheduler.capacity.root.acl_administer_queue";
+ private static final String SPARK_USER_PROPERTY_NAME = "spark_user";
+
+ @Override
+ public CommandReport execute(ConcurrentMap
requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+String clusterName = getExecutionCommand().getClusterName();
+
+Cluster cluster = getClusters().getCluster(clusterName);
+
+Config capacitySchedulerConfig =
cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE);
+Config sparkEnvConfig =
cluster.getDesiredConfigByType(SPARK_ENV_CONFIG_TYPE);
+
+if (capacitySchedulerConfig == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED,"{}",
+ String.format("Source type %s not found",
CAPACITY_SCHEDULER_CONFIG_TYPE), "");
+}
+
+if (sparkEnvConfig == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED,"{}",
+ String.format("Source type %s not found",
SPARK_ENV_CONFIG_TYPE), "");
+}
+
+Map capacitySchedulerProperties =
capacitySchedulerConfig.getProperties();
+Map sparkEnvProperties = sparkEnvConfig.getProperties();
+
+String yarnACLAdminQueue =
capacitySchedulerProperties.get(YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME);
+String sparkUser = sparkEnvProperties.get(SPARK_USER_PROPERTY_NAME);
+
+String message = "";
+if (yarnACLAdminQueue != null && !yarnACLAdminQueue.trim().equals("*")) {
+ yarnACLAdminQueue = yarnACLAdminQueue + "," + sparkUser;
+ capacitySchedulerProperties.put(YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME,
yarnACLAdminQueue);
+ capacitySchedulerConfig.setProperties(capacitySchedulerProperties);
+ capacitySchedulerConfig.save();
+ agentConfigsHolder.updateData(cluster.getClusterId(),
cluster.getHosts().stream().map(Host::getHostId).collect(Collectors.toList()));
+ message = String.format("Spark user %s was successfully added to %s
property value.", sparkUser, YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME);
+}
+
+return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", message, "");
+ }
+}
[ambari] branch trunk updated: [AMBARI-24271] Spark thrift server is not starting on Upgraded cluster.(vbrodetskyi) (#1729)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 95bed16 [AMBARI-24271] Spark thrift server is not starting on
Upgraded cluster.(vbrodetskyi) (#1729)
95bed16 is described below
commit 95bed16dd0252b9882185c6ab01dcb1af5535d54
Author: vbrodetskyi
AuthorDate: Tue Jul 10 02:22:05 2018 +0300
[AMBARI-24271] Spark thrift server is not starting on Upgraded
cluster.(vbrodetskyi) (#1729)
---
.../upgrades/AddSparkUserToYarnACLAdminQueue.java | 79 ++
1 file changed, 79 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
new file mode 100644
index 000..0fdc197
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/AddSparkUserToYarnACLAdminQueue.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.Host;
+
+
+public class AddSparkUserToYarnACLAdminQueue extends
AbstractUpgradeServerAction {
+ private static final String CAPACITY_SCHEDULER_CONFIG_TYPE =
"capacity-scheduler";
+ private static final String SPARK_ENV_CONFIG_TYPE = "spark2-env";
+
+ private static final String YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME =
"yarn.scheduler.capacity.root.acl_administer_queue";
+ private static final String SPARK_USER_PROPERTY_NAME = "spark_user";
+
+ @Override
+ public CommandReport execute(ConcurrentMap
requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+
+String clusterName = getExecutionCommand().getClusterName();
+
+Cluster cluster = getClusters().getCluster(clusterName);
+
+Config capacitySchedulerConfig =
cluster.getDesiredConfigByType(CAPACITY_SCHEDULER_CONFIG_TYPE);
+Config sparkEnvConfig =
cluster.getDesiredConfigByType(SPARK_ENV_CONFIG_TYPE);
+
+if (capacitySchedulerConfig == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED,"{}",
+ String.format("Source type %s not found",
CAPACITY_SCHEDULER_CONFIG_TYPE), "");
+}
+
+if (sparkEnvConfig == null) {
+ return createCommandReport(0, HostRoleStatus.FAILED,"{}",
+ String.format("Source type %s not found",
SPARK_ENV_CONFIG_TYPE), "");
+}
+
+Map capacitySchedulerProperties =
capacitySchedulerConfig.getProperties();
+Map sparkEnvProperties = sparkEnvConfig.getProperties();
+
+String yarnACLAdminQueue =
capacitySchedulerProperties.get(YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME);
+String sparkUser = sparkEnvProperties.get(SPARK_USER_PROPERTY_NAME);
+
+String message = "";
+if (yarnACLAdminQueue != null && !yarnACLAdminQueue.trim().equals("*")) {
+ yarnACLAdminQueue = yarnACLAdminQueue + "," + sparkUser;
+ capacitySchedulerProperties.put(YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME,
yarnACLAdminQueue);
+ capacitySchedulerConfig.setProperties(capacitySchedulerProperties);
+ capacitySchedulerConfig.save();
+ agentConfigsHolder.updateData(cluster.getClusterId(),
cluster.getHosts().stream().map(Host::getHostId).collect(Collectors.toList()));
+ message = String.format("Spark user %s was successfully added to %s
property value.", sparkUser, YARN_ACL_ADMIN_QUEUE_PROPERTY_NAME);
+}
+
+return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", message, "");
+ }
+}
[ambari] branch trunk updated: [AMBARI-24023] Make "/hdp/apps/3.0.0.0-X/spark2/spark2-hdp-hive-archive.tar.gz"(vbrodetskyi) (#1454)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5cdd641 [AMBARI-24023] Make
"/hdp/apps/3.0.0.0-X/spark2/spark2-hdp-hive-archive.tar.gz"(vbrodetskyi) (#1454)
5cdd641 is described below
commit 5cdd641bf2426c241eb737c624c0b5b6acfdfa5f
Author: vbrodetskyi
AuthorDate: Mon Jun 4 16:34:19 2018 +0300
[AMBARI-24023] Make
"/hdp/apps/3.0.0.0-X/spark2/spark2-hdp-hive-archive.tar.gz"(vbrodetskyi) (#1454)
---
.../resource_management/libraries/functions/copy_tarball.py| 10 +-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index e362f1a..6c39aff 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -269,6 +269,13 @@ TARBALL_MAP = {
"dirs":
("/tmp/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN),
"/{0}/apps/{1}/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN)),
"service": "SPARK2"
+ },
+
+ "spark2hive": {
+"dirs":
("/tmp/spark2/spark2-{0}-hive-archive.tar.gz".format(STACK_NAME_PATTERN),
+
"/{0}/apps/{1}/spark2/spark2-{0}-hive-archive.tar.gz".format(STACK_NAME_PATTERN,
STACK_VERSION_PATTERN)),
+
+"service": "SPARK2"
}
}
@@ -283,7 +290,8 @@ SERVICE_TO_CONFIG_MAP = {
"hadoop_streaming": "mapred-env",
"tez_hive2": "hive-env",
"spark": "spark-env",
- "spark2": "spark2-env"
+ "spark2": "spark2-env",
+ "spark2hive": "spark2-env"
}
def get_sysprep_skip_copy_tarballs_hdfs():
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23966] Upgrade Spark/Zeppelin/Livy from HDP 2.6 to HDP 3.0.(vbrodetskyi) (#1397)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8a92f37 [AMBARI-23966] Upgrade Spark/Zeppelin/Livy from HDP 2.6 to
HDP 3.0.(vbrodetskyi) (#1397)
8a92f37 is described below
commit 8a92f3726c8d6d29e84bce97c3489b66f6c52138
Author: vbrodetskyi
AuthorDate: Tue May 29 22:41:14 2018 +0300
[AMBARI-23966] Upgrade Spark/Zeppelin/Livy from HDP 2.6 to HDP
3.0.(vbrodetskyi) (#1397)
---
.../upgrades/CreateZeppelinSiteConfig.java | 90 ++
1 file changed, 90 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/CreateZeppelinSiteConfig.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/CreateZeppelinSiteConfig.java
new file mode 100644
index 000..ed3202f
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/CreateZeppelinSiteConfig.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigFactory;
+import org.apache.ambari.server.state.ConfigHelper;
+
+import com.google.inject.Inject;
+
+public class CreateZeppelinSiteConfig extends AbstractServerAction {
+ public static final String VERSION_TAG = "version1";
+ public static final String ZEPPELIN_SITE_CONFIG = "zeppelin-site";
+ public static final String ZEPPELIN_CONFIG_CONFIG = "zeppelin-config";
+
+ @Inject
+ private Clusters clusters;
+
+ @Inject
+ private ConfigFactory configFactory;
+
+ @Inject
+ private ConfigHelper configHelper;
+
+ @Override
+ public CommandReport execute(ConcurrentMap
requestSharedDataContext)
+ throws AmbariException, InterruptedException {
+String clusterName = getExecutionCommand().getClusterName();
+Cluster cluster = clusters.getCluster(clusterName);
+
+Config zeppelinConfConfig =
cluster.getDesiredConfigByType(ZEPPELIN_CONFIG_CONFIG);
+Config zeppelinSiteConfig =
cluster.getDesiredConfigByType(ZEPPELIN_SITE_CONFIG);
+
+if (zeppelinConfConfig == null) {
+ return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
+ String.format("changes are not required"), "");
+}
+
+String output = "";
+Map zeppelinConfProperties =
zeppelinConfConfig.getProperties();
+
zeppelinConfProperties.put("zeppelin.config.storage.class","org.apache.zeppelin.storage.FileSystemConfigStorage");
+if (zeppelinSiteConfig != null) {
+ output += String.format("copying properties from config %s to %s" +
System.lineSeparator(), ZEPPELIN_CONFIG_CONFIG, ZEPPELIN_SITE_CONFIG);
+ zeppelinConfProperties.putAll(zeppelinSiteConfig.getProperties());
+ zeppelinSiteConfig.setProperties(zeppelinConfProperties);
+ zeppelinSiteConfig.save();
+} else {
+ output += String.format("creating new config %s" +
System.lineSeparator(), ZEPPELIN_SITE_CONFIG);
+ Config zeppelinSite =
configFactory.createNew(cluster.getDesiredStackVersion(), cluster,
ZEPPELIN_SITE_CONFIG, VERSION_TAG,
+ zeppelinConfProperties,
zeppelinConfConfig.getPropertiesAttributes());
+ cluster.addConfig(zeppelinSite);
+}
+
+output += String.format("removing %s config" + System.lineSeparator(),
ZEPPELIN_CONFIG_CONFIG);
+configHelper.removeConfigsByType(cluster, ZEPPELIN_CONFIG_CONFIG);
+
+
+if (output.isEmpty()) {
+ output = "change no
[ambari] branch trunk updated: [AMBARI-23948] Proxy user settings are missing for livy in unsecured clusters.(vbrodetskyi) (#1374)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 34afc06 [AMBARI-23948] Proxy user settings are missing for livy in
unsecured clusters.(vbrodetskyi) (#1374)
34afc06 is described below
commit 34afc06f27382bc7bcf4eb974e071d2533320fbc
Author: vbrodetskyi
AuthorDate: Fri May 25 12:46:44 2018 +0300
[AMBARI-23948] Proxy user settings are missing for livy in unsecured
clusters.(vbrodetskyi) (#1374)
---
ambari-server/src/main/resources/stacks/stack_advisor.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py
b/ambari-server/src/main/resources/stacks/stack_advisor.py
index 23db162..5fb4a83 100644
--- a/ambari-server/src/main/resources/stacks/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -2544,7 +2544,8 @@ class DefaultStackAdvisor(StackAdvisor):
("hive-env", "webhcat_user", {HOSTS_PROPERTY:
["WEBHCAT_SERVER"], GROUPS_PROPERTY: ALL_WILDCARD})],
"YARN": [("yarn-env", "yarn_user", {HOSTS_PROPERTY:
["RESOURCEMANAGER"]}, lambda services, hosts:
len(self.getHostsWithComponent("YARN", "RESOURCEMANAGER", services, hosts)) >
1)],
"FALCON": [("falcon-env", "falcon_user", {HOSTS_PROPERTY: ALL_WILDCARD,
GROUPS_PROPERTY: ALL_WILDCARD})],
- "SPARK": [("livy-env", "livy_user", {HOSTS_PROPERTY: ALL_WILDCARD,
GROUPS_PROPERTY: ALL_WILDCARD})]
+ "SPARK": [("livy-env", "livy_user", {HOSTS_PROPERTY: ALL_WILDCARD,
GROUPS_PROPERTY: ALL_WILDCARD})],
+ "SPARK2": [("livy2-env", "livy2_user", {HOSTS_PROPERTY: ALL_WILDCARD,
GROUPS_PROPERTY: ALL_WILDCARD})]
}
def _getHadoopProxyUsersForService(self, serviceName, serviceUserComponents,
services, hosts, configurations):
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23292] livy.superusers is not getting configured correctly when Spark2 and Zeppelin are added as a service later on (#710)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new b3f7779 [AMBARI-23292] livy.superusers is not getting configured
correctly when Spark2 and Zeppelin are added as a service later on (#710)
b3f7779 is described below
commit b3f77793cd5b45fc4f01c937ca19bf4387018de7
Author: vbrodetskyi
AuthorDate: Tue Mar 20 18:03:35 2018 +0200
[AMBARI-23292] livy.superusers is not getting configured correctly when
Spark2 and Zeppelin are added as a service later on (#710)
---
.../ambari/server/controller/KerberosHelperImpl.java | 19 +++
1 file changed, 19 insertions(+)
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 605767f..d2323c4 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -732,6 +732,25 @@ public class KerberosHelperImpl implements KerberosHelper {
continue;
}
+for (Map.Entry>> config :
requestConfigurations.entrySet()) {
+ for (Map properties : config.getValue().values()) {
+for (Map.Entry property : properties.entrySet()) {
+ String oldValue = property.getValue();
+ String updatedValue =
variableReplacementHelper.replaceVariables(property.getValue(),
existingConfigurations);
+ if (!StringUtils.equals(oldValue, updatedValue) &&
!config.getKey().isEmpty()) {
+property.setValue(updatedValue);
+if (kerberosConfigurations.containsKey(config.getKey())) {
+
kerberosConfigurations.get(config.getKey()).put(property.getKey(),
updatedValue);
+} else {
+ Map kerberosConfigProperties = new HashMap<>();
+ kerberosConfigProperties.put(property.getKey(),
updatedValue);
+ kerberosConfigurations.put(config.getKey(),
kerberosConfigProperties);
+}
+ }
+}
+ }
+}
+
StackAdvisorRequest request =
StackAdvisorRequest.StackAdvisorRequestBuilder
.forStack(stackId.getStackName(), stackId.getStackVersion())
.forServices(services)
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23245] Invalid value for zeppelin.config.fs.dir property (#673)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 799711c [AMBARI-23245] Invalid value for zeppelin.config.fs.dir
property (#673)
799711c is described below
commit 799711ce0c77bbb61ba8a04273ce1ac8e2de5b2f
Author: vbrodetskyi
AuthorDate: Thu Mar 15 23:41:21 2018 +0200
[AMBARI-23245] Invalid value for zeppelin.config.fs.dir property (#673)
---
.../ZEPPELIN/0.7.0/configuration/zeppelin-config.xml| 6 --
.../main/resources/stacks/HDP/2.6/services/stack_advisor.py | 13 +
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
index 017898d..dfecae6 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
@@ -86,12 +86,6 @@
-zeppelin.config.fs.dir
-conf
-Location where interpreter.json should be
installed
-
-
-
zeppelin.interpreter.dir
interpreter
Interpreter implementation base directory
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 4f9ba1c..4fdd281 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -70,11 +70,16 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
cluster_env = self.getServicesSiteProperties(services, "cluster-env")
if cluster_env and "recommendations_full_stack_version" in cluster_env:
full_stack_version = cluster_env["recommendations_full_stack_version"]
- if compare_versions(full_stack_version, '2.6.3.0') >= 0:
+ if full_stack_version and compare_versions(full_stack_version,
'2.6.3.0', format=True) >= 0:
zeppelin_config = self.getServicesSiteProperties(services,
"zeppelin-config")
-if zeppelin_config and
zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+if zeppelin_config:
putZeppelinConfigProperty = self.putProperty(configurations,
'zeppelin-config', services)
- putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
+ if zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
+ if 'zeppelin.config.fs.dir' not in zeppelin_config:
+putZeppelinConfigProperty('zeppelin.config.fs.dir', 'conf')
self.__addZeppelinToLivy2SuperUsers(configurations, services)
@@ -745,4 +750,4 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
_superusers.append(zeppelin_user)
putLivy2ConfProperty = self.putProperty(configurations,
'livy2-conf', services)
- putLivy2ConfProperty('livy.superusers', ','.join(_superusers))
\ No newline at end of file
+ putLivy2ConfProperty('livy.superusers', ','.join(_superusers))
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch branch-2.6 updated: [AMBARI-23245] Invalid value for zeppelin.config.fs.dir property (#672)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 50c9a8e [AMBARI-23245] Invalid value for zeppelin.config.fs.dir
property (#672)
50c9a8e is described below
commit 50c9a8ee4993385e67cf25437273a0f6964d3ce3
Author: vbrodetskyi
AuthorDate: Thu Mar 15 21:55:18 2018 +0200
[AMBARI-23245] Invalid value for zeppelin.config.fs.dir property (#672)
---
.../ZEPPELIN/0.7.0/configuration/zeppelin-config.xml | 6 --
.../main/resources/stacks/HDP/2.6/services/stack_advisor.py | 11 ---
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
index 0fbb380..0deb7ea 100644
---
a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
+++
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml
@@ -86,12 +86,6 @@
-zeppelin.config.fs.dir
-conf
-Location where interpreter.json should be
installed
-
-
-
zeppelin.interpreter.dir
interpreter
Interpreter implementation base directory
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 27558cb..08f1947 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -122,11 +122,16 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
cluster_env = self.getServicesSiteProperties(services, "cluster-env")
if cluster_env and "recommendations_full_stack_version" in cluster_env:
full_stack_version = cluster_env["recommendations_full_stack_version"]
- if compare_versions(full_stack_version, '2.6.3.0') >= 0:
+ if full_stack_version and compare_versions(full_stack_version,
'2.6.3.0', format=True) >= 0:
zeppelin_config = self.getServicesSiteProperties(services,
"zeppelin-config")
-if zeppelin_config and
zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+if zeppelin_config:
putZeppelinConfigProperty = self.putProperty(configurations,
'zeppelin-config', services)
- putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
+ if zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
+ if 'zeppelin.config.fs.dir' not in zeppelin_config:
+putZeppelinConfigProperty('zeppelin.config.fs.dir', 'conf')
self.__addZeppelinToLivy2SuperUsers(configurations, services)
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23161] Invalid storage property value for zeppelin in stack HDP 2.6 (#646)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7a267d2 [AMBARI-23161] Invalid storage property value for zeppelin in
stack HDP 2.6 (#646)
7a267d2 is described below
commit 7a267d21eb1dd636be0e4563bba3046ca0d89dd3
Author: vbrodetskyi
AuthorDate: Wed Mar 14 01:28:03 2018 +0200
[AMBARI-23161] Invalid storage property value for zeppelin in stack HDP 2.6
(#646)
---
.../main/resources/stacks/HDP/2.6/services/stack_advisor.py | 11 +++
1 file changed, 11 insertions(+)
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 6951282..4f9ba1c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -20,6 +20,7 @@ import json
import math
import re
from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import compare_versions
class HDP26StackAdvisor(HDP25StackAdvisor):
@@ -65,6 +66,16 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
:type hosts dict
"""
super(HDP26StackAdvisor,
self).recommendZeppelinConfigurations(configurations, clusterData, services,
hosts)
+
+cluster_env = self.getServicesSiteProperties(services, "cluster-env")
+if cluster_env and "recommendations_full_stack_version" in cluster_env:
+ full_stack_version = cluster_env["recommendations_full_stack_version"]
+ if compare_versions(full_stack_version, '2.6.3.0') >= 0:
+zeppelin_config = self.getServicesSiteProperties(services,
"zeppelin-config")
+if zeppelin_config and
zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+ putZeppelinConfigProperty = self.putProperty(configurations,
'zeppelin-config', services)
+ putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
self.__addZeppelinToLivy2SuperUsers(configurations, services)
def recommendAtlasConfigurations(self, configurations, clusterData,
services, hosts):
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch branch-2.6 updated: [AMBARI-23161] Invalid storage property value for zeppelin in stack H… (#568)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new ef1aace [AMBARI-23161] Invalid storage property value for zeppelin in
stack H… (#568)
ef1aace is described below
commit ef1aace01ed1181c5ede4950d7551cd3aead019c
Author: vbrodetskyi
AuthorDate: Wed Mar 7 16:17:06 2018 +0200
[AMBARI-23161] Invalid storage property value for zeppelin in stack H…
(#568)
* [AMBARI-23161] Invalid storage property value for zeppelin in stack HDP
2.6
* [AMBARI-23161] Invalid storage property value for zeppelin in stack HDP
2.6
---
.../resources/stacks/HDP/2.6/services/stack_advisor.py | 14 +-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 0c6c2c7..27558cb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -21,6 +21,7 @@ import json
import math
import re
from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import compare_versions
class HDP26StackAdvisor(HDP25StackAdvisor):
@@ -72,7 +73,7 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
streamline_bare_principal =
get_bare_principal(_streamline_principal_name)
storm_nimbus_impersonation_acl.replace('{{streamline_bare_principal}}',
streamline_bare_principal)
putStormSiteProperty('nimbus.impersonation.acl',
storm_nimbus_impersonation_acl)
-
+
storm_nimbus_autocred_plugin_classes =
storm_site["nimbus.autocredential.plugins.classes"] if
"nimbus.autocredential.plugins.classes" in storm_site else None
if storm_nimbus_autocred_plugin_classes is not None:
new_storm_nimbus_autocred_plugin_classes =
['org.apache.storm.hdfs.security.AutoHDFS',
@@ -117,6 +118,17 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
:type hosts dict
"""
super(HDP26StackAdvisor,
self).recommendZeppelinConfigurations(configurations, clusterData, services,
hosts)
+
+cluster_env = self.getServicesSiteProperties(services, "cluster-env")
+if cluster_env and "recommendations_full_stack_version" in cluster_env:
+ full_stack_version = cluster_env["recommendations_full_stack_version"]
+ if compare_versions(full_stack_version, '2.6.3.0') >= 0:
+zeppelin_config = self.getServicesSiteProperties(services,
"zeppelin-config")
+if zeppelin_config and
zeppelin_config.get('zeppelin.notebook.storage', None) ==
'org.apache.zeppelin.notebook.repo.VFSNotebookRepo':
+ putZeppelinConfigProperty = self.putProperty(configurations,
'zeppelin-config', services)
+ putZeppelinConfigProperty('zeppelin.notebook.storage',
'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo')
+
+
self.__addZeppelinToLivy2SuperUsers(configurations, services)
def recommendAtlasConfigurations(self, configurations, clusterData,
services, hosts):
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#490)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/trunk by this push: new 5964c7f [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#490) 5964c7f is described below commit 5964c7f4d1fefa06ce06e08cb21f10770bbead09 Author: vbrodetskyi AuthorDate: Fri Mar 2 22:09:32 2018 +0200 [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#490) --- .../ZEPPELIN/0.6.0/configuration/zeppelin-config.xml | 12 .../ZEPPELIN/0.7.0/configuration/zeppelin-config.xml | 12 2 files changed, 24 insertions(+) diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml index 03ad5f7..0deb7ea 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml @@ -139,14 +139,22 @@ zeppelin.ssl.keystore.password change me +PASSWORD Keystore password. Can be obfuscated by the Jetty Password tool + + password + zeppelin.ssl.key.manager.password change me +PASSWORD Key Manager password. Defaults to keystore password. Can be obfuscated. + + password + @@ -167,9 +175,13 @@ zeppelin.ssl.truststore.password change me +PASSWORD Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password + + password + diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml index 171b7eb..0f6f09d 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml @@ -145,14 +145,22 @@ zeppelin.ssl.keystore.password change me +PASSWORD Keystore password. Can be obfuscated by the Jetty Password tool + + password + zeppelin.ssl.key.manager.password change me +PASSWORD Key Manager password. Defaults to keystore password. Can be obfuscated. + + password + @@ -173,9 +181,13 @@ zeppelin.ssl.truststore.password change me +PASSWORD Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password + + password + -- To stop receiving notification emails like this one, please contact vbrodets...@apache.org.
[ambari] branch vbrodetskyi-branch-2.6 deleted (was ee56f64)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a change to branch vbrodetskyi-branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git. was ee56f64 [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository. -- To stop receiving notification emails like this one, please contact vbrodets...@apache.org.
[ambari] branch branch-2.6 updated: [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#481)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/branch-2.6 by this push: new dc5cc66 [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#481) dc5cc66 is described below commit dc5cc669f1a536d87fa96cd0a7c02d2a00ee55f9 Author: vbrodetskyi AuthorDate: Tue Feb 27 13:58:45 2018 +0200 [AMBARI-23091] Zeppelin Notebook SSL credentials in Ambari UI are in plain text rather than being hidden (#481) --- .../ZEPPELIN/0.6.0/configuration/zeppelin-config.xml | 12 .../ZEPPELIN/0.7.0/configuration/zeppelin-config.xml | 12 2 files changed, 24 insertions(+) diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml index 03ad5f7..0deb7ea 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0/configuration/zeppelin-config.xml @@ -139,14 +139,22 @@ zeppelin.ssl.keystore.password change me +PASSWORD Keystore password. Can be obfuscated by the Jetty Password tool + + password + zeppelin.ssl.key.manager.password change me +PASSWORD Key Manager password. Defaults to keystore password. Can be obfuscated. + + password + @@ -167,9 +175,13 @@ zeppelin.ssl.truststore.password change me +PASSWORD Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password + + password + diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml index 29821f2..8fddfc7 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/configuration/zeppelin-config.xml @@ -145,14 +145,22 @@ zeppelin.ssl.keystore.password change me +PASSWORD Keystore password. Can be obfuscated by the Jetty Password tool + + password + zeppelin.ssl.key.manager.password change me +PASSWORD Key Manager password. Defaults to keystore password. Can be obfuscated. + + password + @@ -173,9 +181,13 @@ zeppelin.ssl.truststore.password change me +PASSWORD Truststore password. Can be obfuscated by the Jetty Password tool. Defaults to the keystore password + + password + -- To stop receiving notification emails like this one, please contact vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23043] "Table or view not found error" with livy/livy2 interpreter on upgraded cluster to Fenton-M30 (#465)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2cbb348 [AMBARI-23043] "Table or view not found error" with
livy/livy2 interpreter on upgraded cluster to Fenton-M30 (#465)
2cbb348 is described below
commit 2cbb34854ef4fdd7fdacfd105ce830008cdd8d66
Author: vbrodetskyi
AuthorDate: Sat Feb 24 02:15:47 2018 +0200
[AMBARI-23043] "Table or view not found error" with livy/livy2 interpreter
on upgraded cluster to Fenton-M30 (#465)
---
.../src/test/python/stacks/2.6/ZEPPELIN/interpreter_json_generated.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/interpreter_json_generated.py
b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/interpreter_json_generated.py
index 9bc8905..1a8b7be 100644
---
a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/interpreter_json_generated.py
+++
b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/interpreter_json_generated.py
@@ -20,7 +20,7 @@ limitations under the License.
template = '\n{\n "interpreterSettings": {\n"2CKEKWY8Z": {\n "id":
"2CKEKWY8Z",\n "name": "angular",\n "group": "angular",\n
"properties": {},\n "status": "READY",\n "interpreterGroup": [\n
{\n "name": "angular",\n "class":
"org.apache.zeppelin.angular.AngularInterpreter",\n
"defaultInterpreter": false,\n "editor": {\n
"editOnDblClick": true\n }\n}\n ],\n "dependencies":
[] [...]
-template_after_base = '{\n "interpreterSettings": {\n"2CHS8UYQQ": {\n
"status": "READY", \n "group": "sh", \n "name": "sh", \n "id":
"2CHS8UYQQ", \n "interpreterGroup": [\n{\n "editor": {\n
"editOnDblClick": false, \n"language": "sh"\n },
\n "defaultInterpreter": false, \n "name": "sh", \n
"class": "org.apache.zeppelin.shell.ShellInterpreter"\n}\n ], \n
"dependencie [...]
+template_after_base = '{\n "interpreterSettings": {\n"2CHS8UYQQ": {\n
"status": "READY", \n "group": "sh", \n "name": "sh", \n "id":
"2CHS8UYQQ", \n "interpreterGroup": [\n{\n "editor": {\n
"editOnDblClick": false, \n"language": "sh"\n },
\n "defaultInterpreter": false, \n "name": "sh", \n
"class": "org.apache.zeppelin.shell.ShellInterpreter"\n}\n ], \n
"dependencie [...]
template_after_without_spark_and_livy = '{\n "interpreterSettings": {\n
"2CHS8UYQQ": {\n "status": "READY", \n "group": "sh", \n "name":
"sh", \n "id": "2CHS8UYQQ", \n "interpreterGroup": [\n{\n
"editor": {\n"editOnDblClick": false, \n
"language": "sh"\n }, \n "defaultInterpreter": false, \n
"name": "sh", \n "class":
"org.apache.zeppelin.shell.ShellInterpreter"\n}\n ], \n [...]
--
To stop receiving notification emails like this one, please contact
vbrodets...@apache.org.
[ambari] branch trunk updated: [AMBARI-23043] Table or view not found error with livy/livy2 interpreter on upgraded cluster to Fenton-M30 (#429)
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/trunk by this push: new 9be99ce [AMBARI-23043] Table or view not found error with livy/livy2 interpreter on upgraded cluster to Fenton-M30 (#429) 9be99ce is described below commit 9be99cec9b0ad7aaa0290309ba9c56927498c8f6 Author: vbrodetskyi AuthorDate: Wed Feb 21 21:43:37 2018 +0200 [AMBARI-23043] Table or view not found error with livy/livy2 interpreter on upgraded cluster to Fenton-M30 (#429) --- .../ZEPPELIN/0.7.0/package/scripts/master.py | 33 ++ 1 file changed, 33 insertions(+) diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py index 26a7ba2..c85a0e4 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py @@ -298,6 +298,39 @@ class Master(Script): if setting_key not in interpreter_settings: interpreter_settings[setting_key] = interpreter_json_template[ setting_key] + else: +templateGroups = interpreter_json_template[setting_key]['interpreterGroup'] +groups = interpreter_settings[setting_key]['interpreterGroup'] + +templateProperties = interpreter_json_template[setting_key]['properties'] +properties = interpreter_settings[setting_key]['properties'] + +templateOptions = interpreter_json_template[setting_key]['option'] +options = interpreter_settings[setting_key]['option'] + +# search for difference in groups from current interpreter and template interpreter +# if any group exists in template but doesn't exist in current interpreter, it will be added +group_names = [] +for group in groups: + group_names.append(group['name']) + +for template_group in templateGroups: + if not template_group['name'] in group_names: +groups.append(template_group) + + +# search for difference in properties from current interpreter and template interpreter +# if any property exists in template but doesn't exist in current interpreter, it will be added +for template_property in templateProperties: + if not template_property in properties: +properties[template_property] = templateProperties[template_property] + + +# search for difference in options from current interpreter and template interpreter +# if any option exists in template but doesn't exist in current interpreter, it will be added +for template_option in templateOptions: + if not template_option in options: +options[template_option] = templateOptions[template_option] self.set_interpreter_settings(config_data) -- To stop receiving notification emails like this one, please contact vbrodets...@apache.org.
[ambari] branch branch-2.6 updated: [AMBARI-23043] Table or view not found error' with livy/livy2 interpreter on upgraded cluster to Fenton-M30
This is an automated email from the ASF dual-hosted git repository. vbrodetskyi pushed a commit to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/ambari.git The following commit(s) were added to refs/heads/branch-2.6 by this push: new c386567 [AMBARI-23043] Table or view not found error' with livy/livy2 interpreter on upgraded cluster to Fenton-M30 c386567 is described below commit c3865676200a55fdbbcaf8377068d87162141532 Author: vbrodetskyi AuthorDate: Wed Feb 21 18:59:52 2018 +0200 [AMBARI-23043] Table or view not found error' with livy/livy2 interpreter on upgraded cluster to Fenton-M30 --- .../ZEPPELIN/0.7.0/package/scripts/master.py | 33 ++ 1 file changed, 33 insertions(+) diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py index 56ed31c..6607c4c 100644 --- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py @@ -292,6 +292,39 @@ class Master(Script): if setting_key not in interpreter_settings: interpreter_settings[setting_key] = interpreter_json_template[ setting_key] + else: +templateGroups = interpreter_json_template[setting_key]['interpreterGroup'] +groups = interpreter_settings[setting_key]['interpreterGroup'] + +templateProperties = interpreter_json_template[setting_key]['properties'] +properties = interpreter_settings[setting_key]['properties'] + +templateOptions = interpreter_json_template[setting_key]['option'] +options = interpreter_settings[setting_key]['option'] + +# search for difference in groups from current interpreter and template interpreter +# if any group exists in template but doesn't exist in current interpreter, it will be added +group_names = [] +for group in groups: + group_names.append(group['name']) + +for template_group in templateGroups: + if not template_group['name'] in group_names: +groups.append(template_group) + + +# search for difference in properties from current interpreter and template interpreter +# if any property exists in template but doesn't exist in current interpreter, it will be added +for template_property in templateProperties: + if not template_property in properties: +properties[template_property] = templateProperties[template_property] + + +# search for difference in options from current interpreter and template interpreter +# if any option exists in template but doesn't exist in current interpreter, it will be added +for template_option in templateOptions: + if not template_option in options: +options[template_option] = templateOptions[template_option] self.set_interpreter_settings(config_data) -- To stop receiving notification emails like this one, please contact vbrodets...@apache.org.
[ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-22817] Update backend code to handle new versioning schema. (#155)
This is an automated email from the ASF dual-hosted git repository.
vbrodetskyi pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by
this push:
new 80d840e [AMBARI-22817] Update backend code to handle new versioning
schema. (#155)
80d840e is described below
commit 80d840e5d50614a0f148a650c9bfc7d21363a218
Author: vbrodetskyi
AuthorDate: Thu Jan 25 03:04:45 2018 +0200
[AMBARI-22817] Update backend code to handle new versioning schema. (#155)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
* AMBARI-22817. Update backend code to handle new versioning
schema.(vbrodetskyi)
---
.../libraries/functions/module_version.py | 158
.../libraries/functions/mpack_version.py | 207 +
.../apache/ambari/server/utils/ModuleVersion.java | 162
.../apache/ambari/server/utils/MpackVersion.java | 197
.../ambari/server/utils/TestVersionUtils.java | 142 ++
ambari-server/src/test/python/TestVersion.py | 100 +-
6 files changed, 964 insertions(+), 2 deletions(-)
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py
b/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py
new file mode 100644
index 000..6ec4ba3
--- /dev/null
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/module_version.py
@@ -0,0 +1,158 @@
+"""
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""
+
+import re
+
+"""
+ This class should be used to compare module(service) versions.
+ Base method which should be used is parse(..), This method will validate and
parse
+ version which you will pass as parameter, and return object of current class
with
+ parsed version. Same thing you should do with another version, with which you
are
+ planning to compare previous one. After that, use "==", "<", ">" to get final
result.
+"""
+class ModuleVersion(object):
+ __module_version_pattern =
"(?P[0-9]+).(?P[0-9]+).(?P[0-9]+).(?P[0-9]+)(-h(?P[0-9]+))*-b(?P[0-9]+)"
+ __module_version_regex = re.compile(__module_version_pattern)
+
+ def __init__(self, apache_major, apache_minor, internal_minor,
internal_maint, hotfix, build):
+"""
+:type apache_major int
+:type apache_minor int
+:type internal_maint int
+:type internal_minor int
+:type hotfix int
+:type build int
+"""
+self.__apache_major = int(apache_major)
+self.__apache_minor = int(apache_minor)
+self.__internal_maint = int(internal_maint)
+self.__internal_minor = int(internal_minor)
+self.__hotfix = int(hotfix) if hotfix else 0 # hotfix is optional group
+self.__build = int(build)
+
+ def __repr__(self):
+return "{0}.{1}.{2}.{3}-h{4}-b{5}".format(*self.to_list())
+
+ def to_list(self):
+"""
+Return version elements as list
+
+:rtype list
+"""
+return [
+ self.__apache_major,
+ self.__apache_minor,
+ self.__internal_minor,
+ self.__internal_maint,
+ self.__hotfix,
+ self.__build
+]
+
+ def __cmp__(self, other):
+"""
+:type other ModuleVersion
+
+:raise TypeError
+"""
+if other and not isinstance(other, self.__class__):
+ raise TypeError("Operand type is different from
{0}".format(self.__class__.__name__))
+
+r = 0
+x = self.to_list()
+y = other.to_list()
+
+for i in range(0, len(x))
ambari git commit: AMBARI-22724. Idempotent issue on Ambari Upgrade, renameServiceDeletedColumn failed with column already exists exception.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk e13bfb95c -> cd5c1cad8
AMBARI-22724. Idempotent issue on Ambari Upgrade, renameServiceDeletedColumn
failed with column already exists exception.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cd5c1cad
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cd5c1cad
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cd5c1cad
Branch: refs/heads/trunk
Commit: cd5c1cad8f62c45de62094de2b5e442d161c9b34
Parents: e13bfb9
Author: Vitaly Brodetskyi
Authored: Fri Jan 5 16:29:39 2018 +0200
Committer: Vitaly Brodetskyi
Committed: Fri Jan 5 16:32:21 2018 +0200
--
.../org/apache/ambari/server/upgrade/UpgradeCatalog252.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd5c1cad/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index 1b706d6..62e56fc 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -59,6 +59,7 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog
{
static final String CLUSTERCONFIG_TABLE = "clusterconfig";
static final String SERVICE_DELETED_COLUMN = "service_deleted";
+ static final String UNMAPPED_COLUMN = "unmapped";
private static final String UPGRADE_TABLE = "upgrade";
private static final String UPGRADE_TABLE_FROM_REPO_COLUMN =
"from_repo_version_id";
@@ -141,8 +142,10 @@ public class UpgradeCatalog252 extends
AbstractUpgradeCatalog {
* @throws java.sql.SQLException
*/
private void addServiceDeletedColumnToClusterConfigTable() throws
SQLException {
-dbAccessor.addColumn(CLUSTERCONFIG_TABLE,
-new DBColumnInfo(SERVICE_DELETED_COLUMN, Short.class, null, 0, false));
+if (!dbAccessor.tableHasColumn(CLUSTERCONFIG_TABLE, UNMAPPED_COLUMN)) {
+ dbAccessor.addColumn(CLUSTERCONFIG_TABLE,
+ new DBColumnInfo(SERVICE_DELETED_COLUMN, Short.class, null, 0,
false));
+}
}
/**
ambari git commit: AMBARI-22724. Idempotent issue on Ambari Upgrade, renameServiceDeletedColumn failed with column already exists exception.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 868949acd -> 680f737c9
AMBARI-22724. Idempotent issue on Ambari Upgrade, renameServiceDeletedColumn
failed with column already exists exception.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/680f737c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/680f737c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/680f737c
Branch: refs/heads/branch-2.6
Commit: 680f737c944bf4e10c282266c0bb39e43074cec2
Parents: 868949a
Author: Vitaly Brodetskyi
Authored: Fri Jan 5 16:16:12 2018 +0200
Committer: Vitaly Brodetskyi
Committed: Fri Jan 5 16:16:12 2018 +0200
--
.../org/apache/ambari/server/upgrade/UpgradeCatalog252.java | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/680f737c/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index e6a7829..b8a87ec 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -65,6 +65,7 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog
{
static final String CLUSTERCONFIG_TABLE = "clusterconfig";
static final String SERVICE_DELETED_COLUMN = "service_deleted";
+ static final String UNMAPPED_COLUMN = "unmapped";
private static final String UPGRADE_TABLE = "upgrade";
private static final String UPGRADE_TABLE_FROM_REPO_COLUMN =
"from_repo_version_id";
@@ -149,8 +150,10 @@ public class UpgradeCatalog252 extends
AbstractUpgradeCatalog {
* @throws java.sql.SQLException
*/
private void addServiceDeletedColumnToClusterConfigTable() throws
SQLException {
-dbAccessor.addColumn(CLUSTERCONFIG_TABLE,
-new DBColumnInfo(SERVICE_DELETED_COLUMN, Short.class, null, 0, false));
+if (!dbAccessor.tableHasColumn(CLUSTERCONFIG_TABLE, UNMAPPED_COLUMN)) {
+ dbAccessor.addColumn(CLUSTERCONFIG_TABLE,
+ new DBColumnInfo(SERVICE_DELETED_COLUMN, Short.class, null, 0,
false));
+}
}
/**
ambari git commit: AMBARI-22249. Add service group dependencies.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 6a7f5b1bc -> 765fb1c1a
AMBARI-22249. Add service group dependencies.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/765fb1c1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/765fb1c1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/765fb1c1
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 765fb1c1affb0d83e9cf51d2a52627fccb8b6d71
Parents: 6a7f5b1
Author: Vitaly Brodetskyi
Authored: Tue Dec 26 16:09:38 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Tue Dec 26 16:09:38 2017 +0200
--
.../server/api/services/ServiceGroupKey.java| 10 +
.../ServiceGroupDependencyRequest.java | 35 +-
.../ServiceGroupDependencyResponse.java | 37 +-
.../ServiceGroupDependencyResourceProvider.java | 109 ++--
.../entities/ServiceGroupDependencyEntity.java | 8 +
.../ambari/server/state/ServiceGroupImpl.java | 4 +-
...viceGroupDependencyResourceProviderTest.java | 527 +++
7 files changed, 668 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/765fb1c1/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupKey.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupKey.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupKey.java
index 9389f0c..bb8ca83 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupKey.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupKey.java
@@ -27,6 +27,8 @@ public class ServiceGroupKey {
private String clusterName;
private String serviceGroupName;
+ private Long dependencyId;
+
@JsonProperty("cluster_name")
public String getClusterName() {
return clusterName;
@@ -61,6 +63,14 @@ public class ServiceGroupKey {
this.serviceGroupId = serviceGroupId;
}
+ public Long getDependencyId() {
+return dependencyId;
+ }
+
+ public void setDependencyId(Long dependencyId) {
+this.dependencyId = dependencyId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/765fb1c1/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupDependencyRequest.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupDependencyRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupDependencyRequest.java
index 9f4eea3..f7cd738 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupDependencyRequest.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceGroupDependencyRequest.java
@@ -22,12 +22,17 @@ public class ServiceGroupDependencyRequest {
private String clusterName; // REF
private String serviceGroupName; // GET/CREATE/UPDATE/DELETE
- private Long dependencyServiceGroupId;
+ private String dependentServiceGroupClusterName;
+ private String dependentServiceGroupName;
+ private Long dependencyId;
- public ServiceGroupDependencyRequest(String clusterName, String
serviceGroupName, Long dependencyServiceGroupId) {
+ public ServiceGroupDependencyRequest(String clusterName, String
serviceGroupName, String dependentServiceGroupClusterName,
+ String dependentServiceGroupName, Long
dependencyId) {
this.clusterName = clusterName;
this.serviceGroupName = serviceGroupName;
-this.dependencyServiceGroupId = dependencyServiceGroupId;
+this.dependentServiceGroupName = dependentServiceGroupName;
+this.dependentServiceGroupClusterName = dependentServiceGroupClusterName;
+this.dependencyId = dependencyId;
}
/**
@@ -58,12 +63,28 @@ public class ServiceGroupDependencyRequest {
this.serviceGroupName = serviceGroupName;
}
- public Long getDependencyServiceGroupId() {
-return dependencyServiceGroupId;
+ public Long getDependencyId() {
+return dependencyId;
}
- public void setDependencyServiceGroupId(Long dependencyServiceGroupId) {
-this.dependencyServiceGroupId = dependencyServiceGroupId;
+ public void setDependencyId(Long dependencyId) {
+this.dependencyId = dependencyId;
+ }
+
+ public String getDependentServiceGroupName() {
+return dependentServiceGroupName;
+ }
+
+ public void setDependentServiceGroupName(String dependentServiceGroupName) {
+this.dependentServiceGroupName = dependentSer
ambari git commit: AMBARI-22345. Add service dependencies.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 68c62212c -> 6a7f5b1bc
AMBARI-22345. Add service dependencies.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a7f5b1b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a7f5b1b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a7f5b1b
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 6a7f5b1bc12ab13ecd28cb1bac555962eed1cc69
Parents: 68c6221
Author: Vitaly Brodetskyi
Authored: Thu Dec 21 19:15:16 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Thu Dec 21 19:15:16 2017 +0200
--
.../ambari/server/api/services/ServiceKey.java | 13 +-
.../controller/ServiceDependencyRequest.java| 50 +-
.../controller/ServiceDependencyResponse.java | 12 +-
.../ServiceDependencyResourceProvider.java | 121 ++--
.../apache/ambari/server/state/ServiceImpl.java | 3 +-
.../ServiceDependencyResourceProviderTest.java | 573 +++
6 files changed, 714 insertions(+), 58 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a7f5b1b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceKey.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceKey.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceKey.java
index ace7098..32b87f5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceKey.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceKey.java
@@ -22,6 +22,7 @@ public class ServiceKey {
private Long serviceId;
private Long serviceGroupId;
private Long clusterId;
+ private Long dependencyId;
private String serviceName;
private String serviceGroupName;
private String clusterName;
@@ -29,13 +30,15 @@ public class ServiceKey {
public ServiceKey() {
}
- public ServiceKey(Long clusterId, String clusterName, Long serviceGroupId,
String serviceGroupName, Long serviceId, String serviceName) {
+ public ServiceKey(Long clusterId, String clusterName, Long serviceGroupId,
String serviceGroupName, Long serviceId, String serviceName,
+Long dependencyId) {
this.clusterId = clusterId;
this.clusterName = clusterName;
this.serviceGroupId = serviceGroupId;
this.serviceGroupName = serviceGroupName;
this.serviceId = serviceId;
this.serviceName = serviceName;
+this.dependencyId = dependencyId;
}
public Long getClusterId() {
@@ -86,6 +89,14 @@ public class ServiceKey {
this.serviceName = serviceName;
}
+ public Long getDependencyId() {
+return dependencyId;
+ }
+
+ public void setDependencyId(Long dependencyId) {
+this.dependencyId = dependencyId;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a7f5b1b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceDependencyRequest.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceDependencyRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceDependencyRequest.java
index 6cf46b4..7f1e80c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceDependencyRequest.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/ServiceDependencyRequest.java
@@ -22,13 +22,20 @@ public class ServiceDependencyRequest {
private String clusterName;
private String serviceName;
private String serviceGroupName;
- private Long dependencyServiceId;
+ private String dependentClusterName;
+ private String dependentServiceName;
+ private String dependentServiceGroupName;
+ private Long dependencyId;
- public ServiceDependencyRequest(String clusterName, String serviceName,
String serviceGroupName, Long dependencyServiceId) {
+ public ServiceDependencyRequest(String clusterName, String serviceName,
String serviceGroupName, String dependentClusterName,
+ String dependentServiceGroupName, String
dependentServiceName, Long dependencyId) {
this.clusterName = clusterName;
this.serviceName = serviceName;
this.serviceGroupName = serviceGroupName;
-this.dependencyServiceId = dependencyServiceId;
+this.dependentClusterName = dependentClusterName;
+this.dependentServiceGroupName = dependentServiceGroupName;
+this.dependentServiceName = dependentServiceName;
+this.dependencyId = dependencyId;
}
public String getClusterName() {
@@ -39,12 +46
[1/2] ambari git commit: AMBARI-22614. Fix unit tests in feature branch to make them workable.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 bfeaa1071 -> 481175d12
http://git-wip-us.apache.org/repos/asf/ambari/blob/481175d1/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
index 8546a9b..d43a712 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java
@@ -62,6 +62,7 @@ import org.apache.ambari.server.mpack.MpackManagerFactory;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.HostDAO;
import org.apache.ambari.server.orm.dao.HostRoleCommandDAO;
+import org.apache.ambari.server.resources.RootLevelSettingsManagerFactory;
import org.apache.ambari.server.security.SecurityHelper;
import org.apache.ambari.server.security.encryption.CredentialStoreService;
import org.apache.ambari.server.stack.StackManagerFactory;
@@ -133,6 +134,7 @@ public class StageUtilsTest extends EasyMockSupport {
bind(HostRoleCommandDAO.class).toInstance(createNiceMock(HostRoleCommandDAO.class));
bind(MpackManagerFactory.class).toInstance(createNiceMock(MpackManagerFactory.class));
bind(AmbariManagementController.class).toInstance(createNiceMock(AmbariManagementController.class));
+
bind(RootLevelSettingsManagerFactory.class).toInstance(createNiceMock(RootLevelSettingsManagerFactory.class));
install(new
FactoryModuleBuilder().build(ExecutionCommandWrapperFactory.class));
install(new FactoryModuleBuilder().implement(Config.class,
ConfigImpl.class).build(ConfigFactory.class));
install(new
FactoryModuleBuilder().build(ConfigureClusterTaskFactory.class));
[2/2] ambari git commit: AMBARI-22614. Fix unit tests in feature branch to make them workable.(vbrodetskyi)
AMBARI-22614. Fix unit tests in feature branch to make them
workable.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/481175d1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/481175d1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/481175d1
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 481175d129d110a8529aa7a9d45735e97dc41ca5
Parents: bfeaa10
Author: Vitaly Brodetskyi
Authored: Thu Dec 14 16:28:42 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Thu Dec 14 16:29:51 2017 +0200
--
.../ComponentVersionAlertRunnableTest.java | 2 +
.../api/query/render/DefaultRendererTest.java | 4 +-
.../api/query/render/MinimalRendererTest.java | 4 +-
.../server/checks/ServicesUpCheckTest.java | 6 +--
.../AmbariCustomCommandExecutionHelperTest.java | 53 +++-
.../server/controller/AmbariServerTest.java | 3 ++
.../server/controller/KerberosHelperTest.java | 3 ++
...hYarnCapacitySchedulerReleaseConfigTest.java | 7 +--
.../ActiveWidgetLayoutResourceProviderTest.java | 4 ++
.../ClientConfigResourceProviderTest.java | 10 ++--
...ClusterStackVersionResourceProviderTest.java | 23 +
.../internal/ComponentResourceProviderTest.java | 7 +--
.../HostComponentResourceProviderTest.java | 1 +
.../internal/JMXHostProviderTest.java | 6 ++-
.../RepositoryVersionResourceProviderTest.java | 1 +
...erviceHostComponentResourceProviderTest.java | 2 +
.../internal/ServiceResourceProviderTest.java | 20 +---
.../StackConfigurationResourceProviderTest.java | 9 ++--
.../UserAuthorizationResourceProviderTest.java | 6 ++-
.../internal/UserResourceProviderTest.java | 4 ++
.../timeline/AMSPropertyProviderTest.java | 2 +
.../apache/ambari/server/orm/OrmTestHelper.java | 15 +++---
.../server/orm/dao/AlertDispatchDAOTest.java| 21
.../ambari/server/orm/dao/AlertsDAOTest.java| 13 +++--
.../svccomphost/ServiceComponentHostTest.java | 52 ---
.../server/testing/DBInconsistencyTests.java| 4 +-
.../ambari/server/utils/StageUtilsTest.java | 2 +
27 files changed, 189 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/481175d1/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java
index 62ff3d3..13a866a 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/alerts/ComponentVersionAlertRunnableTest.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
import org.apache.ambari.server.orm.entities.UpgradeEntity;
+import org.apache.ambari.server.resources.RootLevelSettingsManagerFactory;
import org.apache.ambari.server.stack.StackManagerFactory;
import org.apache.ambari.server.state.Alert;
import org.apache.ambari.server.state.AlertState;
@@ -360,6 +361,7 @@ public class ComponentVersionAlertRunnableTest extends
EasyMockSupport {
binder.bind(AmbariMetaInfo.class).toInstance(createNiceMock(AmbariMetaInfo.class));
binder.bind(StackManagerFactory.class).toInstance(createNiceMock(StackManagerFactory.class));
binder.bind(MpackManagerFactory.class).toInstance(createNiceMock(MpackManagerFactory.class));
+
binder.bind(RootLevelSettingsManagerFactory.class).toInstance(createNiceMock(RootLevelSettingsManagerFactory.class));
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/481175d1/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
index 96f80b1..f895bef 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java
@@ -55,6 +55,8 @@ public class DefaultRendererTest {
expect(schemaFactory.getSchema(Resource.Type.Component)).andReturn(schema).anyTimes();
expect(schemaFactory.getSchema
[1/2] ambari git commit: AMBARI-22614. Fix unit tests in feature branch to make them workable.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 490808cb9 -> bcbacf0c7
http://git-wip-us.apache.org/repos/asf/ambari/blob/bcbacf0c/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java
index a52f438..d40679a 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterImplTest.java
@@ -48,6 +48,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponent;
import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.ServiceGroup;
import org.apache.ambari.server.state.StackId;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -214,7 +215,7 @@ public class ClusterImplTest {
String stackVersion = "HDP-2.1.1";
String repoVersion = "2.1.1-1234";
StackId stackId = new StackId(stackVersion);
-ormTestHelper.createStack(stackId);
+ormTestHelper.createStackWithRepoVersion(stackId, "");
clusters.addCluster(clusterName, stackId);
Cluster cluster = clusters.getCluster(clusterName);
@@ -233,7 +234,8 @@ public class ClusterImplTest {
clusters.mapAndPublishHostsToCluster(Sets.newHashSet(hostName1,
hostName2), clusterName);
-Service hdfs = cluster.addService(null, "HDFS", "", repositoryVersion);
+ServiceGroup serviceGroup = cluster.addServiceGroup("CORE");
+Service hdfs = cluster.addService(serviceGroup, "HDFS", "HDFS",
repositoryVersion);
ServiceComponent nameNode = hdfs.addServiceComponent("NAMENODE");
nameNode.addServiceComponentHost(hostName1);
@@ -246,7 +248,7 @@ public class ClusterImplTest {
hdfsClient.addServiceComponentHost(hostName1);
hdfsClient.addServiceComponentHost(hostName2);
-Service tez = cluster.addService(null, serviceToDelete, "",
repositoryVersion);
+Service tez = cluster.addService(serviceGroup, serviceToDelete,
serviceToDelete, repositoryVersion);
ServiceComponent tezClient = tez.addServiceComponent("TEZ_CLIENT");
ServiceComponentHost tezClientHost1 =
tezClient.addServiceComponentHost(hostName1);
@@ -275,7 +277,7 @@ public class ClusterImplTest {
String hostToDelete = hostName2;
StackId stackId = new StackId("HDP-2.1.1");
-ormTestHelper.createStack(stackId);
+ormTestHelper.createStackWithRepoVersion(stackId, "");
clusters.addCluster(clusterName, stackId);
Cluster cluster = clusters.getCluster(clusterName);
@@ -315,7 +317,7 @@ public class ClusterImplTest {
String clusterName = "TEST_CLUSTER_SIZE";
String hostName1 = "host1", hostName2 = "host2";
StackId stackId = new StackId("HDP", "2.1.1");
-ormTestHelper.createStack(stackId);
+ormTestHelper.createStackWithRepoVersion(stackId, "");
clusters.addCluster(clusterName, stackId);
Cluster cluster = clusters.getCluster(clusterName);
http://git-wip-us.apache.org/repos/asf/ambari/blob/bcbacf0c/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
index 4ce7387..bbfc838 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java
@@ -96,6 +96,7 @@ import org.apache.ambari.server.state.ServiceComponentFactory;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.ServiceComponentHostFactory;
import org.apache.ambari.server.state.ServiceFactory;
+import org.apache.ambari.server.state.ServiceGroup;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.State;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
@@ -1076,9 +1077,10 @@ public class ClusterTest {
RepositoryVersionEntity repositoryVersion =
helper.getOrCreateRepositoryVersion(c1);
-c1.addService(null, "MAPREDUCE", "", repositoryVersion);
+ServiceGroup serviceGroup = c1.addServiceGroup("CORE");
+c1.addService(serviceGroup, "MAPREDUCE", "MAPREDUCE", repositoryVersion);
-Service hdfs = c1.addService(null, "HDFS", "", repositoryVersion);
+Service hdfs = c1.addService(serviceGroup, "HDFS", "HDFS",
repositoryVersion);
ServiceComponent nameNode = hdfs.addServiceComponent("NAMENODE");
assertEquals(2, c1.getServi
[2/2] ambari git commit: AMBARI-22614. Fix unit tests in feature branch to make them workable.(vbrodetskyi)
AMBARI-22614. Fix unit tests in feature branch to make them
workable.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bcbacf0c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bcbacf0c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bcbacf0c
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: bcbacf0c77fb8aa10a4d5e83df8ebb4650982582
Parents: 490808c
Author: Vitaly Brodetskyi
Authored: Fri Dec 8 19:03:20 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Fri Dec 8 19:03:20 2017 +0200
--
.../ambari/server/stack/StackManager.java | 3 +
.../ambari/server/state/ServiceFactory.java | 2 +-
.../ExecutionCommandWrapperTest.java| 4 +-
.../TestActionSchedulerThreading.java | 4 +-
.../server/agent/HeartbeatProcessorTest.java| 4 +-
.../server/agent/TestHeartbeatHandler.java | 7 +-
.../server/agent/TestHeartbeatMonitor.java | 16 +-
.../ClusterResourceDefinitionTest.java | 21 +-
.../ServiceResourceDefinitionTest.java | 6 +-
.../checks/ComponentExistsInRepoCheckTest.java | 4 +-
.../configuration/RecoveryConfigHelperTest.java | 19 +-
.../AmbariManagementControllerImplTest.java | 112 ---
.../AmbariManagementControllerTest.java | 32 +-
.../StackDefinedPropertyProviderTest.java | 10 +-
.../internal/UpgradeResourceProviderTest.java | 16 +-
.../UpgradeSummaryResourceProviderTest.java | 4 +-
.../VersionDefinitionResourceProviderTest.java | 4 +-
.../GeneralServiceCalculatedStateTest.java | 4 +-
.../HostVersionOutOfSyncListenerTest.java | 4 +-
.../apache/ambari/server/orm/OrmTestHelper.java | 18 +-
.../ambari/server/state/ConfigHelperTest.java | 14 +-
.../apache/ambari/server/state/ServiceTest.java | 23 +-
.../ambari/server/state/UpgradeHelperTest.java | 45 ++-
.../server/state/cluster/ClusterImplTest.java | 12 +-
.../server/state/cluster/ClusterTest.java | 29 +-
.../server/state/cluster/ClustersTest.java | 14 +-
.../ConcurrentServiceConfigVersionTest.java | 15 +-
...omponentHostConcurrentWriteDeadlockTest.java | 15 +-
.../src/test/resources/cluster-settings.xml | 322 +++
29 files changed, 616 insertions(+), 167 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/bcbacf0c/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
index fc49a24..dc77f19 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackManager.java
@@ -307,6 +307,9 @@ public class StackManager {
stackEntity.setStackName(stackName);
stackEntity.setStackVersion(stackVersion);
+//TODO this is hack which should be removed during merge with BP branch
+stackEntity.setRepoVersion("");
+
stackDao.create(stackEntity);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/bcbacf0c/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceFactory.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceFactory.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceFactory.java
index 69eb21d..e6673a3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceFactory.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceFactory.java
@@ -44,7 +44,7 @@ public interface ServiceFactory {
* @return
*/
Service createNew(Cluster cluster, ServiceGroup serviceGroup,
-@Assisted List serviceDependencies,
+List serviceDependencies,
@Assisted("serviceName") String serviceName,
@Assisted("serviceType") String serviceType,
RepositoryVersionEntity desiredRepositoryVersion);
http://git-wip-us.apache.org/repos/asf/ambari/blob/bcbacf0c/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapperTest.java
index 19f940f..b674234 100644
---
a/ambari-server/src/test/java/org/apac
ambari git commit: AMBARI-22515. Fix validators tests in feature branch.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 99259e500 -> 586ab647e
AMBARI-22515. Fix validators tests in feature branch.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/586ab647
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/586ab647
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/586ab647
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 586ab647e232b30dc8dc7791b9ab8e6390eff4ae
Parents: 99259e5
Author: Vitaly Brodetskyi
Authored: Fri Nov 24 22:40:35 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Fri Nov 24 22:40:35 2017 +0200
--
.../server/upgrade/AbstractUpgradeCatalog.java | 4 ++
.../ClusterConfigTypeValidatorTest.java | 29 +++---
.../validators/HiveServiceValidatorTest.java| 58 +++-
.../RequiredConfigPropertiesValidatorTest.java | 46 +---
.../StackConfigTypeValidatorTest.java | 45 +--
.../topology/validators/UnitValidatorTest.java | 29 ++
.../upgrade/AbstractUpgradeCatalogTest.java | 22 +---
.../server/upgrade/UpgradeCatalog300Test.java | 24 +++-
8 files changed, 163 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/586ab647/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
index ebb0362..0941535 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java
@@ -1156,4 +1156,8 @@ public abstract class AbstractUpgradeCatalog implements
UpgradeCatalog {
private StackId getStackId(Cluster cluster) throws AmbariException {
return
cluster.getServices().values().iterator().next().getDesiredStackId();
}
+
+ protected void setConfiguration(Configuration configuration) {
+this.configuration = configuration;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/586ab647/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java
index 424ed63..e469576 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/ClusterConfigTypeValidatorTest.java
@@ -19,11 +19,12 @@ import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-import org.apache.ambari.server.controller.internal.Stack;
-import org.apache.ambari.server.topology.Blueprint;
+import org.apache.ambari.server.controller.StackV2;
+import org.apache.ambari.server.topology.BlueprintV2;
import org.apache.ambari.server.topology.ClusterTopology;
import org.apache.ambari.server.topology.Configuration;
import org.apache.ambari.server.topology.InvalidTopologyException;
+import org.apache.ambari.server.topology.Service;
import org.easymock.EasyMock;
import org.easymock.EasyMockRule;
import org.easymock.EasyMockSupport;
@@ -45,10 +46,16 @@ public class ClusterConfigTypeValidatorTest extends
EasyMockSupport {
private Configuration clusterConfigurationMock;
@Mock
- private Blueprint blueprintMock;
+ private BlueprintV2 blueprintMock;
@Mock
- private Stack stackMock;
+ private StackV2 stackMock;
+
+ @Mock
+ private Service yarnMock;
+
+ @Mock
+ private Service hdfsMock;
@Mock
private ClusterTopology clusterTopologyMock;
@@ -62,8 +69,12 @@ public class ClusterConfigTypeValidatorTest extends
EasyMockSupport {
public void before() {
EasyMock.expect(clusterTopologyMock.getConfiguration()).andReturn(clusterConfigurationMock).anyTimes();
-
EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(null).anyTimes();
-EasyMock.expect(blueprintMock.getStack()).andReturn(stackMock).anyTimes();
+
EasyMock.expect(clusterTopologyMock.getBlueprint()).andReturn(blueprintMock).anyTimes();
+
EasyMock.expect(blueprintMock.getStackById("1")).andReturn(stackMock).anyTimes();
+EasyMock.expect(yarnMock.getStackId()).andReturn("1").anyTimes();
+EasyMock.expect(yarnMock.getType()).and
[5/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
index 3d8d161..0b07fb7 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/BackgroundCustomCommandExecutionTest.java
@@ -217,8 +217,8 @@ public class BackgroundCustomCommandExecutionTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceRequest r1 = new ServiceRequest(clusterName, serviceName,
-m_repositoryVersion.getId(), dStateStr);
+ServiceRequest r1 = new ServiceRequest(clusterName, "", serviceName,
+m_repositoryVersion.getId(), dStateStr, null);
Set requests = new HashSet<>();
requests.add(r1);
@@ -234,7 +234,7 @@ public class BackgroundCustomCommandExecutionTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceComponentRequest r = new ServiceComponentRequest(clusterName,
+ServiceComponentRequest r = new ServiceComponentRequest(clusterName, "",
serviceName, componentName, dStateStr);
Set requests =
new HashSet<>();
@@ -248,7 +248,7 @@ public class BackgroundCustomCommandExecutionTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName,
+ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName, "",
serviceName, componentName, hostname, dStateStr);
Set requests =
new HashSet<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/ClusterResponseTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/ClusterResponseTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/ClusterResponseTest.java
index c9e91e3..40d12de 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/ClusterResponseTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/ClusterResponseTest.java
@@ -40,7 +40,7 @@ public class ClusterResponseTest {
new ClusterResponse(clusterId, clusterName, provisioningState,
securityType,
hostNames, hostNames.size(), "bar", null);
-Assert.assertEquals(clusterId, r1.getClusterId());
+Assert.assertTrue(clusterId == r1.getClusterId().longValue());
Assert.assertEquals(clusterName, r1.getClusterName());
Assert.assertEquals(provisioningState, r1.getProvisioningState());
Assert.assertEquals(securityType, r1.getSecurityType());
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
index ef186f1..3ce6686 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/RefreshYarnCapacitySchedulerReleaseConfigTest.java
@@ -102,12 +102,12 @@ public class
RefreshYarnCapacitySchedulerReleaseConfigTest {
// Start
ClusterRequest cr = new ClusterRequest(cluster.getClusterId(), "c1",
cluster.getDesiredStackVersion().getStackVersion(), null);
-cr.setDesiredConfig(Collections.singletonList(new
ConfigurationRequest("c1","capacity-scheduler","version2", new HashMap<>(),
null)));
+cr.setDesiredConfig(Collections.singletonList(new
ConfigurationRequest("c1","capacity-scheduler","version2", new HashMap<>(),
null, 1L, 1L)));
controller.updateClusters(Collections.singleton(cr) , null);
-ServiceComponentHostRequest r = new ServiceComponentHostRequest("c1",
null, null, null, null);
+ServiceComponentHostRequest r = new ServiceComponentHostRequest("c1",
null, null, null, null, null);
r.setStaleConfig("true");
Set resps =
controller.getHostComponents(Collections.singleton(r));
Assert.assertEquals(1, resps.size());
@@ -123,12 +123,12 @@ public class
RefreshYarnCapacitySchedulerReleaseConfigTest {
// Start
[7/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1f176845 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1f176845 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1f176845 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 1f176845ec636adbc1dc710c03f8062f979f5405 Parents: 35d704c Author: Vitaly Brodetskyi Authored: Thu Nov 23 12:22:11 2017 +0200 Committer: Vitaly Brodetskyi Committed: Thu Nov 23 12:22:11 2017 +0200 -- ambari-server/pom.xml | 2 +- .../ExecutionCommandWrapperTest.java| 2 +- .../TestActionSchedulerThreading.java | 8 +- .../server/agent/HeartbeatProcessorTest.java| 2 +- .../server/agent/TestHeartbeatHandler.java | 2 +- .../server/agent/TestHeartbeatMonitor.java | 10 +- .../render/ClusterBlueprintRendererTest.java| 8 +- .../api/services/ComponentServiceTest.java | 4 +- .../api/services/ConfigurationServiceTest.java | 2 +- .../server/api/services/ServiceServiceTest.java | 8 +- .../StackAdvisorBlueprintProcessorTest.java | 12 +- .../checks/ComponentsInstallationCheckTest.java | 12 +- .../server/checks/ServicesUpCheckTest.java | 16 +- .../configuration/RecoveryConfigHelperTest.java | 12 +- .../AmbariCustomCommandExecutionHelperTest.java | 13 +- .../AmbariManagementControllerImplTest.java | 58 +- .../AmbariManagementControllerTest.java | 892 +-- .../BackgroundCustomCommandExecutionTest.java | 8 +- .../server/controller/ClusterResponseTest.java | 2 +- ...hYarnCapacitySchedulerReleaseConfigTest.java | 16 +- .../internal/AbstractResourceProviderTest.java | 4 +- .../BlueprintConfigurationProcessorTest.java| 306 +++ .../ClientConfigResourceProviderTest.java | 4 +- .../internal/ClusterResourceProviderTest.java | 4 +- .../internal/ComponentResourceProviderTest.java | 20 +- .../ConfigGroupResourceProviderTest.java| 10 +- .../internal/ExportBlueprintRequestTest.java| 2 +- .../HostComponentResourceProviderTest.java | 14 +- .../internal/HostResourceProviderTest.java | 26 +- .../internal/JMXHostProviderTest.java | 24 +- .../internal/ProvisionClusterRequestTest.java | 2 +- .../internal/RequestResourceProviderTest.java | 2 +- .../internal/ScaleClusterRequestTest.java | 2 +- .../internal/ServiceResourceProviderTest.java | 6 +- .../StackConfigurationResourceProviderTest.java | 14 +- .../StackDefinedPropertyProviderTest.java | 8 +- ...kLevelConfigurationResourceProviderTest.java | 14 +- .../server/controller/internal/StackTest.java | 17 +- .../StackUpgradeConfigurationMergeTest.java | 6 +- .../controller/internal/UnitUpdaterTest.java| 8 +- .../internal/UpgradeResourceProviderTest.java | 20 +- .../UpgradeSummaryResourceProviderTest.java | 2 +- .../VersionDefinitionResourceProviderTest.java | 2 +- .../LoggingSearchPropertyProviderTest.java | 6 +- .../utilities/KerberosIdentityCleanerTest.java | 4 +- .../GeneralServiceCalculatedStateTest.java | 2 +- .../apache/ambari/server/events/EventsTest.java | 4 +- .../HostVersionOutOfSyncListenerTest.java | 8 +- .../apache/ambari/server/orm/OrmTestHelper.java | 5 +- .../apache/ambari/server/orm/TestOrmImpl.java | 14 +- .../server/orm/dao/ServiceConfigDAOTest.java| 30 +- .../orm/dao/TopologyLogicalRequestDAOTest.java | 4 +- .../server/orm/dao/TopologyRequestDAOTest.java | 8 +- .../orm/entities/ServiceConfigEntityTest.java | 2 +- .../ComponentVersionCheckActionTest.java| 4 +- .../upgrades/ConfigureActionTest.java | 5 +- .../PreconfigureKerberosActionTest.java | 2 +- .../upgrades/UpgradeActionTest.java | 4 +- .../ambari/server/state/ConfigGroupTest.java| 3 +- .../ambari/server/state/ConfigHelperTest.java | 11 +- .../ambari/server/state/DesiredConfigTest.java | 4 +- .../server/state/ServiceComponentTest.java | 58 +- .../apache/ambari/server/state/ServiceTest.java | 16 +- .../ambari/server/state/UpgradeHelperTest.java | 33 +- .../state/alerts/AlertEventPublisherTest.java | 4 +- .../state/alerts/InitialAlertEventTest.java | 4 +- .../state/cluster/ClusterDeadlockTest.java | 3 +- .../server/state/cluster/ClusterImplTest.java | 4 +- .../server/state/cluster/ClusterTest.java | 137 +-- .../state/cluster/ClustersDeadlockTest.java | 3 +- .../server/state/cluster/ClustersTest.java | 20 +- .../ConcurrentServiceConfigVersionTest.java | 9 +- ...omponentHostConcurrentWriteDeadlockTest.java | 3 +- .../svccomphost/ServiceComponentHostTest.java | 23 +- .../server/topology
[3/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ComponentResourceProviderTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ComponentResourceProviderTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ComponentResourceProviderTest.java
index 110e359..efa351a 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ComponentResourceProviderTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ComponentResourceProviderTest.java
@@ -244,13 +244,13 @@ public class ComponentResourceProviderTest {
expect(service.getServiceComponents()).andReturn(serviceComponentMap).anyTimes();
expect(serviceComponent1.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component100", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component100", stackId, "", serviceComponentStateCountMap,
true /* recovery enabled */, "Component100 Client", null, null));
expect(serviceComponent2.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component101", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component101", stackId, "", serviceComponentStateCountMap,
false /* recovery not enabled */, "Component101 Client", null,
null));
expect(serviceComponent3.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component102", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component102", stackId, "", serviceComponentStateCountMap,
true /* recovery enabled */, "Component102 Client", "1.1",
RepositoryVersionState.CURRENT));
expect(ambariMetaInfo.getComponent("FOO", "1.0", null,
"Component100")).andReturn(
@@ -423,13 +423,13 @@ public class ComponentResourceProviderTest {
expect(component3Info.getCategory()).andReturn(null);
expect(serviceComponent1.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component101", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component101", stackId, "", serviceComponentStateCountMap,
false /* recovery not enabled */, "Component101 Client", null,
null));
expect(serviceComponent2.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component102", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component102", stackId, "", serviceComponentStateCountMap,
false /* recovery not enabled */, "Component102 Client", null,
null));
expect(serviceComponent3.convertToResponse()).andReturn(
- new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component103", stackId, "", serviceComponentStateCountMap,
+ new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component103", stackId, "", serviceComponentStateCountMap,
false /* recovery not enabled */, "Component103 Client", null,
null));
expect(serviceComponent1.getDesiredState()).andReturn(State.INSTALLED).anyTimes();
expect(serviceComponent2.getDesiredState()).andReturn(State.INSTALLED).anyTimes();
@@ -731,7 +731,7 @@ public class ComponentResourceProviderTest {
expect(component1Info.getCategory()).andReturn(null);
expect(serviceComponent1.convertToResponse()).andReturn(
-new ServiceComponentResponse(100L, "Cluster100", "Service100",
"Component101", stackId, "", serviceComponentStateCountMap,
+new ServiceComponentResponse(100L, "Cluster100", 1L, "", 1L,
"Service100", "", "Component101", stackId, "", serviceComponentStateCountMap,
false /* recovery not enabled */, "Component101 Client", null,
null));
expect(serviceComponent1.getDesiredState()).andReturn(State.INSTALLED).anyTimes();
@@ -897,8 +897,8 @@ public class ComponentResourceProviderTest {
expect(component4Info.getCategory()).andReturn(null);
expect(service.getName()).andReturn("service1").anyTimes();
-expect(service.getServiceComponent("component1")).andThrow(new
ServiceComponentNotFoundException("cluster1", "service1", "component1"));
-expect(service.getServiceComponent("component2")).andThrow(new
ServiceComponentNotFoundException("cluster1", "service1", "component2"));
+expect
[4/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
index 22b0e6b..f9765f2 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java
@@ -43,7 +43,7 @@ import java.util.Set;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.KerberosHelper;
-import org.apache.ambari.server.controller.StackConfigurationResponse;
+import org.apache.ambari.server.controller.StackLevelConfigurationResponse;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.ConfigHelper;
@@ -290,7 +290,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
configProcessor.doUpdateForBlueprintExport();
String updatedVal =
properties.get("yarn-site").get("yarn.resourcemanager.hostname");
@@ -338,7 +338,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
configProcessor.doUpdateForBlueprintExport();
assertEquals(properties.size(), 3);
@@ -372,7 +372,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
// When
configProcessor.doUpdateForBlueprintExport();
@@ -416,7 +416,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
configProcessor.doUpdateForBlueprintExport();
assertEquals("%HOSTGROUP::group1%",
clusterConfig.getPropertyValue("yarn-site", "yarn.resourcemanager.hostname"));
@@ -464,7 +464,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
configProcessor.doUpdateForBlueprintExport();
assertEquals("%HOSTGROUP::group1%",
properties.get("yarn-site").get("yarn.resourcemanager.hostname"));
@@ -499,7 +499,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology, null);
configProcessor.doUpdateForBlueprintExport();
String updatedVal = properties.get("core-site").get("fs.defaultFS");
@@ -532,7 +532,7 @@ public class BlueprintConfigurationProcessorTest extends
EasyMockSupport {
hostGroups.add(group2);
ClusterTopology topology = createClusterTopology(bp, clusterConfig,
hostGroups);
-BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(topology);
+BlueprintConfigurationProcessor configProcessor = new
BlueprintConfigurationProcessor(t
[2/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
index 82d5481..e9da564 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -40,6 +41,7 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.ServiceKey;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.controller.AmbariServer;
import org.apache.ambari.server.controller.ServiceConfigVersionResponse;
@@ -602,7 +604,7 @@ public class UpgradeActionTest {
try {
service = cluster.getService(serviceName);
} catch (ServiceNotFoundException e) {
- service = serviceFactory.createNew(cluster, serviceName,
repositoryVersionEntity);
+ service = serviceFactory.createNew(cluster, null, new
ArrayList(), serviceName, "", repositoryVersionEntity);
cluster.addService(service);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigGroupTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigGroupTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigGroupTest.java
index cc23e9b..1296be9 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigGroupTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigGroupTest.java
@@ -105,8 +105,7 @@ public class ConfigGroupTest {
configs.put(config.getType(), config);
hosts.put(host.getHostId(), host);
-ConfigGroup configGroup = configGroupFactory.createNew(cluster, "HDFS",
"cg-test",
- "HDFS", "New HDFS configs for h1", configs, hosts);
+ConfigGroup configGroup = configGroupFactory.createNew(cluster, 1L, 1L,
"HDFS", "", "New HDFS configs for h1", configs, hosts);
cluster.addConfigGroup(configGroup);
return configGroup;
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index 7e1e2da..ff28eb7 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -156,9 +156,9 @@ public class ConfigHelperTest {
cr2.setType("flume-conf");
cr2.setVersionTag("version1");
- cluster.addService("FLUME", repositoryVersion);
- cluster.addService("OOZIE", repositoryVersion);
- cluster.addService("HDFS", repositoryVersion);
+ cluster.addService(null, "", "FLUME", repositoryVersion);
+ cluster.addService(null, "", "OOZIE", repositoryVersion);
+ cluster.addService(null, "", "HDFS", repositoryVersion);
final ClusterRequest clusterRequest2 =
new ClusterRequest(cluster.getClusterId(), clusterName,
@@ -298,7 +298,10 @@ public class ConfigHelperTest {
configMap.put(config.getType(), config);
}
- ConfigGroup configGroup = configGroupFactory.createNew(cluster, null,
name,
+ long serviceGroupId = 1;
+ long serviceId = 1;
+
+ ConfigGroup configGroup = configGroupFactory.createNew(cluster,
serviceGroupId, serviceId, name,
tag, "", configMap, hostMap);
LOG.info("Config group created with tag " + tag);
configGroup.setTag(tag);
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/state/DesiredConfigTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/state/DesiredConfigTest.java
b/ambari-server/src/test/java/org/apache/ambari/se
[1/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 35d704cb3 -> 1f176845e
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
index d89c8ca..8d58107 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterInstallWithoutStartTest.java
@@ -286,7 +286,7 @@ public class ClusterInstallWithoutStartTest extends
EasyMockSupport {
expect(stack.getExcludedConfigurationTypes("service1")).andReturn(Collections.emptySet()).anyTimes();
expect(stack.getExcludedConfigurationTypes("service2")).andReturn(Collections.emptySet()).anyTimes();
-expect(request.getBlueprint()).andReturn(blueprint).anyTimes();
+expect(request.getBlueprint()).andReturn(null).anyTimes();
expect(request.getClusterId()).andReturn(CLUSTER_ID).anyTimes();
expect(request.getClusterName()).andReturn(CLUSTER_NAME).anyTimes();
expect(request.getDescription()).andReturn("Provision Cluster
Test").anyTimes();
@@ -340,7 +340,7 @@ public class ClusterInstallWithoutStartTest extends
EasyMockSupport {
expect(ambariContext.getPersistedTopologyState()).andReturn(persistedState).anyTimes();
//todo: don't ignore param
-ambariContext.createAmbariResources(isA(ClusterTopology.class),
eq(CLUSTER_NAME), (SecurityType) isNull(), eq("1"), anyLong());
+ambariContext.createAmbariResources(isA(ClusterTopology.class),
eq(CLUSTER_NAME), (SecurityType) isNull());
expectLastCall().once();
expect(ambariContext.getNextRequestId()).andReturn(1L).once();
expect(ambariContext.isClusterKerberosEnabled(CLUSTER_ID)).andReturn(false).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
index a8674b3..d8c266d 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterTopologyImplTest.java
@@ -104,21 +104,21 @@ public class ClusterTopologyImplTest {
hostGroupMap.put("group4", group4);
Set group1Components = new HashSet<>();
-group1Components.add(new ComponentV2("component1", new Service("service1",
stack)));
-group1Components.add(new ComponentV2("component2", new Service("service1",
stack)));
+group1Components.add(null);
+group1Components.add(null);
Set group1ComponentNames = new HashSet<>();
group1ComponentNames.add("component1");
group1ComponentNames.add("component2");
Set group2Components = new HashSet<>();
-group2Components.add(new ComponentV2("component3", new Service("service1",
stack)));
+group2Components.add(null);
Set group3Components = new HashSet<>();
-group3Components.add(new ComponentV2("component4", new Service("service2",
stack)));
+group3Components.add(null);
Set group4Components = new HashSet<>();
-group4Components.add(new ComponentV2("component5", new Service("service2",
stack)));
+group4Components.add(null);
-expect(blueprint.getHostGroups()).andReturn(hostGroupMap).anyTimes();
+expect(blueprint.getHostGroups()).andReturn(null).anyTimes();
expect(blueprint.getHostGroup("group1")).andReturn(group1).anyTimes();
expect(blueprint.getHostGroup("group2")).andReturn(group2).anyTimes();
expect(blueprint.getHostGroup("group3")).andReturn(group3).anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java
index a5265f6..02e4e75 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/topology/LogicalRequestTest.java
@@ -112,7 +112,7 @@ public class LogicalRequestTest extends EasyMockSupport {
expect(clusterTopology.getAmbariContext()).andReturn(ambariContext).anyTimes();
expect(clusterTopolo
[6/7] ambari git commit: AMBARI-22493. Fix build for test classes in feature branch.(vbrodetskyi)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f176845/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 44b46f4..92eced2 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -77,6 +77,7 @@ import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.agent.HeartBeatHandler;
import org.apache.ambari.server.agent.rest.AgentResource;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.ServiceKey;
import org.apache.ambari.server.audit.AuditLogger;
import org.apache.ambari.server.configuration.Configuration;
import
org.apache.ambari.server.controller.internal.ClusterStackVersionResourceProviderTest;
@@ -377,7 +378,7 @@ public class AmbariManagementControllerTest {
dStateStr = desiredState.toString();
}
-ServiceRequest r1 = new ServiceRequest(clusterName, serviceName,
+ServiceRequest r1 = new ServiceRequest(clusterName, "", serviceName,
repositoryVersion.getId(), dStateStr,
null);
@@ -394,7 +395,7 @@ public class AmbariManagementControllerTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceComponentRequest r = new ServiceComponentRequest(clusterName,
+ServiceComponentRequest r = new ServiceComponentRequest(clusterName, "",
serviceName, componentName, dStateStr);
Set requests =
new HashSet<>();
@@ -409,7 +410,7 @@ public class AmbariManagementControllerTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName,
+ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName, "",
serviceName, componentName, hostname, dStateStr);
Set requests =
new HashSet<>();
@@ -424,7 +425,7 @@ public class AmbariManagementControllerTest {
if (desiredState != null) {
dStateStr = desiredState.toString();
}
-ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName,
+ServiceComponentHostRequest r = new
ServiceComponentHostRequest(clusterName, "",
serviceName, componentName, hostname, dStateStr);
Set requests =
new HashSet<>();
@@ -449,10 +450,9 @@ public class AmbariManagementControllerTest {
configMap.put(config.getType(), config);
}
-ConfigGroup configGroup = configGroupFactory.createNew(cluster,
serviceName, name,
- tag, "", configMap, hostMap);
+ConfigGroup configGroup = configGroupFactory.createNew(cluster, 1L, 1L,
serviceName, tag, "", configMap, hostMap);
-configGroup.setServiceName(serviceName);
+//configGroup.setServiceName(serviceName);
cluster.addConfigGroup(configGroup);
@@ -462,7 +462,7 @@ public class AmbariManagementControllerTest {
private long stopService(String clusterName, String serviceName,
boolean runSmokeTests, boolean reconfigureClients) throws
AmbariException, AuthorizationException {
-ServiceRequest r = new ServiceRequest(clusterName, serviceName, null,
State.INSTALLED.toString(), null);
+ServiceRequest r = new ServiceRequest(clusterName, "", serviceName, null,
State.INSTALLED.toString(), null);
Set requests = new HashSet<>();
requests.add(r);
Map mapRequestProps = new HashMap<>();
@@ -495,7 +495,7 @@ public class AmbariManagementControllerTest {
for (ServiceComponent sc : s.getServiceComponents().values()) {
for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) {
ServiceComponentHostRequest schr = new ServiceComponentHostRequest
- (clusterName, serviceName, sc.getName(),
+ (clusterName, "", serviceName, sc.getName(),
sch.getHostName(), State.INSTALLED.name());
requests.add(schr);
}
@@ -527,7 +527,7 @@ public class AmbariManagementControllerTest {
boolean runSmokeTests, boolean reconfigureClients,
MaintenanceStateHelper maintenanceStateHelper)
throws
AmbariException, AuthorizationException {
-ServiceRequest r = new ServiceRequest(clusterName, serviceName,
repositoryVersion02.getId(),
+ServiceRequest r = new ServiceRequest(clusterName, "", serviceName,
repositoryVersion02.getId(),
State.STARTED.toString(), null);
Set requests = new HashSet<>();
requests.add(r);
@@ -582,7 +582,7 @@ public class AmbariManagementCo
ambari git commit: AMBARI-22464. disable hive.auto.convert.sortmerge.join in Hive configs in Ambari.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk 1d1c55644 -> e95d34300 AMBARI-22464. disable hive.auto.convert.sortmerge.join in Hive configs in Ambari.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e95d3430 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e95d3430 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e95d3430 Branch: refs/heads/trunk Commit: e95d34300225b57bdba141ec085173c4e8f41314 Parents: 1d1c556 Author: Vitaly Brodetskyi Authored: Fri Nov 17 15:15:48 2017 +0200 Committer: Vitaly Brodetskyi Committed: Fri Nov 17 15:15:48 2017 +0200 -- .../common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml| 2 +- .../stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml | 2 +- .../stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/e95d3430/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 5520c10..b7af252 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -287,7 +287,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join. http://git-wip-us.apache.org/repos/asf/ambari/blob/e95d3430/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml index 1c8f475..ea6ad25 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml @@ -258,7 +258,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join. http://git-wip-us.apache.org/repos/asf/ambari/blob/e95d3430/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index 2073832..0944252 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -696,7 +696,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join.
ambari git commit: AMBARI-22464. disable hive.auto.convert.sortmerge.join in Hive configs in Ambari.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.6 946356a9c -> c09c69b2f AMBARI-22464. disable hive.auto.convert.sortmerge.join in Hive configs in Ambari.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c09c69b2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c09c69b2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c09c69b2 Branch: refs/heads/branch-2.6 Commit: c09c69b2fd502801c1355fc4dc275f398c81 Parents: 946356a Author: Vitaly Brodetskyi Authored: Fri Nov 17 15:14:01 2017 +0200 Committer: Vitaly Brodetskyi Committed: Fri Nov 17 15:14:01 2017 +0200 -- .../common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml| 2 +- .../stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml | 2 +- .../stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/c09c69b2/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml index 0b11187..89fe471 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml @@ -283,7 +283,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join. http://git-wip-us.apache.org/repos/asf/ambari/blob/c09c69b2/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml index 1c8f475..ea6ad25 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/configuration/hive-site.xml @@ -258,7 +258,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join. http://git-wip-us.apache.org/repos/asf/ambari/blob/c09c69b2/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml index d8cf961..15cd9f2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-site.xml @@ -692,7 +692,7 @@ limitations under the License. hive.auto.convert.sortmerge.join -true +false Will the join be automatically converted to a sort-merge join, if the joined tables pass the criteria for sort-merge join.
ambari git commit: AMBARI-22345. Add service dependencies.Part 2.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 6a78e1a1a -> 19513a85f
AMBARI-22345. Add service dependencies.Part 2.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19513a85
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19513a85
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19513a85
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 19513a85f12fb235a01cecce12e68f405785a177
Parents: 6a78e1a
Author: Vitaly Brodetskyi
Authored: Mon Nov 13 19:45:46 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Mon Nov 13 19:45:46 2017 +0200
--
.../server/api/services/ServiceService.java | 22 +--
.../controller/ServiceDependencyRequest.java| 28 +---
.../ServiceDependencyResourceProvider.java | 102 ++--
.../server/orm/dao/ClusterServiceDAO.java | 16 ++
.../orm/entities/ClusterServiceEntity.java | 29 ++--
.../orm/entities/ServiceDependencyEntity.java | 154 +++
.../org/apache/ambari/server/state/Cluster.java | 5 +-
.../org/apache/ambari/server/state/Service.java | 5 +
.../apache/ambari/server/state/ServiceImpl.java | 110 ++---
.../server/state/cluster/ClusterImpl.java | 61 +++-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 8 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 7 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 7 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql| 7 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 19 ++-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 7 +-
.../src/main/resources/META-INF/persistence.xml | 1 +
17 files changed, 411 insertions(+), 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/19513a85/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
index 1f3332f..76782d5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceService.java
@@ -316,10 +316,10 @@ public class ServiceService extends BaseService {
responseContainer = RESPONSE_CONTAINER_LIST)
@ApiImplicitParams({
@ApiImplicitParam(name = QUERY_FIELDS, value =
QUERY_FILTER_DESCRIPTION,
- defaultValue = "ServiceInfo/service_group_name,
ServiceInfo/service_name, ServiceInfo/cluster_name",
+ defaultValue = "ServiceDependencyInfo/dependency_service_id,
ServiceInfo/service_group_name, ServiceInfo/service_name,
ServiceInfo/cluster_name",
dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_SORT, value = QUERY_SORT_DESCRIPTION,
- defaultValue = "ServiceInfo/service_group_name.asc,
ServiceInfo/service_name.asc, ServiceInfo/cluster_name.asc",
+ defaultValue =
"ServiceDependencyInfo/dependency_service_id.asc,
ServiceInfo/service_group_name.asc, ServiceInfo/service_name.asc,
ServiceInfo/cluster_name.asc",
dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_PAGE_SIZE, value =
QUERY_PAGE_SIZE_DESCRIPTION, defaultValue = DEFAULT_PAGE_SIZE, dataType =
DATA_TYPE_INT, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_FROM, value = QUERY_FROM_DESCRIPTION,
defaultValue = DEFAULT_FROM, dataType = DATA_TYPE_STRING, paramType =
PARAM_TYPE_QUERY),
@@ -336,7 +336,7 @@ public class ServiceService extends BaseService {
}
@GET
- @Path("{serviceName}/dependencies/{dependencyServiceName}")
+ @Path("{serviceName}/dependencies/{dependencyServiceId}")
@Produces(MediaType.TEXT_PLAIN)
@ApiOperation(value = "Get the details of a service dependency",
nickname = "ServiceService#getServiceDependency",
@@ -344,7 +344,7 @@ public class ServiceService extends BaseService {
response = ServiceResponse.ServiceResponseSwagger.class,
responseContainer = RESPONSE_CONTAINER_LIST)
@ApiImplicitParams({
- @ApiImplicitParam(name = QUERY_FIELDS, value =
QUERY_FILTER_DESCRIPTION, defaultValue = "ServiceInfo/*",
+ @ApiImplicitParam(name = QUERY_FIELDS, value =
QUERY_FILTER_DESCRIPTION, defaultValue = "ServiceDependencyInfo/*",
dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUER
ambari git commit: AMBARI-22249. Add service group dependencies. Part 2.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 55e450eea -> 6a78e1a1a
AMBARI-22249. Add service group dependencies. Part 2.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6a78e1a1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6a78e1a1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6a78e1a1
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 6a78e1a1ab5ed046678c9fc4c907ac925cb6bbd5
Parents: 55e450e
Author: Vitaly Brodetskyi
Authored: Mon Nov 13 18:20:50 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Mon Nov 13 18:20:50 2017 +0200
--
.../api/services/ServiceGroupService.java | 38 +++---
.../ServiceGroupDependencyRequest.java | 14 +--
.../ServiceGroupDependencyResourceProvider.java | 66 +--
.../ambari/server/orm/dao/ServiceGroupDAO.java | 35 +-
.../entities/ServiceGroupDependencyEntity.java | 111 ++
.../server/orm/entities/ServiceGroupEntity.java | 31 +++--
.../org/apache/ambari/server/state/Cluster.java | 26 -
.../ambari/server/state/ServiceGroup.java | 12 +-
.../ambari/server/state/ServiceGroupImpl.java | 117 ---
.../server/state/cluster/ClusterImpl.java | 46 +++-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 8 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 7 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 7 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql| 7 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 7 +-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 7 +-
.../src/main/resources/META-INF/persistence.xml | 1 +
17 files changed, 372 insertions(+), 168 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/6a78e1a1/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupService.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupService.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupService.java
index 0638a06..ec63aec 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupService.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ServiceGroupService.java
@@ -298,10 +298,10 @@ public class ServiceGroupService extends BaseService {
responseContainer = RESPONSE_CONTAINER_LIST)
@ApiImplicitParams({
@ApiImplicitParam(name = QUERY_FIELDS, value =
QUERY_FILTER_DESCRIPTION,
- defaultValue = "ServiceGroupInfo/service_group_name,
ServiceGroupInfo/cluster_name",
+ defaultValue =
"ServiceGroupDependencyInfo/dependency_service_group_id,
ServiceGroupInfo/service_group_name, ServiceGroupInfo/cluster_name",
dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_SORT, value = QUERY_SORT_DESCRIPTION,
- defaultValue = "ServiceGroupInfo/service_group_name.asc,
ServiceGroupInfo/cluster_name.asc",
+ defaultValue =
"ServiceGroupDependencyInfo/dependency_service_group_id.asc,
ServiceGroupInfo/service_group_name.asc, ServiceGroupInfo/cluster_name.asc",
dataType = DATA_TYPE_STRING, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_PAGE_SIZE, value =
QUERY_PAGE_SIZE_DESCRIPTION, defaultValue = DEFAULT_PAGE_SIZE, dataType =
DATA_TYPE_INT, paramType = PARAM_TYPE_QUERY),
@ApiImplicitParam(name = QUERY_FROM, value = QUERY_FROM_DESCRIPTION,
defaultValue = DEFAULT_FROM, dataType = DATA_TYPE_STRING, paramType =
PARAM_TYPE_QUERY),
@@ -322,14 +322,14 @@ public class ServiceGroupService extends BaseService {
* Handles URL:
/clusters/{clusterName}/servicegroups/{serviceGroupName}/dependencies/{serviceGroupDependency}
* Get a specific servicegroupdependency.
*
- * @param headers http headers
- * @param ui uri info
- * @param serviceGroupNameservice group name
- * @param serviceGroupDependency service group dependency name
+ * @param headershttp headers
+ * @param ui uri info
+ * @param serviceGroupName service group name
+ * @param serviceGroupDependencyId service group dependency id
* @return servicegroupdependencyresource representation
*/
@GET
- @Path("{serviceGroupName}/dependencies/{serviceGroupDependency}")
+ @Path("{serviceGroupName}/dependencies/{serviceGroupDependencyId}")
@Produces(MediaType.TEXT_PLAIN)
@Ap
[2/2] ambari git commit: AMBARI-22345. Add service dependencies.(vbrodetskyi)
AMBARI-22345. Add service dependencies.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/72bac1f7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/72bac1f7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/72bac1f7
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 72bac1f7f580c3c5ac4160cbc0418002b683bda0
Parents: adc7060
Author: Vitaly Brodetskyi
Authored: Fri Nov 3 00:06:23 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Fri Nov 3 00:06:23 2017 +0200
--
.../resources/ResourceInstanceFactoryImpl.java | 4 +
.../ServiceDependencyResourceDefinition.java| 45 ++
.../resources/ServiceResourceDefinition.java| 1 +
.../ambari/server/api/services/ServiceKey.java | 118 +
.../server/api/services/ServiceService.java | 111
.../server/controller/ControllerModule.java | 2 +
.../controller/ResourceProviderFactory.java | 3 +
.../controller/ServiceDependencyRequest.java| 87
.../controller/ServiceDependencyResponse.java | 197 +++
.../AbstractControllerResourceProvider.java | 2 +
.../internal/DefaultProviderModule.java | 2 +
.../ServiceDependencyResourceProvider.java | 511 +++
.../ambari/server/controller/spi/Resource.java | 2 +
.../orm/entities/ClusterServiceEntity.java | 36 +-
.../org/apache/ambari/server/state/Cluster.java | 6 +
.../org/apache/ambari/server/state/Service.java | 8 +
.../ambari/server/state/ServiceFactory.java | 4 +
.../apache/ambari/server/state/ServiceImpl.java | 87
.../server/state/cluster/ClusterImpl.java | 86 +++-
.../ambari/server/topology/AmbariContext.java | 10 +
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 11 +
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 11 +
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 11 +
.../resources/Ambari-DDL-Postgres-CREATE.sql| 11 +
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 11 +
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 11 +
26 files changed, 1386 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/72bac1f7/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
index 6096fa5..7ac41a5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
@@ -120,6 +120,10 @@ public class ResourceInstanceFactoryImpl implements
ResourceInstanceFactory {
resourceDefinition = new ServiceResourceDefinition();
break;
+ case ServiceDependency:
+resourceDefinition = new ServiceDependencyResourceDefinition();
+break;
+
case Host:
resourceDefinition = mapIds.containsKey(Resource.Type.Cluster) ?
new HostResourceDefinition() : new
DetachedHostResourceDefinition();
http://git-wip-us.apache.org/repos/asf/ambari/blob/72bac1f7/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceDependencyResourceDefinition.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceDependencyResourceDefinition.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceDependencyResourceDefinition.java
new file mode 100644
index 000..2aec7f6
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceDependencyResourceDefinition.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the Lice
[1/2] ambari git commit: AMBARI-22345. Add service dependencies.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-14714 adc7060a6 -> 72bac1f7f http://git-wip-us.apache.org/repos/asf/ambari/blob/72bac1f7/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql index b1990df..70a6543 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql @@ -215,6 +215,17 @@ CREATE TABLE clusterservices ( CONSTRAINT PK_clusterservices PRIMARY KEY (id, service_group_id, cluster_id), CONSTRAINT FK_clusterservices_cluster_id FOREIGN KEY (service_group_id, cluster_id) REFERENCES servicegroups (id, cluster_id)); +CREATE TABLE servicedependencies ( + service_id BIGINT NOT NULL, + service_group_id BIGINT NOT NULL, + service_cluster_id BIGINT NOT NULL, + dependent_service_id BIGINT NOT NULL, + dependent_service_group_id BIGINT NOT NULL, + dependent_service_cluster_id BIGINT NOT NULL, + CONSTRAINT PK_servicedependencies PRIMARY KEY (service_id, service_group_id, service_cluster_id, dependent_service_id, dependent_service_group_id, dependent_service_cluster_id), + CONSTRAINT FK_servicedependencies_service_group_cluster_id FOREIGN KEY (service_id, service_group_id, service_cluster_id) REFERENCES clusterservices (id, service_group_id, cluster_id)); + CONSTRAINT FK_servicedependencies_dependent_service_group_cluster_id FOREIGN KEY (dependent_service_id, dependent_service_group_id, dependent_service_cluster_id) REFERENCES clusterservices (id, service_group_id, cluster_id)); + CREATE TABLE clusterstate ( cluster_id BIGINT NOT NULL, current_cluster_state VARCHAR(255) NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/72bac1f7/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 015d6ac..7c5f0fd 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -197,6 +197,17 @@ CREATE TABLE clusterservices ( CONSTRAINT PK_clusterservices PRIMARY KEY (id, service_group_id, cluster_id), CONSTRAINT FK_clusterservices_cluster_id FOREIGN KEY (service_group_id, cluster_id) REFERENCES servicegroups (id, cluster_id)); +CREATE TABLE servicedependencies ( + service_id NUMBER(19) NOT NULL, + service_group_id NUMBER(19) NOT NULL, + service_cluster_id NUMBER(19) NOT NULL, + dependent_service_id NUMBER(19) NOT NULL, + dependent_service_group_id NUMBER(19) NOT NULL, + dependent_service_cluster_id NUMBER(19) NOT NULL, + CONSTRAINT PK_servicedependencies PRIMARY KEY (service_id, service_group_id, service_cluster_id, dependent_service_id, dependent_service_group_id, dependent_service_cluster_id), + CONSTRAINT FK_servicedependencies_service_group_cluster_id FOREIGN KEY (service_id, service_group_id, service_cluster_id) REFERENCES clusterservices (id, service_group_id, cluster_id)); + CONSTRAINT FK_servicedependencies_dependent_service_group_cluster_id FOREIGN KEY (dependent_service_id, dependent_service_group_id, dependent_service_cluster_id) REFERENCES clusterservices (id, service_group_id, cluster_id)); + CREATE TABLE clusterstate ( cluster_id NUMBER(19) NOT NULL, current_cluster_state VARCHAR2(255) NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/72bac1f7/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index d3c7ff6..dfef1ce 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -197,6 +197,17 @@ CREATE TABLE clusterservices ( CONSTRAINT PK_clusterservices PRIMARY KEY (id, service_group_id, cluster_id), CONSTRAINT FK_clusterservices_cluster_id FOREIGN KEY (service_group_id, cluster_id) REFERENCES servicegroups (id, cluster_id)); +CREATE TABLE servicedependencies ( + service_id BIGINT NOT NULL, + service_group_id BIGINT NOT NULL, + service_cluster_id BIGINT NOT NULL, + dependent_service_id BIGINT NOT NULL, + dependent_service_group_id BIGINT NOT NULL, + dependent_service_cluster_id BIGINT NOT NULL, + CONSTRAINT PK_servicedependencies PRIMARY KEY (service_id, service_group_id, service_cluster_id, dependent_service_id, dependent_service_group_id, dependent_service_cluster_id), + CONSTRAINT FK_servicedependencies_service_group_cluster_id FOREIGN KEY (service_id, service_group_id, ser
[1/2] ambari git commit: AMBARI-22249. Add service group dependencies.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-14714 eaeed81e5 -> 16913b201 http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql index 6b69627..015d6ac 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql @@ -178,6 +178,15 @@ CREATE TABLE servicegroups ( CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); +CREATE TABLE servicegroupdependencies ( + service_group_id NUMBER(19) NOT NULL, + service_group_cluster_id NUMBER(19) NOT NULL, + dependent_service_group_id NUMBER(19) NOT NULL, + dependent_service_group_cluster_id NUMBER(19) NOT NULL, + CONSTRAINT PK_servicegroupdependencies PRIMARY KEY (service_group_id, service_group_cluster_id, dependent_service_group_id, dependent_service_group_cluster_id), + CONSTRAINT FK_servicegroupdependencies_service_group_cluster_id FOREIGN KEY (service_group_id, service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CONSTRAINT FK_servicegroupdependencies_dependent_service_group_cluster_id FOREIGN KEY (dependent_service_group_id, dependent_service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CREATE TABLE clusterservices ( id NUMBER(19) NOT NULL, service_name VARCHAR2(255) NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql index 1708671..d3c7ff6 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql @@ -178,6 +178,15 @@ CREATE TABLE servicegroups ( CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); +CREATE TABLE servicegroupdependencies ( + service_group_id BIGINT NOT NULL, + service_group_cluster_id BIGINT NOT NULL, + dependent_service_group_id BIGINT NOT NULL, + dependent_service_group_cluster_id BIGINT NOT NULL, + CONSTRAINT PK_servicegroupdependencies PRIMARY KEY (service_group_id, service_group_cluster_id, dependent_service_group_id, dependent_service_group_cluster_id), + CONSTRAINT FK_servicegroupdependencies_service_group_cluster_id FOREIGN KEY (service_group_id, service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CONSTRAINT FK_servicegroupdependencies_dependent_service_group_cluster_id FOREIGN KEY (dependent_service_group_id, dependent_service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CREATE TABLE clusterservices ( id BIGINT NOT NULL, service_name VARCHAR(255) NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql -- diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql index 87d9169..6bfe205 100644 --- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql +++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql @@ -176,6 +176,15 @@ CREATE TABLE servicegroups ( CONSTRAINT PK_servicegroups PRIMARY KEY (id, cluster_id), CONSTRAINT FK_servicegroups_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id)); +CREATE TABLE servicegroupdependencies ( + service_group_id NUMBER(19) NOT NULL, + service_group_cluster_id NUMBER(19) NOT NULL, + dependent_service_group_id NUMBER(19) NOT NULL, + dependent_service_group_cluster_id NUMBER(19) NOT NULL, + CONSTRAINT PK_servicegroupdependencies PRIMARY KEY (service_group_id, service_group_cluster_id, dependent_service_group_id, dependent_service_group_cluster_id), + CONSTRAINT FK_servicegroupdependencies_service_group_cluster_id FOREIGN KEY (service_group_id, service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CONSTRAINT FK_servicegroupdependencies_dependent_service_group_cluster_id FOREIGN KEY (dependent_service_group_id, dependent_service_group_cluster_id) REFERENCES servicegroups (id, cluster_id)); + CREATE TABLE clusterservices ( id NUMERIC(19) NOT NULL, service_name VARCHAR(255) NOT NULL, http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/resources/Ambari-
[2/2] ambari git commit: AMBARI-22249. Add service group dependencies.(vbrodetskyi)
AMBARI-22249. Add service group dependencies.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/16913b20
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/16913b20
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/16913b20
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 16913b201706c416cea6cf441f4800dd600568f1
Parents: eaeed81
Author: Vitaly Brodetskyi
Authored: Thu Nov 2 17:59:14 2017 +0200
Committer: Vitaly Brodetskyi
Committed: Thu Nov 2 17:59:14 2017 +0200
--
.../resources/ResourceInstanceFactoryImpl.java | 4 +
...erviceGroupDependencyResourceDefinition.java | 41 ++
.../ServiceGroupResourceDefinition.java | 1 +
.../server/api/services/ServiceGroupKey.java| 89
.../api/services/ServiceGroupService.java | 151 +-
.../server/controller/ControllerModule.java | 2 +
.../controller/ResourceProviderFactory.java | 5 +-
.../ServiceGroupDependencyRequest.java | 75 +++
.../ServiceGroupDependencyResponse.java | 166 +++
.../AbstractControllerResourceProvider.java | 2 +
.../internal/DefaultProviderModule.java | 2 +
.../ServiceGroupDependencyResourceProvider.java | 484 +++
.../ambari/server/controller/spi/Resource.java | 2 +
.../server/orm/entities/ServiceGroupEntity.java | 29 +-
.../org/apache/ambari/server/state/Cluster.java | 4 +
.../ambari/server/state/ServiceGroup.java | 15 +-
.../server/state/ServiceGroupFactory.java | 6 +-
.../ambari/server/state/ServiceGroupImpl.java | 121 -
.../server/state/cluster/ClusterImpl.java | 62 ++-
.../ambari/server/topology/AmbariContext.java | 12 +-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 9 +
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 9 +
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 9 +
.../resources/Ambari-DDL-Postgres-CREATE.sql| 9 +
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 9 +
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 9 +
26 files changed, 1317 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
index 2c5edb7..c009cbb 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceInstanceFactoryImpl.java
@@ -112,6 +112,10 @@ public class ResourceInstanceFactoryImpl implements
ResourceInstanceFactory {
resourceDefinition = new ServiceGroupResourceDefinition();
break;
+ case ServiceGroupDependency:
+resourceDefinition = new ServiceGroupDependencyResourceDefinition();
+break;
+
case Service:
resourceDefinition = new ServiceResourceDefinition();
break;
http://git-wip-us.apache.org/repos/asf/ambari/blob/16913b20/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceGroupDependencyResourceDefinition.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceGroupDependencyResourceDefinition.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceGroupDependencyResourceDefinition.java
new file mode 100644
index 000..5c99161
--- /dev/null
+++
b/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ServiceGroupDependencyResourceDefinition.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.api.re
ambari git commit: AMBARI-22209. HdpCoreMpack: Quick Links should show NN web UI links.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 8d8c1a551 -> cbaa88d2b
AMBARI-22209. HdpCoreMpack: Quick Links should show NN web UI
links.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cbaa88d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cbaa88d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cbaa88d2
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: cbaa88d2bdcc3ee3c3f043098f2b7a98f3563746
Parents: 8d8c1a5
Author: Vitaly Brodetskyi
Authored: Thu Oct 12 23:48:20 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Thu Oct 12 23:48:20 2017 +0300
--
.../apache/ambari/server/state/ServiceInfo.java | 28 +---
1 file changed, 24 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbaa88d2/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
index 1cad4df..0e6e3c3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java
@@ -468,9 +468,11 @@ public class ServiceInfo implements Validable, Cloneable {
clone.serviceOsSpecifics = serviceOsSpecifics;
clone.configDir = configDir;
clone.themesDir = themesDir;
-clone.setThemes(themes);
+clone.setThemes(getThemes());
+clone.setThemesMap(themesMap);
clone.setQuickLinksConfigurationsDir(quickLinksConfigurationsDir);
-clone.setQuickLinksConfigurations(quickLinksConfigurations);
+clone.setQuickLinksConfigurations(getQuickLinksConfigurations());
+clone.setQuickLinksConfigurationsMap(quickLinksConfigurationsMap);
clone.setCommandScript(commandScript);
clone.setRequiredServices(requiredServices);
return clone;
@@ -1029,8 +1031,17 @@ public class ServiceInfo implements Validable, Cloneable
{
this.themesDir = themesDir;
}
+ /*
+ * This method will return data from themesMap if themes are not available.
+ * ThemesMap contain merged themes from all previous stacks.
+ * TODO we should think and fix it for HDP 3.0 because there we don't need
merged data.
+ * */
public List getThemes() {
-return themes;
+List themeList = themes;
+if (themes == null & themesMap != null) {
+ themeList = new ArrayList<>(themesMap.values());
+}
+return themeList;
}
public void setThemes(List themes) {
@@ -1063,8 +1074,17 @@ public class ServiceInfo implements Validable, Cloneable
{
this.quickLinksConfigurationsDir = quickLinksConfigurationsDir;
}
+ /*
+ * This method will return data from quickLinksConfigurationsMap if
quickLinksConfigurations are not available.
+ * quickLinksConfigurationsMap contain merged quickLinksConfigurations from
all previous stacks.
+ * TODO we should think and fix it for HDP 3.0 because there we don't need
merged data.
+ * */
public List getQuickLinksConfigurations() {
-return quickLinksConfigurations;
+List quickLinksConigurationList =
quickLinksConfigurations;
+if (quickLinksConfigurations == null && quickLinksConfigurationsMap !=
null) {
+ quickLinksConigurationList = new
ArrayList<>(quickLinksConfigurationsMap.values());
+}
+return quickLinksConigurationList;
}
public void setQuickLinksConfigurations(List
quickLinksConfigurations) {
[1/2] ambari git commit: AMBARI-22124. Refactor AMS logic in stack advisors to service advisors.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 9adfcdcbd -> 0f32765dc
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
--
diff --git
a/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
new file mode 100644
index 000..05254fe
--- /dev/null
+++
b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
@@ -0,0 +1,596 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import imp
+import json
+import os
+from unittest import TestCase
+
+from mock.mock import patch, MagicMock
+
+
+class TestAMBARI_METRICS010ServiceAdvisor(TestCase):
+
+ testDirectory = os.path.dirname(os.path.abspath(__file__))
+ stack_advisor_path = os.path.join(testDirectory,
'../../../../main/resources/stacks/stack_advisor.py')
+ with open(stack_advisor_path, 'rb') as fp:
+imp.load_module('stack_advisor', fp, stack_advisor_path, ('.py', 'rb',
imp.PY_SOURCE))
+
+ serviceAdvisorPath =
'../../../../main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py'
+ ambariMetrics010ServiceAdvisorPath = os.path.join(testDirectory,
serviceAdvisorPath)
+ with open(ambariMetrics010ServiceAdvisorPath, 'rb') as fp:
+service_advisor_impl = imp.load_module('service_advisor_impl', fp,
ambariMetrics010ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
+
+ def setUp(self):
+serviceAdvisorClass = getattr(self.service_advisor_impl,
'AMBARI_METRICSServiceAdvisor')
+self.serviceAdvisor = serviceAdvisorClass()
+
+
+ def test_recommendAmsConfigurations(self):
+configurations = {
+ "hadoop-env": {
+"properties": {
+ "hdfs_user": "hdfs",
+ "proxyuser_group": "users"
+}
+ }
+}
+
+hosts = {
+ "items": [
+{
+ "href": "/api/v1/hosts/host1",
+ "Hosts": {
+"cpu_count": 1,
+"host_name": "c6401.ambari.apache.org",
+"os_arch": "x86_64",
+"os_type": "centos6",
+"ph_cpu_count": 1,
+"public_host_name": "public.c6401.ambari.apache.org",
+"rack_info": "/default-rack",
+"total_mem": 2097152,
+"disk_info": [{
+ "size": '8000',
+ "mountpoint": "/"
+}]
+ }
+},
+{
+ "href": "/api/v1/hosts/host2",
+ "Hosts": {
+"cpu_count": 1,
+"host_name": "c6402.ambari.apache.org",
+"os_arch": "x86_64",
+"os_type": "centos6",
+"ph_cpu_count": 1,
+"public_host_name": "public.c6402.ambari.apache.org",
+"rack_info": "/default-rack",
+"total_mem": 1048576,
+"disk_info": [{
+ "size": '8',
+ "mountpoint": "/"
+}]
+ }
+}
+ ]}
+
+
+services1 = {
+ "services": [
+{
+ "StackServices": {
+"service_name": "HDFS"
+ }, "components": [
+ {
+"StackServiceComponents": {
+ "component_name": "NAMENODE",
+ "hostnames": ["c6401.ambari.apache.org"]
+}
+ }
+]
+},
+{
+ "StackServices": {
+"service_name": "AMBARI_METRICS"
+ },
+ "components": [
+{
+ "StackServiceComponents": {
+"component_name": "METRICS_COLLECTOR",
+"hostnames": ["c6401.ambari.apache.org",
"c6402.ambari.apache.org"]
+ }
+}, {
+ "StackServiceComponents": {
+"component_name": "METRICS_MONITOR",
+"hostnames": ["c6401.ambari.apache.org",
"c6402.ambari.apache.org"]
+ }
+}
+ ]
+}],
+ "configurations": configurations,
+ "ambari-server-properties": {"ambari-server.user":"ambari_user"}
+}
+
+clusterData = {
+ "totalAvailableRam": 2048
+}
+
+expected = {'ams-env'
[2/2] ambari git commit: AMBARI-22124. Refactor AMS logic in stack advisors to service advisors.(vbrodetskyi)
AMBARI-22124. Refactor AMS logic in stack advisors to service
advisors.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0f32765d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0f32765d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0f32765d
Branch: refs/heads/trunk
Commit: 0f32765dc2250044c7925f4e68e6f61b7a77d8f8
Parents: 9adfcdc
Author: Vitaly Brodetskyi
Authored: Fri Oct 6 10:40:33 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Oct 6 10:40:33 2017 +0300
--
.../AMBARI_METRICS/0.1.0/service_advisor.py | 787 +++
.../ATLAS/0.7.0.3.0/service_advisor.py | 5 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 542 +
.../stacks/HDP/2.2/services/stack_advisor.py| 1 -
.../AMBARI_METRICS/test_service_advisor.py | 596 ++
.../stacks/2.0.6/common/test_stack_advisor.py | 576 --
.../stacks/2.2/common/test_stack_advisor.py | 511
7 files changed, 1388 insertions(+), 1630 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/0f32765d/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
--
diff --git
a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
new file mode 100644
index 000..eae98bf
--- /dev/null
+++
b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
@@ -0,0 +1,787 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import re
+import os
+import sys
+import socket
+import traceback
+from math import ceil, floor, log
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+#split points
+metricsDir = os.path.join(SCRIPT_DIR, 'package')
+print "METRICS_DIR=>" + str(metricsDir)
+serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics')
+customServiceMetricsDir = os.path.join(SCRIPT_DIR,
'../../../dashboards/service-metrics')
+sys.path.append(os.path.join(metricsDir, 'scripts'))
+
+from split_points import FindSplitPointsForAMSRegions
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE,
('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class AMBARI_METRICSServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+self.as_super = super(AMBARI_METRICSServiceAdvisor, self)
+self.as_super.__init__(*args, **kwargs)
+
+# Always call these methods
+self.modifyMastersWithMultipleInstances()
+self.modifyCardinalitiesDict()
+self.modifyHeapSizeProperties()
+self.modifyNotValuableComponents()
+self.modifyComponentsNotPreferableOnServer()
+self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+"""
+Modify the set of masters with multiple instances.
+Must be overriden in child class.
+"""
+# Nothing to do
+pass
+
+ def modifyCardinalitiesDict(self):
+"""
+Modify the dictionary of cardinalities.
+Must be overriden in child class.
+"""
+min_val = 1
+
+self.cardinalitiesDict.update(
+ {
+'METRICS_COLLECTOR': {"min": min_val}
+ }
+)
+
+ def modifyHeapSizeProperties(self):
+"""
+Modify the dic
ambari git commit: AMBARI-22107. Enable new Log Aggregation format for Yarn default configuration in Ambari.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk f92da65f7 -> 19fe4cf1e AMBARI-22107. Enable new Log Aggregation format for Yarn default configuration in Ambari.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/19fe4cf1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/19fe4cf1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/19fe4cf1 Branch: refs/heads/trunk Commit: 19fe4cf1e7450c7230031c660295ec6e04958519 Parents: f92da65 Author: Vitaly Brodetskyi Authored: Mon Oct 2 13:38:41 2017 +0300 Committer: Vitaly Brodetskyi Committed: Mon Oct 2 13:38:41 2017 +0300 -- .../2.6/services/YARN/configuration/yarn-site.xml | 18 ++ 1 file changed, 18 insertions(+) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/19fe4cf1/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index 9094b15..b5eedea 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -117,4 +117,22 @@ + +yarn.log-aggregation.file-formats +IndexedFormat,TFile + + + + +yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + + + + +yarn.log-aggregation.file-controller.IndexedFormat.class + org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController + + +
ambari git commit: AMBARI-22107. Enable new Log Aggregation format for Yarn default configuration in Ambari.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.6 65fff2c32 -> c5233ac33 AMBARI-22107. Enable new Log Aggregation format for Yarn default configuration in Ambari.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c5233ac3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c5233ac3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c5233ac3 Branch: refs/heads/branch-2.6 Commit: c5233ac33ccf967dba78c93e34d112e8e96a7283 Parents: 65fff2c Author: Vitaly Brodetskyi Authored: Mon Oct 2 13:36:10 2017 +0300 Committer: Vitaly Brodetskyi Committed: Mon Oct 2 13:36:10 2017 +0300 -- .../2.6/services/YARN/configuration/yarn-site.xml | 18 ++ 1 file changed, 18 insertions(+) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/c5233ac3/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index 9094b15..b5eedea 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -117,4 +117,22 @@ + +yarn.log-aggregation.file-formats +IndexedFormat,TFile + + + + +yarn.log-aggregation.file-controller.TFile.class + org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController + + + + +yarn.log-aggregation.file-controller.IndexedFormat.class + org.apache.hadoop.yarn.logaggregation.filecontroller.ifile.LogAggregationIndexedFileController + + +
ambari git commit: AMBARI-22068. Remove user to group hardcoded logic in hooks.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk 29ea0ffb9 -> 3889ddfca AMBARI-22068. Remove user to group hardcoded logic in hooks.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3889ddfc Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3889ddfc Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3889ddfc Branch: refs/heads/trunk Commit: 3889ddfca272dc4380f60fb1c7f8915998dc7603 Parents: 29ea0ff Author: Vitaly Brodetskyi Authored: Mon Oct 2 11:17:33 2017 +0300 Committer: Vitaly Brodetskyi Committed: Mon Oct 2 11:17:33 2017 +0300 -- ambari-server/src/main/resources/configuration-schema.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/3889ddfc/ambari-server/src/main/resources/configuration-schema.xsd -- diff --git a/ambari-server/src/main/resources/configuration-schema.xsd b/ambari-server/src/main/resources/configuration-schema.xsd index a67de9a..9350984 100644 --- a/ambari-server/src/main/resources/configuration-schema.xsd +++ b/ambari-server/src/main/resources/configuration-schema.xsd @@ -115,7 +115,7 @@ - +
ambari git commit: AMBARI-22068. Additional Fix.Remove user to group hardcoded logic in hooks.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk f69ccd890 -> 18fd04f2e AMBARI-22068. Additional Fix.Remove user to group hardcoded logic in hooks.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/18fd04f2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/18fd04f2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/18fd04f2 Branch: refs/heads/trunk Commit: 18fd04f2e3c59b3c741ac0d32083a70739203f70 Parents: f69ccd8 Author: Vitaly Brodetskyi Authored: Sun Sep 24 12:20:45 2017 +0300 Committer: Vitaly Brodetskyi Committed: Sun Sep 24 12:21:24 2017 +0300 -- .../resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py | 1 + 1 file changed, 1 insertion(+) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/18fd04f2/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py -- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py index 14d08f6..b111c02 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/hooks/before-ANY/scripts/params.py @@ -20,6 +20,7 @@ limitations under the License. import collections import re import os +import ast import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
ambari git commit: AMBARI-22068. Remove user to group hardcoded logic in hooks.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk d198dd3c4 -> b10b6fd48 AMBARI-22068. Remove user to group hardcoded logic in hooks.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b10b6fd4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b10b6fd4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b10b6fd4 Branch: refs/heads/trunk Commit: b10b6fd4887f5e153f2abfb2dd804f9604273be8 Parents: d198dd3 Author: Vitaly Brodetskyi Authored: Sat Sep 23 11:47:07 2017 +0300 Committer: Vitaly Brodetskyi Committed: Sat Sep 23 11:47:07 2017 +0300 -- .../1.6.1.2.2.0/configuration/accumulo-env.xml | 6 + .../0.1.0/configuration/infra-solr-env.xml | 6 + .../0.1.0/configuration/ams-env.xml | 6 + .../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 6 + .../ATLAS/0.7.0.3.0/configuration/atlas-env.xml | 6 + .../DRUID/0.9.2/configuration/druid-env.xml | 6 + .../0.5.0.2.1/configuration/falcon-env.xml | 6 + .../FLUME/1.4.0.2.0/configuration/flume-env.xml | 6 + .../0.96.0.2.0/configuration/hbase-env.xml | 6 + .../HBASE/2.0.0.3.0/configuration/hbase-env.xml | 6 + .../HDFS/2.1.0.2.0/configuration/hadoop-env.xml | 6 + .../HDFS/3.0.0.3.0/configuration/hadoop-env.xml | 6 + .../HIVE/0.12.0.2.0/configuration/hive-env.xml | 12 ++ .../HIVE/2.1.0.3.0/configuration/hive-env.xml | 12 ++ .../0.10.0.3.0/configuration/kafka-env.xml | 6 + .../KAFKA/0.8.1/configuration/kafka-env.xml | 6 + .../KNOX/0.5.0.2.2/configuration/knox-env.xml | 10 .../KNOX/0.5.0.3.0/configuration/knox-env.xml | 10 .../0.5.0/configuration/logsearch-env.xml | 6 + .../1.0.0.2.3/configuration/mahout-env.xml | 6 + .../OOZIE/4.0.0.2.0/configuration/oozie-env.xml | 10 .../OOZIE/4.2.0.3.0/configuration/oozie-env.xml | 10 .../RANGER/0.4.0/configuration/ranger-env.xml | 10 .../1.0.0.3.0/configuration/ranger-env.xml | 10 .../0.5.0.2.3/configuration/kms-env.xml | 10 .../1.0.0.3.0/configuration/kms-env.xml | 10 .../SPARK/1.2.1/configuration/spark-env.xml | 10 .../SPARK/2.2.0/configuration/livy-env.xml | 10 .../SPARK/2.2.0/configuration/spark-env.xml | 10 .../SPARK2/2.0.0/configuration/spark2-env.xml | 10 .../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 6 + .../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 6 + .../STORM/0.9.1/configuration/storm-env.xml | 6 + .../STORM/1.0.1.3.0/configuration/storm-env.xml | 6 + .../0.15.0/configuration/superset-env.xml | 6 + .../TEZ/0.4.0.2.1/configuration/tez-env.xml | 10 .../TEZ/0.9.0.3.0/configuration/tez-env.xml | 10 .../configuration-mapred/mapred-env.xml | 6 + .../YARN/2.1.0.2.0/configuration/yarn-env.xml | 6 + .../configuration-mapred/mapred-env.xml | 6 + .../YARN/3.0.0.3.0/configuration/yarn-env.xml | 6 + .../0.6.0/configuration/zeppelin-env.xml| 10 .../0.7.0/configuration/zeppelin-env.xml| 10 .../3.4.5/configuration/zookeeper-env.xml | 6 + .../HDP/2.0.6/configuration/cluster-env.xml | 10 .../2.0.6/hooks/before-ANY/scripts/params.py| 20 .../services/HBASE/configuration/hbase-env.xml | 6 + .../services/ATLAS/configuration/atlas-env.xml | 6 + .../services/SPARK/configuration/livy-env.xml | 10 .../services/SPARK/configuration/livy-env.xml | 10 .../ZEPPELIN/configuration/zeppelin-env.xml | 10 .../HDP/3.0/configuration/cluster-env.xml | 10 .../HDP/3.0/hooks/before-ANY/scripts/params.py | 24 .../python/stacks/2.0.6/configs/default.json| 1 + .../2.0.6/hooks/before-ANY/test_before_any.py | 10 55 files changed, 421 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/b10b6fd4/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml -- diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml index e4aa21e..af053b2 100644 --- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml +++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/configuration/accumulo-env.xml @@ -106,6 +106,12 @@ user fa
ambari git commit: AMBARI-21973. Set tez.runtime.shuffle.ssl.enable=false in Ambari for HSI.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk 941bc1685 -> bb6f0d2c8 AMBARI-21973. Set tez.runtime.shuffle.ssl.enable=false in Ambari for HSI.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bb6f0d2c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bb6f0d2c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bb6f0d2c Branch: refs/heads/trunk Commit: bb6f0d2c89adf17894caa995da65e7e09f714434 Parents: 941bc16 Author: Vitaly Brodetskyi Authored: Mon Sep 18 21:09:06 2017 +0300 Committer: Vitaly Brodetskyi Committed: Mon Sep 18 21:09:06 2017 +0300 -- .../services/HIVE/configuration/tez-interactive-site.xml | 11 +++ .../resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 + .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 7 +++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 ++ .../resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml| 4 .../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml| 1 + .../resources/stacks/HDP/2.6/upgrades/config-upgrade.xml | 8 .../stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 6 ++ .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml| 4 9 files changed, 52 insertions(+) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/bb6f0d2c/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index c1a42b0..fb4a6b0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -183,4 +183,15 @@ 8 + +tez.runtime.shuffle.ssl.enable +false + + boolean + false + true + +true + + http://git-wip-us.apache.org/repos/asf/ambari/blob/bb6f0d2c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index bd69f1e..cf42e93 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -512,6 +512,11 @@ + +tez-interactive-site + + + hive-interactive-site http://git-wip-us.apache.org/repos/asf/ambari/blob/bb6f0d2c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml index b25db24..4d514ac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml @@ -302,12 +302,19 @@ + Removing atlas.cluster.name property + + + Update tez.runtime.shuffle.ssl.enable property + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/bb6f0d2c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 12d2832..1af3b18 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -587,6 +587,12 @@ + + + Update tez.runtime.shuffle.ssl.enable property + + + Updating JVM stack size for LLAP http://git-wip-us.apache.org/repos/asf/ambari/blob/bb6f0d2c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml -- diff --git
ambari git commit: AMBARI-21973. Set tez.runtime.shuffle.ssl.enable=false in Ambari for HSI.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.6 6a6d29d58 -> 3b525a7cb AMBARI-21973. Set tez.runtime.shuffle.ssl.enable=false in Ambari for HSI.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3b525a7c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3b525a7c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3b525a7c Branch: refs/heads/branch-2.6 Commit: 3b525a7cb1347347f836d7a07712f8433589c878 Parents: 6a6d29d Author: Vitaly Brodetskyi Authored: Mon Sep 18 21:07:15 2017 +0300 Committer: Vitaly Brodetskyi Committed: Mon Sep 18 21:07:15 2017 +0300 -- .../services/HIVE/configuration/tez-interactive-site.xml | 11 +++ .../resources/stacks/HDP/2.5/upgrades/config-upgrade.xml | 5 + .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml | 7 +++ .../stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 6 ++ .../resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml| 4 .../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml| 1 + .../resources/stacks/HDP/2.6/upgrades/config-upgrade.xml | 8 .../stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 6 ++ .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml| 4 9 files changed, 52 insertions(+) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/3b525a7c/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml index c1a42b0..fb4a6b0 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/tez-interactive-site.xml @@ -183,4 +183,15 @@ 8 + +tez.runtime.shuffle.ssl.enable +false + + boolean + false + true + +true + + http://git-wip-us.apache.org/repos/asf/ambari/blob/3b525a7c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 5618e14..cf73a88 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -520,6 +520,11 @@ + +tez-interactive-site + + + hive-interactive-site http://git-wip-us.apache.org/repos/asf/ambari/blob/3b525a7c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml index 9f77065..c8bafc4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.5.xml @@ -302,12 +302,19 @@ + Removing atlas.cluster.name property + + + Update tez.runtime.shuffle.ssl.enable property + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/3b525a7c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 8824de5..0ab391d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -587,6 +587,12 @@ + + + Update tez.runtime.shuffle.ssl.enable property + + + Updating JVM stack size for LLAP http://git-wip-us.apache.org/repos/asf/ambari/blob/3b525a7c/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.5.xml -- d
ambari git commit: AMBARI-21856. Service repoinfo.xml needs to expose component for Ubuntu repos.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk d8e621e5e -> f8b29df96
AMBARI-21856. Service repoinfo.xml needs to expose component for Ubuntu
repos.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8b29df9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8b29df9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8b29df9
Branch: refs/heads/trunk
Commit: f8b29df9685b443d4a5c06c6e1725e4428c95b49
Parents: d8e621e
Author: Vitaly Brodetskyi
Authored: Fri Sep 15 20:40:52 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 15 20:41:41 2017 +0300
--
.../libraries/functions/repository_util.py | 9 ---
.../ambari/server/agent/CommandRepository.java | 23 ++---
.../AmbariManagementControllerImpl.java | 5 ++--
.../server/controller/RepositoryResponse.java | 23 -
.../internal/RepositoryResourceProvider.java| 6 +
.../VersionDefinitionResourceProvider.java | 4 +++
.../server/orm/entities/RepositoryEntity.java | 22
.../apache/ambari/server/stack/RepoUtil.java| 4 ++-
.../ambari/server/state/RepositoryInfo.java | 27 ++--
.../ambari/server/state/ServiceOsSpecific.java | 16
.../server/state/stack/RepositoryXml.java | 11
.../stack/upgrade/RepositoryVersionHelper.java | 10
.../custom_actions/scripts/update_repo.py | 7 +++--
.../scripts/repo_initialization.py | 3 ++-
.../src/main/resources/version_definition.xsd | 2 ++
.../RepositoryVersionEventCreatorTest.java | 4 +++
.../RepositoryResourceProviderTest.java | 20 +--
.../python/stacks/2.0.6/configs/default.json| 2 +-
.../hooks/before-INSTALL/test_before_install.py | 10
19 files changed, 189 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b29df9/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
index 120d464..f1074ee 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -26,7 +26,7 @@ import ambari_simplejson as json
__all__ = ["create_repo_files", "CommandRepository"]
# components_lits = repoName + postfix
-UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+UBUNTU_REPO_COMPONENTS_POSTFIX = "main"
def create_repo_files(template, command_repository):
@@ -122,6 +122,8 @@ class _CommandRepositoryEntry(object):
def __init__(self, json_dict):
self.repo_id = _find_value(json_dict, 'repoId') # this is the id within
the repo file, not an Ambari artifact
self.repo_name = _find_value(json_dict, 'repoName')
+self.distribution = _find_value(json_dict, 'distribution')
+self.components = _find_value(json_dict, 'components')
self.base_url = _find_value(json_dict, 'baseUrl')
self.mirrors_list = _find_value(json_dict, 'mirrorsList')
self.ambari_managed = _find_value(json_dict, 'ambariManaged')
@@ -129,9 +131,8 @@ class _CommandRepositoryEntry(object):
if self.ambari_managed is None:
self.ambari_managed = True
-# if repoName is changed on the java side, this will fail for ubuntu since
we rely on the
-# name being the same as how the repository was built
-self.ubuntu_components = [self.repo_name] + UBUNTU_REPO_COMPONENTS_POSTFIX
+self.ubuntu_components = [self.distribution if self.distribution else
self.repo_name] + \
+ [self.components.replace(",", " ") if
self.components else UBUNTU_REPO_COMPONENTS_POSTFIX]
http://git-wip-us.apache.org/repos/asf/ambari/blob/f8b29df9/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
index 858a55f..1351713 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
@@ -133,12 +133,16 @@ public class CommandRepository {
@SerializedName("ambariManag
ambari git commit: AMBARI-21856. Service repoinfo.xml needs to expose component for Ubuntu repos.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 8217f4c1d -> 3b55821fc
AMBARI-21856. Service repoinfo.xml needs to expose component for Ubuntu
repos.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3b55821f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3b55821f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3b55821f
Branch: refs/heads/branch-2.6
Commit: 3b55821fc2117d6f97d87ce902eb56e4f1b9497a
Parents: 8217f4c
Author: Vitaly Brodetskyi
Authored: Fri Sep 15 19:35:45 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 15 19:35:45 2017 +0300
--
.../libraries/functions/repository_util.py | 9 ---
.../ambari/server/agent/CommandRepository.java | 23 ++---
.../AmbariManagementControllerImpl.java | 5 ++--
.../server/controller/RepositoryResponse.java | 23 -
.../internal/RepositoryResourceProvider.java| 6 +
.../VersionDefinitionResourceProvider.java | 4 +++
.../server/orm/entities/RepositoryEntity.java | 22
.../apache/ambari/server/stack/RepoUtil.java| 4 ++-
.../ambari/server/state/RepositoryInfo.java | 27 ++--
.../ambari/server/state/ServiceOsSpecific.java | 16
.../server/state/stack/RepositoryXml.java | 12 +
.../stack/upgrade/RepositoryVersionHelper.java | 10
.../custom_actions/scripts/update_repo.py | 7 +++--
.../scripts/repo_initialization.py | 3 ++-
.../src/main/resources/version_definition.xsd | 2 ++
.../RepositoryVersionEventCreatorTest.java | 4 +++
.../RepositoryResourceProviderTest.java | 20 +--
.../python/stacks/2.0.6/configs/default.json| 2 +-
.../hooks/before-INSTALL/test_before_install.py | 10
19 files changed, 190 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b55821f/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
index 32b8038..b2f65d8 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py
@@ -26,7 +26,7 @@ import ambari_simplejson as json
__all__ = ["create_repo_files", "CommandRepository"]
# components_lits = repoName + postfix
-UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+UBUNTU_REPO_COMPONENTS_POSTFIX = "main"
def create_repo_files(template, command_repository):
@@ -122,6 +122,8 @@ class _CommandRepositoryEntry(object):
def __init__(self, json_dict):
self.repo_id = _find_value(json_dict, 'repoId') # this is the id within
the repo file, not an Ambari artifact
self.repo_name = _find_value(json_dict, 'repoName')
+self.distribution = _find_value(json_dict, 'distribution')
+self.components = _find_value(json_dict, 'components')
self.base_url = _find_value(json_dict, 'baseUrl')
self.mirrors_list = _find_value(json_dict, 'mirrorsList')
self.ambari_managed = _find_value(json_dict, 'ambariManaged')
@@ -129,6 +131,5 @@ class _CommandRepositoryEntry(object):
if self.ambari_managed is None:
self.ambari_managed = True
-# if repoName is changed on the java side, this will fail for ubuntu since
we rely on the
-# name being the same as how the repository was built
-self.ubuntu_components = [self.repo_name] + UBUNTU_REPO_COMPONENTS_POSTFIX
+self.ubuntu_components = [self.distribution if self.distribution else
self.repo_name] + \
+ [self.components.replace(",", " ") if
self.components else UBUNTU_REPO_COMPONENTS_POSTFIX]
http://git-wip-us.apache.org/repos/asf/ambari/blob/3b55821f/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
index 858a55f..1351713 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java
@@ -133,12 +133,16 @@ public class CommandRepository {
@SerializedName("ambariMa
ambari git commit: AMBARI-21945. Refresh Include/Exclude breaks parallel operations and can"t be turned off.(dsen via vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk b46cc4f85 -> e53cc68d4
AMBARI-21945. Refresh Include/Exclude breaks parallel operations and can"t be
turned off.(dsen via vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e53cc68d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e53cc68d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e53cc68d
Branch: refs/heads/trunk
Commit: e53cc68d48d3c2f8a10b4f66d802a5ecf7efd5a4
Parents: b46cc4f
Author: Vitaly Brodetskyi
Authored: Fri Sep 15 14:49:02 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 15 14:49:02 2017 +0300
--
.../ambari/server/controller/AmbariManagementControllerImpl.java | 3 +--
.../ambari/server/controller/AmbariManagementControllerTest.java | 1 +
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/e53cc68d/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 34744eb..3289d18 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -3723,7 +3723,6 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
LOG.debug("Refresh include/exclude files action will be executed for " +
serviceMasterMap);
HashMap requestProperties = new HashMap<>();
requestProperties.put("context", "Update Include/Exclude Files for " +
serviceMasterMap.keySet().toString());
-requestProperties.put("exclusive", "true");
HashMap params = new HashMap<>();
params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY,
String.valueOf(isDecommission));
@@ -3744,7 +3743,7 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
//Create request for command
ExecuteActionRequest actionRequest = new ExecuteActionRequest(
clusterName,
AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null,
- resourceFilters, null, params, true);
+ resourceFilters, null, params, false);
//Send action
createAction(actionRequest, requestProperties);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e53cc68d/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index dea4870..07ba786 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -9387,6 +9387,7 @@ public class AmbariManagementControllerTest {
List requestIDs = actionDB.getRequestsByStatus(null, 1, false);
Request request = actionDB.getRequest(requestIDs.get(0));
assertEquals("Update Include/Exclude Files for [HDFS]",
request.getRequestContext());
+assertEquals(false, request.isExclusive());
Type type = new TypeToken>(){}.getType();
Map requestParams =
StageUtils.getGson().fromJson(request.getInputs(), type);
assertEquals(2, requestParams.size());
ambari git commit: AMBARI-21945. Refresh Include/Exclude breaks parallel operations and can"t be turned off.(dsen via vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 f28b9f17e -> d327aecee
AMBARI-21945. Refresh Include/Exclude breaks parallel operations and can"t be
turned off.(dsen via vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d327aece
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d327aece
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d327aece
Branch: refs/heads/branch-2.6
Commit: d327aecee173f0ef88d4e3575d56fccbc5e8fcb5
Parents: f28b9f1
Author: Vitaly Brodetskyi
Authored: Fri Sep 15 14:45:08 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 15 14:45:08 2017 +0300
--
.../ambari/server/controller/AmbariManagementControllerImpl.java | 3 +--
.../ambari/server/controller/AmbariManagementControllerTest.java | 1 +
2 files changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/d327aece/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index c78d88d..89dc126 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -3684,7 +3684,6 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
LOG.debug("Refresh include/exclude files action will be executed for " +
serviceMasterMap);
HashMap requestProperties = new HashMap<>();
requestProperties.put("context", "Update Include/Exclude Files for " +
serviceMasterMap.keySet().toString());
-requestProperties.put("exclusive", "true");
HashMap params = new HashMap<>();
params.put(AmbariCustomCommandExecutionHelper.UPDATE_FILES_ONLY,
String.valueOf(isDecommission));
@@ -3705,7 +3704,7 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
//Create request for command
ExecuteActionRequest actionRequest = new ExecuteActionRequest(
clusterName,
AmbariCustomCommandExecutionHelper.DECOMMISSION_COMMAND_NAME, null,
- resourceFilters, null, params, true);
+ resourceFilters, null, params, false);
//Send action
createAction(actionRequest, requestProperties);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d327aece/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
--
diff --git
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index cf4c98f..e51bb0a 100644
---
a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -9388,6 +9388,7 @@ public class AmbariManagementControllerTest {
List requestIDs = actionDB.getRequestsByStatus(null, 1, false);
Request request = actionDB.getRequest(requestIDs.get(0));
assertEquals("Update Include/Exclude Files for [HDFS]",
request.getRequestContext());
+assertEquals(false, request.isExclusive());
Type type = new TypeToken>(){}.getType();
Map requestParams =
StageUtils.getGson().fromJson(request.getInputs(), type);
assertEquals(2, requestParams.size());
ambari git commit: AMBARI-21936. Do not export YARN_CONF_DIR in "yarn-env.sh" file.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 321f9cfeb -> aaf6f0ae0
AMBARI-21936. Do not export YARN_CONF_DIR in "yarn-env.sh" file.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aaf6f0ae
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aaf6f0ae
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aaf6f0ae
Branch: refs/heads/trunk
Commit: aaf6f0ae0f69ed80c1f8ec9ffaf81cf3a358900a
Parents: 321f9cf
Author: Vitaly Brodetskyi
Authored: Tue Sep 12 20:31:56 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Tue Sep 12 20:31:56 2017 +0300
--
.../common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml | 3 ---
1 file changed, 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/aaf6f0ae/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml
--
diff --git
a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml
index aaa72d1..4e722ee 100644
---
a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml
+++
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/configuration/yarn-env.xml
@@ -179,9 +179,6 @@ fi
# User for YARN daemons
export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
-# resolve links - $0 may be a softlink
-export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}"
-
# some Java parameters
# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
if [ "$JAVA_HOME" != "" ]; then
ambari git commit: AMBARI-21173. Not able to start Yarn services after restoring the configs to initial value.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 d2bc398ca -> b4b5773e8
AMBARI-21173. Not able to start Yarn services after restoring the configs to
initial value.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b4b5773e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b4b5773e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b4b5773e
Branch: refs/heads/branch-2.6
Commit: b4b5773e84f953e12236093814254a20a1dc1df6
Parents: d2bc398
Author: Vitaly Brodetskyi
Authored: Mon Aug 21 23:57:12 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 8 17:41:10 2017 +0300
--
.../ambari/server/state/cluster/ClusterImpl.java | 19 +--
1 file changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/b4b5773e/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index b152928..9fd7663 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1817,22 +1817,29 @@ public class ClusterImpl implements Cluster {
// disable all configs related to service
if (serviceConfigEntity.getGroupId() == null) {
+ // Here was fixed bug with entity changes revert. More you can find here
AMBARI-21173.
+ // This issue reproduces only if you are changing same entity in first
and second loop.
+ // In that case eclipselink will revert changes to cached, if entity has
fluchGroup and it
+ // needs to be refreshed. Actually we don't need to change same antities
in few steps, so i
+ // decided to filter out. duplicates and do not change them. It will be
better for performance and bug will be fixed.
Collection configTypes = serviceConfigTypes.get(serviceName);
List enabledConfigs =
clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
List serviceConfigEntities =
serviceConfigEntity.getClusterConfigEntities();
ArrayList duplicatevalues = new
ArrayList(serviceConfigEntities);
duplicatevalues.retainAll(enabledConfigs);
- enabledConfigs.removeAll(duplicatevalues);
- serviceConfigEntities.removeAll(duplicatevalues);
for (ClusterConfigEntity enabledConfig : enabledConfigs) {
-enabledConfig.setSelected(false);
-clusterDAO.merge(enabledConfig);
+if (!duplicatevalues.contains(enabledConfig)) {
+ enabledConfig.setSelected(false);
+ clusterDAO.merge(enabledConfig);
+}
}
for (ClusterConfigEntity configEntity : serviceConfigEntities) {
-configEntity.setSelected(true);
-clusterDAO.merge(configEntity);
+if (!duplicatevalues.contains(configEntity)) {
+ configEntity.setSelected(true);
+ clusterDAO.merge(configEntity);
+}
}
} else {
Long configGroupId = serviceConfigEntity.getGroupId();
ambari git commit: AMBARI-21173. Not able to start Yarn services after restoring the configs to initial value.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 b238b367e -> d2bc398ca
AMBARI-21173. Not able to start Yarn services after restoring the configs to
initial value.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2bc398c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2bc398c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2bc398c
Branch: refs/heads/branch-2.6
Commit: d2bc398caf9ce9e8637397681685bc90dd1fe1b3
Parents: b238b36
Author: Vitaly Brodetskyi
Authored: Fri Aug 11 03:03:55 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Sep 8 17:40:03 2017 +0300
--
.../org/apache/ambari/server/state/cluster/ClusterImpl.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2bc398c/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 68e557a..b152928 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1819,12 +1819,18 @@ public class ClusterImpl implements Cluster {
if (serviceConfigEntity.getGroupId() == null) {
Collection configTypes = serviceConfigTypes.get(serviceName);
List enabledConfigs =
clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
+ List serviceConfigEntities =
serviceConfigEntity.getClusterConfigEntities();
+ ArrayList duplicatevalues = new
ArrayList(serviceConfigEntities);
+ duplicatevalues.retainAll(enabledConfigs);
+ enabledConfigs.removeAll(duplicatevalues);
+ serviceConfigEntities.removeAll(duplicatevalues);
+
for (ClusterConfigEntity enabledConfig : enabledConfigs) {
enabledConfig.setSelected(false);
clusterDAO.merge(enabledConfig);
}
- for (ClusterConfigEntity configEntity :
serviceConfigEntity.getClusterConfigEntities()) {
+ for (ClusterConfigEntity configEntity : serviceConfigEntities) {
configEntity.setSelected(true);
clusterDAO.merge(configEntity);
}
ambari git commit: AMBARI-21891. Fix AmbariServerTest.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 8de9b06d9 -> f3232d204
AMBARI-21891. Fix AmbariServerTest.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f3232d20
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f3232d20
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f3232d20
Branch: refs/heads/trunk
Commit: f3232d20436bf3944ae0b8359e71d840a6d132ac
Parents: 8de9b06
Author: Vitaly Brodetskyi
Authored: Wed Sep 6 17:27:51 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Sep 6 17:29:02 2017 +0300
--
.../ambari/server/checks/DatabaseConsistencyCheckHelper.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3232d20/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index d52ea1f..6fef3b8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -314,7 +314,7 @@ public class DatabaseConsistencyCheckHelper {
} else if (tableRowCount != -1 && tableRowCount <
TABLE_ROW_COUNT_LIMIT) {
LOG.info(String.format("The database table %s currently has %d
rows and is within normal limits (%d)", tableName, tableRowCount,
TABLE_ROW_COUNT_LIMIT));
} else {
-throw new SQLException();
+warning("Unable to get size for table {}!", tableName);
}
} catch (SQLException ex) {
error(String.format("Failed to get %s row count: ", tableName), e);
ambari git commit: AMBARI-21891. Fix AmbariServerTest.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 a86e16185 -> 345031f52
AMBARI-21891. Fix AmbariServerTest.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/345031f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/345031f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/345031f5
Branch: refs/heads/branch-2.6
Commit: 345031f526c7bf675866856683863ec368d69d1c
Parents: a86e161
Author: Vitaly Brodetskyi
Authored: Wed Sep 6 16:03:53 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Sep 6 16:03:53 2017 +0300
--
.../ambari/server/checks/DatabaseConsistencyCheckHelper.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/345031f5/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index f5a7203..97da8b1 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -326,7 +326,7 @@ public class DatabaseConsistencyCheckHelper {
} else if (tableRowCount != -1 && tableRowCount <
TABLE_ROW_COUNT_LIMIT) {
LOG.info(String.format("The database table %s currently has %d
rows and is within normal limits (%d)", tableName, tableRowCount,
TABLE_ROW_COUNT_LIMIT));
} else {
-throw new SQLException();
+warning("Unable to get size for table {}!", tableName);
}
} catch (SQLException ex) {
error(String.format("Failed to get %s row count: ", tableName), e);
ambari git commit: AMBARI-21876. Remove hive.custom-extensions.root from Hive Properties.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk e1f4d2e65 -> 202eaed6f
AMBARI-21876. Remove hive.custom-extensions.root from Hive
Properties.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/202eaed6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/202eaed6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/202eaed6
Branch: refs/heads/trunk
Commit: 202eaed6fb98da59d17d4f41d35ce8c576baf48a
Parents: e1f4d2e
Author: Vitaly Brodetskyi
Authored: Wed Sep 6 13:15:05 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Sep 6 13:15:05 2017 +0300
--
.../2.0.6/hooks/before-START/scripts/custom_extensions.py| 3 +--
.../stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml | 8
2 files changed, 1 insertion(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/202eaed6/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py
index 63f6f70..04299ba 100644
---
a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py
+++
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py
@@ -92,8 +92,7 @@ def setup_extensions_hive():
hive_custom_extensions_enabled =
default("/configurations/hive-site/hive.custom-extensions.enabled", False)
hive_custom_extensions_owner =
default("/configurations/hive-site/hive.custom-extensions.owner",
params.hdfs_user)
- hive_custom_extensions_hdfs_dir =
get_config_formatted_value(default("/configurations/hive-site/hive.custom-extensions.root",
-
DEFAULT_HADOOP_HIVE_EXTENSION_DIR.format(params.major_stack_version)))
+ hive_custom_extensions_hdfs_dir =
DEFAULT_HADOOP_HIVE_EXTENSION_DIR.format(params.major_stack_version)
hive_custom_extensions_local_dir =
"{0}/current/ext/hive".format(Script.get_stack_root())
http://git-wip-us.apache.org/repos/asf/ambari/blob/202eaed6/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
index c0800ed..485f829 100644
---
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
+++
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
@@ -33,12 +33,4 @@ limitations under the License.
-
-hive.custom-extensions.root
-/hdp/ext/{{major_stack_version}}/hive
-
-Root directory for hive extensions in HDFS
-
-
-
ambari git commit: AMBARI-21876. Remove hive.custom-extensions.root from Hive Properties.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.6 4a8b85df3 -> 48ad866e3 AMBARI-21876. Remove hive.custom-extensions.root from Hive Properties.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/48ad866e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/48ad866e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/48ad866e Branch: refs/heads/branch-2.6 Commit: 48ad866e37016b60001226a4712b4b3b7fa5ca18 Parents: 4a8b85d Author: Vitaly Brodetskyi Authored: Wed Sep 6 13:06:17 2017 +0300 Committer: Vitaly Brodetskyi Committed: Wed Sep 6 13:06:17 2017 +0300 -- .../stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml | 5 - .../4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 4 +--- .../stacks/BigInsights/4.2/upgrades/config-upgrade.xml | 5 - .../4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 4 +--- .../2.0.6/hooks/before-START/scripts/custom_extensions.py| 3 +-- .../stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml | 8 6 files changed, 3 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/48ad866e/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml -- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml index 4b53c7c..3c6f533 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml @@ -282,11 +282,6 @@ - -hive-site - - - hive-env http://git-wip-us.apache.org/repos/asf/ambari/blob/48ad866e/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml -- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index 48d1d92..ecf57f0 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -344,9 +344,7 @@ - - - + http://git-wip-us.apache.org/repos/asf/ambari/blob/48ad866e/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml -- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml index 29fcac0..c9f6f53 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml @@ -300,11 +300,6 @@ - -hive-site - - - hive-env http://git-wip-us.apache.org/repos/asf/ambari/blob/48ad866e/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml -- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml index 1d4a6f4..a169eac 100644 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml @@ -291,9 +291,7 @@ - - - + http://git-wip-us.apache.org/repos/asf/ambari/blob/48ad866e/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py -- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/custom_extensions.py index 63f6f70..04299ba 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ST
ambari git commit: AMBARI-21843. Database consistency check fails after upgrade to ambari 2.6.0.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 5c3c6280d -> 1c506ff7d
AMBARI-21843. Database consistency check fails after upgrade to ambari
2.6.0.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1c506ff7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1c506ff7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1c506ff7
Branch: refs/heads/trunk
Commit: 1c506ff7da1dc1763bc24a7665077cf920b4a2bc
Parents: 5c3c628
Author: Vitaly Brodetskyi
Authored: Wed Aug 30 02:19:52 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Aug 30 02:21:05 2017 +0300
--
.../server/checks/DatabaseConsistencyCheckHelper.java | 14 +++---
1 file changed, 7 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c506ff7/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index 1e2dac5..d52ea1f 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -317,7 +317,7 @@ public class DatabaseConsistencyCheckHelper {
throw new SQLException();
}
} catch (SQLException ex) {
- LOG.error(String.format("Failed to get %s row count: ", tableName),
e);
+ error(String.format("Failed to get %s row count: ", tableName), e);
}
} finally {
if (rs != null) {
@@ -376,7 +376,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- LOG.error("Exception occurred during check for config selected more than
ones procedure: ", e);
+ error("Exception occurred during check for config selected more than
once procedure: ", e);
} finally {
if (rs != null) {
try {
@@ -425,7 +425,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- LOG.error("Exception occurred during check for host without state
procedure: ", e);
+ error("Exception occurred during check for host without state procedure:
", e);
} finally {
if (rs != null) {
try {
@@ -458,7 +458,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- LOG.error("Exception occurred during topology request tables check: ",
e);
+ error("Exception occurred during topology request tables check: ", e);
} finally {
if (rs != null) {
try {
@@ -539,7 +539,7 @@ public class DatabaseConsistencyCheckHelper {
}
} catch (SQLException e) {
- LOG.error("Exception occurred during check for same count of host
component states and host component desired states: ", e);
+ error("Exception occurred during check for same count of host component
states and host component desired states: ", e);
} finally {
if (rs != null) {
try {
@@ -775,7 +775,7 @@ public class DatabaseConsistencyCheckHelper {
}
}
} catch (SQLException e) {
- LOG.error("Exception occurred during checking MySQL engine to be innodb:
", e);
+ error("Exception occurred during checking MySQL engine to be innodb: ",
e);
} finally {
if (rs != null) {
try {
@@ -1025,7 +1025,7 @@ public class DatabaseConsistencyCheckHelper {
}
}
} catch (SQLException | AmbariException e) {
- LOG.error("Exception occurred during complex service check procedure: ",
e);
+ error("Exception occurred during complex service check procedure: ", e);
} finally {
if (rs != null) {
try {
ambari git commit: AMBARI-21843. Database consistency check fails after upgrade to ambari 2.6.0.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 63989a2d6 -> 43432ae30
AMBARI-21843. Database consistency check fails after upgrade to ambari
2.6.0.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/43432ae3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/43432ae3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/43432ae3
Branch: refs/heads/branch-2.6
Commit: 43432ae30355765fbf982553d7360aa00d7534e1
Parents: 63989a2
Author: Vitaly Brodetskyi
Authored: Wed Aug 30 01:23:31 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Aug 30 01:23:31 2017 +0300
--
.../checks/DatabaseConsistencyCheckHelper.java | 87 +---
.../DatabaseConsistencyCheckHelperTest.java | 56 ++---
2 files changed, 29 insertions(+), 114 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/43432ae3/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index aa96875..f5a7203 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -187,7 +187,6 @@ public class DatabaseConsistencyCheckHelper {
checkSchemaName();
checkMySQLEngine();
checkForConfigsNotMappedToService();
- checkForNotMappedConfigsToCluster();
checkForConfigsSelectedMoreThanOnce();
checkForHostsWithoutState();
checkHostComponentStatesCountEqualsHostComponentsDesiredStates();
@@ -244,49 +243,6 @@ public class DatabaseConsistencyCheckHelper {
LOG.info("DB store version is compatible");
}
- static void checkForNotMappedConfigsToCluster() {
-LOG.info("Checking for configs not mapped to any cluster");
-
-String GET_NOT_MAPPED_CONFIGS_QUERY = "select type_name from clusterconfig
where type_name not in (select type_name from clusterconfigmapping)";
-Set nonSelectedConfigs = new HashSet<>();
-ResultSet rs = null;
-Statement statement = null;
-
-ensureConnection();
-
-try {
- statement = connection.createStatement(ResultSet.TYPE_SCROLL_SENSITIVE,
ResultSet.CONCUR_UPDATABLE);
- rs = statement.executeQuery(GET_NOT_MAPPED_CONFIGS_QUERY);
- if (rs != null) {
-while (rs.next()) {
- nonSelectedConfigs.add(rs.getString("type_name"));
-}
- }
- if (!nonSelectedConfigs.isEmpty()) {
-warning("You have config(s): {} that is(are) not mapped (in
clusterconfigmapping table) to any cluster!",
-nonSelectedConfigs);
- }
-} catch (SQLException e) {
- LOG.error("Exception occurred during check for not mapped configs to
cluster procedure: ", e);
-} finally {
- if (rs != null) {
-try {
- rs.close();
-} catch (SQLException e) {
- LOG.error("Exception occurred during result set closing procedure:
", e);
-}
- }
-
- if (statement != null) {
-try {
- statement.close();
-} catch (SQLException e) {
- LOG.error("Exception occurred during statement closing procedure: ",
e);
-}
- }
-}
- }
-
/**
* This method checks if ambari database has tables with too big size
(according to limit).
* First of all we are trying to get table size from schema information, but
if it's not possible,
@@ -373,7 +329,7 @@ public class DatabaseConsistencyCheckHelper {
throw new SQLException();
}
} catch (SQLException ex) {
- LOG.error(String.format("Failed to get %s row count: ", tableName),
e);
+ error(String.format("Failed to get %s row count: ", tableName), e);
}
} finally {
if (rs != null) {
@@ -397,7 +353,7 @@ public class DatabaseConsistencyCheckHelper {
}
/**
- * This method checks if any config type in clusterconfigmapping table, has
+ * This method checks if any config type in clusterconfig table, has
* more than one versions selected. If config version is selected(in selected
column = 1),
* it means that this version of config is actual. So, if any config type has
more
* than one selected version it's a bug and we are showing error message for
user.
@@ -405,10 +361,10 @@ public class DatabaseConsistencyCheckHelper {
static vo
ambari git commit: AMBARI-21795. Auto fix service config mapping issue discovered during ambari-upgrade.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 698293d2d -> ca5fd7877
AMBARI-21795. Auto fix service config mapping issue discovered during
ambari-upgrade.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca5fd787
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca5fd787
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca5fd787
Branch: refs/heads/trunk
Commit: ca5fd78778345d74d885cd547c44917cf0feb03f
Parents: 698293d
Author: Vitaly Brodetskyi
Authored: Fri Aug 25 22:54:13 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Aug 25 22:54:13 2017 +0300
--
.../checks/DatabaseConsistencyCheckHelper.java | 4 +--
.../orm/entities/ClusterConfigEntity.java | 12 +++
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../server/upgrade/UpgradeCatalog260.java | 38
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 2 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 2 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql| 2 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 2 +-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 2 +-
.../server/upgrade/UpgradeCatalog260Test.java | 9 +
11 files changed, 62 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index aae34ae..1e2dac5 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -570,7 +570,7 @@ public class DatabaseConsistencyCheckHelper {
List notMappedClusterConfigs =
getNotMappedClusterConfigsToService();
for (ClusterConfigEntity clusterConfigEntity : notMappedClusterConfigs){
- if (!clusterConfigEntity.isServiceDeleted()){
+ if (!clusterConfigEntity.isUnmapped()){
continue; // skip clusterConfigs that did not leave after service
deletion
}
List types = new ArrayList<>();
@@ -605,7 +605,7 @@ public class DatabaseConsistencyCheckHelper {
Set nonMappedConfigs = new HashSet<>();
for (ClusterConfigEntity clusterConfigEntity : notMappedClasterConfigs) {
- if (!clusterConfigEntity.isServiceDeleted()){
+ if (!clusterConfigEntity.isUnmapped()){
nonMappedConfigs.add(clusterConfigEntity.getType() + '-' +
clusterConfigEntity.getTag());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
index 3a74367..0c5276c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
@@ -132,8 +132,8 @@ public class ClusterConfigEntity {
@ManyToMany(mappedBy = "clusterConfigEntities")
private Collection serviceConfigEntities;
- @Column(name = "service_deleted", nullable = false, insertable = true,
updatable = true)
- private short serviceDeleted = 0;
+ @Column(name = "unmapped", nullable = false, insertable = true, updatable =
true)
+ private short unmapped = 0;
/**
* Unidirectional one-to-one association to {@link StackEntity}
@@ -142,12 +142,12 @@ public class ClusterConfigEntity {
@JoinColumn(name = "stack_id", unique = false, nullable = false, insertable
= true, updatable = true)
private StackEntity stack;
- public boolean isServiceDeleted() {
-return serviceDeleted != 0;
+ public boolean isUnmapped() {
+return unmapped != 0;
}
- public void setServiceDeleted(boolean serviceDeleted) {
-this.serviceDeleted = (short)(serviceDeleted ? 1 : 0);
+ public void setUnmapped(boolean unmapped) {
+this.unmapped = (short)(unmapped ? 1 : 0);
}
public Long getConfigId() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/ca5fd787/ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java
---
ambari git commit: AMBARI-21795. Auto fix service config mapping issue discovered during ambari-upgrade.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 3946ff677 -> aff9629ae
AMBARI-21795. Auto fix service config mapping issue discovered during
ambari-upgrade.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aff9629a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aff9629a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aff9629a
Branch: refs/heads/branch-2.6
Commit: aff9629ae8e03b4fe635456b6ac332a5afceb6ae
Parents: 3946ff6
Author: Vitaly Brodetskyi
Authored: Fri Aug 25 22:00:03 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Aug 25 22:00:03 2017 +0300
--
.../checks/DatabaseConsistencyCheckHelper.java | 4 +--
.../orm/entities/ClusterConfigEntity.java | 12 +++
.../apache/ambari/server/state/ServiceImpl.java | 2 +-
.../server/upgrade/UpgradeCatalog260.java | 38 +++-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 2 +-
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 2 +-
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 2 +-
.../resources/Ambari-DDL-Postgres-CREATE.sql| 2 +-
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 2 +-
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 2 +-
.../server/upgrade/UpgradeCatalog260Test.java | 9 +
11 files changed, 61 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/aff9629a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
index 9592209..aa96875 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/checks/DatabaseConsistencyCheckHelper.java
@@ -667,7 +667,7 @@ public class DatabaseConsistencyCheckHelper {
List notMappedClusterConfigs =
getNotMappedClusterConfigsToService();
for (ClusterConfigEntity clusterConfigEntity : notMappedClusterConfigs){
- if (!clusterConfigEntity.isServiceDeleted()){
+ if (!clusterConfigEntity.isUnmapped()){
continue; // skip clusterConfigs that did not leave after service
deletion
}
LOG.info("Removing config that is not mapped to any service {}",
clusterConfigEntity);
@@ -698,7 +698,7 @@ public class DatabaseConsistencyCheckHelper {
Set nonMappedConfigs = new HashSet<>();
for (ClusterConfigEntity clusterConfigEntity : notMappedClasterConfigs) {
- if (!clusterConfigEntity.isServiceDeleted()){
+ if (!clusterConfigEntity.isUnmapped()){
nonMappedConfigs.add(clusterConfigEntity.getType() + '-' +
clusterConfigEntity.getTag());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/aff9629a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
index 34f3034..9bf03b3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ClusterConfigEntity.java
@@ -129,8 +129,8 @@ public class ClusterConfigEntity {
@ManyToMany(mappedBy = "clusterConfigEntities")
private Collection serviceConfigEntities;
- @Column(name = "service_deleted", nullable = false, insertable = true,
updatable = true)
- private short serviceDeleted = 0;
+ @Column(name = "unmapped", nullable = false, insertable = true, updatable =
true)
+ private short unmapped = 0;
/**
* Unidirectional one-to-one association to {@link StackEntity}
@@ -139,12 +139,12 @@ public class ClusterConfigEntity {
@JoinColumn(name = "stack_id", unique = false, nullable = false, insertable
= true, updatable = true)
private StackEntity stack;
- public boolean isServiceDeleted() {
-return serviceDeleted != 0;
+ public boolean isUnmapped() {
+return unmapped != 0;
}
- public void setServiceDeleted(boolean serviceDeleted) {
-this.serviceDeleted = (short)(serviceDeleted ? 1 : 0);
+ public void setUnmapped(boolean unmapped) {
+this.unmapped = (short)(unmapped ? 1 : 0);
}
public Long getConfigId() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/aff9629a/ambari
ambari git commit: AMBARI-21173. Not able to start Yarn services after restoring the configs to initial value.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk d69084461 -> 5895040c8
AMBARI-21173. Not able to start Yarn services after restoring the configs to
initial value.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5895040c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5895040c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5895040c
Branch: refs/heads/trunk
Commit: 5895040c8a3374012c11f0e351c8ba631a689ae7
Parents: d690844
Author: Vitaly Brodetskyi
Authored: Mon Aug 21 23:57:12 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Aug 21 23:57:12 2017 +0300
--
.../ambari/server/state/cluster/ClusterImpl.java | 19 +--
1 file changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/5895040c/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 84b411c..26be06e 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1797,22 +1797,29 @@ public class ClusterImpl implements Cluster {
// disable all configs related to service
if (serviceConfigEntity.getGroupId() == null) {
+ // Here was fixed bug with entity changes revert. More you can find here
AMBARI-21173.
+ // This issue reproduces only if you are changing same entity in first
and second loop.
+ // In that case eclipselink will revert changes to cached, if entity has
fluchGroup and it
+ // needs to be refreshed. Actually we don't need to change same antities
in few steps, so i
+ // decided to filter out. duplicates and do not change them. It will be
better for performance and bug will be fixed.
Collection configTypes = serviceConfigTypes.get(serviceName);
List enabledConfigs =
clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
List serviceConfigEntities =
serviceConfigEntity.getClusterConfigEntities();
ArrayList duplicatevalues = new
ArrayList<>(serviceConfigEntities);
duplicatevalues.retainAll(enabledConfigs);
- enabledConfigs.removeAll(duplicatevalues);
- serviceConfigEntities.removeAll(duplicatevalues);
for (ClusterConfigEntity enabledConfig : enabledConfigs) {
-enabledConfig.setSelected(false);
-clusterDAO.merge(enabledConfig);
+if (!duplicatevalues.contains(enabledConfig)) {
+ enabledConfig.setSelected(false);
+ clusterDAO.merge(enabledConfig);
+}
}
for (ClusterConfigEntity configEntity : serviceConfigEntities) {
-configEntity.setSelected(true);
-clusterDAO.merge(configEntity);
+if (!duplicatevalues.contains(configEntity)) {
+ configEntity.setSelected(true);
+ clusterDAO.merge(configEntity);
+}
}
} else {
Long configGroupId = serviceConfigEntity.getGroupId();
ambari git commit: AMBARI-21525. Test Fix. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 496cfe04e -> 1db9c4e99
AMBARI-21525. Test Fix. EU failure: Oozie service check fails if Falcon is not
installed.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1db9c4e9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1db9c4e9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1db9c4e9
Branch: refs/heads/branch-2.6
Commit: 1db9c4e99c68860fa85ef48c743e383c66b62ef7
Parents: 496cfe0
Author: Vitaly Brodetskyi
Authored: Mon Aug 21 14:34:28 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Aug 21 14:34:28 2017 +0300
--
.../stacks/2.5/common/test_stack_advisor.py | 107 ++-
1 file changed, 105 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/1db9c4e9/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
--
diff --git
a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 9f75f9d..fd96a13 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -5892,12 +5892,32 @@ class TestHDP25StackAdvisor(TestCase):
clusterData = {
"components" : []
}
+services = {
+ "services": [
+{
+ "StackServices": {
+"service_name": "OOZIE"
+ }, "components": []
+},
+],
+ "configurations": configurations,
+ "forced-configurations": []
+}
expected = {
- "oozie-site": {"properties":{}},
+ "oozie-site": {"properties":{}, 'property_attributes':
+{'oozie.service.ELService.ext.functions.workflow': {'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-job-submit-instances':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-start':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-job-submit-data':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-sla-submit': {'delete':
'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-create':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-create-inst':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-sla-create': {'delete':
'true'},
+ 'oozie.service.HadoopAccessorService.supported.filesystems':
{'delete': 'true'}}},
"oozie-env": {"properties":{}}
}
-self.stackAdvisor.recommendOozieConfigurations(configurations,
clusterData, {"configurations":{}}, None)
+self.stackAdvisor.recommendOozieConfigurations(configurations,
clusterData, services, None)
self.assertEquals(configurations, expected)
@@ -5928,6 +5948,89 @@ class TestHDP25StackAdvisor(TestCase):
expected = {
"oozie-site": {
"properties": {
+ "oozie.service.ELService.ext.functions.coord-action-create" :
'now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, \
+
today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, \
+
yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, \
+
currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_currentWeek, \
+
lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_lastWeek, \
+
currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth, \
+
lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth, \
+
currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear, \
+
lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear, \
+
latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, \
+
future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, \
+
formatTime=org.apache.oozie.coord.CoordELFunc
ambari git commit: AMBARI-21525. Test Fix. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 6fb371957 -> 13589bbdf
AMBARI-21525. Test Fix. EU failure: Oozie service check fails if Falcon is not
installed.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/13589bbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/13589bbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/13589bbd
Branch: refs/heads/trunk
Commit: 13589bbdf9c22f91740519ef63fcfbf9c2aacc4b
Parents: 6fb3719
Author: Vitaly Brodetskyi
Authored: Mon Aug 21 14:20:48 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Aug 21 14:21:45 2017 +0300
--
.../stacks/2.5/common/test_stack_advisor.py | 107 ++-
1 file changed, 105 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/13589bbd/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
--
diff --git
a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index fc5f220..51d1678 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -5892,12 +5892,32 @@ class TestHDP25StackAdvisor(TestCase):
clusterData = {
"components" : []
}
+services = {
+ "services": [
+{
+ "StackServices": {
+"service_name": "OOZIE"
+ }, "components": []
+},
+],
+ "configurations": configurations,
+ "forced-configurations": []
+}
expected = {
- "oozie-site": {"properties":{}},
+ "oozie-site": {"properties":{}, 'property_attributes':
+{'oozie.service.ELService.ext.functions.workflow': {'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-job-submit-instances':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-start':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-job-submit-data':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-sla-submit': {'delete':
'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-create':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-action-create-inst':
{'delete': 'true'},
+ 'oozie.service.ELService.ext.functions.coord-sla-create': {'delete':
'true'},
+ 'oozie.service.HadoopAccessorService.supported.filesystems':
{'delete': 'true'}}},
"oozie-env": {"properties":{}}
}
-self.stackAdvisor.recommendOozieConfigurations(configurations,
clusterData, {"configurations":{}}, None)
+self.stackAdvisor.recommendOozieConfigurations(configurations,
clusterData, services, None)
self.assertEquals(configurations, expected)
@@ -5928,6 +5948,89 @@ class TestHDP25StackAdvisor(TestCase):
expected = {
"oozie-site": {
"properties": {
+ "oozie.service.ELService.ext.functions.coord-action-create" :
'now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, \
+
today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, \
+
yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, \
+
currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_currentWeek, \
+
lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_lastWeek, \
+
currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth, \
+
lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth, \
+
currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear, \
+
lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear, \
+
latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, \
+
future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, \
+
formatTime=org.apache.oozie.coord.CoordELFunc
ambari git commit: AMBARI-21525. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk 7ffda34d6 -> 09fed274b AMBARI-21525. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/09fed274 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/09fed274 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/09fed274 Branch: refs/heads/trunk Commit: 09fed274ba869e7fb38a91d2795d54f08c132ca5 Parents: 7ffda34 Author: Vitaly Brodetskyi Authored: Fri Aug 18 14:21:56 2017 +0300 Committer: Vitaly Brodetskyi Committed: Fri Aug 18 14:21:56 2017 +0300 -- .../0.5.0.2.1/configuration/oozie-site.xml | 208 --- .../FALCON/configuration/oozie-site.xml | 198 -- .../FALCON/configuration/oozie-site.xml | 197 -- .../stacks/HDP/2.3/services/stack_advisor.py| 113 ++ .../FALCON/configuration/oozie-site.xml | 47 - .../stacks/HDP/2.5/services/stack_advisor.py| 18 ++ 6 files changed, 131 insertions(+), 650 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/09fed274/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml -- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml deleted file mode 100644 index d46069d..000 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - oozie.service.ELService.ext.functions.coord-job-submit-instances - - now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo, - today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo, - currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph1_currentWeek_echo, - lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph1_lastWeek_echo, - currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo, - lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo, - currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo, - lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo, - formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, - latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, - future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo - - - EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. - This property is a convenience property to add extensions to the built in executors without having to - include all the built in ones. - - - custom - - - - -oozie.service.ELService.ext.functions.coord-action-create-inst - - now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst, - today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst, - currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_currentWeek_inst, - lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_lastWeek_inst, - currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst, - lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst, - currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst, - lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst, - latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, - future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, - formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, - user=org.apache.oozie.coord.CoordELFunctions#coord_user - - - EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. - This property is a convenience property to add extensions to the built in executors without having to - include all the built in ones. - - - custom - - - - -oozie.service.ELService.ext.functions.coord-action-create - - now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, - today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, - currentW
ambari git commit: AMBARI-21525. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.6 6dd0feb20 -> 2f4fe911b AMBARI-21525. EU failure: Oozie service check fails if Falcon is not installed.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2f4fe911 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2f4fe911 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2f4fe911 Branch: refs/heads/branch-2.6 Commit: 2f4fe911b80c558df7fed8fcacf342f702bc8c17 Parents: 6dd0feb Author: Vitaly Brodetskyi Authored: Fri Aug 18 14:15:32 2017 +0300 Committer: Vitaly Brodetskyi Committed: Fri Aug 18 14:15:32 2017 +0300 -- .../0.5.0.2.1/configuration/oozie-site.xml | 208 --- .../FALCON/configuration/oozie-site.xml | 198 -- .../FALCON/configuration/oozie-site.xml | 197 -- .../stacks/HDP/2.3/services/stack_advisor.py| 113 ++ .../FALCON/configuration/oozie-site.xml | 47 - .../stacks/HDP/2.5/services/stack_advisor.py| 18 ++ 6 files changed, 131 insertions(+), 650 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/2f4fe911/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml -- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml deleted file mode 100644 index d46069d..000 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/oozie-site.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - oozie.service.ELService.ext.functions.coord-job-submit-instances - - now=org.apache.oozie.extensions.OozieELExtensions#ph1_now_echo, - today=org.apache.oozie.extensions.OozieELExtensions#ph1_today_echo, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph1_yesterday_echo, - currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph1_currentWeek_echo, - lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph1_lastWeek_echo, - currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_currentMonth_echo, - lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph1_lastMonth_echo, - currentYear=org.apache.oozie.extensions.OozieELExtensions#ph1_currentYear_echo, - lastYear=org.apache.oozie.extensions.OozieELExtensions#ph1_lastYear_echo, - formatTime=org.apache.oozie.coord.CoordELFunctions#ph1_coord_formatTime_echo, - latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, - future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo - - - EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. - This property is a convenience property to add extensions to the built in executors without having to - include all the built in ones. - - - custom - - - - -oozie.service.ELService.ext.functions.coord-action-create-inst - - now=org.apache.oozie.extensions.OozieELExtensions#ph2_now_inst, - today=org.apache.oozie.extensions.OozieELExtensions#ph2_today_inst, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday_inst, - currentWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_currentWeek_inst, - lastWeek=org.apache.oozie.extensions.OozieELExtensions#ph2_lastWeek_inst, - currentMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_currentMonth_inst, - lastMonth=org.apache.oozie.extensions.OozieELExtensions#ph2_lastMonth_inst, - currentYear=org.apache.oozie.extensions.OozieELExtensions#ph2_currentYear_inst, - lastYear=org.apache.oozie.extensions.OozieELExtensions#ph2_lastYear_inst, - latest=org.apache.oozie.coord.CoordELFunctions#ph2_coord_latest_echo, - future=org.apache.oozie.coord.CoordELFunctions#ph2_coord_future_echo, - formatTime=org.apache.oozie.coord.CoordELFunctions#ph2_coord_formatTime, - user=org.apache.oozie.coord.CoordELFunctions#coord_user - - - EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD. - This property is a convenience property to add extensions to the built in executors without having to - include all the built in ones. - - - custom - - - - -oozie.service.ELService.ext.functions.coord-action-create - - now=org.apache.oozie.extensions.OozieELExtensions#ph2_now, - today=org.apache.oozie.extensions.OozieELExtensions#ph2_today, - yesterday=org.apache.oozie.extensions.OozieELExtensions#ph2_yesterday, - currentW
ambari git commit: AMBARI-21173. Not able to start Yarn services after restoring the configs to initial value.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk d0168787e -> 86c7f92c3
AMBARI-21173. Not able to start Yarn services after restoring the configs to
initial value.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/86c7f92c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/86c7f92c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/86c7f92c
Branch: refs/heads/trunk
Commit: 86c7f92c31fda6b89eb7c9389072a94abe35d917
Parents: d016878
Author: Vitaly Brodetskyi
Authored: Fri Aug 11 03:03:55 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Aug 11 03:04:36 2017 +0300
--
.../org/apache/ambari/server/state/cluster/ClusterImpl.java | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/86c7f92c/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index c950d67..c191358 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -1806,12 +1806,18 @@ public class ClusterImpl implements Cluster {
if (serviceConfigEntity.getGroupId() == null) {
Collection configTypes = serviceConfigTypes.get(serviceName);
List enabledConfigs =
clusterDAO.getEnabledConfigsByTypes(clusterId, configTypes);
+ List serviceConfigEntities =
serviceConfigEntity.getClusterConfigEntities();
+ ArrayList duplicatevalues = new
ArrayList(serviceConfigEntities);
+ duplicatevalues.retainAll(enabledConfigs);
+ enabledConfigs.removeAll(duplicatevalues);
+ serviceConfigEntities.removeAll(duplicatevalues);
+
for (ClusterConfigEntity enabledConfig : enabledConfigs) {
enabledConfig.setSelected(false);
clusterDAO.merge(enabledConfig);
}
- for (ClusterConfigEntity configEntity :
serviceConfigEntity.getClusterConfigEntities()) {
+ for (ClusterConfigEntity configEntity : serviceConfigEntities) {
configEntity.setSelected(true);
clusterDAO.merge(configEntity);
}
ambari git commit: AMBARI-21570. Migrate custom extension support.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk d9c300bc1 -> 2bd4803c4
AMBARI-21570. Migrate custom extension support.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2bd4803c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2bd4803c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2bd4803c
Branch: refs/heads/trunk
Commit: 2bd4803c45588dbf7ba4f2483c72fcea1b3b6e50
Parents: d9c300b
Author: Vitaly Brodetskyi
Authored: Fri Jul 28 17:43:49 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Jul 28 17:45:10 2017 +0300
--
.../libraries/functions/constants.py| 3 +-
.../libraries/functions/version.py | 17 +-
.../0.1.0/package/scripts/params_linux.py | 6 +
.../0.96.0.2.0/package/scripts/params_linux.py | 3 +
.../2.1.0.2.0/package/scripts/params_linux.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 4 +-
.../4.0.0.2.0/package/scripts/params_linux.py | 3 +
.../RANGER/0.4.0/package/scripts/params.py | 5 +-
.../0.5.0.2.3/package/scripts/params.py | 5 +-
.../SPARK/1.2.1/package/scripts/params.py | 6 +-
.../SPARK2/2.0.0/package/scripts/params.py | 3 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 4 +-
.../0.6.0.2.5/package/scripts/params.py | 3 +-
.../2.0.6/hooks/after-INSTALL/scripts/params.py | 3 +-
.../2.0.6/hooks/before-ANY/scripts/params.py| 4 +
.../before-START/scripts/custom_extensions.py | 174 +++
.../2.0.6/hooks/before-START/scripts/hook.py| 3 +
.../2.0.6/hooks/before-START/scripts/params.py | 12 +-
.../HDP/2.0.6/properties/stack_features.json| 5 +
.../services/HBASE/configuration/hbase-env.xml | 87 ++
.../services/HBASE/configuration/hbase-site.xml | 8 +
.../services/HDFS/configuration/core-site.xml | 29
.../services/HDFS/configuration/hadoop-env.xml | 5 +
.../services/HIVE/configuration/hive-env.xml| 3 +-
.../services/HIVE/configuration/hive-site.xml | 9 +
.../YARN/configuration-mapred/mapred-site.xml | 9 +
.../services/YARN/configuration/yarn-site.xml | 2 +-
27 files changed, 402 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bd4803c/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 3b941ec..6ae71ef 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -50,6 +50,7 @@ class StackFeature:
FALCON_EXTENSIONS = "falcon_extensions"
DATANODE_NON_ROOT = "datanode_non_root"
SECURE_ZOOKEEPER = "secure_zookeeper"
+ HADOOP_CUSTOM_EXTENSIONS = "hadoop_custom_extensions"
REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env"
RANGER = "ranger"
RANGER_TAGSYNC_COMPONENT = "ranger_tagsync_component"
@@ -119,4 +120,4 @@ class StackFeature:
SECURE_RANGER_SSL_PASSWORD = "secure_ranger_ssl_password"
RANGER_KMS_SSL = "ranger_kms_ssl"
KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support"
- ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
\ No newline at end of file
+ ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
http://git-wip-us.apache.org/repos/asf/ambari/blob/2bd4803c/ambari-common/src/main/python/resource_management/libraries/functions/version.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/version.py
b/ambari-common/src/main/python/resource_management/libraries/functions/version.py
index 406bd95..2cf5a1b 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/version.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/version.py
@@ -83,4 +83,19 @@ def compare_versions(version1, version2, format=False):
v2 = version2 if not format else format_stack_version(version2)
max_segments = max(len(v1.split(".")), len(v2.split(".")))
- return cmp(_normalize(v1, desired_segments=max_segments), _normalize(v2,
desired_segments=max_segments))
\ No newline at end of file
+ return cmp(_normalize(v1, desired_segments=max_segments), _normalize(v2,
desired_segments=max_segments))
+
+def get_major_version(full_version):
+ """
+ :param in
ambari git commit: AMBARI-21570. Migrate custom extension support.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 9f6a5a66a -> 0a78ceab8
AMBARI-21570. Migrate custom extension support.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0a78ceab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0a78ceab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0a78ceab
Branch: refs/heads/branch-2.5
Commit: 0a78ceab8922cde53d7fc9b5a477bbad8e870a81
Parents: 9f6a5a6
Author: Vitaly Brodetskyi
Authored: Fri Jul 28 14:40:59 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Jul 28 14:40:59 2017 +0300
--
.../libraries/functions/constants.py| 3 +-
.../libraries/functions/version.py | 17 +-
.../0.1.0/package/scripts/params_linux.py | 6 +
.../0.96.0.2.0/package/scripts/params_linux.py | 2 +
.../2.1.0.2.0/package/scripts/params_linux.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 4 +-
.../4.0.0.2.0/package/scripts/params_linux.py | 2 +
.../RANGER/0.4.0/package/scripts/params.py | 5 +-
.../0.5.0.2.3/package/scripts/params.py | 5 +-
.../SPARK/1.2.1/package/scripts/params.py | 5 +
.../SPARK2/2.0.0/package/scripts/params.py | 3 +-
.../TITAN/1.0.0/package/scripts/params.py | 3 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 4 +-
.../0.6.0.2.5/package/scripts/params.py | 3 +-
.../2.0.6/hooks/after-INSTALL/scripts/params.py | 3 +-
.../2.0.6/hooks/before-ANY/scripts/params.py| 4 +
.../before-START/scripts/custom_extensions.py | 174 +++
.../2.0.6/hooks/before-START/scripts/hook.py| 3 +
.../2.0.6/hooks/before-START/scripts/params.py | 12 +-
.../HDP/2.0.6/properties/stack_features.json| 5 +
.../services/HBASE/configuration/hbase-env.xml | 87 ++
.../services/HBASE/configuration/hbase-site.xml | 8 +
.../services/HDFS/configuration/core-site.xml | 29
.../services/HDFS/configuration/hadoop-env.xml | 5 +
.../services/HIVE/configuration/hive-env.xml| 3 +-
.../services/HIVE/configuration/hive-site.xml | 9 +
.../YARN/configuration-mapred/mapred-site.xml | 9 +
.../services/YARN/configuration/yarn-site.xml | 2 +-
28 files changed, 402 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a78ceab/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 3b941ec..6ae71ef 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -50,6 +50,7 @@ class StackFeature:
FALCON_EXTENSIONS = "falcon_extensions"
DATANODE_NON_ROOT = "datanode_non_root"
SECURE_ZOOKEEPER = "secure_zookeeper"
+ HADOOP_CUSTOM_EXTENSIONS = "hadoop_custom_extensions"
REMOVE_RANGER_HDFS_PLUGIN_ENV = "remove_ranger_hdfs_plugin_env"
RANGER = "ranger"
RANGER_TAGSYNC_COMPONENT = "ranger_tagsync_component"
@@ -119,4 +120,4 @@ class StackFeature:
SECURE_RANGER_SSL_PASSWORD = "secure_ranger_ssl_password"
RANGER_KMS_SSL = "ranger_kms_ssl"
KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support"
- ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
\ No newline at end of file
+ ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a78ceab/ambari-common/src/main/python/resource_management/libraries/functions/version.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/version.py
b/ambari-common/src/main/python/resource_management/libraries/functions/version.py
index 2500430..7dfa6b8 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/version.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/version.py
@@ -74,4 +74,19 @@ def compare_versions(version1, version2, format=False):
v2 = version2 if not format else format_stack_version(version2)
max_segments = max(len(v1.split(".")), len(v2.split(".")))
- return cmp(_normalize(v1, desired_segments=max_segments), _normalize(v2,
desired_segments=max_segments))
\ No newline at end of file
+ return cmp(_normalize(v1, desired_segments=max_segments), _normalize(v2,
desired_segments=max_segments))
+
+def
ambari git commit: AMBARI-21520. Ambari server logs NPE with no additional stack trace on any host component start/stop command.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 4b189a113 -> 30cd71579
AMBARI-21520. Ambari server logs NPE with no additional stack trace on any host
component start/stop command.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/30cd7157
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/30cd7157
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/30cd7157
Branch: refs/heads/trunk
Commit: 30cd715793183e0c00ea7fdb900482cfcdc13a8a
Parents: 4b189a1
Author: Vitaly Brodetskyi
Authored: Wed Jul 19 17:18:27 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Jul 19 17:19:11 2017 +0300
--
.../AmbariManagementControllerImpl.java | 3 ++
.../ambari/server/state/ConfigHelper.java | 29 ++--
2 files changed, 30 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/30cd7157/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 38842fa..fac7b94 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2707,6 +2707,9 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
return requestStages;
}
+// check all stack configs are present in desired configs
+configHelper.checkAllStageConfigsPresentInDesiredConfigs(cluster);
+
// caching upgrade suspended
boolean isUpgradeSuspended = cluster.isUpgradeSuspended();
http://git-wip-us.apache.org/repos/asf/ambari/blob/30cd7157/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 2a70ee1..e8250fe 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -699,7 +699,9 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType =
fileNameToConfigType(stackProperty.getFilename());
-result.put(stackProperty,
actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+if (actualConfigs.containsKey(stackPropertyConfigType)) {
+ result.put(stackProperty,
actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+}
}
}
@@ -776,13 +778,36 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType =
fileNameToConfigType(stackProperty.getFilename());
-
result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+if (actualConfigs.containsKey(stackPropertyConfigType)) {
+
result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+}
}
}
return result;
}
+ public void checkAllStageConfigsPresentInDesiredConfigs(Cluster cluster)
throws AmbariException {
+StackId stackId = cluster.getDesiredStackVersion();
+Set stackConfigTypes =
ambariMetaInfo.getStack(stackId.getStackName(),
+stackId.getStackVersion()).getConfigTypeAttributes().keySet();
+Map actualConfigs = new HashMap<>();
+Map desiredConfigs = cluster.getDesiredConfigs();
+
+for (Map.Entry desiredConfigEntry :
desiredConfigs.entrySet()) {
+ String configType = desiredConfigEntry.getKey();
+ DesiredConfig desiredConfig = desiredConfigEntry.getValue();
+ actualConfigs.put(configType, cluster.getConfig(configType,
desiredConfig.getTag()));
+}
+
+for (String stackConfigType : stackConfigTypes) {
+ if (!actualConfigs.containsKey(stackConfigType)) {
+LOG.error(String.format("Unable to find stack configuration %s in
ambari configs!", stackConfigType));
+ }
+}
+
+ }
+
/***
* Fetch all the config values of a given PropertyType. For eg: Fetch all
stack confi
ambari git commit: AMBARI-21520. Ambari server logs NPE with no additional stack trace on any host component start/stop command.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 7542bc391 -> 2b24cde43
AMBARI-21520. Ambari server logs NPE with no additional stack trace on any host
component start/stop command.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2b24cde4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2b24cde4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2b24cde4
Branch: refs/heads/branch-2.5
Commit: 2b24cde4315212a782caa88294d98e3338c86ce4
Parents: 7542bc3
Author: Vitaly Brodetskyi
Authored: Wed Jul 19 16:38:13 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Jul 19 16:38:13 2017 +0300
--
.../AmbariManagementControllerImpl.java | 4 +++
.../ambari/server/state/ConfigHelper.java | 29 ++--
2 files changed, 31 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b24cde4/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index b80ee98..cccb4b2 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2727,6 +2727,10 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
return requestStages;
}
+// check all stack configs are present in desired configs
+configHelper.checkAllStageConfigsPresentInDesiredConfigs(cluster);
+
+
// caching effective cluster version
ClusterVersionEntity effectiveClusterVersion =
cluster.getEffectiveClusterVersion();
http://git-wip-us.apache.org/repos/asf/ambari/blob/2b24cde4/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index a4c5511..6e60890 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -698,7 +698,9 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType =
fileNameToConfigType(stackProperty.getFilename());
-result.put(stackProperty,
actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+if (actualConfigs.containsKey(stackPropertyConfigType)) {
+ result.put(stackProperty,
actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+}
}
}
@@ -764,13 +766,36 @@ public class ConfigHelper {
for (PropertyInfo stackProperty : stackProperties) {
if (stackProperty.getPropertyTypes().contains(propertyType)) {
String stackPropertyConfigType =
fileNameToConfigType(stackProperty.getFilename());
-
result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+if (actualConfigs.containsKey(stackPropertyConfigType)) {
+
result.add(actualConfigs.get(stackPropertyConfigType).getProperties().get(stackProperty.getName()));
+}
}
}
return result;
}
+ public void checkAllStageConfigsPresentInDesiredConfigs(Cluster cluster)
throws AmbariException {
+StackId stackId = cluster.getDesiredStackVersion();
+Set stackConfigTypes =
ambariMetaInfo.getStack(stackId.getStackName(),
+stackId.getStackVersion()).getConfigTypeAttributes().keySet();
+Map actualConfigs = new HashMap<>();
+Map desiredConfigs = cluster.getDesiredConfigs();
+
+for (Map.Entry desiredConfigEntry :
desiredConfigs.entrySet()) {
+ String configType = desiredConfigEntry.getKey();
+ DesiredConfig desiredConfig = desiredConfigEntry.getValue();
+ actualConfigs.put(configType, cluster.getConfig(configType,
desiredConfig.getTag()));
+}
+
+for (String stackConfigType : stackConfigTypes) {
+ if (!actualConfigs.containsKey(stackConfigType)) {
+LOG.error(String.format("Unable to find stack configuration %s in
ambari configs!", stackConfigType));
+ }
+}
+
+ }
+
/***
* Fetch all the config values of a give
ambari git commit: AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk f450eba5c -> 274a9951c
AMBARI-21504. Restart of MR2 History Server failed due to null in
immutable_paths.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/274a9951
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/274a9951
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/274a9951
Branch: refs/heads/trunk
Commit: 274a9951c34721a867e8e541b4fa73bdf03aa5d0
Parents: f450eba
Author: Vitaly Brodetskyi
Authored: Tue Jul 18 17:14:10 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Tue Jul 18 17:14:10 2017 +0300
--
.../AmbariCustomCommandExecutionHelper.java | 7 +--
.../AmbariManagementControllerImpl.java | 3 ++-
.../internal/ClientConfigResourceProvider.java | 3 ++-
.../ambari/server/state/ConfigHelper.java | 17 +++
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4
.../YARN/2.1.0.2.0/package/scripts/service.py | 4
.../AmbariManagementControllerImplTest.java | 12 ++-
.../ambari/server/state/ConfigHelperTest.java | 22
8 files changed, 67 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index aeb5a9c..5180870 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -88,6 +88,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.Service;
@@ -431,7 +432,8 @@ public class AmbariCustomCommandExecutionHelper {
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set notManagedHdfsPathSet =
configHelper.getPropertyValuesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map notManagedHdfsPathMap =
configHelper.getPropertiesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set notManagedHdfsPathSet =
configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap,
NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -1494,7 +1496,8 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(AGENT_STACK_RETRY_COUNT,
configs.getAgentStackRetryOnInstallCount());
Map desiredConfigs = cluster.getDesiredConfigs();
-Set notManagedHdfsPathSet =
configHelper.getPropertyValuesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+Map notManagedHdfsPathMap =
configHelper.getPropertiesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+Set notManagedHdfsPathSet =
configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap,
NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/274a9951/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 433ed56..38842fa 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2522,7 +2522,8 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
String groupList = gson.toJson(groupSet);
hostParams.put(GROUP_L
ambari git commit: AMBARI-21504. Restart of MR2 History Server failed due to null in immutable_paths.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 d8a5bad1b -> 74cab5067
AMBARI-21504. Restart of MR2 History Server failed due to null in
immutable_paths.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/74cab506
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/74cab506
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/74cab506
Branch: refs/heads/branch-2.5
Commit: 74cab506783515f8b6ce62936b990277a6d26a95
Parents: d8a5bad
Author: Vitaly Brodetskyi
Authored: Tue Jul 18 16:46:54 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Tue Jul 18 16:46:54 2017 +0300
--
.../AmbariCustomCommandExecutionHelper.java | 7 +--
.../AmbariManagementControllerImpl.java | 3 ++-
.../internal/ClientConfigResourceProvider.java | 3 ++-
.../ambari/server/state/ConfigHelper.java | 17 +++
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 4
.../YARN/2.1.0.2.0/package/scripts/service.py | 4
.../AmbariManagementControllerImplTest.java | 12 ++-
.../ambari/server/state/ConfigHelperTest.java | 22
8 files changed, 67 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index e321559..2eca76c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -90,6 +90,7 @@ import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.HostComponentAdminState;
import org.apache.ambari.server.state.HostState;
import org.apache.ambari.server.state.MaintenanceState;
+import org.apache.ambari.server.state.PropertyInfo;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.RepositoryInfo;
import org.apache.ambari.server.state.RepositoryVersionState;
@@ -421,7 +422,8 @@ public class AmbariCustomCommandExecutionHelper {
String groupList = gson.toJson(groupSet);
hostLevelParams.put(GROUP_LIST, groupList);
- Set notManagedHdfsPathSet =
configHelper.getPropertyValuesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Map notManagedHdfsPathMap =
configHelper.getPropertiesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+ Set notManagedHdfsPathSet =
configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap,
NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
@@ -1342,7 +1344,8 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(AGENT_STACK_RETRY_COUNT,
configs.getAgentStackRetryOnInstallCount());
Map desiredConfigs = cluster.getDesiredConfigs();
-Set notManagedHdfsPathSet =
configHelper.getPropertyValuesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+Map notManagedHdfsPathMap =
configHelper.getPropertiesWithPropertyType(stackId,
PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredConfigs);
+Set notManagedHdfsPathSet =
configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap,
NOT_MANAGED_HDFS_PATH_LIST);
String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet);
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/74cab506/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index a34422d..b80ee98 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2541,7 +2541,8 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
String groupList = gson.toJson(group
ambari git commit: AMBARI-21444. Hive warehouse fixes.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 383b8c7d8 -> 31b9d7774
AMBARI-21444. Hive warehouse fixes.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31b9d777
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31b9d777
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31b9d777
Branch: refs/heads/trunk
Commit: 31b9d7774b22f59a4d7120c9836c73a5216fd529
Parents: 383b8c7
Author: Vitaly Brodetskyi
Authored: Wed Jul 12 15:35:53 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Jul 12 15:35:53 2017 +0300
--
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 2 ++
.../services/HIVE/configuration/hive-site.xml | 35
.../stacks/2.0.6/HIVE/test_hive_server.py | 2 ++
4 files changed, 41 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
--
diff --git
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 36725c3..8e176b6 100644
---
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -224,7 +224,8 @@ def setup_hiveserver2():
type="directory",
action="create_on_execute",
owner=params.hive_user,
- mode=0777
+ group=params.user_group,
+ mode=params.hive_apps_whs_mode
)
else:
Logger.info(format("Not creating warehouse directory
'{hive_apps_whs_dir}', as the location is not in DFS."))
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
--
diff --git
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 078076a..21b3d8b 100644
---
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -505,6 +505,8 @@ hive_env_sh_template =
config['configurations']['hive-env']['content']
hive_hdfs_user_dir = format("/user/{hive_user}")
hive_hdfs_user_mode = 0755
+#Parameter for custom warehouse directory permissions. Permissions are in
octal format and need to be converted to decimal
+hive_apps_whs_mode =
int(default('/configurations/hive-site/custom.hive.warehouse.mode', '0777'), 8)
hive_apps_whs_dir =
config['configurations']['hive-site']["hive.metastore.warehouse.dir"]
whs_dir_protocol = urlparse(hive_apps_whs_dir).scheme
hive_exec_scratchdir =
config['configurations']['hive-site']["hive.exec.scratchdir"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
new file mode 100644
index 000..a07c16f
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
@@ -0,0 +1,35 @@
+
+
+
+
+
+hive.warehouse.subdir.inherit.perms
+true
+Set this to true if table directories should inherit the
permissions of the warehouse or database directory instead of being created
with permissions derived from dfs umask
+
+
+
+
+
+hive.start.cleanup.scratchdir
+false
+To cleanup the hive scratchdir while starting the hive server.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/31b9d777/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
--
diff --git
a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_serv
ambari git commit: AMBARI-21444. Hive warehouse fixes.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 37192c9b9 -> 88d93b924
AMBARI-21444. Hive warehouse fixes.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/88d93b92
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/88d93b92
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/88d93b92
Branch: refs/heads/branch-2.5
Commit: 88d93b92429225e03f46cb739a738b9033d45b84
Parents: 37192c9
Author: Vitaly Brodetskyi
Authored: Wed Jul 12 15:07:09 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Wed Jul 12 15:07:09 2017 +0300
--
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 3 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 2 ++
.../services/HIVE/configuration/hive-site.xml | 35
.../stacks/2.0.6/HIVE/test_hive_server.py | 2 ++
4 files changed, 41 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/88d93b92/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
--
diff --git
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 1c53804..0d6e6dc 100644
---
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -188,7 +188,8 @@ def hive(name=None):
type="directory",
action="create_on_execute",
owner=params.hive_user,
-mode=0777
+group=params.user_group,
+mode=params.hive_apps_whs_mode
)
else:
Logger.info(format("Not creating warehouse directory
'{hive_apps_whs_dir}', as the location is not in DFS."))
http://git-wip-us.apache.org/repos/asf/ambari/blob/88d93b92/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
--
diff --git
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index cf116c5..4b595a8 100644
---
a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -506,6 +506,8 @@ hive_env_sh_template =
config['configurations']['hive-env']['content']
hive_hdfs_user_dir = format("/user/{hive_user}")
hive_hdfs_user_mode = 0755
+#Parameter for custom warehouse directory permissions. Permissions are in
octal format and need to be converted to decimal
+hive_apps_whs_mode =
int(default('/configurations/hive-site/custom.hive.warehouse.mode', '0777'), 8)
hive_apps_whs_dir =
config['configurations']['hive-site']["hive.metastore.warehouse.dir"]
whs_dir_protocol = urlparse(hive_apps_whs_dir).scheme
hive_exec_scratchdir =
config['configurations']['hive-site']["hive.exec.scratchdir"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/88d93b92/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
new file mode 100644
index 000..a07c16f
--- /dev/null
+++
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/HIVE/configuration/hive-site.xml
@@ -0,0 +1,35 @@
+
+
+
+
+
+hive.warehouse.subdir.inherit.perms
+true
+Set this to true if table directories should inherit the
permissions of the warehouse or database directory instead of being created
with permissions derived from dfs umask
+
+
+
+
+
+hive.start.cleanup.scratchdir
+false
+To cleanup the hive scratchdir while starting the hive server.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/88d93b92/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
--
diff --git
a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
b/ambari-server/src/test/python/stacks/2.0.6/H
ambari git commit: AMBARI-21427. Assigning hosts concurrently to same config group may fail with "org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config group already exist". (
Repository: ambari
Updated Branches:
refs/heads/trunk 70cf77e40 -> 639f4523f
AMBARI-21427. Assigning hosts concurrently to same config group may fail with
"org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config
group already exist". (stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/639f4523
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/639f4523
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/639f4523
Branch: refs/heads/trunk
Commit: 639f4523fdf49c8e0dddf79074cdb7eb4e43940c
Parents: 70cf77e
Author: Vitaly Brodetskyi
Authored: Tue Jul 11 00:55:59 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Tue Jul 11 00:55:59 2017 +0300
--
.../ambari/server/topology/AmbariContext.java | 81 +++-
1 file changed, 62 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/639f4523/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index 106d7c8..dee0e6c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,6 +30,7 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
import javax.annotation.Nullable;
import javax.inject.Inject;
@@ -69,9 +70,11 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigFactory;
+import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.SecurityType;
+import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
@@ -79,6 +82,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Striped;
+import com.google.inject.Provider;
/**
@@ -99,6 +104,12 @@ public class AmbariContext {
@Inject
ConfigFactory configFactory;
+ /**
+ * Used for getting configuration property values from stack and services.
+ */
+ @Inject
+ private Provider configHelper;
+
private static AmbariManagementController controller;
private static ClusterController clusterController;
//todo: task id's. Use existing mechanism for getting next task id sequence
@@ -112,6 +123,16 @@ public class AmbariContext {
private final static Logger LOG =
LoggerFactory.getLogger(AmbariContext.class);
+
+ /**
+ * When config groups are created using Blueprints these are created when
+ * hosts join a hostgroup and are added to the corresponding config group.
+ * Since hosts join in parallel there might be a race condition in creating
+ * the config group a host is to be added to. Thus we need to synchronize
+ * the creation of config groups with the same name.
+ */
+ private Striped configGroupCreateLock = Striped.lazyWeakLock(1);
+
public boolean isClusterKerberosEnabled(long clusterId) {
Cluster cluster;
try {
@@ -167,9 +188,10 @@ public class AmbariContext {
public void createAmbariResources(ClusterTopology topology, String
clusterName, SecurityType securityType, String repoVersion) {
Stack stack = topology.getBlueprint().getStack();
+StackId stackId = new StackId(stack.getName(), stack.getVersion());
createAmbariClusterResource(clusterName, stack.getName(),
stack.getVersion(), securityType, repoVersion);
-createAmbariServiceAndComponentResources(topology, clusterName);
+createAmbariServiceAndComponentResources(topology, clusterName, stackId,
repoVersion);
}
public void createAmbariClusterResource(String clusterName, String
stackName, String stackVersion, SecurityType securityType, String repoVersion) {
@@ -196,7 +218,8 @@ public class AmbariContext {
}
}
- public void createAmbariServiceAndComponentResources(ClusterTopology
topology, String clusterName) {
+ public vo
ambari git commit: Revert "AMBARI-21427. Assigning hosts concurrently to same config group may fail with "org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config group already e
Repository: ambari
Updated Branches:
refs/heads/trunk 15dd999ff -> 70cf77e40
Revert "AMBARI-21427. Assigning hosts concurrently to same config group may
fail with
"org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config
group already exist". (stoader)"
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/70cf77e4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/70cf77e4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/70cf77e4
Branch: refs/heads/trunk
Commit: 70cf77e4087840e89fab50a741d36bf8747ba416
Parents: 15dd999
Author: Vitaly Brodetskyi
Authored: Mon Jul 10 23:11:38 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Jul 10 23:19:34 2017 +0300
--
.../ambari/server/topology/AmbariContext.java | 81 +---
1 file changed, 19 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/70cf77e4/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index dee0e6c..106d7c8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,7 +30,6 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
import javax.annotation.Nullable;
import javax.inject.Inject;
@@ -70,11 +69,9 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigFactory;
-import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.SecurityType;
-import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
@@ -82,8 +79,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Striped;
-import com.google.inject.Provider;
/**
@@ -104,12 +99,6 @@ public class AmbariContext {
@Inject
ConfigFactory configFactory;
- /**
- * Used for getting configuration property values from stack and services.
- */
- @Inject
- private Provider configHelper;
-
private static AmbariManagementController controller;
private static ClusterController clusterController;
//todo: task id's. Use existing mechanism for getting next task id sequence
@@ -123,16 +112,6 @@ public class AmbariContext {
private final static Logger LOG =
LoggerFactory.getLogger(AmbariContext.class);
-
- /**
- * When config groups are created using Blueprints these are created when
- * hosts join a hostgroup and are added to the corresponding config group.
- * Since hosts join in parallel there might be a race condition in creating
- * the config group a host is to be added to. Thus we need to synchronize
- * the creation of config groups with the same name.
- */
- private Striped configGroupCreateLock = Striped.lazyWeakLock(1);
-
public boolean isClusterKerberosEnabled(long clusterId) {
Cluster cluster;
try {
@@ -188,10 +167,9 @@ public class AmbariContext {
public void createAmbariResources(ClusterTopology topology, String
clusterName, SecurityType securityType, String repoVersion) {
Stack stack = topology.getBlueprint().getStack();
-StackId stackId = new StackId(stack.getName(), stack.getVersion());
createAmbariClusterResource(clusterName, stack.getName(),
stack.getVersion(), securityType, repoVersion);
-createAmbariServiceAndComponentResources(topology, clusterName, stackId,
repoVersion);
+createAmbariServiceAndComponentResources(topology, clusterName);
}
public void createAmbariClusterResource(String clusterName, String
stackName, String stackVersion, SecurityType securityType, String repoVersion) {
@@ -218,8 +196,7 @@ public class AmbariContext {
}
}
- public void createAmbariServiceAndComponentResources(ClusterTopology
topology, String clusterName,
-
ambari git commit: Revert "AMBARI-21427. Assigning hosts concurrently to same config group may fail with "org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config group already e
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 a21fa124b -> e92b50359
Revert "AMBARI-21427. Assigning hosts concurrently to same config group may
fail with
"org.apache.ambari.server.controller.spi.ResourceAlreadyExistsException: Config
group already exist". (stoader)"
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e92b5035
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e92b5035
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e92b5035
Branch: refs/heads/branch-2.5
Commit: e92b50359653636de49718df8cf2eab36b186b37
Parents: a21fa12
Author: Vitaly Brodetskyi
Authored: Mon Jul 10 23:11:38 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Jul 10 23:11:38 2017 +0300
--
.../ambari/server/topology/AmbariContext.java | 81 +---
1 file changed, 19 insertions(+), 62 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/e92b5035/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
index dee0e6c..106d7c8 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,7 +30,6 @@ import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
import javax.annotation.Nullable;
import javax.inject.Inject;
@@ -70,11 +69,9 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigFactory;
-import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.DesiredConfig;
import org.apache.ambari.server.state.Host;
import org.apache.ambari.server.state.SecurityType;
-import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.RetryHelper;
import org.slf4j.Logger;
@@ -82,8 +79,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.Striped;
-import com.google.inject.Provider;
/**
@@ -104,12 +99,6 @@ public class AmbariContext {
@Inject
ConfigFactory configFactory;
- /**
- * Used for getting configuration property values from stack and services.
- */
- @Inject
- private Provider configHelper;
-
private static AmbariManagementController controller;
private static ClusterController clusterController;
//todo: task id's. Use existing mechanism for getting next task id sequence
@@ -123,16 +112,6 @@ public class AmbariContext {
private final static Logger LOG =
LoggerFactory.getLogger(AmbariContext.class);
-
- /**
- * When config groups are created using Blueprints these are created when
- * hosts join a hostgroup and are added to the corresponding config group.
- * Since hosts join in parallel there might be a race condition in creating
- * the config group a host is to be added to. Thus we need to synchronize
- * the creation of config groups with the same name.
- */
- private Striped configGroupCreateLock = Striped.lazyWeakLock(1);
-
public boolean isClusterKerberosEnabled(long clusterId) {
Cluster cluster;
try {
@@ -188,10 +167,9 @@ public class AmbariContext {
public void createAmbariResources(ClusterTopology topology, String
clusterName, SecurityType securityType, String repoVersion) {
Stack stack = topology.getBlueprint().getStack();
-StackId stackId = new StackId(stack.getName(), stack.getVersion());
createAmbariClusterResource(clusterName, stack.getName(),
stack.getVersion(), securityType, repoVersion);
-createAmbariServiceAndComponentResources(topology, clusterName, stackId,
repoVersion);
+createAmbariServiceAndComponentResources(topology, clusterName);
}
public void createAmbariClusterResource(String clusterName, String
stackName, String stackVersion, SecurityType securityType, String repoVersion) {
@@ -218,8 +196,7 @@ public class AmbariContext {
}
}
- public void createAmbariServiceAndComponentResources(ClusterTopology
topology, String cluster
ambari git commit: AMBARI-21317. Config update API should not need to have a unique tag, BE can auto add the tag when it is missing.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 86347182a -> 40e6352b0
AMBARI-21317. Config update API should not need to have a unique tag, BE can
auto add the tag when it is missing.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40e6352b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40e6352b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40e6352b
Branch: refs/heads/trunk
Commit: 40e6352b0e704ca1af7b0e88a267b03bde5cea59
Parents: 8634718
Author: Vitaly Brodetskyi
Authored: Tue Jun 27 22:44:28 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Tue Jun 27 22:44:28 2017 +0300
--
.../controller/AmbariManagementController.java | 4 +++
.../AmbariManagementControllerImpl.java | 29 +++---
.../server/controller/ConfigGroupResponse.java | 10 +++
.../internal/ConfigGroupResourceProvider.java | 31 ++--
.../apache/ambari/server/state/ConfigImpl.java | 3 +-
.../AmbariManagementControllerImplTest.java | 16 --
.../ConfigGroupResourceProviderTest.java| 2 ++
7 files changed, 85 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/40e6352b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
index bb1c95e..f0f13e1 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementController.java
@@ -911,5 +911,9 @@ public interface AmbariManagementController {
*/
QuickLinkVisibilityController getQuicklinkVisibilityController();
+ ConfigGroupResponse getConfigGroupUpdateResults(ConfigGroupRequest
configGroupRequest);
+
+ void saveConfigGroupUpdate(ConfigGroupRequest configGroupRequest,
ConfigGroupResponse configGroupResponse);
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/40e6352b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 6781f65..77883e3 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -65,6 +65,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.TimeUnit;
import javax.persistence.RollbackException;
@@ -346,6 +347,8 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
private Cache clusterUpdateCache =
CacheBuilder.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build();
+ private Cache
configGroupUpdateCache =
+ CacheBuilder.newBuilder().expireAfterWrite(5,
TimeUnit.MINUTES).build();
@Inject
private AmbariCustomCommandExecutionHelper customCommandExecutionHelper;
@@ -1632,6 +1635,15 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
cluster = clusters.getClusterById(request.getClusterId());
}
+List desiredConfigs = request.getDesiredConfig();
+if (desiredConfigs != null) {
+ for (ConfigurationRequest configurationRequest : desiredConfigs) {
+if (StringUtils.isEmpty(configurationRequest.getVersionTag())) {
+ configurationRequest.setVersionTag(UUID.randomUUID().toString());
+}
+ }
+}
+
// Ensure the user has access to update this cluster
AuthorizationHelper.verifyAuthorization(ResourceType.CLUSTER,
cluster.getResourceId(), RoleAuthorization.AUTHORIZATIONS_UPDATE_CLUSTER);
@@ -1640,7 +1652,7 @@ public class AmbariManagementControllerImpl implements
AmbariManagementControlle
new LinkedList<>();
ServiceConfigVersionResponse serviceConfigVersionResponse = null;
-if (request.getDesiredConfig() != null &&
request.getServiceConfigVersionRequest() != null) {
+if (desiredConfigs != null && request.getServiceConfigVersionRequest() !=
null) {
String msg = "Unable to set desired configs and rollback at same time,
request = " + request;
ambari git commit: AMBARI-21263. Ambari 2.5.1 upgrade fails with HDPSearch mpack installed.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/trunk aa33c1b5b -> 31058596b AMBARI-21263. Ambari 2.5.1 upgrade fails with HDPSearch mpack installed.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/31058596 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/31058596 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/31058596 Branch: refs/heads/trunk Commit: 31058596b4ed0cf5b78c464fec2585940051c597 Parents: aa33c1b Author: Vitaly Brodetskyi Authored: Tue Jun 20 22:21:01 2017 +0300 Committer: Vitaly Brodetskyi Committed: Tue Jun 20 22:21:01 2017 +0300 -- .../main/python/ambari_server/serverUpgrade.py | 22 +--- .../src/test/python/TestAmbariServer.py | 35 +++- 2 files changed, 52 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/31058596/ambari-server/src/main/python/ambari_server/serverUpgrade.py -- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index b488ca6..a4a4ae2 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -405,6 +405,7 @@ def find_and_copy_custom_services(resources_dir, services_search_path, old_dir_n old_dir_mask, base_service_dir): services = glob.glob(os.path.join(resources_dir, services_search_path)) managed_services = [] + is_common_services_base_dir = "common-services" in base_service_dir for service in services: if os.path.isdir(service) and not os.path.basename(service) in managed_services: managed_services.append(os.path.basename(service)) @@ -425,10 +426,23 @@ def find_and_copy_custom_services(resources_dir, services_search_path, old_dir_n continue # process dirs only - if os.path.isdir(backup_service) and not os.path.islink(backup_service): -service_name = os.path.basename(backup_service) -if not service_name in managed_services: - shutil.copytree(backup_service, os.path.join(current_base_service_dir,service_name)) + if is_common_services_base_dir: +version_dirs_in_backup_service_dir = glob.glob(os.path.join(backup_service,"*")) +if os.path.isdir(backup_service) and not os.path.islink(backup_service): + service_name = os.path.basename(backup_service) + current_service_dir_path = os.path.join(current_base_service_dir, service_name) + if not service_name in managed_services: +if not os.path.exists(current_service_dir_path): + os.makedirs(current_service_dir_path) +for version_dir_path in version_dirs_in_backup_service_dir: + if not os.path.islink(version_dir_path): +version_dir = os.path.basename(version_dir_path) +shutil.copytree(version_dir_path, os.path.join(current_service_dir_path, version_dir)) + else: +if os.path.isdir(backup_service) and not os.path.islink(backup_service): + service_name = os.path.basename(backup_service) + if not service_name in managed_services: +shutil.copytree(backup_service, os.path.join(current_base_service_dir,service_name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/31058596/ambari-server/src/test/python/TestAmbariServer.py -- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 7f0cb93..8c135c3 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -114,7 +114,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur download_and_install_jdk, prompt_db_properties, setup, \ AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file from ambari_server.serverUpgrade import upgrade, change_objects_owner, \ - run_schema_upgrade, move_user_custom_actions + run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \ generate_random_string, get_cert_info, COMMON_NAME_ATTR, is_valid_cert_exp, NOT_AFTER_ATTR, NOT_BEFORE_ATTR, \ SSL_DATE_FORMAT, import_cert_and_key, is_valid_cert_host, setup_truststor
ambari git commit: AMBARI-21263. Ambari 2.5.1 upgrade fails with HDPSearch mpack installed.(vbrodetskyi)
Repository: ambari Updated Branches: refs/heads/branch-2.5 3dc1811e1 -> e239c8cf2 AMBARI-21263. Ambari 2.5.1 upgrade fails with HDPSearch mpack installed.(vbrodetskyi) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e239c8cf Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e239c8cf Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e239c8cf Branch: refs/heads/branch-2.5 Commit: e239c8cf27f84ad7f3478895cf29f3b1179e234d Parents: 3dc1811 Author: Vitaly Brodetskyi Authored: Tue Jun 20 20:20:31 2017 +0300 Committer: Vitaly Brodetskyi Committed: Tue Jun 20 20:20:31 2017 +0300 -- .../main/python/ambari_server/serverUpgrade.py | 22 +--- .../src/test/python/TestAmbariServer.py | 35 +++- 2 files changed, 52 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/e239c8cf/ambari-server/src/main/python/ambari_server/serverUpgrade.py -- diff --git a/ambari-server/src/main/python/ambari_server/serverUpgrade.py b/ambari-server/src/main/python/ambari_server/serverUpgrade.py index 160c91d..8263062 100644 --- a/ambari-server/src/main/python/ambari_server/serverUpgrade.py +++ b/ambari-server/src/main/python/ambari_server/serverUpgrade.py @@ -545,6 +545,7 @@ def find_and_copy_custom_services(resources_dir, services_search_path, old_dir_n old_dir_mask, base_service_dir): services = glob.glob(os.path.join(resources_dir, services_search_path)) managed_services = [] + is_common_services_base_dir = "common-services" in base_service_dir for service in services: if os.path.isdir(service) and not os.path.basename(service) in managed_services: managed_services.append(os.path.basename(service)) @@ -565,10 +566,23 @@ def find_and_copy_custom_services(resources_dir, services_search_path, old_dir_n continue # process dirs only - if os.path.isdir(backup_service) and not os.path.islink(backup_service): -service_name = os.path.basename(backup_service) -if not service_name in managed_services: - shutil.copytree(backup_service, os.path.join(current_base_service_dir,service_name)) + if is_common_services_base_dir: +version_dirs_in_backup_service_dir = glob.glob(os.path.join(backup_service,"*")) +if os.path.isdir(backup_service) and not os.path.islink(backup_service): + service_name = os.path.basename(backup_service) + current_service_dir_path = os.path.join(current_base_service_dir, service_name) + if not service_name in managed_services: +if not os.path.exists(current_service_dir_path): + os.makedirs(current_service_dir_path) +for version_dir_path in version_dirs_in_backup_service_dir: + if not os.path.islink(version_dir_path): +version_dir = os.path.basename(version_dir_path) +shutil.copytree(version_dir_path, os.path.join(current_service_dir_path, version_dir)) + else: +if os.path.isdir(backup_service) and not os.path.islink(backup_service): + service_name = os.path.basename(backup_service) + if not service_name in managed_services: +shutil.copytree(backup_service, os.path.join(current_base_service_dir,service_name)) http://git-wip-us.apache.org/repos/asf/ambari/blob/e239c8cf/ambari-server/src/test/python/TestAmbariServer.py -- diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py index 6bf5e43..1ac77ab2 100644 --- a/ambari-server/src/test/python/TestAmbariServer.py +++ b/ambari-server/src/test/python/TestAmbariServer.py @@ -114,7 +114,7 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur download_and_install_jdk, prompt_db_properties, setup, \ AmbariUserChecks, AmbariUserChecksLinux, AmbariUserChecksWindows, JDKSetup, reset, setup_jce_policy, expand_jce_zip_file from ambari_server.serverUpgrade import upgrade, upgrade_local_repo, change_objects_owner, upgrade_stack, \ - run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade, move_user_custom_actions + run_stack_upgrade, run_metainfo_upgrade, run_schema_upgrade, move_user_custom_actions, find_and_copy_custom_services from ambari_server.setupHttps import is_valid_https_port, setup_https, import_cert_and_key_action, get_fqdn, \ generate_random_string, get_cert_info, COMMON_NAME_ATTR, is_valid_cert_exp
ambari git commit: AMBARI-21148. Add a flag to indicate NN restart is rolling.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk bd918763c -> 8873e9928
AMBARI-21148. Add a flag to indicate NN restart is rolling.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8873e992
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8873e992
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8873e992
Branch: refs/heads/trunk
Commit: 8873e99288bfb929a0e279b8366e1bfb135a5f91
Parents: bd91876
Author: Vitaly Brodetskyi
Authored: Fri Jun 16 00:32:13 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Jun 16 00:32:13 2017 +0300
--
.../AmbariCustomCommandExecutionHelper.java | 26 +++-
.../AmbariManagementControllerImpl.java | 2 +-
.../internal/UpgradeResourceProvider.java | 8 +++---
.../apache/ambari/server/utils/StageUtils.java | 8 --
.../2.1.0.2.0/package/scripts/hdfs_namenode.py | 10 +---
.../HDFS/2.1.0.2.0/package/scripts/namenode.py | 2 +-
.../2.1.0.2.0/package/scripts/params_linux.py | 4 ++-
.../3.0.0.3.0/package/scripts/hdfs_namenode.py | 10 +---
.../HDFS/3.0.0.3.0/package/scripts/namenode.py | 2 +-
.../3.0.0.3.0/package/scripts/params_linux.py | 2 ++
.../HDP/2.0.6/configuration/cluster-env.xml | 12 +
.../HDP/3.0/configuration/cluster-env.xml | 12 +
12 files changed, 74 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/8873e992/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index 07f6e30..0473690 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -84,6 +84,7 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.CommandScriptDefinition;
import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigHelper;
import org.apache.ambari.server.state.CustomCommandDefinition;
import org.apache.ambari.server.state.DesiredConfig;
@@ -270,8 +271,8 @@ public class AmbariCustomCommandExecutionHelper {
* @throws AmbariException
*/
private void addCustomCommandAction(final ActionExecutionContext
actionExecutionContext,
- final RequestResourceFilter resourceFilter, Stage stage,
- Map additionalCommandParams, String commandDetail)
throws AmbariException {
+ final RequestResourceFilter resourceFilter, Stage stage, Map additionalCommandParams,
+ String commandDetail, Map requestParams) throws
AmbariException {
final String serviceName = resourceFilter.getServiceName();
final String componentName = resourceFilter.getComponentName();
final String commandName = actionExecutionContext.getActionName();
@@ -479,6 +480,19 @@ public class AmbariCustomCommandExecutionHelper {
commandTimeout = Math.max(60, commandTimeout);
}
+ if (requestParams != null && requestParams.containsKey("context")) {
+String requestContext = requestParams.get("context");
+if (StringUtils.isNotEmpty(requestContext) &&
requestContext.toLowerCase().contains("rolling-restart")) {
+ Config clusterEnvConfig =
cluster.getDesiredConfigByType("cluster-env");
+ if (clusterEnvConfig != null) {
+String componentRollingRestartTimeout =
clusterEnvConfig.getProperties().get("namenode_rolling_restart_timeout");
+if (StringUtils.isNotEmpty(componentRollingRestartTimeout)) {
+ commandTimeout =
Integer.parseInt(componentRollingRestartTimeout);
+}
+ }
+}
+ }
+
commandParams.put(COMMAND_TIMEOUT, "" + commandTimeout);
commandParams.put(SERVICE_PACKAGE_FOLDER,
serviceInfo.getServicePackageFolder());
commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
@@ -1038,7 +1052,7 @@ public class AmbariCustomCommandExecutionHelper {
if (!serviceName.equals(Service.Type.HBASE.name()) ||
hostName.equals(primaryCandidate)) {
commandParams.put(UPDATE_EXCLUDE_FILE_ONLY, "false");
-addCustomCommandAction(commandContext, commandFilter, stage,
commandParams, commandDetail.toString
ambari git commit: AMBARI-21237. Add validation and recommendation if hive user value is different than hive.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk a06b6e1d2 -> 874b1f833
AMBARI-21237. Add validation and recommendation if hive user value is different
than hive.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/874b1f83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/874b1f83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/874b1f83
Branch: refs/heads/trunk
Commit: 874b1f8334ba214b4be1eb4ad7394e5ce6abdd73
Parents: a06b6e1
Author: Vitaly Brodetskyi
Authored: Thu Jun 15 14:15:34 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Thu Jun 15 14:15:34 2017 +0300
--
.../services/YARN/configuration/yarn-site.xml | 6
.../stacks/HDP/2.6/services/stack_advisor.py| 36 ++--
2 files changed, 40 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/874b1f83/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index cab0e65..5f8de49 100644
---
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -109,6 +109,12 @@
yarn.nodemanager.kill-escape.user
hive
+
+
+hive-env
+hive_user
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/874b1f83/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 488562b..f8bbca5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -180,6 +180,26 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations,
clusterData, services, hosts)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site",
services)
putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
+servicesList = [service["StackServices"]["service_name"] for service in
services["services"]]
+
+if 'HIVE' in servicesList and "yarn-site" in services["configurations"]
and "yarn.nodemanager.kill-escape.user" in \
+services["configurations"]["yarn-site"]["properties"] and
'hive-env' in services['configurations'] and \
+'hive_user' in
services['configurations']['hive-env']['properties']:
+ hive_user_name =
services['configurations']['hive-env']['properties']['hive_user']
+ old_hive_user_name = getOldValue(self, services, "hive-env", "hive_user")
+ yarn_nm_kill_escape_user =
services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.kill-escape.user"]
+ if not hive_user_name in yarn_nm_kill_escape_user:
+if not yarn_nm_kill_escape_user or yarn_nm_kill_escape_user.strip() ==
"":
+ yarn_nm_kill_escape_user = hive_user_name
+else:
+ escape_user_names = yarn_nm_kill_escape_user.split(",")
+ if old_hive_user_name in escape_user_names:
+escape_user_names.remove(old_hive_user_name)
+ escape_user_names.append(hive_user_name)
+ yarn_nm_kill_escape_user = ",".join(escape_user_names)
+
+putYarnSiteProperty("yarn.nodemanager.kill-escape.user",
yarn_nm_kill_escape_user)
+
if "yarn-site" in services["configurations"] and \
"yarn.resourcemanager.scheduler.monitor.enable" in
services["configurations"]["yarn-site"]["properties"]:
@@ -391,6 +411,18 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
def validateYarnSiteConfigurations(self, properties, recommendedDefaults,
configurations, services, hosts):
validationItems = []
siteProperties = services["configurations"]["yarn-site"]["properties"]
+ servicesList = [service["StackServices"]["serv
ambari git commit: AMBARI-21237. Add validation and recommendation if hive user value is different than hive.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 8fee561b5 -> f0395176c
AMBARI-21237. Add validation and recommendation if hive user value is different
than hive.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f0395176
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f0395176
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f0395176
Branch: refs/heads/branch-2.5
Commit: f0395176c9159d29c3adf112c657e03e8ffd82c9
Parents: 8fee561
Author: Vitaly Brodetskyi
Authored: Thu Jun 15 14:10:01 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Thu Jun 15 14:10:44 2017 +0300
--
.../services/YARN/configuration/yarn-site.xml | 6
.../stacks/HDP/2.6/services/stack_advisor.py| 36 ++--
2 files changed, 40 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0395176/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
index cab0e65..5f8de49 100644
---
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
+++
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml
@@ -109,6 +109,12 @@
yarn.nodemanager.kill-escape.user
hive
+
+
+hive-env
+hive_user
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0395176/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
index 9cd4acf..2b4c14c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py
@@ -181,6 +181,26 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
super(HDP26StackAdvisor, self).recommendYARNConfigurations(configurations,
clusterData, services, hosts)
putYarnSiteProperty = self.putProperty(configurations, "yarn-site",
services)
putYarnEnvProperty = self.putProperty(configurations, "yarn-env", services)
+servicesList = [service["StackServices"]["service_name"] for service in
services["services"]]
+
+if 'HIVE' in servicesList and "yarn-site" in services["configurations"]
and "yarn.nodemanager.kill-escape.user" in \
+services["configurations"]["yarn-site"]["properties"] and
'hive-env' in services['configurations'] and \
+'hive_user' in
services['configurations']['hive-env']['properties']:
+ hive_user_name =
services['configurations']['hive-env']['properties']['hive_user']
+ old_hive_user_name = getOldValue(self, services, "hive-env", "hive_user")
+ yarn_nm_kill_escape_user =
services["configurations"]["yarn-site"]["properties"]["yarn.nodemanager.kill-escape.user"]
+ if not hive_user_name in yarn_nm_kill_escape_user:
+if not yarn_nm_kill_escape_user or yarn_nm_kill_escape_user.strip() ==
"":
+ yarn_nm_kill_escape_user = hive_user_name
+else:
+ escape_user_names = yarn_nm_kill_escape_user.split(",")
+ if old_hive_user_name in escape_user_names:
+escape_user_names.remove(old_hive_user_name)
+ escape_user_names.append(hive_user_name)
+ yarn_nm_kill_escape_user = ",".join(escape_user_names)
+
+putYarnSiteProperty("yarn.nodemanager.kill-escape.user",
yarn_nm_kill_escape_user)
+
if "yarn-site" in services["configurations"] and \
"yarn.resourcemanager.scheduler.monitor.enable" in
services["configurations"]["yarn-site"]["properties"]:
@@ -392,6 +412,18 @@ class HDP26StackAdvisor(HDP25StackAdvisor):
def validateYarnSiteConfigurations(self, properties, recommendedDefaults,
configurations, services, hosts):
validationItems = []
siteProperties = services["configurations"]["yarn-site"]["properties"]
+ servicesList = [service["StackServices&q
ambari git commit: AMBARI-21167. mpack install fails when using https links.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 219f22345 -> c7b774364
AMBARI-21167. mpack install fails when using https links.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c7b77436
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c7b77436
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c7b77436
Branch: refs/heads/trunk
Commit: c7b77436477117050103b35d1e579d02a096d341
Parents: 219f223
Author: Vitaly Brodetskyi
Authored: Mon Jun 12 14:05:12 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Jun 12 14:05:12 2017 +0300
--
.../main/python/ambari_commons/inet_utils.py| 32
.../main/python/ambari_server/setupMpacks.py| 4 +--
2 files changed, 34 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b77436/ambari-common/src/main/python/ambari_commons/inet_utils.py
--
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py
b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 22eaaf5..5fd3e88 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -23,10 +23,18 @@ import time
import sys
import urllib2
import socket
+from ambari_commons import OSCheck
from functools import wraps
from exceptions import FatalException, NonFatalException, TimeoutError
+if OSCheck.is_windows_family():
+ from ambari_commons.os_windows import os_run_os_command
+else:
+ # MacOS not supported
+ from ambari_commons.os_linux import os_run_os_command
+ pass
+
from logging_utils import *
from os_check import OSCheck
@@ -59,6 +67,30 @@ def download_file(link, destination, chunk_size=16 * 1024,
progress_func = None)
force_download_file(link, destination, chunk_size, progress_func =
progress_func)
+def download_file_anyway(link, destination, chunk_size=16 * 1024,
progress_func = None):
+ print_info_msg("Trying to download {0} to {1} with python lib
[urllib2].".format(link, destination))
+ if os.path.exists(destination):
+print_warning_msg("File {0} already exists, assuming it was downloaded
before".format(destination))
+return
+ try:
+force_download_file(link, destination, chunk_size, progress_func =
progress_func)
+ except:
+print_error_msg("Download {0} with python lib [urllib2] failed with error:
{1}".format(link, str(sys.exc_info(
+
+ if not os.path.exists(destination):
+print "Trying to download {0} to {1} with [curl] command.".format(link,
destination)
+#print_info_msg("Trying to download {0} to {1} with [curl]
command.".format(link, destination))
+curl_command = "curl --fail -k -o %s %s" % (destination, link)
+retcode, out, err = os_run_os_command(curl_command)
+if retcode != 0:
+ print_error_msg("Download file {0} with [curl] command failed with
error: {1}".format(link, out + err))
+
+
+ if not os.path.exists(destination):
+print_error_msg("Unable to download file {0}!".format(link))
+print "ERROR: unable to donwload file %s!" % (link)
+
+
def download_progress(file_name, downloaded_size, blockSize, totalSize):
percent = int(downloaded_size * 100 / totalSize)
status = "\r" + file_name
http://git-wip-us.apache.org/repos/asf/ambari/blob/c7b77436/ambari-server/src/main/python/ambari_server/setupMpacks.py
--
diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py
b/ambari-server/src/main/python/ambari_server/setupMpacks.py
index 625e428..6f232f4 100755
--- a/ambari-server/src/main/python/ambari_server/setupMpacks.py
+++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py
@@ -27,7 +27,7 @@ import logging
from ambari_server.serverClassPath import ServerClassPath
from ambari_commons.exceptions import FatalException
-from ambari_commons.inet_utils import download_file
+from ambari_commons.inet_utils import download_file, download_file_anyway
from ambari_commons.logging_utils import print_info_msg, print_error_msg,
print_warning_msg
from ambari_commons.os_utils import copy_file, run_os_command, change_owner,
set_file_permissions
from ambari_server.serverConfiguration import get_ambari_properties,
get_ambari_version, get_stack_location, \
@@ -137,7 +137,7 @@ def download_mpack(mpack_path):
copy_file(mpack_path, tmp_archive_path)
else:
# remote path
-download_file(mpack_path, tmp_archive_path)
+download_file_anyway(mpack_path, tmp_archive_path)
return tmp_archive_path
def expand_mpack(archive_path):
ambari git commit: AMBARI-21167. mpack install fails when using https links.(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 4b87ad4ea -> e1ca8d7ca
AMBARI-21167. mpack install fails when using https links.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1ca8d7c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1ca8d7c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1ca8d7c
Branch: refs/heads/branch-2.5
Commit: e1ca8d7ca0b3a4b3c65e17a51cc72dd7d36ec11a
Parents: 4b87ad4
Author: Vitaly Brodetskyi
Authored: Mon Jun 12 13:55:15 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Mon Jun 12 13:55:15 2017 +0300
--
.../main/python/ambari_commons/inet_utils.py| 32
.../main/python/ambari_server/setupMpacks.py| 4 +--
2 files changed, 34 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/e1ca8d7c/ambari-common/src/main/python/ambari_commons/inet_utils.py
--
diff --git a/ambari-common/src/main/python/ambari_commons/inet_utils.py
b/ambari-common/src/main/python/ambari_commons/inet_utils.py
index 66f6544..0da7f32 100644
--- a/ambari-common/src/main/python/ambari_commons/inet_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/inet_utils.py
@@ -22,10 +22,18 @@ import os
import sys
import urllib2
import socket
+from ambari_commons import OSCheck
from functools import wraps
from exceptions import FatalException, NonFatalException, TimeoutError
+if OSCheck.is_windows_family():
+ from ambari_commons.os_windows import os_run_os_command
+else:
+ # MacOS not supported
+ from ambari_commons.os_linux import os_run_os_command
+ pass
+
from logging_utils import *
from os_check import OSCheck
@@ -58,6 +66,30 @@ def download_file(link, destination, chunk_size=16 * 1024,
progress_func = None)
force_download_file(link, destination, chunk_size, progress_func =
progress_func)
+def download_file_anyway(link, destination, chunk_size=16 * 1024,
progress_func = None):
+ print_info_msg("Trying to download {0} to {1} with python lib
[urllib2].".format(link, destination))
+ if os.path.exists(destination):
+print_warning_msg("File {0} already exists, assuming it was downloaded
before".format(destination))
+return
+ try:
+force_download_file(link, destination, chunk_size, progress_func =
progress_func)
+ except:
+print_error_msg("Download {0} with python lib [urllib2] failed with error:
{1}".format(link, str(sys.exc_info(
+
+ if not os.path.exists(destination):
+print "Trying to download {0} to {1} with [curl] command.".format(link,
destination)
+#print_info_msg("Trying to download {0} to {1} with [curl]
command.".format(link, destination))
+curl_command = "curl --fail -k -o %s %s" % (destination, link)
+retcode, out, err = os_run_os_command(curl_command)
+if retcode != 0:
+ print_error_msg("Download file {0} with [curl] command failed with
error: {1}".format(link, out + err))
+
+
+ if not os.path.exists(destination):
+print_error_msg("Unable to download file {0}!".format(link))
+print "ERROR: unable to donwload file %s!" % (link)
+
+
def download_progress(file_name, downloaded_size, blockSize, totalSize):
percent = int(downloaded_size * 100 / totalSize)
status = "\r" + file_name
http://git-wip-us.apache.org/repos/asf/ambari/blob/e1ca8d7c/ambari-server/src/main/python/ambari_server/setupMpacks.py
--
diff --git a/ambari-server/src/main/python/ambari_server/setupMpacks.py
b/ambari-server/src/main/python/ambari_server/setupMpacks.py
index 917dd9c..6f048d1 100755
--- a/ambari-server/src/main/python/ambari_server/setupMpacks.py
+++ b/ambari-server/src/main/python/ambari_server/setupMpacks.py
@@ -27,7 +27,7 @@ import logging
from ambari_server.serverClassPath import ServerClassPath
from ambari_commons.exceptions import FatalException
-from ambari_commons.inet_utils import download_file
+from ambari_commons.inet_utils import download_file, download_file_anyway
from ambari_commons.logging_utils import print_info_msg, print_error_msg,
print_warning_msg
from ambari_commons.os_utils import copy_file, run_os_command, change_owner,
set_file_permissions
from ambari_server.serverConfiguration import get_ambari_properties,
get_ambari_version, get_stack_location, \
@@ -137,7 +137,7 @@ def download_mpack(mpack_path):
copy_file(mpack_path, tmp_archive_path)
else:
# remote path
-download_file(mpack_path, tmp_archive_path)
+download_file_anyway(mpack_path, tmp_archive_path)
return tmp_archive_path
def expand_mpack(archive_path):
ambari git commit: AMBARI-21176. Oozie server start failed during cluster deploy with "NameError: global name "out" is not defined".(vbrodetskyi)
Repository: ambari
Updated Branches:
refs/heads/trunk 201c6d01b -> 9acb7a644
AMBARI-21176. Oozie server start failed during cluster deploy with "NameError:
global name "out" is not defined".(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9acb7a64
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9acb7a64
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9acb7a64
Branch: refs/heads/trunk
Commit: 9acb7a6d5cc1e0ebc648f0e6413f6f949ac1
Parents: 201c6d0
Author: Vitaly Brodetskyi
Authored: Fri Jun 2 22:01:12 2017 +0300
Committer: Vitaly Brodetskyi
Committed: Fri Jun 2 22:03:22 2017 +0300
--
.../OOZIE/4.0.0.2.0/package/scripts/oozie.py| 9 ++---
.../OOZIE/4.2.0.3.0/package/scripts/oozie.py| 9 ++---
2 files changed, 4 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acb7a64/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
--
diff --git
a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index def0545..d916d3b 100644
---
a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -38,7 +38,7 @@ from resource_management.libraries.resources.xml_config
import XmlConfig
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.security_commons import
update_credential_provider_path
from resource_management.core.resources.packaging import Package
-from resource_management.core.shell import as_user, as_sudo, call
+from resource_management.core.shell import as_user, as_sudo, call, checked_call
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.setup_atlas_hook import
has_atlas_in_cluster, setup_atlas_hook
@@ -409,12 +409,7 @@ def
copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
# This can return over 100 files, so take the first 5 lines after "Available
ShareLib"
# Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env
does not export OOZIE_URL
command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie
{oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5')
- Execute(command,
- user=params.oozie_user,
- tries=10,
- try_sleep=5,
- logoutput=True,
- )
+ code, out = checked_call(command, user=params.oozie_user, tries=10,
try_sleep=5, logoutput=True)
hive_sharelib_dir = __parse_sharelib_from_output(out)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9acb7a64/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
--
diff --git
a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
index def0545..d916d3b 100644
---
a/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
+++
b/ambari-server/src/main/resources/common-services/OOZIE/4.2.0.3.0/package/scripts/oozie.py
@@ -38,7 +38,7 @@ from resource_management.libraries.resources.xml_config
import XmlConfig
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.security_commons import
update_credential_provider_path
from resource_management.core.resources.packaging import Package
-from resource_management.core.shell import as_user, as_sudo, call
+from resource_management.core.shell import as_user, as_sudo, call, checked_call
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.setup_atlas_hook import
has_atlas_in_cluster, setup_atlas_hook
@@ -409,12 +409,7 @@ def
copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
# This can return over 100 files, so take the first 5 lines after "Available
ShareLib"
# Use -oozie http(s):localhost:{oozie_server_admin_port}/oozie as oozie-env
does not export OOZIE_URL
command = format(r'source {conf_dir}/oozie-env.sh ; oozie admin -oozie
{oozie_base_url} -shareliblist hive | grep "\[Available ShareLib\]" -A 5')
- Execute(command,
- user=params.oozie_user,
- tries=10,
- try
