ambari git commit: Revert "AMBARI-22476. Modify bootstrap API to pass Ambari repo URL from UI and retrieve Host OS types (Sinia Garudi via ncole)"
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-21674 51a7baeb3 -> d9aaa07bc
Revert "AMBARI-22476. Modify bootstrap API to pass Ambari repo URL from UI and
retrieve Host OS types (Sinia Garudi via ncole)"
This reverts commit 95164bccec30af3ab21382040376b76ac05433b4.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9aaa07b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9aaa07b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9aaa07b
Branch: refs/heads/branch-feature-AMBARI-21674
Commit: d9aaa07bc6b0943bed1366096dee5f010b44f7d9
Parents: 51a7bae
Author: Nate Cole
Authored: Wed Dec 20 14:30:58 2017 -0500
Committer: Nate Cole
Committed: Wed Dec 20 14:30:58 2017 -0500
--
.../ambari/server/bootstrap/BSRunner.java | 3 +-
.../ambari/server/bootstrap/SshHostInfo.java| 11 ---
ambari-server/src/main/python/bootstrap.py | 8 +-
.../ambari/server/bootstrap/BootStrapTest.java | 2 -
ambari-server/src/test/python/TestBootstrap.py | 64 +++---
.../app/controllers/wizard/step3_controller.js | 59 +
.../test/controllers/wizard/step3_test.js | 91 +---
7 files changed, 40 insertions(+), 198 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9aaa07b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java
index 2cd061c..c7976ee 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java
@@ -202,7 +202,7 @@ class BSRunner extends Thread {
sshPort = DEFAULT_SSHPORT;
}
-String command[] = new String[14];
+String command[] = new String[13];
BSStat stat = BSStat.RUNNING;
String scriptlog = "";
try {
@@ -242,7 +242,6 @@ class BSRunner extends Thread {
command[10] = this.serverPort+"";
command[11] = userRunAs;
command[12] = (this.passwordFile==null) ? "null" :
this.passwordFile.toString();
- command[13] = this.sshHostInfo.getAmbariRepoUrls();
Map envVariables = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9aaa07b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java
index 6218fb6..86888f4 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java
@@ -57,9 +57,6 @@ public class SshHostInfo {
@XmlElement
private String userRunAs;
- @XmlElement
- private String ambariRepoUrls;
-
public String getSshKey() {
return sshKey;
}
@@ -122,12 +119,4 @@ public class SshHostInfo {
}
return ret.toString();
}
-
- public String getAmbariRepoUrls() {
-return ambariRepoUrls;
- }
-
- public void setAmbariRepoUrls(String ambariRepoUrls) {
-this.ambariRepoUrls = ambariRepoUrls;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9aaa07b/ambari-server/src/main/python/bootstrap.py
--
diff --git a/ambari-server/src/main/python/bootstrap.py
b/ambari-server/src/main/python/bootstrap.py
index b949bc0..f1c53ce 100755
--- a/ambari-server/src/main/python/bootstrap.py
+++ b/ambari-server/src/main/python/bootstrap.py
@@ -844,7 +844,7 @@ class PBootstrap:
class SharedState:
def __init__(self, user, sshPort, sshkey_file, script_dir, boottmpdir,
setup_agent_file,
ambari_server, cluster_os_type, ambari_version, server_port,
- user_run_as, ambariRepoUrls, password_file = None):
+ user_run_as, password_file = None):
self.hostlist_to_remove_password_file = None
self.user = user
self.sshPort = sshPort
@@ -859,7 +859,6 @@ class SharedState:
self.password_file = password_file
self.statuses = None
self.server_port = server_port
-self.ambariRepoUrls = ambariRepoUrls
self.remote_files = {}
self.ret = {}
pass
@@ -889,7 +888,6 @@ def main(argv=None):
server_port = onlyargs[9]
user_run_as = onlyargs[10]
passwordFile = onlyargs[11]
- ambariRepoUrls = onlyargs[12]
if not OSCheck.is_windows_family():
# ssh doesn't like open files
@@ -902,10 +900,10
ambari git commit: Revert "UI changes for prompting user during registration failure for new OS family (Sonia Garudi via ncole)"
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-21674 d9aaa07bc -> 37361fd3f
Revert "UI changes for prompting user during registration failure for new OS
family (Sonia Garudi via ncole)"
This reverts commit d43882d1aa55dc2f7e358fec176641b57a6c71f6.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/37361fd3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/37361fd3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/37361fd3
Branch: refs/heads/branch-feature-AMBARI-21674
Commit: 37361fd3fa3232de10cd9d4614c4f036dafb2668
Parents: d9aaa07
Author: Nate Cole
Authored: Wed Dec 20 14:31:19 2017 -0500
Committer: Nate Cole
Committed: Wed Dec 20 14:31:19 2017 -0500
--
.../ambari/server/bootstrap/BSHostStatus.java | 9 --
.../server/bootstrap/BSHostStatusCollector.java | 7 -
.../app/controllers/wizard/step3_controller.js | 114 ---
ambari-web/app/messages.js | 3 -
ambari-web/app/styles/wizard.less | 22 ---
ambari-web/app/templates/wizard/step3.hbs | 47 ---
.../test/controllers/wizard/step3_test.js | 140 ---
7 files changed, 342 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/37361fd3/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatus.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatus.java
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatus.java
index a91a1f2..3d1b31c 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatus.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatus.java
@@ -41,16 +41,7 @@ public class BSHostStatus {
private String statusAction;
@XmlElement
private String log;
- @XmlElement
- private String osType;
-
- public String getOsType() {
-return osType;
- }
- public void setOsType(String osType) {
-this.osType = osType;
- }
public void setStatus(String status) {
this.status = status;
http://git-wip-us.apache.org/repos/asf/ambari/blob/37361fd3/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatusCollector.java
--
diff --git
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatusCollector.java
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatusCollector.java
index 45e0870..b72ca20 100644
---
a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatusCollector.java
+++
b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSHostStatusCollector.java
@@ -79,13 +79,6 @@ class BSHostStatusCollector {
if (statusCode.equals("0")) {
status.setStatus("DONE");
}
- //the status code 44 is returned if ambari-repo property isn't set
for the host's os_type in ambari.properties file.
- //'44:' is written to the .done file for the respective
host.
- if (statusCode.startsWith("44")) {
-String[] sc = statusCode.split(":");
-status.setOsType(sc[1]);
-statusCode = sc[0];
- }
updateStatus(status, statusCode);
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/37361fd3/ambari-web/app/controllers/wizard/step3_controller.js
--
diff --git a/ambari-web/app/controllers/wizard/step3_controller.js
b/ambari-web/app/controllers/wizard/step3_controller.js
index 5ff2cdc..29393e3 100644
--- a/ambari-web/app/controllers/wizard/step3_controller.js
+++ b/ambari-web/app/controllers/wizard/step3_controller.js
@@ -184,9 +184,6 @@ App.WizardStep3Controller =
Em.Controller.extend(App.ReloadPopupMixin, App.Check
this.set('isLoaded', false);
this.set('isSubmitDisabled', true);
this.set('stopChecking', false);
-this.set('newAmbariOsTypes', []);
-this.set('promptAmbariRepoUrl', false);
-this.set('bootstrapInProgress', false);
},
/**
@@ -304,24 +301,6 @@ App.WizardStep3Controller =
Em.Controller.extend(App.ReloadPopupMixin, App.Check
return App.showConfirmationPopup(function () {
App.router.send('removeHosts', hosts);
self.hosts.removeObjects(hosts);
- hosts.forEach(function(_host) {
-var ambariOsTypeIndex = -1;
-self.newAmbariOsTypes.some(function(os, index) {
- if (os.hosts.contains(_host.name)){
-ambariOsTypeIndex = index;
-return true;
- }
-});
-if (ambariOsTypeIndex != -1) {
-
[1/2] ambari git commit: AMBARI-22630. Update the default storm log search configuration (use wildcards) (oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-2.6 e1d95ff63 -> 0cf1defb1
AMBARI-22630. Update the default storm log search configuration (use wildcards)
(oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0cf1defb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0cf1defb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0cf1defb
Branch: refs/heads/branch-2.6
Commit: 0cf1defb1a3fc62a752ca1287393d33290b6d8f6
Parents: ab71b5a
Author: Oliver Szabo
Authored: Tue Dec 12 16:59:33 2017 +0100
Committer: Oliver Szabo
Committed: Wed Dec 20 18:03:23 2017 +0100
--
.../configuration/storm-logsearch-conf.xml | 104 +--
1 file changed, 75 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/0cf1defb/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
--
diff --git
a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
index d485017..0c3e38d 100644
---
a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
+++
b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml
@@ -65,46 +65,92 @@
"type":"storm_ui",
"rowtype":"service",
"path":"{{default('/configurations/storm-env/storm_log_dir',
'/var/log/storm')}}/ui.log"
-},
-{
+}
+{#
+,{
"type":"storm_worker",
"rowtype":"service",
- "path":"{{default('/configurations/storm-env/storm_log_dir',
'/var/log/storm')}}/*worker*.log"
+ "path":"{{default('/configurations/storm-env/storm_log_dir',
'/var/log/storm')}}/workers-artifacts/*/*/worker.log",
+ "init_default_fields": "true"
}
+#}
],
- "filter":[
-{
- "filter":"grok",
- "conditions":{
-"fields":{
- "type":[
-"storm_drpc",
-"storm_logviewer",
-"storm_nimbus",
-"storm_supervisor",
-"storm_ui",
-"storm_worker"
- ]
+ "filter":[
+ {
+ "filter":"grok",
+ "sort_order": 1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_nimbus",
+ "storm_supervisor",
+ "storm_logviewer",
+ "storm_drpc",
+ "storm_ui",
+ "storm_worker"
+ ]
+ }
+ },
+ "log4j_format":"",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+
"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}\\s%{GREEDYDATA:thread_name}\\s\\[%{LOGLEVEL:level}\\]\\s%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"-MM-dd HH:mm:ss.SSS"
+ }
+ }
}
},
- "log4j_format":"",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
-
"message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}",
- "post_map_values":{
-"logtime":{
- "map_date":{
-"target_date_pattern":"-MM-dd HH:mm:ss.SSS"
- }
- }
+ {
+ "filter":"grok",
+ "sort_order": 2,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "thread_name",
+ "remove_source_field": "false",
+
"message_pattern":"(Thread\\-[0-9]+\\-[0-9]+\\-%{DATA:sdi_streamline_component_name}\\-executor%{DATA}|%{DATA:thread_name})"
+ },
+ {
+ "filter":"grok",
+ "sort_order": 3,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "path",
+ "remove_source_field": "false",
+
"message_pattern":"{{default('/configurations/storm-env/storm_log_dir',
'/var/log/storm')}}/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/worker\\.log"
+ },
+ {
+ "filter":"grok",
+ "sort_order": 4,
+ "conditions":{
+ "fields":{
+ "type":[
+ "storm_worker"
+ ]
+ }
+ },
+ "source_field": "sdi_storm_topology_id",
+ "remove_source_field": "false",
+
"mess
[2/2] ambari git commit: AMBARI-22633. MapDate provides the date incorrectly when Filter is cloned and used in multi-threads (Jungtaek Lim via oleewere)
AMBARI-22633. MapDate provides the date incorrectly when Filter is cloned and
used in multi-threads (Jungtaek Lim via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ab71b5a4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ab71b5a4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ab71b5a4
Branch: refs/heads/branch-2.6
Commit: ab71b5a4153c10936aacdbf581a8481ad7929432
Parents: e1d95ff
Author: Jungtaek Lim
Authored: Tue Dec 12 14:50:39 2017 +0100
Committer: Oliver Szabo
Committed: Wed Dec 20 18:03:23 2017 +0100
--
ambari-logsearch/ambari-logsearch-logfeeder/pom.xml | 5 +
.../apache/ambari/logfeeder/mapper/MapperDate.java| 14 +++---
2 files changed, 12 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab71b5a4/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
--
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index a06ae90..db2923e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -88,6 +88,11 @@
commons-logging
1.1.1
+
+ org.apache.commons
+ commons-lang3
+ 3.6
+
com.google.guava
http://git-wip-us.apache.org/repos/asf/ambari/blob/ab71b5a4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
--
diff --git
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 6a7fad7..dca3949 100644
---
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -19,7 +19,6 @@
package org.apache.ambari.logfeeder.mapper;
-import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Map;
@@ -28,15 +27,16 @@ import
org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
public class MapperDate extends Mapper {
private static final Logger LOG = Logger.getLogger(MapperDate.class);
- private SimpleDateFormat targetDateFormatter = null;
+ private FastDateFormat targetDateFormatter = null;
private boolean isEpoch = false;
- private SimpleDateFormat srcDateFormatter=null;
+ private FastDateFormat srcDateFormatter=null;
@Override
public boolean init(String inputDesc, String fieldName, String mapClassCode,
Object mapConfigs) {
@@ -61,9 +61,9 @@ public class MapperDate extends Mapper {
return true;
} else {
try {
- targetDateFormatter = new SimpleDateFormat(targetDateFormat);
+ targetDateFormatter = FastDateFormat.getInstance(targetDateFormat);
if (!StringUtils.isEmpty(srcDateFormat)) {
-srcDateFormatter = new SimpleDateFormat(srcDateFormat);
+srcDateFormatter = FastDateFormat.getInstance(srcDateFormat);
}
return true;
} catch (Throwable ex) {
@@ -86,7 +86,7 @@ public class MapperDate extends Mapper {
if (srcDateFormatter != null) {
Date srcDate = srcDateFormatter.parse(value.toString());
//set year in src_date when src_date does not have year component
-if (!srcDateFormatter.toPattern().contains("yy")) {
+if (!srcDateFormatter.getPattern().contains("yy")) {
Calendar currentCalendar = Calendar.getInstance();
Calendar logDateCalendar = Calendar.getInstance();
logDateCalendar.setTimeInMillis(srcDate.getTime());
@@ -110,7 +110,7 @@ public class MapperDate extends Mapper {
jsonObj.put(fieldName, value);
} catch (Throwable t) {
LogFeederUtil.logErrorMessageByInterval(this.getClass().getSimpleName() +
":apply", "Error applying date transformation." +
-" isEpoch=" + isEpoch + ", targetateFormat=" +
(targetDateFormatter!=null ?targetDateFormatter.toPattern():"")
+" isEpoch=" + isEpoch + ", targetDateFormat=" +
(targetDateFormatter!=null ?targetDateFormatter.getPattern():"")
+ ", value=" + value + ". " + thi
ambari git commit: AMBARI-22677. Addendum - Don't call pprint.pformat unnecessarily in Ambari agent (amagyar)
Repository: ambari
Updated Branches:
refs/heads/trunk 4e731d31f -> 9ea3fa154
AMBARI-22677. Addendum - Don't call pprint.pformat unnecessarily in Ambari
agent (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9ea3fa15
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9ea3fa15
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9ea3fa15
Branch: refs/heads/trunk
Commit: 9ea3fa154adb1328000bfb8170ea93b7faa66783
Parents: 4e731d3
Author: Attila Magyar
Authored: Wed Dec 20 17:34:45 2017 +0100
Committer: Attila Magyar
Committed: Wed Dec 20 17:34:45 2017 +0100
--
ambari-agent/src/main/python/ambari_agent/Controller.py | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/9ea3fa15/ambari-agent/src/main/python/ambari_agent/Controller.py
--
diff --git a/ambari-agent/src/main/python/ambari_agent/Controller.py
b/ambari-agent/src/main/python/ambari_agent/Controller.py
index ef50440..e4f148f 100644
--- a/ambari-agent/src/main/python/ambari_agent/Controller.py
+++ b/ambari-agent/src/main/python/ambari_agent/Controller.py
@@ -164,14 +164,12 @@ class Controller(threading.Thread):
while not self.isRegistered:
try:
data = json.dumps(self.register.build(self.version))
-prettyData = pprint.pformat(data)
-
try:
server_ip = socket.gethostbyname(self.hostname)
- logger.info("Registering with %s (%s) (agent=%s)", self.hostname,
server_ip, prettyData)
+ logger.info("Registering with %s (%s) (agent=%s)", self.hostname,
server_ip, data)
except socket.error:
logger.warn("Unable to determine the IP address of '%s', agent
registration may fail (agent=%s)",
- self.hostname, prettyData)
+ self.hostname, data)
ret = self.sendRequest(self.registerUrl, data)
[2/5] ambari git commit: AMBARI-22644 - Node Managers fail to start after Spark2 is patched due to CNF YarnShuffleService (jonathanhurley)
AMBARI-22644 - Node Managers fail to start after Spark2 is patched due to CNF
YarnShuffleService (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cd880fd1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cd880fd1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cd880fd1
Branch: refs/heads/branch-2.6
Commit: cd880fd1f25394518c9e1643cc6d9f62bb43c174
Parents: edac2c6
Author: Jonathan Hurley
Authored: Wed Dec 13 12:33:26 2017 -0500
Committer: Jonathan Hurley
Committed: Wed Dec 13 12:33:26 2017 -0500
--
.../2.1.0.2.0/package/scripts/params_linux.py| 19 ---
.../services/YARN/configuration/yarn-site.xml| 4 ++--
.../stacks/HDP/2.6/upgrades/config-upgrade.xml | 2 --
3 files changed, 14 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd880fd1/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
--
diff --git
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 6b5559c..1efe53b 100644
---
a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -89,16 +89,20 @@ def get_spark_version(service_name, component_name,
yarn_version):
"""
Attempts to calculate the correct version placeholder value for spark or
spark2 based on
what is installed in the cluster. If Spark is not installed, then this value
will need to be
- that of YARN so it can still find the correct shuffle class.
+ that of YARN so it can still find the correct spark class.
On cluster installs, we have not yet calcualted any versions and all known
values could be None.
This doesn't affect daemons, but it does affect client-only hosts where they
will never receive
a start command after install. Therefore, this function will attempt to use
stack-select as a
last resort to get a value value.
+
+ ATS needs this since it relies on packages installed by Spark. Some classes,
like the shuffle
+ classes, are not provided by spark, but by a dependent RPM to YARN, so they
do not use this
+ value.
:param service_name: the service name (SPARK, SPARK2, etc)
:param component_name: the component name (SPARK_CLIENT, etc)
:param yarn_version: the default version of Yarn to use if no spark is
installed
- :return: a value for the version placeholder in shuffle classpath properties
+ :return: a value for the version placeholder in spark classpath properties
"""
# start off seeing if we need to populate a default value for YARN
if yarn_version is None:
@@ -106,18 +110,19 @@ def get_spark_version(service_name, component_name,
yarn_version):
component_name = "YARN_CLIENT")
# now try to get the version of spark/spark2, defaulting to the version if
YARN
- shuffle_classpath_version =
component_version.get_component_repository_version(service_name = service_name,
+ spark_classpath_version =
component_version.get_component_repository_version(service_name = service_name,
component_name = component_name, default_value = yarn_version)
# even with the default of using YARN's version, on an install this might be
None since we haven't
# calculated the version of YARN yet - use stack_select as a last ditch
effort
- if shuffle_classpath_version is None:
+ if spark_classpath_version is None:
try:
- shuffle_classpath_version =
stack_select.get_role_component_current_stack_version()
+ spark_classpath_version =
stack_select.get_role_component_current_stack_version()
except:
- Logger.exception("Unable to query for the correct shuffle classpath")
+ Logger.exception("Unable to query for the correct spark version to use
when building classpaths")
+
+ return spark_classpath_version
- return shuffle_classpath_version
# these are used to render the classpath for picking up Spark classes
# in the event that spark is not installed, then we must default to the
vesrion of YARN installed
http://git-wip-us.apache.org/repos/asf/ambari/blob/cd880fd1/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
--
diff --git
a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
index 29833fb..b6fadcb 100644
---
a/ambari-server/src/main/r
[4/5] ambari git commit: AMBARI-22522 - Livy server fails to start during downgrade due to absence of 'conf' directory (Dmitro Lisnichenko via jonathanhurley)
AMBARI-22522 - Livy server fails to start during downgrade due to absence of
'conf' directory (Dmitro Lisnichenko via jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b7c6ea96
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b7c6ea96
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b7c6ea96
Branch: refs/heads/branch-2.6
Commit: b7c6ea96e60927020a54883dc319bcb5ce88c10e
Parents: 87ef706
Author: Jonathan Hurley
Authored: Sun Dec 17 20:48:31 2017 -0500
Committer: Jonathan Hurley
Committed: Sun Dec 17 20:48:31 2017 -0500
--
.../custom_actions/scripts/install_packages.py | 37 +++-
1 file changed, 36 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/b7c6ea96/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
--
diff --git
a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index c8497cd..cb1b7b0 100644
---
a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++
b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -31,7 +31,6 @@ from resource_management.core.resources import Package
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.stack_select import
get_stack_versions
-from resource_management.libraries.functions.version import
format_stack_version
from resource_management.libraries.functions.repo_version_history \
import read_actual_version_from_history_file,
write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE
from resource_management.core.providers import get_provider
@@ -153,11 +152,47 @@ class InstallPackages(Script):
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
+self._fix_default_links("livy", "livy-server")
+self._fix_default_links("livy2", "livy2-server")
+
# if installing a version of HDP that needs some symlink love, then create
them
if is_package_install_successful and 'actual_version' in
self.structured_output:
self._relink_configurations_with_conf_select(stack_id,
self.structured_output['actual_version'])
+ def _fix_default_links(self, package_name, component_name):
+"""
+If a prior version of Ambari did not correctly reverse the conf symlinks,
then they would
+be put into a bad state when distributing a new stack. For example:
+
+/etc/component/conf (directory)
+/v1/component/conf -> /etc/component/conf
+
+When distributing v2, we'd detect the /etc/component/conf problems and
would try to adjust it:
+/etc/component/conf -> /current/component/conf
+/v2/component/conf -> /etc/component/v2/0
+
+The problem is that v1 never gets changed (since the stack being
distributed is v2), and
+we end up with a circular link:
+/etc/component/conf -> /current/component/conf
+/v1/component/conf -> /etc/component/conf
+
+:return: None
+"""
+from resource_management.libraries.functions import stack_select
+package_dirs = conf_select.get_package_dirs()
+if package_name in package_dirs:
+ Logger.info("Attempting to fix the default conf links for
{0}".format(package_name))
+
+ directories = package_dirs[package_name]
+ Logger.info("The following directories will be fixed for {0}:
{1}".format(package_name,
+str(directories)))
+
+ stack_version =
stack_select.get_stack_version_before_install(component_name)
+ if stack_version:
+conf_select.convert_conf_directories_to_symlinks(package_name,
stack_version, directories)
+
+
def _relink_configurations_with_conf_select(self, stack_id, stack_version):
"""
Sets up the required structure for /etc//conf symlinks and
/current
[3/5] ambari git commit: AMBARI-22655 - Livy/Livy2 Unable To Start Due to Address Already In Use (jonathanhurley)
AMBARI-22655 - Livy/Livy2 Unable To Start Due to Address Already In Use
(jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/87ef706a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/87ef706a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/87ef706a
Branch: refs/heads/branch-2.6
Commit: 87ef706a63a7e7e3a43f6d96e275521369d28f59
Parents: cd880fd
Author: Jonathan Hurley
Authored: Thu Dec 14 21:59:19 2017 -0500
Committer: Jonathan Hurley
Committed: Thu Dec 14 22:36:56 2017 -0500
--
.../SPARK/1.2.1/package/scripts/livy_service.py | 22 +---
.../2.0.0/package/scripts/livy2_service.py | 27 +---
.../SPARK2/2.0.0/package/scripts/setup_livy2.py | 2 ++
.../HDP/2.0.6/properties/stack_packages.json| 8 +-
.../python/stacks/2.5/SPARK/test_spark_livy.py | 3 ++-
.../stacks/2.6/SPARK2/test_spark_livy2.py | 5 +++-
6 files changed, 40 insertions(+), 27 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/87ef706a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
--
diff --git
a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
index cce2148..add5a77 100644
---
a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
+++
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py
@@ -21,28 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
-'ls {livy_server_pid_file} >/dev/null 2>&1 && ps -p `cat
{livy_server_pid_file}` >/dev/null 2>&1')
+ # use the livy user to get the PID (it is protected on non-root systems)
+ livy_server_pid = get_user_call_output.get_user_call_output(format("cat
{livy_server_pid_file}"),
+user=params.livy_user, is_checked_call=False)[1]
+
+ livy_server_pid = livy_server_pid.replace("\n", " ")
+
+ process_id_exists_command = format("ls {livy_server_pid_file} >/dev/null
2>&1 && ps -p {livy_server_pid} >/dev/null 2>&1")
+
if action == 'start':
Execute(format('{livy_server_start}'),
user=params.livy_user,
environment={'JAVA_HOME': params.java_home},
-not_if=livyserver_no_op_test)
+not_if=process_id_exists_command
+)
elif action == 'stop':
-pid = get_user_call_output.get_user_call_output(
- format("! test -f {livy_server_pid_file} || cat
{livy_server_pid_file}"), user=params.livy_user)[1]
-pid = pid.replace("\n", " ")
Execute(format('{livy_server_stop}'),
user=params.livy_user,
-only_if=livyserver_no_op_test,
+only_if=process_id_exists_command,
timeout=10,
-on_timeout=format("! ( {livyserver_no_op_test} ) || {sudo} -H -E
kill -9 {pid}"),
+on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H
-E kill -9 {livy_server_pid}"),
environment={'JAVA_HOME': params.java_home}
)
+
File(params.livy_server_pid_file, action="delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/87ef706a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
--
diff --git
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
index dfadd84..0180a31 100644
---
a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
+++
b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py
@@ -21,35 +21,32 @@ limitations under the License.
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
from resource_management.libraries.functions import get_user_call_output
-import threading
def livy2_service(name, upgrade_type=None, action=None):
import params
- livyserver_no_op_test = format(
-'ls {livy2_server_pid_file} >/dev/null 2>&1 && ps -p `cat
{livy2_server_pid_file}` >/dev/null 2>&1')
+ # use the livy2 user to get the PID (it is protected on non-root systems)
+ livy2
[5/5] ambari git commit: AMBARI-22665 - Livy2 Does Not Start On HDP 2.6.0 to 2.6.3 (jonathanhurley)
AMBARI-22665 - Livy2 Does Not Start On HDP 2.6.0 to 2.6.3 (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1d95ff6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1d95ff6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1d95ff6
Branch: refs/heads/branch-2.6
Commit: e1d95ff6337fc280e242750d9d0e25270f9692a5
Parents: b7c6ea9
Author: Jonathan Hurley
Authored: Mon Dec 18 12:23:06 2017 -0500
Committer: Jonathan Hurley
Committed: Mon Dec 18 12:35:54 2017 -0500
--
.../resource_management/libraries/functions/conf_select.py| 7 +++
.../main/resources/custom_actions/scripts/install_packages.py | 4 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/e1d95ff6/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
--
diff --git
a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index d38e273..9de2f8f 100644
---
a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -211,6 +211,13 @@ def convert_conf_directories_to_symlinks(package, version,
dirs):
# determine which directories would be created, if any are needed
dry_run_directory = create(stack_name, package, version, dry_run = True)
+ # if the dry run reported an error, then we must assume that the package
does not exist in
+ # the conf-select tool
+ if len(dry_run_directory) == 0:
+Logger.info("The conf-select tool reported an error for the package {0}.
The configuration linking will be skipped.".format(package))
+return
+
+
need_dirs = []
for d in dry_run_directory:
if not os.path.exists(d):
http://git-wip-us.apache.org/repos/asf/ambari/blob/e1d95ff6/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
--
diff --git
a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
index cb1b7b0..5cff342 100644
---
a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
+++
b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py
@@ -182,10 +182,10 @@ class InstallPackages(Script):
from resource_management.libraries.functions import stack_select
package_dirs = conf_select.get_package_dirs()
if package_name in package_dirs:
- Logger.info("Attempting to fix the default conf links for
{0}".format(package_name))
+ Logger.info("Determining if the default conf links for {0} need to be
fixed".format(package_name))
directories = package_dirs[package_name]
- Logger.info("The following directories will be fixed for {0}:
{1}".format(package_name,
+ Logger.info("The following directories will be checked for {0}:
{1}".format(package_name,
str(directories)))
stack_version =
stack_select.get_stack_version_before_install(component_name)
[1/5] ambari git commit: AMBARI-22640 - HBase Cannot Find LZO Classes After Being Patched (jonathanhurley)
Repository: ambari Updated Branches: refs/heads/branch-2.6 207a8e3c6 -> e1d95ff63 AMBARI-22640 - HBase Cannot Find LZO Classes After Being Patched (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/edac2c62 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/edac2c62 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/edac2c62 Branch: refs/heads/branch-2.6 Commit: edac2c6270e9fbd0c65008c99936636e5460ba67 Parents: 207a8e3 Author: Jonathan Hurley Authored: Tue Dec 12 16:18:47 2017 -0500 Committer: Jonathan Hurley Committed: Wed Dec 13 10:45:40 2017 -0500 -- .../HBASE/0.96.0.2.0/package/scripts/hbase.py| 19 +-- 1 file changed, 17 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/ambari/blob/edac2c62/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py -- diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py index ac71ce4..6490b59 100644 --- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py +++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py @@ -18,11 +18,23 @@ limitations under the License. """ from urlparse import urlparse + import os -from resource_management import * -import sys + from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from ambari_commons import OSConst +from resource_management.core.resources import Directory +from resource_management.core.resources import Execute +from resource_management.core.resources import File +from resource_management.core.resources import Package +from resource_management.core.resources import ServiceConfig +from resource_management.core.source import InlineTemplate +from resource_management.core.source import Template +from resource_management.libraries import Script +from resource_management.libraries.functions import format +from resource_management.libraries.functions import lzo_utils +from resource_management.libraries.resources import TemplateConfig +from resource_management.libraries.resources import XmlConfig from resource_management.libraries.functions.constants import StackFeature from resource_management.libraries.functions.stack_features import check_stack_feature @@ -48,6 +60,9 @@ def hbase(name=None): def hbase(name=None): import params + # ensure that matching LZO libraries are installed for HBase + lzo_utils.install_lzo_if_needed() + Directory( params.etc_prefix_dir, mode=0755 )
ambari git commit: AMBARI-22677. Don't call pprint.pformat unnecessarily in Ambari agent (amagyar)
Repository: ambari
Updated Branches:
refs/heads/trunk f3dc1ca39 -> 4e731d31f
AMBARI-22677. Don't call pprint.pformat unnecessarily in Ambari agent (amagyar)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4e731d31
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4e731d31
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4e731d31
Branch: refs/heads/trunk
Commit: 4e731d31f7c59bceab3c91565686384645251546
Parents: f3dc1ca
Author: Attila Magyar
Authored: Wed Dec 20 14:16:36 2017 +0100
Committer: Attila Magyar
Committed: Wed Dec 20 14:16:36 2017 +0100
--
.../src/main/python/ambari_agent/ActionQueue.py | 21 ++--
.../python/ambari_agent/CommandStatusDict.py| 2 +-
.../src/main/python/ambari_agent/Controller.py | 17
.../ambari_agent/CustomServiceOrchestrator.py | 4 ++--
.../src/main/python/ambari_agent/DataCleaner.py | 4 ++--
.../src/main/python/ambari_agent/Heartbeat.py | 17
.../src/main/python/ambari_agent/LiveStatus.py | 3 +--
.../main/python/ambari_agent/PythonExecutor.py | 15 +++---
.../ambari_agent/PythonReflectiveExecutor.py| 3 ++-
.../main/python/ambari_agent/RecoveryManager.py | 17
.../ambari_agent/StatusCommandsExecutor.py | 3 ++-
.../python/ambari_agent/alerts/ams_alert.py | 8 +---
.../python/ambari_agent/alerts/metric_alert.py | 3 ++-
.../src/main/python/ambari_agent/security.py| 8
14 files changed, 66 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/ambari/blob/4e731d31/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
--
diff --git a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
index f1b3a42..da4dc4c 100644
--- a/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
+++ b/ambari-agent/src/main/python/ambari_agent/ActionQueue.py
@@ -121,7 +121,8 @@ class ActionQueue(threading.Thread):
for command in commands:
logger.info("Canceling command with taskId = {tid}".format(tid =
str(command['target_task_id'])))
- logger.debug(pprint.pformat(command))
+ if logger.isEnabledFor(logging.DEBUG):
+logger.debug(pprint.pformat(command))
task_id = command['target_task_id']
reason = command['reason']
@@ -200,7 +201,7 @@ class ActionQueue(threading.Thread):
def process_command(self, command):
# make sure we log failures
commandType = command['commandType']
-logger.debug("Took an element of Queue (command type = %s)." % commandType)
+logger.debug("Took an element of Queue (command type = %s).", commandType)
try:
if commandType in [self.EXECUTION_COMMAND,
self.BACKGROUND_EXECUTION_COMMAND, self.AUTO_EXECUTION_COMMAND]:
try:
@@ -211,7 +212,7 @@ class ActionQueue(threading.Thread):
if self.controller.recovery_manager.enabled():
self.controller.recovery_manager.stop_execution_command()
else:
-logger.error("Unrecognized command " + pprint.pformat(command))
+logger.error("Unrecognized command %s", pprint.pformat(command))
except Exception:
logger.exception("Exception while processing {0}
command".format(commandType))
@@ -475,14 +476,14 @@ class ActionQueue(threading.Thread):
self.customServiceOrchestrator
def on_background_command_complete_callback(self, process_condensed_result,
handle):
-logger.debug('Start callback: %s' % process_condensed_result)
-logger.debug('The handle is: %s' % handle)
+logger.debug('Start callback: %s', process_condensed_result)
+logger.debug('The handle is: %s', handle)
status = self.COMPLETED_STATUS if handle.exitCode == 0 else
self.FAILED_STATUS
aborted_postfix =
self.customServiceOrchestrator.command_canceled_reason(handle.command['taskId'])
if aborted_postfix:
status = self.FAILED_STATUS
- logger.debug('Set status to: %s , reason = %s' % (status,
aborted_postfix))
+ logger.debug('Set status to: %s , reason = %s', status, aborted_postfix)
else:
aborted_postfix = ''
@@ -556,11 +557,9 @@ class ActionQueue(threading.Thread):
result['extra'] = component_extra
- logger.debug("Got live status for component " + component + \
- " of service " + str(service) + \
- " of cluster " + str(cluster))
-
- logger.debug(pprint.pformat(result))
+ if logger.isEnabledFor(logging.DEBUG):
+logger.debug("Got live status for component %s of service %s of
cluster %s", component, service, cluster)
+logger.debug(pprint.pformat(result))
if result is not None:
sel
