[02/32] ambari git commit: AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)

2016-12-01 Thread jonathanhurley 
AMBARI-18792. Update some configuration properties for hive interactive for the 
HDP 2.6 stack (Siddharth Seth via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca7e071f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca7e071f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca7e071f

Branch: refs/heads/branch-feature-AMBARI-18456
Commit: ca7e071f044d70186c7bf9d18bf24c29f3767730
Parents: 2cc4c9e
Author: Sumit Mohanty 
Authored: Tue Nov 29 11:39:53 2016 -0800
Committer: Sumit Mohanty 
Committed: Tue Nov 29 15:28:46 2016 -0800

--
 .../server/upgrade/UpgradeCatalog250.java   | 52 +
 .../configuration/hive-interactive-site.xml | 58 +++
 .../HIVE/configuration/tez-interactive-site.xml | 78 
 3 files changed, 188 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index e81568c..3b2cdd3 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -134,6 +135,7 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 addNewConfigurationsFromXml();
 updateAMSConfigs();
 updateKafkaConfigs();
+updateHiveLlapConfigs();
   }
 
   protected void updateHostVersionTable() throws SQLException {
@@ -151,6 +153,56 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", 
"group_type");
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+AmbariManagementController ambariManagementController = 
injector.getInstance(AmbariManagementController.class);
+Clusters clusters = ambariManagementController.getClusters();
+
+if (clusters != null) {
+  Map clusterMap = clusters.getClusters();
+
+  if (clusterMap != null && !clusterMap.isEmpty()) {
+for (final Cluster cluster : clusterMap.values()) {
+  Set installedServices = cluster.getServices().keySet();
+
+  if (installedServices.contains("HIVE")) {
+Config hiveSite = 
cluster.getDesiredConfigByType("hive-interactive-site");
+if (hiveSite != null) {
+  Map hiveSiteProperties = 
hiveSite.getProperties();
+  String schedulerDelay = 
hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+  if (schedulerDelay != null) {
+// Property exists. Change to new default if set to -1.
+if (schedulerDelay.length() != 0) {
+  try {
+int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+if (schedulerDelayInt == -1) {
+  // Old default. Set to new default.
+  updateConfigurationProperties("hive-interactive-site", 
Collections
+
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+false);
+}
+  } catch (NumberFormatException e) {
+// Invalid existing value. Set to new default.
+updateConfigurationProperties("hive-interactive-site", 
Collections
+  
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+  false);
+  }
+}
+  }
+  updateConfigurationProperties("hive-interactive-site",
+
Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+false);
+  updateConfigurationProperties("tez-interactive-site",
+
Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), 
true,
+false);
+  //

[26/50] ambari git commit: AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)

2016-12-01 Thread swapan 
AMBARI-18792. Update some configuration properties for hive interactive for the 
HDP 2.6 stack (Siddharth Seth via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8bfa056
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8bfa056
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8bfa056

Branch: refs/heads/branch-feature-AMBARI-18901
Commit: f8bfa056574d1f8dc695615eea70fae74b156fc2
Parents: aba9640
Author: Sumit Mohanty 
Authored: Tue Nov 29 11:39:53 2016 -0800
Committer: Sumit Mohanty 
Committed: Tue Nov 29 11:39:53 2016 -0800

--
 .../server/upgrade/UpgradeCatalog250.java   | 53 -
 .../configuration/hive-interactive-site.xml | 58 +++
 .../HIVE/configuration/tez-interactive-site.xml | 78 
 3 files changed, 188 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f8bfa056/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index bc3c120..44ce895 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -123,7 +124,7 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
*/
   @Override
   protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
+updateHiveLlapConfigs();
   }
 
   /**
@@ -201,6 +202,56 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 return content;
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+AmbariManagementController ambariManagementController = 
injector.getInstance(AmbariManagementController.class);
+Clusters clusters = ambariManagementController.getClusters();
+
+if (clusters != null) {
+  Map clusterMap = clusters.getClusters();
+
+  if (clusterMap != null && !clusterMap.isEmpty()) {
+for (final Cluster cluster : clusterMap.values()) {
+  Set installedServices = cluster.getServices().keySet();
+
+  if (installedServices.contains("HIVE")) {
+Config hiveSite = 
cluster.getDesiredConfigByType("hive-interactive-site");
+if (hiveSite != null) {
+  Map hiveSiteProperties = 
hiveSite.getProperties();
+  String schedulerDelay = 
hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+  if (schedulerDelay != null) {
+// Property exists. Change to new default if set to -1.
+if (schedulerDelay.length() != 0) {
+  try {
+int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+if (schedulerDelayInt == -1) {
+  // Old default. Set to new default.
+  updateConfigurationProperties("hive-interactive-site", 
Collections
+  
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+  false);
+}
+  } catch (NumberFormatException e) {
+// Invalid existing value. Set to new default.
+updateConfigurationProperties("hive-interactive-site", 
Collections
+
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+false);
+  }
+}
+  }
+  updateConfigurationProperties("hive-interactive-site",
+  
Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+  false);
+  updateConfigurationProperties("tez-interactive-site",
+  
Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), 
true,
+  false);
+  // Explicitly skipping hive.llap.allow.permanent.fns during 
upgrades, since it's related to security,
+  // and we don't know if the value is set by the user or as a 
result of the previous default.
+}
+  }
+}
+  }
+}
+  }
+
   /**
* Creates the

ambari git commit: AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)

2016-11-29 Thread smohanty 
Repository: ambari
Updated Branches:
  refs/heads/trunk 2cc4c9ed7 -> ca7e071f0


AMBARI-18792. Update some configuration properties for hive interactive for the 
HDP 2.6 stack (Siddharth Seth via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ca7e071f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ca7e071f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ca7e071f

Branch: refs/heads/trunk
Commit: ca7e071f044d70186c7bf9d18bf24c29f3767730
Parents: 2cc4c9e
Author: Sumit Mohanty 
Authored: Tue Nov 29 11:39:53 2016 -0800
Committer: Sumit Mohanty 
Committed: Tue Nov 29 15:28:46 2016 -0800

--
 .../server/upgrade/UpgradeCatalog250.java   | 52 +
 .../configuration/hive-interactive-site.xml | 58 +++
 .../HIVE/configuration/tez-interactive-site.xml | 78 
 3 files changed, 188 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/ca7e071f/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index e81568c..3b2cdd3 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -134,6 +135,7 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 addNewConfigurationsFromXml();
 updateAMSConfigs();
 updateKafkaConfigs();
+updateHiveLlapConfigs();
   }
 
   protected void updateHostVersionTable() throws SQLException {
@@ -151,6 +153,56 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 dbAccessor.addUniqueConstraint(GROUPS_TABLE, "UNQ_groups_0", "group_name", 
"group_type");
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+AmbariManagementController ambariManagementController = 
injector.getInstance(AmbariManagementController.class);
+Clusters clusters = ambariManagementController.getClusters();
+
+if (clusters != null) {
+  Map clusterMap = clusters.getClusters();
+
+  if (clusterMap != null && !clusterMap.isEmpty()) {
+for (final Cluster cluster : clusterMap.values()) {
+  Set installedServices = cluster.getServices().keySet();
+
+  if (installedServices.contains("HIVE")) {
+Config hiveSite = 
cluster.getDesiredConfigByType("hive-interactive-site");
+if (hiveSite != null) {
+  Map hiveSiteProperties = 
hiveSite.getProperties();
+  String schedulerDelay = 
hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+  if (schedulerDelay != null) {
+// Property exists. Change to new default if set to -1.
+if (schedulerDelay.length() != 0) {
+  try {
+int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+if (schedulerDelayInt == -1) {
+  // Old default. Set to new default.
+  updateConfigurationProperties("hive-interactive-site", 
Collections
+
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+false);
+}
+  } catch (NumberFormatException e) {
+// Invalid existing value. Set to new default.
+updateConfigurationProperties("hive-interactive-site", 
Collections
+  
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+  false);
+  }
+}
+  }
+  updateConfigurationProperties("hive-interactive-site",
+
Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+false);
+  updateConfigurationProperties("tez-interactive-site",
+
Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), 
true,
+

ambari git commit: AMBARI-18792. Update some configuration properties for hive interactive for the HDP 2.6 stack (Siddharth Seth via smohanty)

2016-11-29 Thread smohanty 
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 aba964062 -> f8bfa0565


AMBARI-18792. Update some configuration properties for hive interactive for the 
HDP 2.6 stack (Siddharth Seth via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8bfa056
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8bfa056
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8bfa056

Branch: refs/heads/branch-2.5
Commit: f8bfa056574d1f8dc695615eea70fae74b156fc2
Parents: aba9640
Author: Sumit Mohanty 
Authored: Tue Nov 29 11:39:53 2016 -0800
Committer: Sumit Mohanty 
Committed: Tue Nov 29 11:39:53 2016 -0800

--
 .../server/upgrade/UpgradeCatalog250.java   | 53 -
 .../configuration/hive-interactive-site.xml | 58 +++
 .../HIVE/configuration/tez-interactive-site.xml | 78 
 3 files changed, 188 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/f8bfa056/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
--
diff --git 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index bc3c120..44ce895 100644
--- 
a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ 
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.CommandExecutionType;
@@ -123,7 +124,7 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
*/
   @Override
   protected void executePreDMLUpdates() throws AmbariException, SQLException {
-
+updateHiveLlapConfigs();
   }
 
   /**
@@ -201,6 +202,56 @@ public class UpgradeCatalog250 extends 
AbstractUpgradeCatalog {
 return content;
   }
 
+  protected void updateHiveLlapConfigs() throws AmbariException {
+AmbariManagementController ambariManagementController = 
injector.getInstance(AmbariManagementController.class);
+Clusters clusters = ambariManagementController.getClusters();
+
+if (clusters != null) {
+  Map clusterMap = clusters.getClusters();
+
+  if (clusterMap != null && !clusterMap.isEmpty()) {
+for (final Cluster cluster : clusterMap.values()) {
+  Set installedServices = cluster.getServices().keySet();
+
+  if (installedServices.contains("HIVE")) {
+Config hiveSite = 
cluster.getDesiredConfigByType("hive-interactive-site");
+if (hiveSite != null) {
+  Map hiveSiteProperties = 
hiveSite.getProperties();
+  String schedulerDelay = 
hiveSiteProperties.get("hive.llap.task.scheduler.locality.delay");
+  if (schedulerDelay != null) {
+// Property exists. Change to new default if set to -1.
+if (schedulerDelay.length() != 0) {
+  try {
+int schedulerDelayInt = Integer.parseInt(schedulerDelay);
+if (schedulerDelayInt == -1) {
+  // Old default. Set to new default.
+  updateConfigurationProperties("hive-interactive-site", 
Collections
+  
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+  false);
+}
+  } catch (NumberFormatException e) {
+// Invalid existing value. Set to new default.
+updateConfigurationProperties("hive-interactive-site", 
Collections
+
.singletonMap("hive.llap.task.scheduler.locality.delay", "8000"), true,
+false);
+  }
+}
+  }
+  updateConfigurationProperties("hive-interactive-site",
+  
Collections.singletonMap("hive.mapjoin.hybridgrace.hashtable", "true"), true,
+  false);
+  updateConfigurationProperties("tez-interactive-site",
+  
Collections.singletonMap("tez.session.am.dag.submit.timeout.secs", "1209600"), 
true,
+  false);
+  // Explicitly skipping hive.llap.allow.permanent.fns during 
upgrades, since it's related to security,
+  // and we don't know if the value is set by the user or as a 
result of the previous default.
+}
+  }
+