smallyao commented on code in PR #3695:
URL: https://github.com/apache/ambari/pull/3695#discussion_r1204931846
##########
ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/SPARK/service_advisor.py:
##########
@@ -163,115 +163,100 @@ def isKerberosEnabled(services, configurations):
:rtype: bool
:return: True or False
"""
- if configurations and "spark2-defaults" in configurations and \
- "spark.history.kerberos.enabled" in
configurations["spark2-defaults"]["properties"]:
- return
configurations["spark2-defaults"]["properties"]["spark.history.kerberos.enabled"].lower()
== "true"
- elif services and "spark2-defaults" in services["configurations"] and \
- "spark.history.kerberos.enabled" in
services["configurations"]["spark2-defaults"]["properties"]:
- return
services["configurations"]["spark2-defaults"]["properties"]["spark.history.kerberos.enabled"].lower()
== "true"
+ if configurations and "spark-defaults" in configurations and \
+ "spark.history.kerberos.enabled" in
configurations["spark-defaults"]["properties"]:
+ return
configurations["spark-defaults"]["properties"]["spark.history.kerberos.enabled"].lower()
== "true"
+ elif services and "spark-defaults" in services["configurations"] and \
+ "spark.history.kerberos.enabled" in
services["configurations"]["spark-defaults"]["properties"]:
+ return
services["configurations"]["spark-defaults"]["properties"]["spark.history.kerberos.enabled"].lower()
== "true"
else:
return False
-class Spark2Recommender(service_advisor.ServiceAdvisor):
+class SparkRecommender(service_advisor.ServiceAdvisor):
"""
- Spark2 Recommender suggests properties when adding the service for the first
time or modifying configs via the UI.
+ Spark Recommender suggests properties when adding the service for the first
time or modifying configs via the UI.
"""
def __init__(self, *args, **kwargs):
- self.as_super = super(Spark2Recommender, self)
+ self.as_super = super(SparkRecommender, self)
self.as_super.__init__(*args, **kwargs)
- def recommendSpark2ConfigurationsFromHDP25(self, configurations,
clusterData, services, hosts):
+ def recommendSparkConfigurationsFromHDP25(self, configurations, clusterData,
services, hosts):
"""
:type configurations dict
:type clusterData dict
:type services dict
:type hosts dict
"""
- putSparkProperty = self.putProperty(configurations, "spark2-defaults",
services)
- putSparkThriftSparkConf = self.putProperty(configurations,
"spark2-thrift-sparkconf", services)
-
- spark_queue = self.recommendYarnQueue(services, "spark2-defaults",
"spark.yarn.queue")
+ putSparkProperty = self.putProperty(configurations, "spark-defaults",
services)
+ spark_queue = self.recommendYarnQueue(services, "spark-defaults",
"spark.yarn.queue")
if spark_queue is not None:
putSparkProperty("spark.yarn.queue", spark_queue)
- spark_thrift_queue = self.recommendYarnQueue(services,
"spark2-thrift-sparkconf", "spark.yarn.queue")
- if spark_thrift_queue is not None:
- putSparkThriftSparkConf("spark.yarn.queue", spark_thrift_queue)
-
-
- def recommendSPARK2ConfigurationsFromHDP26(self, configurations,
clusterData, services, hosts):
+ def recommendSPARKConfigurationsFromHDP26(self, configurations, clusterData,
services, hosts):
"""
:type configurations dict
:type clusterData dict
:type services dict
:type hosts dict
"""
- if Spark2ServiceAdvisor.isKerberosEnabled(services, configurations):
+ if SparkServiceAdvisor.isKerberosEnabled(services, configurations):
- spark2_defaults = self.getServicesSiteProperties(services,
"spark2-defaults")
+ spark_defaults = self.getServicesSiteProperties(services,
"spark-defaults")
- if spark2_defaults:
- putSpark2DafaultsProperty = self.putProperty(configurations,
"spark2-defaults", services)
- putSpark2DafaultsProperty('spark.acls.enable', 'true')
- putSpark2DafaultsProperty('spark.admin.acls', '')
- putSpark2DafaultsProperty('spark.history.ui.acls.enable', 'true')
- putSpark2DafaultsProperty('spark.history.ui.admin.acls', '')
+ if spark_defaults:
+ putSparkDafaultsProperty = self.putProperty(configurations,
"spark-defaults", services)
+ putSparkDafaultsProperty('spark.acls.enable', 'true')
+ putSparkDafaultsProperty('spark.admin.acls', '')
+ putSparkDafaultsProperty('spark.history.ui.acls.enable', 'true')
+ putSparkDafaultsProperty('spark.history.ui.admin.acls', '')
- self.__addZeppelinToLivy2SuperUsers(configurations, services)
-
- def recommendSPARK2ConfigurationsFromHDP30(self, configurations,
clusterData, services, hosts):
+ def recommendSPARKConfigurationsFromHDP30(self, configurations, clusterData,
services, hosts):
# SAC
- if "spark2-atlas-application-properties-override" in
services["configurations"]:
- spark2_atlas_application_properties_override =
self.getServicesSiteProperties(services,
"spark2-atlas-application-properties-override")
- spark2_defaults_properties = self.getServicesSiteProperties(services,
"spark2-defaults")
- spark2_thriftspark_conf_properties =
self.getServicesSiteProperties(services, "spark2-thrift-sparkconf")
- putSpark2DefautlsProperty = self.putProperty(configurations,
"spark2-defaults", services)
- putSpark2DefaultsPropertyAttribute =
self.putPropertyAttribute(configurations,"spark2-defaults")
- putSpark2ThriftSparkConfProperty = self.putProperty(configurations,
"spark2-thrift-sparkconf", services)
- putSpark2AtlasHookProperty = self.putProperty(configurations,
"spark2-atlas-application-properties-override", services)
- putSpark2AtlasHookPropertyAttribute =
self.putPropertyAttribute(configurations,"spark2-atlas-application-properties-override")
- spark2_sac_enabled = None
- if
self.checkSiteProperties(spark2_atlas_application_properties_override,
"atlas.spark.enabled"):
- spark2_sac_enabled =
spark2_atlas_application_properties_override["atlas.spark.enabled"]
- spark2_sac_enabled = str(spark2_sac_enabled).upper() == 'TRUE'
-
- if spark2_sac_enabled:
-
- self.setOrAddValueToProperty(putSpark2DefautlsProperty,
spark2_defaults_properties, "spark.driver.extraClassPath",
"/usr/hdp/current/spark-atlas-connector/*", ":")
- self.setOrAddValueToProperty(putSpark2DefautlsProperty,
spark2_defaults_properties, "spark.yarn.dist.files",
"/etc/spark2/conf/atlas-application.properties.yarn#atlas-application.properties",
",")
- self.setOrAddValueToProperty(putSpark2ThriftSparkConfProperty,
spark2_thriftspark_conf_properties, "spark.driver.extraClassPath",
"/usr/hdp/current/spark-atlas-connector/*", ":")
-
- self.setOrAddValueToProperty(putSpark2DefautlsProperty,
spark2_defaults_properties, "spark.extraListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
- self.setOrAddValueToProperty(putSpark2DefautlsProperty,
spark2_defaults_properties, "spark.sql.queryExecutionListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
- self.setOrAddValueToProperty(putSpark2ThriftSparkConfProperty,
spark2_thriftspark_conf_properties, "spark.extraListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
- self.setOrAddValueToProperty(putSpark2ThriftSparkConfProperty,
spark2_thriftspark_conf_properties, "spark.sql.queryExecutionListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
-
- self.setOrAddValueToProperty(putSpark2DefautlsProperty,
spark2_defaults_properties, "spark.sql.streaming.streamingQueryListeners",
"com.hortonworks.spark.atlas.SparkAtlasStreamingQueryEventTracker", ",")
- self.setOrAddValueToProperty(putSpark2ThriftSparkConfProperty,
spark2_thriftspark_conf_properties,
"spark.sql.streaming.streamingQueryListeners",
"com.hortonworks.spark.atlas.SparkAtlasStreamingQueryEventTracker", ",")
-
- putSpark2AtlasHookProperty("atlas.client.checkModelInStart", "false")
+ if "spark-atlas-application-properties-override" in
services["configurations"]:
+ spark_atlas_application_properties_override =
self.getServicesSiteProperties(services,
"spark-atlas-application-properties-override")
+ spark_defaults_properties = self.getServicesSiteProperties(services,
"spark-defaults")
+ putSparkDefautlsProperty = self.putProperty(configurations,
"spark-defaults", services)
+ putSparkDefaultsPropertyAttribute =
self.putPropertyAttribute(configurations,"spark-defaults")
+ putSparkAtlasHookProperty = self.putProperty(configurations,
"spark-atlas-application-properties-override", services)
+ putSparkAtlasHookPropertyAttribute =
self.putPropertyAttribute(configurations,"spark-atlas-application-properties-override")
+ spark_sac_enabled = None
+ if self.checkSiteProperties(spark_atlas_application_properties_override,
"atlas.spark.enabled"):
+ spark_sac_enabled =
spark_atlas_application_properties_override["atlas.spark.enabled"]
+ spark_sac_enabled = str(spark_sac_enabled).upper() == 'TRUE'
+
+ if spark_sac_enabled:
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.driver.extraClassPath",
"/usr/bigtop/current/spark-atlas-connector/*", ":")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.yarn.dist.files",
"/etc/spark/conf/atlas-application.properties.yarn#atlas-application.properties",
",")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.driver.extraClassPath",
"/usr/bigtop/current/spark-atlas-connector/*", ":")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.extraListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.sql.queryExecutionListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.extraListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.sql.queryExecutionListeners",
"com.hortonworks.spark.atlas.SparkAtlasEventTracker", ",")
+
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.sql.streaming.streamingQueryListeners",
"com.hortonworks.spark.atlas.SparkAtlasStreamingQueryEventTracker", ",")
+ self.setOrAddValueToProperty(putSparkDefautlsProperty,
spark_defaults_properties, "spark.sql.streaming.streamingQueryListeners",
"com.hortonworks.spark.atlas.SparkAtlasStreamingQueryEventTracker", ",")
Review Comment:
`com.hortonworks.spark.atlas.SparkAtlasStreamingQueryEventTracker
com.hortonworks.spark.atlas.SparkAtlasEventTracker
`
Should we keep these classes here in bigtop stack?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
