This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 789a510 [SPARK-38249][CORE][GRAPHX] Cleanup unused private
methods/fields
789a510 is described below
commit 789a510c78ca81db0137bba1687102e2d9acd149
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Sat Feb 19 09:04:01 2022 -0600
[SPARK-38249][CORE][GRAPHX] Cleanup unused private methods/fields
### What changes were proposed in this pull request?
This pr aims to cleanup unused ยท`private methods/fields`.
### Why are the changes needed?
Code clean.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA
Closes #35566 from LuciferYang/never-used.
Authored-by: yangjie01 <yangji...@baidu.com>
Signed-off-by: Sean Owen <sro...@gmail.com>
---
core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 9 +--------
.../org/apache/spark/executor/CoarseGrainedExecutorBackend.scala | 5 -----
core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala | 1 -
.../src/main/scala/org/apache/spark/resource/ResourceUtils.scala | 1 -
.../src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala | 9 ---------
5 files changed, 1 insertion(+), 24 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 7dbf6b9..775b27b 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.{ArrayBuffer, HashMap,
HashSet}
import scala.util.Random
import org.apache.spark.{SecurityManager, SparkConf, SparkException}
-import org.apache.spark.deploy.{ApplicationDescription, DriverDescription,
ExecutorState, SparkHadoopUtil}
+import org.apache.spark.deploy.{ApplicationDescription, DriverDescription,
ExecutorState}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.DriverState.DriverState
import org.apache.spark.deploy.master.MasterMessages._
@@ -53,8 +53,6 @@ private[deploy] class Master(
private val forwardMessageThread =
ThreadUtils.newDaemonSingleThreadScheduledExecutor("master-forward-message-thread")
- private val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
-
// For application IDs
private def createDateFormat = new SimpleDateFormat("yyyyMMddHHmmss",
Locale.US)
@@ -95,11 +93,6 @@ private[deploy] class Master(
// After onStart, webUi will be set
private var webUi: MasterWebUI = null
- private val masterPublicAddress = {
- val envVar = conf.getenv("SPARK_PUBLIC_DNS")
- if (envVar != null) envVar else address.host
- }
-
private val masterUrl = address.toSparkURL
private var masterWebUiUrl: String = _
diff --git
a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index fb7b4e6..a94e636 100644
---
a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -42,7 +42,6 @@ import org.apache.spark.resource.ResourceUtils._
import org.apache.spark.rpc._
import org.apache.spark.scheduler.{ExecutorLossMessage, ExecutorLossReason,
TaskDescription}
import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages._
-import org.apache.spark.serializer.SerializerInstance
import org.apache.spark.util.{ChildFirstURLClassLoader, MutableURLClassLoader,
SignalUtils, ThreadUtils, Utils}
private[spark] class CoarseGrainedExecutorBackend(
@@ -65,10 +64,6 @@ private[spark] class CoarseGrainedExecutorBackend(
var executor: Executor = null
@volatile var driver: Option[RpcEndpointRef] = None
- // If this CoarseGrainedExecutorBackend is changed to support multiple
threads, then this may need
- // to be changed so that we don't share the serializer instance across
threads
- private[this] val ser: SerializerInstance =
env.closureSerializer.newInstance()
-
private var _resources = Map.empty[String, ResourceInformation]
/**
diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
index c6959a5..596298b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala
@@ -244,7 +244,6 @@ class NewHadoopRDD[K, V](
}
private var havePair = false
- private var recordsSinceMetricsUpdate = 0
override def hasNext: Boolean = {
if (!finished && !havePair) {
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
index 837b2d8..3f0a0d3 100644
--- a/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
+++ b/core/src/main/scala/org/apache/spark/resource/ResourceUtils.scala
@@ -386,7 +386,6 @@ private[spark] object ResourceUtils extends Logging {
val resourcePlugins =
Utils.loadExtensions(classOf[ResourceDiscoveryPlugin], pluginClasses,
sparkConf)
// apply each plugin until one of them returns the information for this
resource
- var riOption: Optional[ResourceInformation] = Optional.empty()
resourcePlugins.foreach { plugin =>
val riOption = plugin.discoverResource(resourceRequest, sparkConf)
if (riOption.isPresent()) {
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
index 8564597..4a79087 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
@@ -21,7 +21,6 @@ import scala.reflect.{classTag, ClassTag}
import org.apache.spark.HashPartitioner
import org.apache.spark.graphx._
-import org.apache.spark.graphx.util.BytecodeUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.storage.StorageLevel
@@ -265,14 +264,6 @@ class GraphImpl[VD: ClassTag, ED: ClassTag] protected (
}
}
- /** Test whether the closure accesses the attribute with name `attrName`. */
- private def accessesVertexAttr(closure: AnyRef, attrName: String): Boolean =
{
- try {
- BytecodeUtils.invokedMethod(closure, classOf[EdgeTriplet[VD, ED]],
attrName)
- } catch {
- case _: ClassNotFoundException => true // if we don't know, be
conservative
- }
- }
} // end of class GraphImpl
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
