This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fb96b1a8d648 [SPARK-47723][CORE][TESTS] Introduce a tool that can sort
alphabetically enumeration field in `LogEntry` automatically
fb96b1a8d648 is described below
commit fb96b1a8d6480612ca61ec39f62c8db0b341327b
Author: panbingkun <panbing...@baidu.com>
AuthorDate: Thu Apr 4 17:04:53 2024 -0700
[SPARK-47723][CORE][TESTS] Introduce a tool that can sort alphabetically
enumeration field in `LogEntry` automatically
### What changes were proposed in this pull request?
The pr aims to `introduce` a `tool` that can `sort alphabetically`
enumeration field in `LogEntry` automatically.
### Why are the changes needed?
Enable developers to more conveniently write the enumeration values in
`LogEntry` in alphabetical order according to the requirements of structured
log development documents.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Manually test.
```
SPARK_GENERATE_GOLDEN_FILES=1
build/sbt "common-utils/testOnly *LogKeySuite -- -t \"LogKey enumeration
fields are correctly sorted\""
```
- Pass GA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45867 from panbingkun/SPARK-47723.
Lead-authored-by: panbingkun <panbing...@baidu.com>
Co-authored-by: panbingkun <pbk1...@gmail.com>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
.../scala/org/apache/spark/util/LogKeySuite.scala | 71 ++++++++++++++++++++--
1 file changed, 67 insertions(+), 4 deletions(-)
diff --git
a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
b/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
index 1f3c2d77d35f..24a24538ad72 100644
--- a/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
+++ b/common/utils/src/test/scala/org/apache/spark/util/LogKeySuite.scala
@@ -17,17 +17,80 @@
package org.apache.spark.util
+import java.nio.charset.StandardCharsets
+import java.nio.file.{Files, Path}
+import java.util.{ArrayList => JList}
+
+import scala.jdk.CollectionConverters._
+
+import org.apache.commons.io.FileUtils
import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
import org.apache.spark.internal.{Logging, LogKey}
+import org.apache.spark.internal.LogKey.LogKey
+// scalastyle:off line.size.limit
+/**
+ * To re-generate the LogKey class file, run:
+ * {{{
+ * SPARK_GENERATE_GOLDEN_FILES=1 build/sbt "common-utils/testOnly
org.apache.spark.util.LogKeySuite"
+ * }}}
+ */
+// scalastyle:on line.size.limit
class LogKeySuite
extends AnyFunSuite // scalastyle:ignore funsuite
with Logging {
- test("LogKey enumeration fields must be sorted alphabetically") {
- val keys = LogKey.values.toSeq
- assert(keys === keys.sortBy(_.toString),
- "LogKey enumeration fields must be sorted alphabetically")
+ /**
+ * Get a Path relative to the root project. It is assumed that a spark home
is set.
+ */
+ protected final def getWorkspaceFilePath(first: String, more: String*): Path
= {
+ if (!(sys.props.contains("spark.test.home") ||
sys.env.contains("SPARK_HOME"))) {
+ fail("spark.test.home or SPARK_HOME is not set.")
+ }
+ val sparkHome = sys.props.getOrElse("spark.test.home",
sys.env("SPARK_HOME"))
+ java.nio.file.Paths.get(sparkHome, first +: more: _*)
+ }
+
+ private val regenerateGoldenFiles: Boolean =
System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1"
+
+ private val logKeyFilePath = getWorkspaceFilePath("common", "utils", "src",
"main", "scala",
+ "org", "apache", "spark", "internal", "LogKey.scala")
+
+ // regenerate the file `LogKey.scala` with its enumeration fields sorted
alphabetically
+ private def regenerateLogKeyFile(
+ originalKeys: Seq[LogKey], sortedKeys: Seq[LogKey]): Unit = {
+ if (originalKeys != sortedKeys) {
+ val logKeyFile = logKeyFilePath.toFile
+ logInfo(s"Regenerating LogKey file $logKeyFile")
+ val originalContents = FileUtils.readLines(logKeyFile,
StandardCharsets.UTF_8)
+ val sortedContents = new JList[String]()
+ var firstMatch = false
+ originalContents.asScala.foreach { line =>
+ if (line.trim.startsWith("val ") && line.trim.endsWith(" = Value")) {
+ if (!firstMatch) {
+ sortedKeys.foreach { logKey =>
+ sortedContents.add(s" val ${logKey.toString} = Value")
+ }
+ firstMatch = true
+ }
+ } else {
+ sortedContents.add(line)
+ }
+ }
+ Files.delete(logKeyFile.toPath)
+ FileUtils.writeLines(logKeyFile, StandardCharsets.UTF_8.name(),
sortedContents)
+ }
+ }
+
+ test("LogKey enumeration fields are correctly sorted") {
+ val originalKeys = LogKey.values.toSeq
+ val sortedKeys = originalKeys.sortBy(_.toString)
+ if (regenerateGoldenFiles) {
+ regenerateLogKeyFile(originalKeys, sortedKeys)
+ } else {
+ assert(originalKeys === sortedKeys,
+ "LogKey enumeration fields must be sorted alphabetically")
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
