This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new 0efbfd57401 [SPARK-39904][SQL][FOLLOW-UP] Rename CSV option
`prefersDate` as `preferDate`
0efbfd57401 is described below
commit 0efbfd574011dc19cac74ee25043b0a1d57e8497
Author: Gengliang Wang <gengli...@apache.org>
AuthorDate: Thu Feb 16 17:23:15 2023 -0800
[SPARK-39904][SQL][FOLLOW-UP] Rename CSV option `prefersDate` as
`preferDate`
### What changes were proposed in this pull request?
Rename the CSV data source option `prefersDate` as `preferDate`.
### Why are the changes needed?
All the CSV data source options doesn't have a `s` on the verb in the
naming. For example, `inferSchema`, `ignoreLeadingWhiteSpace` and
`ignoreTrailingWhiteSpace`.
The renaming makes the naming consistent.
Also, the title of JIRA https://issues.apache.org/jira/browse/SPARK-39904
uses `preferDate` as well.
### Does this PR introduce _any_ user-facing change?
No, the data source option is not released yet.
### How was this patch tested?
Existing UT
Closes #40043 from gengliangwang/renameCSVOption.
Authored-by: Gengliang Wang <gengli...@apache.org>
Signed-off-by: Gengliang Wang <gengli...@apache.org>
(cherry picked from commit 6ead12e4ac08cef6c1346df3d380e85e5937a842)
Signed-off-by: Gengliang Wang <gengli...@apache.org>
---
docs/sql-data-sources-csv.md | 2 +-
.../org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala | 2 +-
.../scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala | 10 +++++-----
.../apache/spark/sql/execution/datasources/csv/CSVSuite.scala | 8 ++++----
4 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/docs/sql-data-sources-csv.md b/docs/sql-data-sources-csv.md
index 42b117bea46..be53f0301c0 100644
--- a/docs/sql-data-sources-csv.md
+++ b/docs/sql-data-sources-csv.md
@@ -109,7 +109,7 @@ Data source options of CSV can be set via:
<td>read</td>
</tr>
<tr>
- <td><code>prefersDate</code></td>
+ <td><code>preferDate</code></td>
<td>true</td>
<td>During schema inference (<code>inferSchema</code>), attempts to infer
string columns that contain dates as <code>Date</code> if the values satisfy
the <code>dateFormat</code> option or default date format. For columns that
contain a mixture of dates and timestamps, try inferring them as
<code>TimestampType</code> if timestamp format not specified, otherwise infer
them as <code>StringType</code>.</td>
<td>read</td>
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index bdfa4ac3f0f..51586a0065e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -183,7 +183,7 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
private def tryParseDouble(field: String): DataType = {
if ((allCatch opt field.toDouble).isDefined || isInfOrNan(field)) {
DoubleType
- } else if (options.prefersDate) {
+ } else if (options.preferDate) {
tryParseDate(field)
} else {
tryParseTimestampNTZ(field)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
index a66070aa853..1a9de5bc35e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala
@@ -159,19 +159,19 @@ class CSVOptions(
* Not compatible with legacyTimeParserPolicy == LEGACY since legacy date
parser will accept
* extra trailing characters. Thus, disabled when legacyTimeParserPolicy ==
LEGACY
*/
- val prefersDate = {
+ val preferDate = {
if (SQLConf.get.legacyTimeParserPolicy == LegacyBehaviorPolicy.LEGACY) {
false
} else {
- getBool(PREFERS_DATE, true)
+ getBool(PREFER_DATE, true)
}
}
val dateFormatOption: Option[String] = parameters.get(DATE_FORMAT)
- // Provide a default value for dateFormatInRead when prefersDate. This
ensures that the
+ // Provide a default value for dateFormatInRead when preferDate. This
ensures that the
// Iso8601DateFormatter (with strict date parsing) is used for date inference
val dateFormatInRead: Option[String] =
- if (prefersDate) {
+ if (preferDate) {
Option(dateFormatOption.getOrElse(DateFormatter.defaultPattern))
} else {
dateFormatOption
@@ -335,7 +335,7 @@ object CSVOptions extends DataSourceOptions {
val INFER_SCHEMA = newOption("inferSchema")
val IGNORE_LEADING_WHITESPACE = newOption("ignoreLeadingWhiteSpace")
val IGNORE_TRAILING_WHITESPACE = newOption("ignoreTrailingWhiteSpace")
- val PREFERS_DATE = newOption("prefersDate")
+ val PREFER_DATE = newOption("preferDate")
val ESCAPE_QUOTES = newOption("escapeQuotes")
val QUOTE_ALL = newOption("quoteAll")
val ENFORCE_SCHEMA = newOption("enforceSchema")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index c46298e558c..d27d9454680 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2843,7 +2843,7 @@ abstract class CSVSuite
.load(path.getAbsolutePath)
val expected = if (SQLConf.get.legacyTimeParserPolicy ==
LegacyBehaviorPolicy.LEGACY) {
- // When legacy parser is enabled, `prefersDate` will be disabled
+ // When legacy parser is enabled, `preferDate` will be disabled
Seq(
Row("2001-09-08"),
Row("1941-01-02"),
@@ -2940,7 +2940,7 @@ abstract class CSVSuite
.csv(path.getAbsolutePath)
if (SQLConf.get.legacyTimeParserPolicy != LegacyBehaviorPolicy.LEGACY) {
- // When legacy parser is enabled, `prefersDate` will be disabled
+ // When legacy parser is enabled, `preferDate` will be disabled
checkAnswer(
output,
Seq(
@@ -3108,7 +3108,7 @@ abstract class CSVSuite
assert(CSVOptions.isValidOption("inferSchema"))
assert(CSVOptions.isValidOption("ignoreLeadingWhiteSpace"))
assert(CSVOptions.isValidOption("ignoreTrailingWhiteSpace"))
- assert(CSVOptions.isValidOption("prefersDate"))
+ assert(CSVOptions.isValidOption("preferDate"))
assert(CSVOptions.isValidOption("escapeQuotes"))
assert(CSVOptions.isValidOption("quoteAll"))
assert(CSVOptions.isValidOption("enforceSchema"))
@@ -3149,7 +3149,7 @@ abstract class CSVSuite
assert(CSVOptions.getAlternativeOption("charset").contains("encoding"))
assert(CSVOptions.getAlternativeOption("compression").contains("codec"))
assert(CSVOptions.getAlternativeOption("codec").contains("compression"))
- assert(CSVOptions.getAlternativeOption("prefersDate").isEmpty)
+ assert(CSVOptions.getAlternativeOption("preferDate").isEmpty)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
