gengliangwang commented on code in PR #47425:
URL: https://github.com/apache/spark/pull/47425#discussion_r1717670601
##########
connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala:
##########
@@ -170,4 +180,46 @@ private[sql] object AvroOptions extends DataSourceOptions {
// When STABLE_ID_FOR_UNION_TYPE is enabled, the option allows to configure
the prefix for fields
// of Avro Union type.
val STABLE_ID_PREFIX_FOR_UNION_TYPE =
newOption("stableIdentifierPrefixForUnionType")
+
+ /**
+ * Adds support for recursive fields. If this option is not specified or is
set to 0, recursive
+ * fields are not permitted. Setting it to 1 drops all recursive fields, 2
allows recursive
+ * fields to be recursed once, and 3 allows it to be recursed twice and so
on, up to 15.
+ * Values larger than 15 are not allowed in order to avoid inadvertently
creating very large
+ * schemas. If an avro message has depth beyond this limit, the Spark struct
returned is
+ * truncated after the recursion limit.
+ *
+ * Examples: Consider an Avro schema with a recursive field:
+ * {"type" : "record", "name" : "Node", "fields" : [{"name": "Id", "type":
"int"},
+ * {"name": "Next", "type": ["null", "Node"]}]}
+ * The following lists the parsed schema with different values for this
setting.
+ * 1: `struct<Id: int>`
+ * 2: `struct<Id: int, Next: struct<Id: int>>`
+ * 3: `struct<Id: int, Next: struct<Id: int, Next: struct<Id: int>>>`
+ * and so on.
+ */
+ val RECURSIVE_FIELD_MAX_DEPTH = newOption("recursiveFieldMaxDepth")
+
+ val RECURSIVE_FIELD_MAX_DEPTH_LIMIT: Int = 15
+}
+
+abstract class AvroOptionsException(
+ errorClass: String,
+ messageParameters: Map[String, String],
+ cause: Throwable)
+ extends SparkRuntimeException(
+ errorClass,
+ messageParameters,
+ cause)
+
+object AvroOptionsError {
Review Comment:
Let's move this to QueryCompilationErrors. There are some Avro errors in it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
