tanishqgandhi1908 commented on code in PR #6502:
URL: https://github.com/apache/texera/pull/6502#discussion_r3769959433
##########
common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/dataset/FileListerSourceOpExec.scala:
##########
@@ -28,13 +29,34 @@ import
org.apache.texera.dao.jooq.generated.tables.Dataset.DATASET
import
org.apache.texera.dao.jooq.generated.tables.DatasetVersion.DATASET_VERSION
import org.apache.texera.dao.jooq.generated.tables.User.USER
+object FileListerSourceOpExec {
+
+ /**
+ * Parses a dataset version path
(/datasets/ownerEmail/datasetName/versionName) into its
+ * (ownerEmail, datasetName, versionName) components.
+ *
+ * @throws IllegalArgumentException if the path is not a well-formed
dataset version path
+ */
+ private[dataset] def parseDatasetVersionPath(
+ datasetVersionPath: String
+ ): (String, String, String) = {
+ val segments = datasetVersionPath.split("/").filter(_.nonEmpty)
+ require(
+ segments.length >= 4 && ResourceType.isValidPrefix(segments.head),
+ s"Invalid dataset version path '$datasetVersionPath'; " +
+ "expected /datasets/ownerEmail/datasetName/versionName"
+ )
+ (segments(1), segments(2), segments(3))
+ }
+}
+
class FileListerSourceOpExec private[dataset] (descString: String) extends
SourceOperatorExecutor {
private val desc: FileListerSourceOpDesc =
objectMapper.readValue(descString, classOf[FileListerSourceOpDesc])
override def produceTuple(): Iterator[TupleLike] = {
- val Seq(_, ownerEmail, datasetName, versionName, _*) =
- desc.datasetVersionPath.split("/").toSeq
+ val (ownerEmail, datasetName, versionName) =
+ FileListerSourceOpExec.parseDatasetVersionPath(desc.datasetVersionPath)
Review Comment:
Fixed. parseDatasetVersionPath now also returns the validated prefix, and
emitted paths are built from the parsed components via a new
canonicalVersionPath helper instead of the raw desc.datasetVersionPath. So
/datasets/alice/ds/v2/extra/ now emits /datasets/alice/ds/v2/<object> instead
of leaking the extra segment. Added tests for the canonical rebuild and the
extra-segment case.
I kept the lenient parse rather than requiring exactly four segments, to
avoid rejecting previously-working saved configurations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
