cloud-fan commented on code in PR #56527: URL: https://github.com/apache/spark/pull/56527#discussion_r3424580432
########## sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/TextTarArchiveReadSuite.scala: ########## @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.datasources + +import java.io.{File, FileOutputStream, OutputStream} +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.Locale +import java.util.zip.GZIPOutputStream + +import org.apache.commons.compress.archivers.tar.{TarArchiveEntry, TarArchiveOutputStream} + +import org.apache.spark.{SparkConf, SparkException} +import org.apache.spark.sql.{DataFrame, QueryTest, Row} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.util.Utils + +/** + * Reads of text files packed in tar archives (`.tar`/`.tar.gz`/`.tgz`), streamed through the + * [[ArchiveReader]] path. Entries are streamed (never unpacked to disk), and the central contract + * verified throughout is parity with reading the same files from a directory. + * + * Unlike CSV/JSON this does not reuse [[ArchiveReadSuiteBase]]: the text data source has a single + * fixed `value` column (one row per line, or per entry with `wholetext`) and no schema inference, + * so the structured, two-column tests there do not apply. + */ +class TextTarArchiveReadSuite extends QueryTest with SharedSparkSession { + + override def sparkConf: SparkConf = + super.sparkConf.set(SQLConf.ARCHIVE_FORMAT_READER_ENABLED.key, "true") + + /** Archive extensions to exercise; the head is the default. */ + private val archiveExtensions: Seq[String] = Seq("tar", "tar.gz", "tgz") + + private def textBytes(s: String): Array[Byte] = s.getBytes(StandardCharsets.UTF_8) + + /** Writes `entries` (name -> bytes) into the archive at `dest`; compression follows the ext. */ + private def writeArchive(dest: File, entries: Seq[(String, Array[Byte])]): Unit = { Review Comment: Optional / non-blocking: `writeArchive`, `archiveExtensions` (line 50), and the inlined corrupt-archive write (line 176) are byte-identical to `TarArchiveReadBase.writeArchive` / `archiveExtensions` / `writeCorruptArchive`. The CSV/JSON tar suites get these for free by mixing in `TarArchiveReadBase`, but text can't — `TarArchiveReadBase extends ArchiveReadSuiteBase`, whose two-column / schema-inference tests don't fit text's single `value` column (your comment above correctly explains why). Consider extracting just the tar-*writing* helpers into a small standalone trait that both `TarArchiveReadBase` and this suite mix in, so the container logic lives in one place and can't silently drift from the CSV/JSON suites if tar writing is ever changed. Low priority. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
