jackluo923 commented on code in PR #13782:
URL: https://github.com/apache/pinot/pull/13782#discussion_r1717394353
##########
pinot-common/src/main/java/org/apache/pinot/common/utils/TarCompressionUtils.java:
##########
@@ -66,106 +67,122 @@ public class TarGzCompressionUtils {
*/
- private TarGzCompressionUtils() {
+ private TarCompressionUtils() {
}
public static final String TAR_GZ_FILE_EXTENSION = ".tar.gz";
+ public static final String TAR_LZ4_FILE_EXTENSION = ".tar.lz4";
+ public static final String TAR_ZST_FILE_EXTENSION = ".tar.zst";
+ public static final Map<String, String> COMPRESSOR_NAME_BY_FILE_EXTENSIONS =
Map.of(
+ TAR_GZ_FILE_EXTENSION, CompressorStreamFactory.GZIP,
+ TAR_LZ4_FILE_EXTENSION, CompressorStreamFactory.LZ4_FRAMED,
+ TAR_ZST_FILE_EXTENSION, CompressorStreamFactory.ZSTANDARD
+ );
+ private static final CompressorStreamFactory COMPRESSOR_STREAM_FACTORY = new
CompressorStreamFactory();
private static final char ENTRY_NAME_SEPARATOR = '/';
/**
- * Creates a tar.gz file from the input file/directory to the output file.
The output file must have ".tar.gz" as the
- * file extension.
+ * Creates a compressed tar file from the input file/directory to the output
file. The output file must have
+ * a supported compressed tar file extension as the file extension such as
".tar.gz" or ".tar.zst"
*/
- public static void createTarGzFile(File inputFile, File outputFile)
- throws IOException {
- createTarGzFile(new File[]{inputFile}, outputFile);
+ public static void createCompressedTarFile(File inputFile, File outputFile)
+ throws IOException, CompressorException {
+ createCompressedTarFile(new File[]{inputFile}, outputFile);
}
/**
- * Creates a tar.gz file from a list of input file/directories to the output
file. The output file must have
- * ".tar.gz" as the file extension.
+ * Creates a compressed tar file from a list of input file/directories to
the output file. The output file must have
+ * a supported file extension such as "tar.gz" or "tar.zst"
*/
- public static void createTarGzFile(File[] inputFiles, File outputFile)
- throws IOException {
-
Preconditions.checkArgument(outputFile.getName().endsWith(TAR_GZ_FILE_EXTENSION),
- "Output file: %s does not have '.tar.gz' file extension", outputFile);
+ public static void createCompressedTarFile(File[] inputFiles, File
outputFile)
+ throws IOException, CompressorException {
+ String compressorName = null;
+ for (String supportedCompressorExtension :
COMPRESSOR_NAME_BY_FILE_EXTENSIONS.keySet()) {
+ if (outputFile.getName().endsWith(supportedCompressorExtension)) {
+ compressorName =
COMPRESSOR_NAME_BY_FILE_EXTENSIONS.get(supportedCompressorExtension);
+ break;
+ }
+ }
+ Preconditions.checkState(null != compressorName,
+ "Output file: %s does not have a supported compressed tar file
extension", outputFile);
try (OutputStream fileOut = Files.newOutputStream(outputFile.toPath());
- BufferedOutputStream bufferedOut = new BufferedOutputStream(fileOut);
- OutputStream gzipOut = new GzipCompressorOutputStream(bufferedOut);
- TarArchiveOutputStream tarGzOut = new TarArchiveOutputStream(gzipOut))
{
- tarGzOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
- tarGzOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
+ BufferedOutputStream bufferedOut = new BufferedOutputStream(fileOut);
+ OutputStream compressorOut =
+
COMPRESSOR_STREAM_FACTORY.createCompressorOutputStream(compressorName,
bufferedOut);
+ TarArchiveOutputStream tarOut = new
TarArchiveOutputStream(compressorOut)) {
+ tarOut.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
+ tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
for (File inputFile : inputFiles) {
- addFileToTarGz(tarGzOut, inputFile, "");
+ addFileToCompressedTar(tarOut, inputFile, "");
}
}
}
/**
- * Helper method to write a file into the tar.gz file output stream. The
base entry name is the relative path of the
- * file to the root directory.
+ * Helper method to write a file into the compressed tar file output stream.
The base entry name is the relative
+ * path of the file to the root directory.
*/
- private static void addFileToTarGz(ArchiveOutputStream tarGzOut, File file,
String baseEntryName)
+ private static void addFileToCompressedTar(ArchiveOutputStream tarOut, File
file, String baseEntryName)
throws IOException {
String entryName = baseEntryName + file.getName();
TarArchiveEntry entry = new TarArchiveEntry(file, entryName);
- tarGzOut.putArchiveEntry(entry);
+ tarOut.putArchiveEntry(entry);
if (file.isFile()) {
try (InputStream in = Files.newInputStream(file.toPath())) {
- IOUtils.copy(in, tarGzOut);
+ IOUtils.copy(in, tarOut);
}
- tarGzOut.closeArchiveEntry();
+ tarOut.closeArchiveEntry();
} else {
- tarGzOut.closeArchiveEntry();
+ tarOut.closeArchiveEntry();
File[] children = file.listFiles();
assert children != null;
String baseEntryNameForChildren = entryName + ENTRY_NAME_SEPARATOR;
for (File child : children) {
- addFileToTarGz(tarGzOut, child, baseEntryNameForChildren);
+ addFileToCompressedTar(tarOut, child, baseEntryNameForChildren);
}
}
}
/**
- * Un-tars a tar.gz file into a directory, returns all the untarred
files/directories.
+ * Un-tars a compressed tar file into a directory, returns all the untarred
files/directories.
* <p>For security reason, the untarred files must reside in the output
directory.
*/
public static List<File> untar(File inputFile, File outputDir)
- throws IOException {
+ throws IOException, CompressorException {
try (InputStream fileIn = Files.newInputStream(inputFile.toPath())) {
return untar(fileIn, outputDir);
}
}
/**
- * Un-tars an inputstream of a tar.gz file into a directory, returns all the
untarred files/directories.
+ * Un-tars an inputstream of a compressed tar file into a directory, returns
all the untarred files/directories.
* <p>For security reason, the untarred files must reside in the output
directory.
*/
public static List<File> untar(InputStream inputStream, File outputDir)
- throws IOException {
+ throws IOException, CompressorException {
return untarWithRateLimiter(inputStream, outputDir,
NO_DISK_WRITE_RATE_LIMIT);
}
/**
- * Un-tars an inputstream of a tar.gz file into a directory, returns all the
untarred files/directories.
+ * Un-tars an inputstream of a compressed tar file into a directory, returns
all the untarred files/directories.
* RateLimit limits the untar rate
* <p>For security reason, the untarred files must reside in the output
directory.
*/
public static List<File> untarWithRateLimiter(InputStream inputStream, File
outputDir, long maxStreamRateInByte)
- throws IOException {
+ throws IOException, CompressorException {
String outputDirCanonicalPath = outputDir.getCanonicalPath();
// Prevent partial path traversal
if (!outputDirCanonicalPath.endsWith(File.separator)) {
outputDirCanonicalPath += File.separator;
}
List<File> untarredFiles = new ArrayList<>();
try (InputStream bufferedIn = new BufferedInputStream(inputStream);
Review Comment:
Switch to using Pinot Style and adjusted style in various files.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
