QiangCai commented on a change in pull request #3535: [WIP] Refactory data
loading for partition table
URL: https://github.com/apache/carbondata/pull/3535#discussion_r361888306
##########
File path: core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
##########
@@ -2730,39 +2734,66 @@ public static String encodeToString(byte[] bytes)
throws UnsupportedEncodingExce
return
Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
}
+ public static void copyCarbonDataFileToCarbonStorePath(String localFilePath,
+ String carbonDataDirectoryPath, long fileSizeInBytes,
+ OutputFilesInfoHolder outputFilesInfoHolder) throws
CarbonDataWriterException {
+ if (carbonDataDirectoryPath.endsWith(".tmp") && localFilePath
+ .endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
+ // for partition case, write carbondata file directly to final path,
keep index in temp path.
+ // This can improve the commit job performance on s3a.
+ carbonDataDirectoryPath =
+ carbonDataDirectoryPath.substring(0,
carbonDataDirectoryPath.lastIndexOf("/"));
+ if (outputFilesInfoHolder != null) {
+ outputFilesInfoHolder.addToPartitionPath(carbonDataDirectoryPath);
+ }
+ }
+ long targetSize = copyCarbonDataFileToCarbonStorePath(localFilePath,
carbonDataDirectoryPath,
+ fileSizeInBytes);
+ if (outputFilesInfoHolder != null) {
+ // Storing the number of files written by each task.
+ outputFilesInfoHolder.incrementCount();
+ // Storing the files written by each task.
+ outputFilesInfoHolder.addToOutputFiles(carbonDataDirectoryPath +
localFilePath
+ .substring(localFilePath.lastIndexOf(File.separator)) + ":" +
targetSize);
+ }
+ }
+
/**
* This method will copy the given file to carbon store location
*
* @param localFilePath local file name with full path
* @throws CarbonDataWriterException
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
