Indhumathi27 commented on a change in pull request #3584: [CARBONDATA-3718]
Support SegmentLevel MinMax for better Pruning and less driver memory usage for
cache
URL: https://github.com/apache/carbondata/pull/3584#discussion_r394837462
##########
File path:
integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
##########
@@ -248,21 +258,31 @@ class CarbonTableCompactor(carbonLoadModel:
CarbonLoadModel,
} else {
// Get the segment files each updated segment in case of IUD compaction
if (compactionType == CompactionType.IUD_UPDDEL_DELTA) {
- val segmentFilesList = loadsToMerge.asScala.map{seg =>
+ val segmentFilesList = loadsToMerge.asScala.map { seg =>
+ val segmentMetaDataInfo = new
SegmentFileStore(carbonLoadModel.getTablePath,
+ seg.getSegmentFile).getSegmentFile.getSegmentMetaDataInfo
val file = SegmentFileStore.writeSegmentFile(
carbonTable,
seg.getLoadName,
- carbonLoadModel.getFactTimeStamp.toString)
+ carbonLoadModel.getFactTimeStamp.toString,
+ segmentMetaDataInfo)
new Segment(seg.getLoadName, file)
}.filter(_.getSegmentFileName != null).asJava
segmentFilesForIUDCompact = new
util.ArrayList[Segment](segmentFilesList)
} else {
+ // get segmentMetadata info from accumulator
+ val segmentMetaDataInfo =
CarbonDataRDDFactory.getSegmentMetaDataInfoFromAccumulator(
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
