akashrn5 commented on a change in pull request #4072: URL: https://github.com/apache/carbondata/pull/4072#discussion_r579051553
########## File path: docs/clean-files.md ########## @@ -64,4 +64,41 @@ The stale_inprogress option with force option will delete Marked for delete, Com ``` CLEAN FILES FOR TABLE TABLE_NAME options('stale_inprogress'='true', 'force'='true') - ``` \ No newline at end of file + ``` +### DRY RUN OPTION +Clean files also support a dry run option which will let the user know how much space fill we freed +during the actual clean files operation. The dry run operation will not delete any data but will just give +size based statistics on the data which will be cleaned in clean files. Dry run operation will return two columns where the first will +show how much space will be freed by that clean files operation and the second column will show the +remaining stale data(data which can be deleted but has not yet expired as per the ```max.query.execution.time``` and ``` carbon.trash.retention.days``` values +). By default the value of ```dryrun``` option is ```false```. + +Dry Run Operation is supported with four types of commands: + ``` + CLEAN FILES FOR TABLE TABLE_NAME options('dryrun'='true') + ``` + ``` + CLEAN FILES FOR TABLE TABLE_NAME options('force'='true', 'dryrun'='true') + ``` + ``` + CLEAN FILES FOR TABLE TABLE_NAME options('stale_inprogress'='true','dryrun'='true') + ``` + + ``` + CLEAN FILES FOR TABLE TABLE_NAME options('stale_inprogress'='true', 'force'='true','dryrun'='true') + ``` + +**NOTE**: + * Since the dry run operation will calculate size and will access File level API's, the operation can + be a costly and a time consuming operation in case of tables with large number of segments. + * When dry run is true, the statistics option will not matter. + +### SHOW STATISTICS +Clean files operation tells how much size is freed during that operation to the user. By default, the clean files operation +will show the size freed statistics. Since calculating and showing statistics can be a costly operation and reduce the performance of the +clean files operation, the user can disable that option by using ```statistics = false``` in the clean files options. + + ``` + CLEAN FILES FOR TABLE TABLE_NAME options('statistics`='false') Review comment: ```suggestion CLEAN FILES FOR TABLE TABLE_NAME options('statistics'='false') ``` ########## File path: integration/spark/src/main/scala/org/apache/carbondata/trash/DataTrashManager.scala ########## @@ -87,13 +104,48 @@ object DataTrashManager { } } - private def checkAndCleanTrashFolder(carbonTable: CarbonTable, isForceDelete: Boolean): Unit = { + /** + * Checks the size of the segment files as well as datafiles, this method is used before and after + * clean files operation to check how much space is actually freed, during the operation. + */ + def getSizeScreenshot(carbonTable: CarbonTable): Long = { + val metadataDetails = SegmentStatusManager.readLoadMetadata(carbonTable.getMetadataPath) + var size: Long = FileFactory.getDirectorySize(CarbonTablePath.getSegmentFilesLocation( Review comment: ```suggestion var segmentFilesLocationSize: Long = FileFactory.getDirectorySize(CarbonTablePath.getSegmentFilesLocation( ``` ########## File path: integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala ########## @@ -41,6 +42,26 @@ case class CarbonCleanFilesCommand( extends DataCommand { val LOGGER: Logger = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + val isDryRun: Boolean = options.getOrElse("dryrun", "false").toBoolean + var showStats: Boolean = options.getOrElse("statistics", "true").toBoolean + if (isInternalCleanCall) { + showStats = false + } + + override def output: Seq[AttributeReference] = { + if (isDryRun) { + // dry run operation + Seq( + AttributeReference("Size Freed", LongType, nullable = false)(), + AttributeReference("Trash Data Remaining", LongType, nullable = false)()) Review comment: please return the return values of size in readable format, if giving in MB, have UNIT in column header as (MB) ########## File path: docs/clean-files.md ########## @@ -64,4 +64,41 @@ The stale_inprogress option with force option will delete Marked for delete, Com ``` CLEAN FILES FOR TABLE TABLE_NAME options('stale_inprogress'='true', 'force'='true') - ``` \ No newline at end of file + ``` +### DRY RUN OPTION +Clean files also support a dry run option which will let the user know how much space fill we freed Review comment: ```suggestion Clean files also support a dry run option which will let the user know how much space will we freed ``` ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org