[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533390359
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath.DataFileUtil;
+
+import javafx.util.Pair;
+
+import org.apache.log4j.Logger;
+
+/**
+ *This util provide clean stale data methods for clean files command
+ */
+public class CleanFilesUtil {
+
+ private static final Logger LOGGER =
+ LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+ /**
+ * This method will clean all the stale segments for a table, delete the
source folder after
+ * copying the data to the trash and also remove the .segment files of the
stale segments
+ */
+ public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+Pair, List> staleSegmentFiles =
getStaleSegmentFiles(carbonTable);
+for (String staleSegmentFile : staleSegmentFiles.getKey()) {
+ String segmentNumber =
DataFileUtil.getSegmentNoFromSegmentFile(staleSegmentFile);
+ SegmentFileStore fileStore = new
SegmentFileStore(carbonTable.getTablePath(),
+ staleSegmentFile);
+ Map locationMap =
fileStore.getSegmentFile()
+ .getLocationMap();
+ if (locationMap != null) {
+if (locationMap.entrySet().iterator().next().getValue().isRelative()) {
+ CarbonFile segmentPath =
FileFactory.getCarbonFile(CarbonTablePath.getSegmentPath(
+ carbonTable.getTablePath(), segmentNumber));
+ // copy the complete segment to the trash folder
+ TrashUtil.copySegmentToTrash(segmentPath,
TrashUtil.getCompleteTrashFolderPath(
+ carbonTable.getTablePath(), timeStampForTrashFolder,
segmentNumber));
+ // Deleting the stale Segment folders and the segment file.
+ try {
+CarbonUtil.deleteFoldersAndFiles(segmentPath);
+// delete the segment file as well
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+staleSegmentFile));
+for (String duplicateStaleSegmentFile :
staleSegmentFiles.getValue()) {
+ if
(DataFileUtil.getSegmentNoFromSegmentFile(duplicateStaleSegmentFile)
+ .equals(segmentNumber)) {
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable
+.getTablePath(), duplicateStaleSegmentFile));
+ }
+}
+ } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentPath + "
from after moving" +
+" it to the trash folder. Please delete them manually : " +
e.getMessage(), e);
+ }
+}
+ }
+}
+ }
+
+ /**
+ * This method will clean all the stale segments for partition table, delete
the source folders
+ * after copying the data to the trash and also remove the .segment files of
the stale segments
+ */
+ public static void cleanStaleSegmentsForPartitionTable(CarbonTable
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+Pair, List> staleSeg
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533390359
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath.DataFileUtil;
+
+import javafx.util.Pair;
+
+import org.apache.log4j.Logger;
+
+/**
+ *This util provide clean stale data methods for clean files command
+ */
+public class CleanFilesUtil {
+
+ private static final Logger LOGGER =
+ LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+ /**
+ * This method will clean all the stale segments for a table, delete the
source folder after
+ * copying the data to the trash and also remove the .segment files of the
stale segments
+ */
+ public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+Pair, List> staleSegmentFiles =
getStaleSegmentFiles(carbonTable);
+for (String staleSegmentFile : staleSegmentFiles.getKey()) {
+ String segmentNumber =
DataFileUtil.getSegmentNoFromSegmentFile(staleSegmentFile);
+ SegmentFileStore fileStore = new
SegmentFileStore(carbonTable.getTablePath(),
+ staleSegmentFile);
+ Map locationMap =
fileStore.getSegmentFile()
+ .getLocationMap();
+ if (locationMap != null) {
+if (locationMap.entrySet().iterator().next().getValue().isRelative()) {
+ CarbonFile segmentPath =
FileFactory.getCarbonFile(CarbonTablePath.getSegmentPath(
+ carbonTable.getTablePath(), segmentNumber));
+ // copy the complete segment to the trash folder
+ TrashUtil.copySegmentToTrash(segmentPath,
TrashUtil.getCompleteTrashFolderPath(
+ carbonTable.getTablePath(), timeStampForTrashFolder,
segmentNumber));
+ // Deleting the stale Segment folders and the segment file.
+ try {
+CarbonUtil.deleteFoldersAndFiles(segmentPath);
+// delete the segment file as well
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+staleSegmentFile));
+for (String duplicateStaleSegmentFile :
staleSegmentFiles.getValue()) {
+ if
(DataFileUtil.getSegmentNoFromSegmentFile(duplicateStaleSegmentFile)
+ .equals(segmentNumber)) {
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable
+.getTablePath(), duplicateStaleSegmentFile));
+ }
+}
+ } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentPath + "
from after moving" +
+" it to the trash folder. Please delete them manually : " +
e.getMessage(), e);
+ }
+}
+ }
+}
+ }
+
+ /**
+ * This method will clean all the stale segments for partition table, delete
the source folders
+ * after copying the data to the trash and also remove the .segment files of
the stale segments
+ */
+ public static void cleanStaleSegmentsForPartitionTable(CarbonTable
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+Pair, List> staleSeg
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533028217
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.log4j.Logger;
+
+/**
+ *This util provide clean stale data methods for clean files command
+ */
+public class CleanFilesUtil {
+
+ private static final Logger LOGGER =
+ LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+ /**
+ * This method will clean all the stale segments for a table, delete the
source folder after
+ * copying the data to the trash and also remove the .segment files of the
stale segments
+ */
+ public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegmentFiles = getStaleSegmentFiles(carbonTable);
+for (String staleSegmentFile : staleSegmentFiles) {
+ String segmentNumber =
CarbonTablePath.DataFileUtil.getSegmentNoFromSegmentFile(
+ staleSegmentFile);
+ SegmentFileStore fileStore = new
SegmentFileStore(carbonTable.getTablePath(),
+ staleSegmentFile);
+ Map locationMap =
fileStore.getSegmentFile()
+ .getLocationMap();
+ if (locationMap != null) {
+// segmentLocation is the location in Fact/part0 where the segment is
stored.
+CarbonFile segmentLocation =
FileFactory.getCarbonFile(carbonTable.getTablePath() +
Review comment:
CarbonFile segmentPath =
CarbonTablePath.getSegmentPath(carbonTable.getTablePath(), segmentNo)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533026831
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,49 @@ public int getMaxSIRepairLimit(String dbName, String
tableName) {
return Math.abs(Integer.parseInt(thresholdValue));
}
+ /**
+ * The below method returns the time(in milliseconds) for which timestamp
folder retention in
+ * trash folder will take place.
+ */
+ public long getTrashFolderRetentionTime() {
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long)validateTrashFolderRetentionTime() * milliSecondsInADay;
Review comment:
get configuration directly without invoke
validateTrashFolderRetentionTime
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533026831
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,49 @@ public int getMaxSIRepairLimit(String dbName, String
tableName) {
return Math.abs(Integer.parseInt(thresholdValue));
}
+ /**
+ * The below method returns the time(in milliseconds) for which timestamp
folder retention in
+ * trash folder will take place.
+ */
+ public long getTrashFolderRetentionTime() {
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long)validateTrashFolderRetentionTime() * milliSecondsInADay;
Review comment:
get configuration directly
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533026365
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,49 @@ public int getMaxSIRepairLimit(String dbName, String
tableName) {
return Math.abs(Integer.parseInt(thresholdValue));
}
+ /**
+ * The below method returns the time(in milliseconds) for which timestamp
folder retention in
+ * trash folder will take place.
+ */
+ public long getTrashFolderRetentionTime() {
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long)validateTrashFolderRetentionTime() * milliSecondsInADay;
+ }
+
+ /**
+ * The below method returns the time(in days) for which timestamp folder
retention in trash
+ * folder will take place
+ */
+ private int validateTrashFolderRetentionTime() {
Review comment:
invoke this method in validateAndLoadDefaultProperties
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r533016768
##
File path:
core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
##
@@ -183,16 +186,18 @@ private static boolean
checkIfLoadCanBeDeleted(LoadMetadataDetails oneLoad,
return true;
}
long deletionTime = oneLoad.getModificationOrDeletionTimestamp();
-
+ if (isCleanFilesOperation) {
Review comment:
this option is not required.
Only clean files can do clean data.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532010590
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for
clean files
+ * operation.
+ */
+public class CleanFilesUtil {
+
+ private static final Logger LOGGER =
+ LogServiceFactory.getLogService(CleanFilesUtil.class.getName());
+
+ /**
+ * This method will clean all the stale segments for a table, delete the
source folder after
+ * copying the data to the trash and also remove the .segment files of the
stale segments
+ */
+ public static void cleanStaleSegments(CarbonTable carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+ for (String staleSegment : staleSegments) {
+String segmentNumber =
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+SegmentFileStore fileStore = new
SegmentFileStore(carbonTable.getTablePath(),
+staleSegment);
+Map locationMap =
fileStore.getSegmentFile()
+.getLocationMap();
+if (locationMap != null) {
+ CarbonFile segmentLocation =
FileFactory.getCarbonFile(carbonTable.getTablePath() +
+ CarbonCommonConstants.FILE_SEPARATOR +
fileStore.getSegmentFile().getLocationMap()
+ .entrySet().iterator().next().getKey());
+ // copy the complete segment to the trash folder
+ TrashUtil.copySegmentToTrash(segmentLocation,
CarbonTablePath.getTrashFolderPath(
+ carbonTable.getTablePath()) +
CarbonCommonConstants.FILE_SEPARATOR +
+ timeStampForTrashFolder + CarbonCommonConstants.FILE_SEPARATOR +
CarbonTablePath
+ .SEGMENT_PREFIX + segmentNumber);
+ // Deleting the stale Segment folders.
+ try {
+CarbonUtil.deleteFoldersAndFiles(segmentLocation);
+ } catch (IOException | InterruptedException e) {
+LOGGER.error("Unable to delete the segment: " + segmentNumber + "
from after moving" +
+" it to the trash folder. Please delete them manually : " +
e.getMessage(), e);
+ }
+ // delete the segment file as well
+
FileFactory.deleteFile(CarbonTablePath.getSegmentFilePath(carbonTable.getTablePath(),
+ staleSegment));
+}
+ }
+ staleSegments.clear();
+}
+ }
+
+ /**
+ * This method will clean all the stale segments for partition table, delete
the source folders
+ * after copying the data to the trash and also remove the .segment files of
the stale segments
+ */
+ public static void cleanStaleSegmentsForPartitionTable(CarbonTable
carbonTable)
+throws IOException {
+long timeStampForTrashFolder = CarbonUpdateUtil.readCurrentTime();
+List staleSegments = getStaleSegments(carbonTable);
+if (staleSegments.size() > 0) {
+ for (String staleSegment : staleSegments) {
+String segmentNumber =
staleSegment.split(CarbonCommonConstants.UNDERSCORE)[0];
+// for ea
[GitHub] [carbondata] QiangCai commented on a change in pull request #4005: [CARBONDATA-3978] Trash Folder support in carbondata
QiangCai commented on a change in pull request #4005:
URL: https://github.com/apache/carbondata/pull/4005#discussion_r532004928
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
##
@@ -2086,6 +2087,41 @@ public int getMaxSIRepairLimit(String dbName, String
tableName) {
return Math.abs(Integer.parseInt(thresholdValue));
}
+ /**
+ * The below method returns the time(in milliseconds) for which timestamp
folder retention in
+ * trash folder will take place.
+ */
+ public long getTrashFolderRetentionTime() {
+String propertyValue =
getProperty(CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS, Integer
+.toString(CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT));
+int configuredValue = 0;
+try {
+ configuredValue = Integer.parseInt(propertyValue);
+ if (configuredValue < 0 || configuredValue > CarbonCommonConstants
+ .CARBON_TRASH_RETENTION_DAYS_MAXIMUM) {
+LOGGER.warn("Value of " +
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS + " is" +
+" invalid, taking default value instead");
+configuredValue =
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT;
+ }
+} catch (NumberFormatException e) {
+ LOGGER.error("Invalid value configured for " + CarbonCommonConstants
+ .CARBON_TRASH_RETENTION_DAYS + ", considering the default value");
+ configuredValue =
CarbonCommonConstants.CARBON_TRASH_RETENTION_DAYS_DEFAULT;
+}
+long milliSecondsInADay = TimeUnit.DAYS.toMillis(1);
+return (long) configuredValue * milliSecondsInADay;
+ }
Review comment:
please move validate logical to validateTrashFolderRetentionTime method,
getTrashFolderRetentionTime should return the value and not write any log.
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
+import org.apache.carbondata.core.statusmanager.SegmentStatus;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/**
+ * Mantains the clean files command in carbondata. This class has methods for
clean files
+ * operation.
Review comment:
This util provide clean stale data methods for clean files command
##
File path:
core/src/main/java/org/apache/carbondata/core/util/CleanFilesUtil.java
##
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.util;
+
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cons
