[GitHub] [carbondata] ShreelekhyaG commented on a diff in pull request #4287: [CARBONDATA-4346] Remove list files while query and invalid cache

[GitBox]

ShreelekhyaG commented on code in PR #4287:
URL: https://github.com/apache/carbondata/pull/4287#discussion_r917549877


##########
integration/spark/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala:
##########
@@ -190,6 +192,10 @@ private[sql] case class CarbonProjectForUpdateCommand(
             updateTableModel,
             executionErrors)
 
+          // clear invalid segments from cache
+          IndexStoreManager.getInstance()

Review Comment:
   Done



##########
core/src/main/java/org/apache/carbondata/core/util/BlockletIndexUtil.java:
##########
@@ -193,7 +154,14 @@ private static BlockMetaInfo createBlockMetaInfo(
         CarbonFile carbonFile = FileFactory.getCarbonFile(carbonDataFile);
         return new BlockMetaInfo(new String[] { "localhost" }, 
carbonFile.getSize());
       default:
-        return 
fileNameToMetaInfoMapping.get(FileFactory.getFormattedPath(carbonDataFile));
+        if (!FileFactory.isFileExist(carbonDataFile)) {
+          return null;
+        }
+        CarbonFile file = 
FileFactory.getCarbonFile(FileFactory.getFormattedPath(carbonDataFile));
+        String[] location = file.getLocations();
+        long len = file.getSize();
+        BlockMetaInfo blockMetaInfo = new BlockMetaInfo(location, len);
+        return blockMetaInfo;

Review Comment:
   Done



##########
core/src/main/java/org/apache/carbondata/core/util/BlockletIndexUtil.java:
##########
@@ -143,38 +135,7 @@ public static Map<String, BlockMetaInfo> 
getBlockMetaInfoMap(
     return blockMetaInfoMap;
   }
 
-  /**
-   * This method will create file name to block Meta Info Mapping. This method 
will reduce the
-   * number of nameNode calls and using this method one namenode will fetch 
1000 entries
-   *
-   * @param segmentFilePath
-   * @return
-   * @throws IOException
-   */
-  public static Map<String, BlockMetaInfo> 
createCarbonDataFileBlockMetaInfoMapping(
-      String segmentFilePath, Configuration configuration) throws IOException {
-    Map<String, BlockMetaInfo> fileNameToMetaInfoMapping = new TreeMap();
-    CarbonFile carbonFile = FileFactory.getCarbonFile(segmentFilePath, 
configuration);
-    if (carbonFile instanceof AbstractDFSCarbonFile && !(carbonFile instanceof 
S3CarbonFile)) {
-      PathFilter pathFilter = new PathFilter() {
-        @Override
-        public boolean accept(Path path) {
-          return CarbonTablePath.isCarbonDataFile(path.getName());
-        }
-      };
-      CarbonFile[] carbonFiles = carbonFile.locationAwareListFiles(pathFilter);

Review Comment:
   ok



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: dev-unsubscr...@carbondata.apache.org

For queries about this service, please contact Infrastructure at:
us...@infra.apache.org