[CARBONDATA-2538] added filter while listing files from writer path 1. Added filter to list only index and carbondata files. So even if the lock files are present proper exception can be thrown 2. Updated complex type docs
This closes #2344 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d7773187 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d7773187 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d7773187 Branch: refs/heads/spark-2.3 Commit: d7773187f72c73b7f9514f13bce17de3c552247c Parents: 8b80b12 Author: kunal642 <kunalkapoor...@gmail.com> Authored: Fri May 25 16:21:45 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Tue May 29 10:49:59 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/schema/table/CarbonTable.java | 2 +- .../readcommitter/LatestFilesReadCommittedScope.java | 9 ++++++++- .../java/org/apache/carbondata/core/util/CarbonUtil.java | 11 ++++++++--- docs/supported-data-types-in-carbondata.md | 2 ++ .../command/table/CarbonDescribeFormattedCommand.scala | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index 8528d6f..b1ed981 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -892,7 +892,7 @@ public class CarbonTable implements Serializable { public long size() throws IOException { - Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this); + Map<String, Long> dataIndexSize = CarbonUtil.calculateDataIndexSize(this, true); Long dataSize = dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE); if (dataSize == null) { dataSize = 0L; http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java index 6106174..14bba65 100644 --- a/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java +++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/LatestFilesReadCommittedScope.java @@ -26,6 +26,7 @@ import org.apache.carbondata.common.annotations.InterfaceAudience; import org.apache.carbondata.common.annotations.InterfaceStability; import org.apache.carbondata.core.datamap.Segment; import org.apache.carbondata.core.datastore.filesystem.CarbonFile; +import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter; import org.apache.carbondata.core.datastore.impl.FileFactory; import org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore; import org.apache.carbondata.core.mutate.UpdateVO; @@ -138,7 +139,13 @@ public class LatestFilesReadCommittedScope implements ReadCommittedScope { @Override public void takeCarbonIndexFileSnapShot() throws IOException { // Read the current file Path get the list of indexes from the path. CarbonFile file = FileFactory.getCarbonFile(carbonFilePath); - if (file.listFiles().length == 0) { + CarbonFile[] files = file.listFiles(new CarbonFileFilter() { + @Override public boolean accept(CarbonFile file) { + return file.getName().endsWith(CarbonTablePath.INDEX_FILE_EXT) || file.getName() + .endsWith(CarbonTablePath.CARBON_DATA_EXT); + } + }); + if (files.length == 0) { // For nonTransactional table, files can be removed at any point of time. // So cannot assume files will be present throw new IOException("No files are present in the table location :" + carbonFilePath); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 23d02ef..9ccd772 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -2550,7 +2550,8 @@ public final class CarbonUtil { /** * This method will calculate the data size and index size for carbon table */ - public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable) + public static Map<String, Long> calculateDataIndexSize(CarbonTable carbonTable, + Boolean updateSize) throws IOException { Map<String, Long> dataIndexSizeMap = new HashMap<String, Long>(); long dataSize = 0L; @@ -2565,7 +2566,11 @@ public final class CarbonUtil { SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); try { - if (carbonLock.lockWithRetries()) { + boolean lockAcquired = true; + if (updateSize) { + lockAcquired = carbonLock.lockWithRetries(); + } + if (lockAcquired) { LOGGER.info("Acquired lock for table for table status updation"); String metadataPath = carbonTable.getMetadataPath(); LoadMetadataDetails[] loadMetadataDetails = @@ -2593,7 +2598,7 @@ public final class CarbonUtil { } } // If it contains old segment, write new load details - if (needUpdate) { + if (needUpdate && updateSize) { SegmentStatusManager.writeLoadDetailsIntoFile( CarbonTablePath.getTableStatusFilePath(identifier.getTablePath()), loadMetadataDetails); http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/docs/supported-data-types-in-carbondata.md ---------------------------------------------------------------------- diff --git a/docs/supported-data-types-in-carbondata.md b/docs/supported-data-types-in-carbondata.md index 6c21508..7260afe 100644 --- a/docs/supported-data-types-in-carbondata.md +++ b/docs/supported-data-types-in-carbondata.md @@ -38,6 +38,8 @@ * Complex Types * arrays: ARRAY``<data_type>`` * structs: STRUCT``<col_name : data_type COMMENT col_comment, ...>`` + + **NOTE**: Only 2 level complex type schema is supported for now. * Other Types * BOOLEAN \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/d7773187/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala index 69bb91f..7d15cc1 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala @@ -96,7 +96,7 @@ private[sql] case class CarbonDescribeFormattedCommand( val tableComment = tblProps.asScala.getOrElse(CarbonCommonConstants.TABLE_COMMENT, "") results ++= Seq(("Comment", tableComment, "")) results ++= Seq(("Table Block Size ", carbonTable.getBlockSizeInMB + " MB", "")) - val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable) + val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable, false) if (!dataIndexSize.isEmpty) { results ++= Seq((CarbonCommonConstants.TABLE_DATA_SIZE, dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).toString, ""))