Repository: carbondata Updated Branches: refs/heads/master 81038f55e -> a16289786
[CARBONDATA-2724][DataMap]Unsupported create datamap on table with V1 or V2 format data block creating datamap on carbon table with V1 or V2 format Currently the version info is read from carbon data file This closes #2488 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a1628978 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a1628978 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a1628978 Branch: refs/heads/master Commit: a162897862c92947ea8fd63713b7dbe6098f3b13 Parents: 81038f5 Author: ndwangsen <luffy.w...@huawei.com> Authored: Wed Jul 11 17:41:25 2018 +0800 Committer: xuchuanyin <xuchuan...@hust.edu.cn> Committed: Tue Jul 17 23:35:50 2018 +0800 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 51 ++++++++++++++++++++ .../datamap/CarbonCreateDataMapCommand.scala | 8 ++- 2 files changed, 58 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 9796696..642fe8e 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -88,6 +88,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath; import org.apache.carbondata.format.BlockletHeader; import org.apache.carbondata.format.DataChunk2; import org.apache.carbondata.format.DataChunk3; +import org.apache.carbondata.format.FileHeader; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -3184,4 +3185,54 @@ public final class CarbonUtil { } return columnLocalDictGenMap; } + + /** + * This method get the carbon file format version + * + * @param carbonTable + * carbon Table + */ + public static ColumnarFormatVersion getFormatVersion(CarbonTable carbonTable) + throws IOException { + String storePath = null; + // if the carbontable is support flat folder + boolean supportFlatFolder = carbonTable.isSupportFlatFolder(); + if (supportFlatFolder) { + storePath = carbonTable.getTablePath(); + } else { + // get the valid segments + SegmentStatusManager segmentStatusManager = + new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier()); + SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = + segmentStatusManager.getValidAndInvalidSegments(); + List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments(); + CarbonProperties carbonProperties = CarbonProperties.getInstance(); + if (validSegments.isEmpty()) { + return carbonProperties.getFormatVersion(); + } + storePath = carbonTable.getSegmentPath(validSegments.get(0).getSegmentNo()); + } + + CarbonFile[] carbonFiles = FileFactory + .getCarbonFile(storePath) + .listFiles(new CarbonFileFilter() { + @Override + public boolean accept(CarbonFile file) { + if (file == null) { + return false; + } + return file.getName().endsWith("carbondata"); + } + }); + if (carbonFiles == null || carbonFiles.length < 1) { + return CarbonProperties.getInstance().getFormatVersion(); + } + + CarbonFile carbonFile = carbonFiles[0]; + // get the carbon file header + CarbonHeaderReader headerReader = new CarbonHeaderReader(carbonFile.getCanonicalPath()); + FileHeader fileHeader = headerReader.readHeader(); + int version = fileHeader.getVersion(); + return ColumnarFormatVersion.valueOf((short)version); + } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/a1628978/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala index 7600160..336793e 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala @@ -26,9 +26,10 @@ import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandExcept import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager} import org.apache.carbondata.core.datamap.status.DataMapStatusManager +import org.apache.carbondata.core.metadata.ColumnarFormatVersion import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty} import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema} -import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil} import org.apache.carbondata.datamap.{DataMapManager, IndexDataMapProvider} import org.apache.carbondata.events._ @@ -79,6 +80,11 @@ case class CarbonCreateDataMapCommand( s"$dmProviderName datamap") } + if (mainTable !=null && CarbonUtil.getFormatVersion(mainTable) != ColumnarFormatVersion.V3) { + throw new MalformedCarbonCommandException(s"Unsupported operation on table with " + + s"V1 or V2 format data") + } + dataMapSchema = new DataMapSchema(dataMapName, dmProviderName) val property = dmProperties.map(x => (x._1.trim, x._2.trim)).asJava
