Repository: carbondata Updated Branches: refs/heads/master 75e0bd419 -> 956833e55
http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala index 1addd03..ef89771 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.types._ import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier} import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier} -import org.apache.carbondata.core.metadata.datatype.DataType +import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes} import org.apache.carbondata.core.metadata.encoder.Encoding import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension import org.apache.carbondata.core.util.DataTypeUtil @@ -165,34 +165,34 @@ case class CarbonDictionaryDecoder( """.stripMargin val caseCode = getDictionaryColumnIds(index)._3.getDataType match { - case DataType.INT => + case CarbonDataTypes.INT => s""" |int $value = Integer.parseInt(new String($valueIntern, |org.apache.carbondata.core.constants.CarbonCommonConstants |.DEFAULT_CHARSET_CLASS)); """.stripMargin - case DataType.SHORT => + case CarbonDataTypes.SHORT => s""" |short $value = |Short.parseShort(new String($valueIntern, |org.apache.carbondata.core.constants.CarbonCommonConstants |.DEFAULT_CHARSET_CLASS)); """.stripMargin - case DataType.DOUBLE => + case CarbonDataTypes.DOUBLE => s""" |double $value = |Double.parseDouble(new String($valueIntern, |org.apache.carbondata.core.constants.CarbonCommonConstants |.DEFAULT_CHARSET_CLASS)); """.stripMargin - case DataType.LONG => + case CarbonDataTypes.LONG => s""" |long $value = |Long.parseLong(new String($valueIntern, |org.apache.carbondata.core.constants.CarbonCommonConstants |.DEFAULT_CHARSET_CLASS)); """.stripMargin - case DataType.DECIMAL => + case CarbonDataTypes.DECIMAL => s""" |org.apache.spark.sql.types.Decimal $value = |Decimal.apply(new java.math.BigDecimal( @@ -382,13 +382,13 @@ object CarbonDictionaryDecoder { def convertCarbonToSparkDataType(carbonDimension: CarbonDimension, relation: CarbonRelation): types.DataType = { carbonDimension.getDataType match { - case DataType.STRING => StringType - case DataType.SHORT => ShortType - case DataType.INT => IntegerType - case DataType.LONG => LongType - case DataType.DOUBLE => DoubleType - case DataType.BOOLEAN => BooleanType - case DataType.DECIMAL => + case CarbonDataTypes.STRING => StringType + case CarbonDataTypes.SHORT => ShortType + case CarbonDataTypes.INT => IntegerType + case CarbonDataTypes.LONG => LongType + case CarbonDataTypes.DOUBLE => DoubleType + case CarbonDataTypes.BOOLEAN => BooleanType + case CarbonDataTypes.DECIMAL => val scale: Int = carbonDimension.getColumnSchema.getScale val precision: Int = carbonDimension.getColumnSchema.getPrecision if (scale == 0 && precision == 0) { @@ -396,12 +396,12 @@ object CarbonDictionaryDecoder { } else { DecimalType(precision, scale) } - case DataType.TIMESTAMP => TimestampType - case DataType.DATE => DateType - case DataType.STRUCT => + case CarbonDataTypes.TIMESTAMP => TimestampType + case CarbonDataTypes.DATE => DateType + case CarbonDataTypes.STRUCT => CarbonMetastoreTypes .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>") - case DataType.ARRAY => + case CarbonDataTypes.ARRAY => CarbonMetastoreTypes .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>") } @@ -467,13 +467,13 @@ class CarbonDecoderRDD( def convertCarbonToSparkDataType(carbonDimension: CarbonDimension, relation: CarbonRelation): types.DataType = { carbonDimension.getDataType match { - case DataType.STRING => StringType - case DataType.SHORT => ShortType - case DataType.INT => IntegerType - case DataType.LONG => LongType - case DataType.DOUBLE => DoubleType - case DataType.BOOLEAN => BooleanType - case DataType.DECIMAL => + case CarbonDataTypes.STRING => StringType + case CarbonDataTypes.SHORT => ShortType + case CarbonDataTypes.INT => IntegerType + case CarbonDataTypes.LONG => LongType + case CarbonDataTypes.DOUBLE => DoubleType + case CarbonDataTypes.BOOLEAN => BooleanType + case CarbonDataTypes.DECIMAL => val scale: Int = carbonDimension.getColumnSchema.getScale val precision: Int = carbonDimension.getColumnSchema.getPrecision if (scale == 0 && precision == 0) { @@ -481,12 +481,12 @@ class CarbonDecoderRDD( } else { DecimalType(precision, scale) } - case DataType.TIMESTAMP => TimestampType - case DataType.DATE => DateType - case DataType.STRUCT => + case CarbonDataTypes.TIMESTAMP => TimestampType + case CarbonDataTypes.DATE => DateType + case CarbonDataTypes.STRUCT => CarbonMetastoreTypes .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>") - case DataType.ARRAY => + case CarbonDataTypes.ARRAY => CarbonMetastoreTypes .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala index f245df6..6bac0da 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala @@ -16,6 +16,7 @@ */ package org.apache.spark.sql.hive +import java.util import java.util.LinkedHashSet import scala.Array.canBuildFrom @@ -28,7 +29,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Stati import org.apache.spark.sql.types._ import org.apache.carbondata.core.datastore.impl.FileFactory -import org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL +import org.apache.carbondata.core.metadata.datatype.DataTypes.DECIMAL import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension} import org.apache.carbondata.core.statusmanager.SegmentStatusManager import org.apache.carbondata.core.util.path.CarbonStorePath @@ -45,7 +46,7 @@ case class CarbonRelation( extends LeafNode with MultiInstanceRelation { def recursiveMethod(dimName: String, childDim: CarbonDimension): String = { - childDim.getDataType.toString.toLowerCase match { + childDim.getDataType.getName.toLowerCase match { case "array" => s"${ childDim.getColName.substring(dimName.length + 1) }:array<${ getArrayChildren(childDim.getColName) }>" @@ -58,7 +59,7 @@ case class CarbonRelation( def getArrayChildren(dimName: String): String = { metaData.carbonTable.getChildren(dimName).asScala.map(childDim => { - childDim.getDataType.toString.toLowerCase match { + childDim.getDataType.getName.toLowerCase match { case "array" => s"array<${ getArrayChildren(childDim.getColName) }>" case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>" case dType => addDecimalScaleAndPrecision(childDim, dType) @@ -68,7 +69,7 @@ case class CarbonRelation( def getStructChildren(dimName: String): String = { metaData.carbonTable.getChildren(dimName).asScala.map(childDim => { - childDim.getDataType.toString.toLowerCase match { + childDim.getDataType.getName.toLowerCase match { case "array" => s"${ childDim.getColName.substring(dimName.length + 1) }:array<${ getArrayChildren(childDim.getColName) }>" @@ -95,8 +96,7 @@ case class CarbonRelation( sett.asScala.toSeq.map(dim => { val dimval = metaData.carbonTable .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName) - val output: DataType = dimval.getDataType - .toString.toLowerCase match { + val output: DataType = dimval.getDataType.getName.toLowerCase match { case "array" => CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>") case "struct" => @@ -118,14 +118,17 @@ case class CarbonRelation( new LinkedHashSet( tableMeta.carbonTable. getMeasureByTableName(tableMeta.carbonTable.getFactTableName). - asScala.asJava).asScala.toSeq - .map(x => AttributeReference(x.getColName, CarbonMetastoreTypes.toDataType( - metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.toString - .toLowerCase match { - case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")" - case others => others - }), - nullable = true)()) + asScala.asJava).asScala.toSeq.map { x => + val metastoreType = metaData.carbonTable.getMeasureByName(factTable, x.getColName) + .getDataType.getName.toLowerCase match { + case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")" + case others => others + } + AttributeReference( + x.getColName, + CarbonMetastoreTypes.toDataType(metastoreType), + nullable = true)() + } } override val output = { @@ -134,7 +137,7 @@ case class CarbonRelation( // convert each column to Attribute columns.filter(!_.isInvisible).map { column => if (column.isDimension()) { - val output: DataType = column.getDataType.toString.toLowerCase match { + val output: DataType = column.getDataType.getName.toLowerCase match { case "array" => CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>") case "struct" => @@ -147,8 +150,7 @@ case class CarbonRelation( qualifier = Option(tableName + "." + column.getColName)) } else { val output = CarbonMetastoreTypes.toDataType { - column.getDataType.toString - .toLowerCase match { + column.getDataType.getName.toLowerCase match { case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column .getColumnSchema.getScale + ")" case others => others http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala index 73c9760..dbc807d 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.CarbonContainsWith import org.apache.spark.sql.CarbonEndsWith -import org.apache.carbondata.core.metadata.datatype.DataType +import org.apache.carbondata.core.metadata.datatype.{DataTypes => CarbonDataTypes} import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression} @@ -134,9 +134,9 @@ object CarbonFilters { def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = { val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)) val dataType = if (Option(value).isDefined - && dataTypeOfAttribute == DataType.STRING + && dataTypeOfAttribute == CarbonDataTypes.STRING && value.isInstanceOf[Double]) { - DataType.DOUBLE + CarbonDataTypes.DOUBLE } else { dataTypeOfAttribute } @@ -396,24 +396,6 @@ object CarbonFilters { } } - private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = { - var carbonColumn: CarbonColumn = - carbonTable.getDimensionByName(carbonTable.getFactTableName, column) - val dataType = if (carbonColumn != null) { - carbonColumn.getDataType - } else { - carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column) - carbonColumn.getDataType match { - case DataType.INT => DataType.INT - case DataType.SHORT => DataType.SHORT - case DataType.LONG => DataType.LONG - case DataType.DECIMAL => DataType.DECIMAL - case _ => DataType.DOUBLE - } - } - CarbonScalaUtil.convertCarbonToSparkDataType(dataType) - } - // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is // not able find the classes inside scala list and gives ClassNotFoundException. private def convertToJavaList( http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java index e90fd4a..a340ab1 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java +++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/StructDataType.java @@ -176,8 +176,8 @@ public class StructDataType implements GenericDataType<StructObject> { } /* - * parse bytearray and bit pack - */ + * parse bytearray and bit pack + */ @Override public void parseAndBitPack(ByteBuffer byteArrayInput, DataOutputStream dataOutputStream, KeyGenerator[] generator) throws IOException, KeyGenException { http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java index 2efbe26..193d3e6 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java @@ -25,6 +25,8 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary; import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier; import org.apache.carbondata.core.dictionary.client.DictionaryClient; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; @@ -108,35 +110,37 @@ public class FieldEncoderFactory { Cache<DictionaryColumnUniqueIdentifier, Dictionary> cache, CarbonTableIdentifier carbonTableIdentifier, DictionaryClient client, Boolean useOnePass, String storePath, Map<Object, Integer> localCache) { - switch (carbonColumn.getDataType()) { - case ARRAY: - List<CarbonDimension> listOfChildDimensions = - ((CarbonDimension) carbonColumn).getListOfChildDimensions(); - // Create array parser with complex delimiter - ArrayDataType arrayDataType = - new ArrayDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId()); - for (CarbonDimension dimension : listOfChildDimensions) { - arrayDataType.addChildren(createComplexType(dimension, carbonColumn.getColName(), cache, - carbonTableIdentifier, client, useOnePass, storePath, localCache)); - } - return arrayDataType; - case STRUCT: - List<CarbonDimension> dimensions = - ((CarbonDimension) carbonColumn).getListOfChildDimensions(); - // Create struct parser with complex delimiter - StructDataType structDataType = - new StructDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId()); - for (CarbonDimension dimension : dimensions) { - structDataType.addChildren(createComplexType(dimension, carbonColumn.getColName(), cache, - carbonTableIdentifier, client, useOnePass, storePath, localCache)); - } - return structDataType; - case MAP: - throw new UnsupportedOperationException("Complex type Map is not supported yet"); - default: - return new PrimitiveDataType(carbonColumn.getColName(), parentName, - carbonColumn.getColumnId(), (CarbonDimension) carbonColumn, cache, - carbonTableIdentifier, client, useOnePass, storePath, localCache); + DataType dataType = carbonColumn.getDataType(); + if (dataType == DataTypes.ARRAY) { + List<CarbonDimension> listOfChildDimensions = + ((CarbonDimension) carbonColumn).getListOfChildDimensions(); + // Create array parser with complex delimiter + ArrayDataType arrayDataType = + new ArrayDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId()); + for (CarbonDimension dimension : listOfChildDimensions) { + arrayDataType.addChildren( + createComplexType(dimension, carbonColumn.getColName(), cache, carbonTableIdentifier, + client, useOnePass, storePath, localCache)); + } + return arrayDataType; + } else if (dataType == DataTypes.STRUCT) { + List<CarbonDimension> dimensions = + ((CarbonDimension) carbonColumn).getListOfChildDimensions(); + // Create struct parser with complex delimiter + StructDataType structDataType = + new StructDataType(carbonColumn.getColName(), parentName, carbonColumn.getColumnId()); + for (CarbonDimension dimension : dimensions) { + structDataType.addChildren( + createComplexType(dimension, carbonColumn.getColName(), cache, carbonTableIdentifier, + client, useOnePass, storePath, localCache)); + } + return structDataType; + } else if (dataType == DataTypes.MAP) { + throw new UnsupportedOperationException("Complex type Map is not supported yet"); + } else { + return new PrimitiveDataType(carbonColumn.getColName(), parentName, + carbonColumn.getColumnId(), (CarbonDimension) carbonColumn, cache, carbonTableIdentifier, + client, useOnePass, storePath, localCache); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java index 8170680..eac9d69 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/NonDictionaryFieldConverterImpl.java @@ -19,6 +19,7 @@ package org.apache.carbondata.processing.loading.converter.impl; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.util.DataTypeUtil; import org.apache.carbondata.processing.loading.DataField; @@ -52,7 +53,7 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter { @Override public void convert(CarbonRow row, BadRecordLogHolder logHolder) { String dimensionValue = row.getString(index); - if (null == dimensionValue && column.getDataType() != DataType.STRING) { + if (null == dimensionValue && column.getDataType() != DataTypes.STRING) { logHolder.setReason( CarbonDataProcessorUtil.prepareFailureReason(column.getColName(), column.getDataType())); updateWithNullValue(row); @@ -81,7 +82,7 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter { } private void updateWithNullValue(CarbonRow row) { - if (dataType == DataType.STRING) { + if (dataType == DataTypes.STRING) { row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index); } else { row.update(CarbonCommonConstants.EMPTY_BYTE_ARRAY, index); http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java index 0ee1d90..608d0a9 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/CarbonParserFactory.java @@ -18,6 +18,8 @@ package org.apache.carbondata.processing.loading.parser; import java.util.List; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.processing.loading.parser.impl.ArrayParserImpl; @@ -51,30 +53,29 @@ public final class CarbonParserFactory { */ private static GenericParser createParser(CarbonColumn carbonColumn, String[] complexDelimiters, String nullFormat, int depth) { - switch (carbonColumn.getDataType()) { - case ARRAY: - List<CarbonDimension> listOfChildDimensions = - ((CarbonDimension) carbonColumn).getListOfChildDimensions(); - // Create array parser with complex delimiter - ArrayParserImpl arrayParser = new ArrayParserImpl(complexDelimiters[depth], nullFormat); - for (CarbonDimension dimension : listOfChildDimensions) { - arrayParser - .addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1)); - } - return arrayParser; - case STRUCT: - List<CarbonDimension> dimensions = - ((CarbonDimension) carbonColumn).getListOfChildDimensions(); - // Create struct parser with complex delimiter - StructParserImpl parser = new StructParserImpl(complexDelimiters[depth], nullFormat); - for (CarbonDimension dimension : dimensions) { - parser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1)); - } - return parser; - case MAP: - throw new UnsupportedOperationException("Complex type Map is not supported yet"); - default: - return new PrimitiveParserImpl(); + DataType dataType = carbonColumn.getDataType(); + if (dataType == DataTypes.ARRAY) { + List<CarbonDimension> listOfChildDimensions = + ((CarbonDimension) carbonColumn).getListOfChildDimensions(); + // Create array parser with complex delimiter + ArrayParserImpl arrayParser = new ArrayParserImpl(complexDelimiters[depth], nullFormat); + for (CarbonDimension dimension : listOfChildDimensions) { + arrayParser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1)); + } + return arrayParser; + } else if (dataType == DataTypes.STRUCT) { + List<CarbonDimension> dimensions = + ((CarbonDimension) carbonColumn).getListOfChildDimensions(); + // Create struct parser with complex delimiter + StructParserImpl parser = new StructParserImpl(complexDelimiters[depth], nullFormat); + for (CarbonDimension dimension : dimensions) { + parser.addChildren(createParser(dimension, complexDelimiters, nullFormat, depth + 1)); + } + return parser; + } else if (dataType == DataTypes.MAP) { + throw new UnsupportedOperationException("Complex type Map is not supported yet"); + } else { + return new PrimitiveParserImpl(); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java index 06bd716..a41f734 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/partition/impl/HashPartitionerImpl.java @@ -19,6 +19,8 @@ package org.apache.carbondata.processing.loading.partition.impl; import java.util.List; +import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; import org.apache.carbondata.processing.loading.partition.Partitioner; @@ -36,19 +38,14 @@ public class HashPartitionerImpl implements Partitioner<Object[]> { this.numberOfBuckets = numberOfBuckets; hashes = new Hash[indexes.size()]; for (int i = 0; i < indexes.size(); i++) { - switch (columnSchemas.get(i).getDataType()) { - case SHORT: - case INT: - case LONG: - hashes[i] = new IntegralHash(indexes.get(i)); - break; - case DOUBLE: - case FLOAT: - case DECIMAL: - hashes[i] = new DecimalHash(indexes.get(i)); - break; - default: - hashes[i] = new StringHash(indexes.get(i)); + DataType dataType = columnSchemas.get(i).getDataType(); + if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) { + hashes[i] = new IntegralHash(indexes.get(i)); + } else if (dataType == DataTypes.DOUBLE || dataType == DataTypes.FLOAT || + dataType == DataTypes.DECIMAL) { + hashes[i] = new DecimalHash(indexes.get(i)); + } else { + hashes[i] = new StringHash(indexes.get(i)); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java index 14ab838..ff42e2d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeCarbonRowPage.java @@ -28,6 +28,7 @@ import org.apache.carbondata.core.memory.MemoryBlock; import org.apache.carbondata.core.memory.UnsafeMemoryManager; import org.apache.carbondata.core.memory.UnsafeSortMemoryManager; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.DataTypeUtil; /** @@ -125,41 +126,35 @@ public class UnsafeCarbonRowPage { for (int mesCount = 0; mesCount < measureSize; mesCount++) { Object value = row[mesCount + dimensionSize]; if (null != value) { - switch (measureDataType[mesCount]) { - case SHORT: - Short sval = (Short) value; - CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval); - size += 2; - break; - case INT: - Integer ival = (Integer) value; - CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival); - size += 4; - break; - case LONG: - Long val = (Long) value; - CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val); - size += 8; - break; - case DOUBLE: - Double doubleVal = (Double) value; - CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal); - size += 8; - break; - case DECIMAL: - BigDecimal decimalVal = (BigDecimal) value; - byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal); - CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, - (short) bigDecimalInBytes.length); - size += 2; - CarbonUnsafe.getUnsafe() - .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, - address + size, bigDecimalInBytes.length); - size += bigDecimalInBytes.length; - break; - default: - throw new IllegalArgumentException("unsupported data type:" + - measureDataType[mesCount]); + DataType dataType = measureDataType[mesCount]; + if (dataType == DataTypes.SHORT) { + Short sval = (Short) value; + CarbonUnsafe.getUnsafe().putShort(baseObject, address + size, sval); + size += 2; + } else if (dataType == DataTypes.INT) { + Integer ival = (Integer) value; + CarbonUnsafe.getUnsafe().putInt(baseObject, address + size, ival); + size += 4; + } else if (dataType == DataTypes.LONG) { + Long val = (Long) value; + CarbonUnsafe.getUnsafe().putLong(baseObject, address + size, val); + size += 8; + } else if (dataType == DataTypes.DOUBLE) { + Double doubleVal = (Double) value; + CarbonUnsafe.getUnsafe().putDouble(baseObject, address + size, doubleVal); + size += 8; + } else if (dataType == DataTypes.DECIMAL) { + BigDecimal decimalVal = (BigDecimal) value; + byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(decimalVal); + CarbonUnsafe.getUnsafe() + .putShort(baseObject, address + size, (short) bigDecimalInBytes.length); + size += 2; + CarbonUnsafe.getUnsafe() + .copyMemory(bigDecimalInBytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, baseObject, + address + size, bigDecimalInBytes.length); + size += bigDecimalInBytes.length; + } else { + throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]); } set(nullSetWords, mesCount); } else { @@ -213,39 +208,33 @@ public class UnsafeCarbonRowPage { for (int mesCount = 0; mesCount < measureSize; mesCount++) { if (isSet(nullSetWords, mesCount)) { - switch (measureDataType[mesCount]) { - case SHORT: - Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); - size += 2; - rowToFill[dimensionSize + mesCount] = sval; - break; - case INT: - Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size); - size += 4; - rowToFill[dimensionSize + mesCount] = ival; - break; - case LONG: - Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size); - size += 8; - rowToFill[dimensionSize + mesCount] = val; - break; - case DOUBLE: - Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size); - size += 8; - rowToFill[dimensionSize + mesCount] = doubleVal; - break; - case DECIMAL: - short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); - byte[] bigDecimalInBytes = new byte[aShort]; - size += 2; - CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes, - CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length); - size += bigDecimalInBytes.length; - rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + - measureDataType[mesCount]); + DataType dataType = measureDataType[mesCount]; + if (dataType == DataTypes.SHORT) { + Short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); + size += 2; + rowToFill[dimensionSize + mesCount] = sval; + } else if (dataType == DataTypes.INT) { + Integer ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size); + size += 4; + rowToFill[dimensionSize + mesCount] = ival; + } else if (dataType == DataTypes.LONG) { + Long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size); + size += 8; + rowToFill[dimensionSize + mesCount] = val; + } else if (dataType == DataTypes.DOUBLE) { + Double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size); + size += 8; + rowToFill[dimensionSize + mesCount] = doubleVal; + } else if (dataType == DataTypes.DECIMAL) { + short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); + byte[] bigDecimalInBytes = new byte[aShort]; + size += 2; + CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes, + CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length); + size += bigDecimalInBytes.length; + rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes); + } else { + throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]); } } else { rowToFill[dimensionSize + mesCount] = null; @@ -301,40 +290,34 @@ public class UnsafeCarbonRowPage { for (int mesCount = 0; mesCount < measureSize; mesCount++) { if (isSet(nullSetWords, mesCount)) { - switch (measureDataType[mesCount]) { - case SHORT: - short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); - size += 2; - stream.writeShort(sval); - break; - case INT: - int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size); - size += 4; - stream.writeInt(ival); - break; - case LONG: - long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size); - size += 8; - stream.writeLong(val); - break; - case DOUBLE: - double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size); - size += 8; - stream.writeDouble(doubleVal); - break; - case DECIMAL: - short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); - byte[] bigDecimalInBytes = new byte[aShort]; - size += 2; - CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes, - CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length); - size += bigDecimalInBytes.length; - stream.writeShort(aShort); - stream.write(bigDecimalInBytes); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + - measureDataType[mesCount]); + DataType dataType = measureDataType[mesCount]; + if (dataType == DataTypes.SHORT) { + short sval = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); + size += 2; + stream.writeShort(sval); + } else if (dataType == DataTypes.INT) { + int ival = CarbonUnsafe.getUnsafe().getInt(baseObject, address + size); + size += 4; + stream.writeInt(ival); + } else if (dataType == DataTypes.LONG) { + long val = CarbonUnsafe.getUnsafe().getLong(baseObject, address + size); + size += 8; + stream.writeLong(val); + } else if (dataType == DataTypes.DOUBLE) { + double doubleVal = CarbonUnsafe.getUnsafe().getDouble(baseObject, address + size); + size += 8; + stream.writeDouble(doubleVal); + } else if (dataType == DataTypes.DECIMAL) { + short aShort = CarbonUnsafe.getUnsafe().getShort(baseObject, address + size); + byte[] bigDecimalInBytes = new byte[aShort]; + size += 2; + CarbonUnsafe.getUnsafe().copyMemory(baseObject, address + size, bigDecimalInBytes, + CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length); + size += bigDecimalInBytes.length; + stream.writeShort(aShort); + stream.write(bigDecimalInBytes); + } else { + throw new IllegalArgumentException("unsupported data type:" + measureDataType[mesCount]); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java index 5fed2ea..404a521 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java @@ -33,6 +33,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.DataTypeUtil; @@ -325,28 +326,23 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder { for (int mesCount = 0; mesCount < measureCount; mesCount++) { if (UnsafeCarbonRowPage.isSet(words, mesCount)) { - switch (measureDataType[mesCount]) { - case SHORT: - row[dimensionCount + mesCount] = stream.readShort(); - break; - case INT: - row[dimensionCount + mesCount] = stream.readInt(); - break; - case LONG: - row[dimensionCount + mesCount] = stream.readLong(); - break; - case DOUBLE: - row[dimensionCount + mesCount] = stream.readDouble(); - break; - case DECIMAL: - short aShort = stream.readShort(); - byte[] bigDecimalInBytes = new byte[aShort]; - stream.readFully(bigDecimalInBytes); - row[dimensionCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + - measureDataType[mesCount]); + DataType dataType = measureDataType[mesCount]; + if (dataType == DataTypes.SHORT) { + row[dimensionCount + mesCount] = stream.readShort(); + } else if (dataType == DataTypes.INT) { + row[dimensionCount + mesCount] = stream.readInt(); + } else if (dataType == DataTypes.LONG) { + row[dimensionCount + mesCount] = stream.readLong(); + } else if (dataType == DataTypes.DOUBLE) { + row[dimensionCount + mesCount] = stream.readDouble(); + } else if (dataType == DataTypes.DECIMAL) { + short aShort = stream.readShort(); + byte[] bigDecimalInBytes = new byte[aShort]; + stream.readFully(bigDecimalInBytes); + row[dimensionCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes); + } else { + throw new IllegalArgumentException( + "unsupported data type:" + measureDataType[mesCount]); } } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java index 4303ec8..9f7d6c3 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateFileMerger.java @@ -32,6 +32,7 @@ import java.util.concurrent.Callable; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage; import org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder; @@ -315,31 +316,26 @@ public class UnsafeIntermediateFileMerger implements Callable<Void> { for (int mesCount = 0; mesCount < measureSize; mesCount++) { Object value = row[mesCount + dimensionSize]; if (null != value) { - switch (type[mesCount]) { - case SHORT: - rowData.putShort(size, (Short) value); - size += 2; - break; - case INT: - rowData.putInt(size, (Integer) value); - size += 4; - break; - case LONG: - rowData.putLong(size, (Long) value); - size += 8; - break; - case DOUBLE: - rowData.putDouble(size, (Double) value); - size += 8; - break; - case DECIMAL: - byte[] bigDecimalInBytes = (byte[]) value; - rowData.putShort(size, (short)bigDecimalInBytes.length); - size += 2; - for (int i = 0; i < bigDecimalInBytes.length; i++) { - rowData.put(size++, bigDecimalInBytes[i]); - } - break; + DataType dataType = type[mesCount]; + if (dataType == DataTypes.SHORT) { + rowData.putShort(size, (Short) value); + size += 2; + } else if (dataType == DataTypes.INT) { + rowData.putInt(size, (Integer) value); + size += 4; + } else if (dataType == DataTypes.LONG) { + rowData.putLong(size, (Long) value); + size += 8; + } else if (dataType == DataTypes.DOUBLE) { + rowData.putDouble(size, (Double) value); + size += 8; + } else if (dataType == DataTypes.DECIMAL) { + byte[] bigDecimalInBytes = (byte[]) value; + rowData.putShort(size, (short) bigDecimalInBytes.length); + size += 2; + for (int i = 0; i < bigDecimalInBytes.length; i++) { + rowData.put(size++, bigDecimalInBytes[i]); + } } UnsafeCarbonRowPage.set(nullSetWords, mesCount); } else { http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java index edffae9..0e20ef5 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java @@ -27,6 +27,7 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties; import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException; import org.apache.carbondata.core.datastore.row.CarbonRow; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; @@ -256,14 +257,13 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor { * @return */ private Object getConvertedMeasureValue(Object value, DataType type) { - switch (type) { - case DECIMAL: - if (value != null) { - value = ((Decimal) value).toJavaBigDecimal(); - } - return value; - default: - return value; + if (type == DataTypes.DECIMAL) { + if (value != null) { + value = ((Decimal) value).toJavaBigDecimal(); + } + return value; + } else { + return value; } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java index d4a8dd6..3671316 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/IntermediateFileMerger.java @@ -30,6 +30,7 @@ import java.util.concurrent.Callable; import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.CarbonUtil; import org.apache.carbondata.core.util.NonDictionaryUtil; import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException; @@ -345,26 +346,21 @@ public class IntermediateFileMerger implements Callable<Void> { for (int counter = 0; counter < mergerParameters.getMeasureColCount(); counter++) { if (null != NonDictionaryUtil.getMeasure(fieldIndex, row)) { stream.write((byte) 1); - switch (aggType[counter]) { - case SHORT: - stream.writeShort((short)NonDictionaryUtil.getMeasure(fieldIndex, row)); - break; - case INT: - stream.writeInt((int)NonDictionaryUtil.getMeasure(fieldIndex, row)); - break; - case LONG: - stream.writeLong((long)NonDictionaryUtil.getMeasure(fieldIndex, row)); - break; - case DOUBLE: - stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row)); - break; - case DECIMAL: - byte[] bigDecimalInBytes = (byte[]) NonDictionaryUtil.getMeasure(fieldIndex, row); - stream.writeInt(bigDecimalInBytes.length); - stream.write(bigDecimalInBytes); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + aggType[counter]); + DataType dataType = aggType[counter]; + if (dataType == DataTypes.SHORT) { + stream.writeShort((short) NonDictionaryUtil.getMeasure(fieldIndex, row)); + } else if (dataType == DataTypes.INT) { + stream.writeInt((int) NonDictionaryUtil.getMeasure(fieldIndex, row)); + } else if (dataType == DataTypes.LONG) { + stream.writeLong((long) NonDictionaryUtil.getMeasure(fieldIndex, row)); + } else if (dataType == DataTypes.DOUBLE) { + stream.writeDouble((Double) NonDictionaryUtil.getMeasure(fieldIndex, row)); + } else if (dataType == DataTypes.DECIMAL) { + byte[] bigDecimalInBytes = (byte[]) NonDictionaryUtil.getMeasure(fieldIndex, row); + stream.writeInt(bigDecimalInBytes.length); + stream.write(bigDecimalInBytes); + } else { + throw new IllegalArgumentException("unsupported data type:" + aggType[counter]); } } else { stream.write((byte) 0); http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java index 11df276..5cc96c5 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortDataRows.java @@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonThreadFactory; import org.apache.carbondata.core.util.CarbonUtil; @@ -293,27 +294,22 @@ public class SortDataRows { Object value = row[mesCount + dimColCount]; if (null != value) { stream.write((byte) 1); - switch (type[mesCount]) { - case SHORT: - stream.writeShort((Short) value); - break; - case INT: - stream.writeInt((Integer) value); - break; - case LONG: - stream.writeLong((Long) value); - break; - case DOUBLE: - stream.writeDouble((Double) value); - break; - case DECIMAL: - BigDecimal val = (BigDecimal) value; - byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val); - stream.writeInt(bigDecimalInBytes.length); - stream.write(bigDecimalInBytes); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + type[mesCount]); + DataType dataType = type[mesCount]; + if (dataType == DataTypes.SHORT) { + stream.writeShort((Short) value); + } else if (dataType == DataTypes.INT) { + stream.writeInt((Integer) value); + } else if (dataType == DataTypes.LONG) { + stream.writeLong((Long) value); + } else if (dataType == DataTypes.DOUBLE) { + stream.writeDouble((Double) value); + } else if (dataType == DataTypes.DECIMAL) { + BigDecimal val = (BigDecimal) value; + byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val); + stream.writeInt(bigDecimalInBytes.length); + stream.write(bigDecimalInBytes); + } else { + throw new IllegalArgumentException("unsupported data type:" + type[mesCount]); } } else { stream.write((byte) 0); http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java index 3e56605..5d339c7 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/sort/sortdata/SortTempFileChunkHolder.java @@ -32,6 +32,7 @@ import org.apache.carbondata.common.logging.LogService; import org.apache.carbondata.common.logging.LogServiceFactory; import org.apache.carbondata.core.constants.CarbonCommonConstants; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer; import org.apache.carbondata.core.util.CarbonProperties; import org.apache.carbondata.core.util.CarbonThreadFactory; @@ -343,27 +344,22 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold // read measure values for (int i = 0; i < this.measureCount; i++) { if (stream.readByte() == 1) { - switch (aggType[i]) { - case SHORT: - measures[index++] = stream.readShort(); - break; - case INT: - measures[index++] = stream.readInt(); - break; - case LONG: - measures[index++] = stream.readLong(); - break; - case DOUBLE: - measures[index++] = stream.readDouble(); - break; - case DECIMAL: - int len = stream.readInt(); - byte[] buff = new byte[len]; - stream.readFully(buff); - measures[index++] = DataTypeUtil.byteToBigDecimal(buff); - break; - default: - throw new IllegalArgumentException("unsupported data type:" + aggType[i]); + DataType dataType = aggType[i]; + if (dataType == DataTypes.SHORT) { + measures[index++] = stream.readShort(); + } else if (dataType == DataTypes.INT) { + measures[index++] = stream.readInt(); + } else if (dataType == DataTypes.LONG) { + measures[index++] = stream.readLong(); + } else if (dataType == DataTypes.DOUBLE) { + measures[index++] = stream.readDouble(); + } else if (dataType == DataTypes.DECIMAL) { + int len = stream.readInt(); + byte[] buff = new byte[len]; + stream.readFully(buff); + measures[index++] = DataTypeUtil.byteToBigDecimal(buff); + } else { + throw new IllegalArgumentException("unsupported data type:" + aggType[i]); } } else { measures[index++] = null; http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java index 78f1637..fdf44cf 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java @@ -45,6 +45,7 @@ import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.CarbonMetadata; import org.apache.carbondata.core.metadata.ColumnarFormatVersion; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.util.CarbonProperties; @@ -503,7 +504,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler { new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); DataType[] type = model.getMeasureDataType(); for (int j = 0; j < type.length; j++) { - if (type[j] != DataType.BYTE && type[j] != DataType.DECIMAL) { + if (type[j] != DataTypes.BYTE && type[j] != DataTypes.DECIMAL) { otherMeasureIndexList.add(j); } else { customMeasureIndexList.add(j); http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java index 287de0a..d2cf1c4 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java @@ -46,6 +46,7 @@ import org.apache.carbondata.core.datastore.row.WriteStepRowUtil; import org.apache.carbondata.core.keygenerator.KeyGenException; import org.apache.carbondata.core.memory.MemoryException; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.processing.datatypes.GenericDataType; import org.apache.spark.sql.types.Decimal; @@ -91,14 +92,14 @@ public class TablePage { dictDimensionPages = new ColumnPage[numDictDimension]; for (int i = 0; i < dictDimensionPages.length; i++) { TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i); - ColumnPage page = ColumnPage.newPage(spec, DataType.BYTE_ARRAY, pageSize); - page.setStatsCollector(KeyPageStatsCollector.newInstance(DataType.BYTE_ARRAY)); + ColumnPage page = ColumnPage.newPage(spec, DataTypes.BYTE_ARRAY, pageSize); + page.setStatsCollector(KeyPageStatsCollector.newInstance(DataTypes.BYTE_ARRAY)); dictDimensionPages[i] = page; } noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()]; for (int i = 0; i < noDictDimensionPages.length; i++) { TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i + numDictDimension); - ColumnPage page = ColumnPage.newPage(spec, DataType.STRING, pageSize); + ColumnPage page = ColumnPage.newPage(spec, DataTypes.STRING, pageSize); page.setStatsCollector(LVStringStatsCollector.newInstance()); noDictDimensionPages[i] = page; } @@ -113,7 +114,7 @@ public class TablePage { for (int i = 0; i < measurePages.length; i++) { TableSpec.MeasureSpec spec = model.getTableSpec().getMeasureSpec(i); ColumnPage page; - if (spec.getSchemaDataType() == DataType.DECIMAL) { + if (spec.getSchemaDataType() == DataTypes.DECIMAL) { page = ColumnPage.newDecimalPage(spec, dataTypes[i], pageSize); } else { page = ColumnPage.newPage(spec, dataTypes[i], pageSize); @@ -182,7 +183,7 @@ public class TablePage { // in compaction flow the measure with decimal type will come as Spark decimal. // need to convert it to byte array. - if (measurePages[i].getDataType() == DataType.DECIMAL && + if (measurePages[i].getDataType() == DataTypes.DECIMAL && model.isCompactionFlow() && value != null) { value = ((Decimal) value).toJavaBigDecimal(); http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java index 79e49ef..1c7f9e7 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java @@ -41,6 +41,7 @@ import org.apache.carbondata.core.datastore.impl.FileFactory.FileType; import org.apache.carbondata.core.metadata.CarbonMetadata; import org.apache.carbondata.core.metadata.CarbonTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.schema.table.CarbonTable; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; @@ -278,8 +279,8 @@ public final class CarbonDataProcessorUtil { StringBuilder dimString = new StringBuilder(); for (int i = 0; i < dataFields.length; i++) { DataField dataField = dataFields[i]; - if (dataField.getColumn().getDataType().equals(DataType.ARRAY) || dataField.getColumn() - .getDataType().equals(DataType.STRUCT)) { + if (dataField.getColumn().getDataType() == DataTypes.ARRAY || + dataField.getColumn().getDataType() == DataTypes.STRUCT) { addAllComplexTypeChildren((CarbonDimension) dataField.getColumn(), dimString, ""); dimString.append(CarbonCommonConstants.SEMICOLON_SPC_CHARACTER); } @@ -321,22 +322,19 @@ public final class CarbonDataProcessorUtil { for (int i = 0; i < hierarchies.length; i++) { String[] levels = hierarchies[i].split(CarbonCommonConstants.HASH_SPC_CHARACTER); String[] levelInfo = levels[0].split(CarbonCommonConstants.COLON_SPC_CHARACTER); - GenericDataType g = levelInfo[1].equals(CarbonCommonConstants.ARRAY) ? + GenericDataType g = levelInfo[1].toLowerCase().contains(CarbonCommonConstants.ARRAY) ? new ArrayDataType(levelInfo[0], "", levelInfo[3]) : new StructDataType(levelInfo[0], "", levelInfo[3]); complexTypesMap.put(levelInfo[0], g); for (int j = 1; j < levels.length; j++) { levelInfo = levels[j].split(CarbonCommonConstants.COLON_SPC_CHARACTER); - switch (levelInfo[1]) { - case CarbonCommonConstants.ARRAY: - g.addChildren(new ArrayDataType(levelInfo[0], levelInfo[2], levelInfo[3])); - break; - case CarbonCommonConstants.STRUCT: - g.addChildren(new StructDataType(levelInfo[0], levelInfo[2], levelInfo[3])); - break; - default: - g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3], - Integer.parseInt(levelInfo[4]))); + if (levelInfo[1].toLowerCase().contains(CarbonCommonConstants.ARRAY)) { + g.addChildren(new ArrayDataType(levelInfo[0], levelInfo[2], levelInfo[3])); + } else if (levelInfo[1].toLowerCase().contains(CarbonCommonConstants.STRUCT)) { + g.addChildren(new StructDataType(levelInfo[0], levelInfo[2], levelInfo[3])); + } else { + g.addChildren(new PrimitiveDataType(levelInfo[0], levelInfo[2], levelInfo[3], + Integer.parseInt(levelInfo[4]))); } } } @@ -396,7 +394,7 @@ public final class CarbonDataProcessorUtil { String tableName) { DataType[] type = new DataType[measureCount]; for (int i = 0; i < type.length; i++) { - type[i] = DataType.DOUBLE; + type[i] = DataTypes.DOUBLE; } CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable( databaseName + CarbonCommonConstants.UNDERSCORE + tableName); @@ -458,7 +456,7 @@ public final class CarbonDataProcessorUtil { int measureCount) { DataType[] type = new DataType[measureCount]; for (int i = 0; i < type.length; i++) { - type[i] = DataType.DOUBLE; + type[i] = DataTypes.DOUBLE; } List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(tableName); for (int i = 0; i < measureCount; i++) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/956833e5/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java ---------------------------------------------------------------------- diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java index 61771ea..37331c9 100644 --- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java +++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java @@ -50,6 +50,7 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier; import org.apache.carbondata.core.metadata.converter.SchemaConverter; import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl; import org.apache.carbondata.core.metadata.datatype.DataType; +import org.apache.carbondata.core.metadata.datatype.DataTypes; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.SchemaEvolution; import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry; @@ -183,7 +184,7 @@ public class StoreCreator { ColumnSchema id = new ColumnSchema(); id.setColumnName("ID"); id.setColumnar(true); - id.setDataType(DataType.INT); + id.setDataType(DataTypes.INT); id.setEncodingList(encodings); id.setColumnUniqueId(UUID.randomUUID().toString()); id.setDimensionColumn(true); @@ -193,7 +194,7 @@ public class StoreCreator { ColumnSchema date = new ColumnSchema(); date.setColumnName("date"); date.setColumnar(true); - date.setDataType(DataType.STRING); + date.setDataType(DataTypes.STRING); date.setEncodingList(encodings); date.setColumnUniqueId(UUID.randomUUID().toString()); date.setDimensionColumn(true); @@ -203,7 +204,7 @@ public class StoreCreator { ColumnSchema country = new ColumnSchema(); country.setColumnName("country"); country.setColumnar(true); - country.setDataType(DataType.STRING); + country.setDataType(DataTypes.STRING); country.setEncodingList(encodings); country.setColumnUniqueId(UUID.randomUUID().toString()); country.setDimensionColumn(true); @@ -213,7 +214,7 @@ public class StoreCreator { ColumnSchema name = new ColumnSchema(); name.setColumnName("name"); name.setColumnar(true); - name.setDataType(DataType.STRING); + name.setDataType(DataTypes.STRING); name.setEncodingList(encodings); name.setColumnUniqueId(UUID.randomUUID().toString()); name.setDimensionColumn(true); @@ -223,7 +224,7 @@ public class StoreCreator { ColumnSchema phonetype = new ColumnSchema(); phonetype.setColumnName("phonetype"); phonetype.setColumnar(true); - phonetype.setDataType(DataType.STRING); + phonetype.setDataType(DataTypes.STRING); phonetype.setEncodingList(encodings); phonetype.setColumnUniqueId(UUID.randomUUID().toString()); phonetype.setDimensionColumn(true); @@ -233,7 +234,7 @@ public class StoreCreator { ColumnSchema serialname = new ColumnSchema(); serialname.setColumnName("serialname"); serialname.setColumnar(true); - serialname.setDataType(DataType.STRING); + serialname.setDataType(DataTypes.STRING); serialname.setEncodingList(encodings); serialname.setColumnUniqueId(UUID.randomUUID().toString()); serialname.setDimensionColumn(true); @@ -243,7 +244,7 @@ public class StoreCreator { ColumnSchema salary = new ColumnSchema(); salary.setColumnName("salary"); salary.setColumnar(true); - salary.setDataType(DataType.INT); + salary.setDataType(DataTypes.INT); salary.setEncodingList(new ArrayList<Encoding>()); salary.setColumnUniqueId(UUID.randomUUID().toString()); salary.setDimensionColumn(false);