Github user sounakr commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2417#discussion_r201226564
--- Diff:
core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoder.java
---
@@ -147,10 +160,48 @@ protected void fillLegacyFields(DataChunk2 dataChunk)
public static EncodedColumnPage encodedColumn(ColumnPage page)
throws IOException, MemoryException {
- ColumnPageEncoder encoder = new
DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);
- return encoder.encode(page);
+ ColumnPageEncoder pageEncoder = createCodecForDimension(page);
+ if (pageEncoder == null) {
+ ColumnPageEncoder encoder = new
DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);
+ return encoder.encode(page);
+ } else {
+ LOGGER.info("Encoder result ---> Source data type: " +
pageEncoder.getEncoderMeta(page)
+ .getColumnSpec().getSchemaDataType() + " Destination data type:
" + pageEncoder
+ .getEncoderMeta(page).getStoreDataType() + " for the column: " +
pageEncoder
+ .getEncoderMeta(page).getColumnSpec().getFieldName());
+ // TODO: remove the Sout after testing,
+ // currently added as executor Info logs doesn't come in IDE
+ System.out.println("Encoder result ---> Source data type: " +
pageEncoder.getEncoderMeta(page)
+ .getColumnSpec().getSchemaDataType() + " Destination data type:
" + pageEncoder
+ .getEncoderMeta(page).getStoreDataType() + " for the column: " +
pageEncoder
+ .getEncoderMeta(page).getColumnSpec().getFieldName());
+ return pageEncoder.encode(page);
+ }
}
+ private static ColumnPageEncoder createCodecForDimension(ColumnPage
inputPage) {
+ TableSpec.ColumnSpec columnSpec = inputPage.getColumnSpec();
+ if (columnSpec.getColumnType() == ColumnType.COMPLEX_PRIMITIVE) {
+ if (inputPage.getDataType() == DataTypes.BYTE_ARRAY
+ || inputPage.getDataType() == DataTypes.STRING) {
+ // use legacy encoder
+ return null;
+ } else if ((inputPage.getDataType() == DataTypes.BYTE) ||
(inputPage.getDataType()
+ == DataTypes.SHORT) || (inputPage.getDataType() ==
DataTypes.INT) || (
+ inputPage.getDataType() == DataTypes.LONG)) {
+ return
selectCodecByAlgorithmForIntegral(inputPage.getStatistics(), true)
+ .createEncoder(null);
+ } else if ((inputPage.getDataType() == DataTypes.FLOAT) ||
(inputPage.getDataType()
+ == DataTypes.DOUBLE)) {
+ return
selectCodecByAlgorithmForFloating(inputPage.getStatistics(), true)
+ .createEncoder(null);
+ }
+ }
+ // use legacy encoder
+ return null;
--- End diff --
As part of Refactoring in CARBONDATA-2713, this will be handled.
---
