Repository: incubator-carbondata Updated Branches: refs/heads/master 2f24b523f -> 32f2d2786
Fixed data loading performance issue Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9a3b2218 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9a3b2218 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9a3b2218 Branch: refs/heads/master Commit: 9a3b22183a63324bad3dc2990b304e689adc7c35 Parents: 2f24b52 Author: kumarvishal <kumarvishal.1...@gmail.com> Authored: Wed Nov 16 18:32:25 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Fri Nov 18 08:32:10 2016 +0800 ---------------------------------------------------------------------- .../csvbased/CarbonCSVBasedSeqGenStep.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9a3b2218/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java index 23bf6d9..b1ef7fc 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java +++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java @@ -267,7 +267,10 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { private GenericDataType[] complexTypes; private DirectDictionaryGenerator[] directDictionaryGenerators; - + /** + * dimension column ids + */ + private String[] dimensionColumnIds; /** * Constructor * @@ -350,7 +353,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { if (null != getInputRowMeta()) { data.setOutputRowMeta((RowMetaInterface) getInputRowMeta().clone()); } - + this.dimensionColumnIds = meta.getDimensionColumnIds(); ColumnsInfo columnsInfo = new ColumnsInfo(); columnsInfo.setDims(meta.dims); columnsInfo.setDimColNames(meta.dimColNames); @@ -373,7 +376,7 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { columnsInfo.setPrimaryKeyMap(meta.getPrimaryKeyMap()); columnsInfo.setMeasureColumns(meta.measureColumn); columnsInfo.setComplexTypesMap(meta.getComplexTypes()); - columnsInfo.setDimensionColumnIds(meta.getDimensionColumnIds()); + columnsInfo.setDimensionColumnIds(this.dimensionColumnIds); columnsInfo.setColumnSchemaDetailsWrapper(meta.getColumnSchemaDetailsWrapper()); columnsInfo.setColumnProperties(meta.getColumnPropertiesMap()); updateBagLogFileName(); @@ -945,7 +948,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep { int inputColumnsSize = metaColumnNames.length; boolean isGenerated = false; int generatedSurrogate = -1; - String[] dimensionColumnIds = meta.getDimensionColumnIds(); //If CSV Exported from DB and we enter one row down then that row become empty. // In that case it will have first value empty and other values will be null