This is an automated email from the ASF dual-hosted git repository. kunalkapoor pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push: new 39af9a6 [CARBONDATA-3437] Changed implementation of Map datatype 39af9a6 is described below commit 39af9a630752b9a4c7dd4f6bdafc812e4f03eb26 Author: manishnalla1994 <manish.nalla1...@gmail.com> AuthorDate: Fri Jun 14 12:47:11 2019 +0530 [CARBONDATA-3437] Changed implementation of Map datatype Problem: Insert into map should override the old values in case of new duplicate value, which it was not doing. Solution: Changed the implementation to work fine in case of duplicate values inside map to show the latest values. This closes #3288 --- .../createTable/TestCreateDDLForComplexMapType.scala | 14 +++++++------- .../processing/loading/parser/impl/MapParserImpl.java | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala index 4d50e1f..473c9c6 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateDDLForComplexMapType.scala @@ -392,7 +392,7 @@ class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll { | 'header' = 'false') """.stripMargin) checkAnswer(sql("select * from carbon"), Seq( - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")) )) } @@ -434,13 +434,13 @@ class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll { sql("alter table carbon compact 'minor'") sql("show segments for table carbon").show(false) checkAnswer(sql("select * from carbon"), Seq( - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")), - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")), - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")), - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")) )) sql("DROP TABLE IF EXISTS carbon") @@ -482,8 +482,8 @@ class TestCreateDDLForComplexMapType extends QueryTest with BeforeAndAfterAll { """.stripMargin) sql("INSERT INTO carbon SELECT * FROM carbon") checkAnswer(sql("select * from carbon"), Seq( - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), - Row(Map(1 -> "Nalla", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), + Row(Map(1 -> "Gupta", 2 -> "Singh", 4 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")), Row(Map(10 -> "Nallaa", 20 -> "Sissngh", 100 -> "Gusspta", 40 -> "Kumar")) )) diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java index e6814f8..06a4e1a 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/parser/impl/MapParserImpl.java @@ -18,8 +18,8 @@ package org.apache.carbondata.processing.loading.parser.impl; import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; +import java.util.HashMap; +import java.util.Map; import org.apache.carbondata.processing.loading.complexobjects.ArrayObject; @@ -36,7 +36,7 @@ public class MapParserImpl extends ArrayParserImpl { } //The Key for Map will always be a PRIMITIVE type so Set<Object> here will work fine - //Only the first occurance of key will be added and the remaining will be skipped/ignored + //The last occurance of the key, value pair will be added and all others will be overwritten @Override public ArrayObject parse(Object data) { if (data != null) { String value = data.toString(); @@ -44,12 +44,13 @@ public class MapParserImpl extends ArrayParserImpl { String[] split = pattern.split(value, -1); if (ArrayUtils.isNotEmpty(split)) { ArrayList<Object> array = new ArrayList<>(); - Set<Object> set = new HashSet<>(); + Map<Object, String> map = new HashMap<>(); for (int i = 0; i < split.length; i++) { Object currKey = split[i].split(keyValueDelimiter)[0]; - if (set.add(currKey)) { - array.add(child.parse(split[i])); - } + map.put(currKey, split[i]); + } + for (Map.Entry<Object, String> entry : map.entrySet()) { + array.add(child.parse(entry.getValue())); } return new ArrayObject(array.toArray()); }