[40/50] [abbrv] carbondata git commit: [CARBONDATA-2420][32K] Support string longer than 32000 characters

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
new file mode 100644
index 000..419b306
--- /dev/null
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -0,0 +1,279 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.carbondata.spark.testsuite.longstring
+
+import java.io.{File, PrintWriter}
+
+import org.apache.commons.lang3.RandomStringUtils
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach 
with BeforeAndAfterAll {
+  private val longStringTable = "long_string_table"
+  private val inputDir = 
s"$resourcesPath${File.separator}varchartype${File.separator}"
+  private val fileName = s"longStringData.csv"
+  private val inputFile = s"$inputDir$fileName"
+  private val fileName_2g_column_page = 
s"longStringData_exceed_2gb_column_page.csv"
+  private val inputFile_2g_column_page = s"$inputDir$fileName_2g_column_page"
+  private val lineNum = 1000
+  private var content: Content = _
+  private var originMemorySize = CarbonProperties.getInstance().getProperty(
+CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
+
+  case class Content(head: Int, desc_line_head: String, note_line_head: String,
+  mid: Int, desc_line_mid: String, note_line_mid: String,
+  tail: Int, desc_line_tail: String, note_line_tail: String)
+
+  override def beforeAll(): Unit = {
+// for one 32000 lines * 32000 characters column page, it use about 1GB 
memory, but here we have only 1000 lines
+CarbonProperties.getInstance().addProperty(
+  CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
+  CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
+deleteFile(inputFile)
+if (!new File(inputDir).exists()) {
+  new File(inputDir).mkdir()
+}
+content = createFile(inputFile, line = lineNum)
+  }
+
+  override def afterAll(): Unit = {
+CarbonProperties.getInstance().addProperty(
+  CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, originMemorySize)
+deleteFile(inputFile)
+deleteFile(inputFile_2g_column_page)
+if (new File(inputDir).exists()) {
+  new File(inputDir).delete()
+}
+  }
+
+  override def beforeEach(): Unit = {
+sql(s"drop table if exists $longStringTable")
+  }
+
+  override def afterEach(): Unit = {
+sql(s"drop table if exists $longStringTable")
+  }
+
+  private def prepareTable(): Unit = {
+sql(
+  s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='description, note', 
'SORT_COLUMNS'='name')
+ |""".stripMargin)
+sql(
+  s"""
+ | LOAD DATA LOCAL INPATH '$inputFile' INTO TABLE $longStringTable
+ | OPTIONS('header'='false')
+   """.stripMargin)
+  }
+
+  private def checkQuery(): Unit = {
+// query without long_string_column
+checkAnswer(sql(s"SELECT id, name, address FROM $longStringTable where id 
= ${content.tail}"),
+  Row(content.tail, s"name_${content.tail}", s"address_${content.tail}"))
+// query return long_string_column in the middle position
+checkAnswer(sql(s"SELECT id, name, description, address FROM 
$longStringTable where id = ${content.head}"),
+  Row(content.head, s"name_${content.head}", 

[48/50] [abbrv] carbondata git commit: [CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long string columns

[jackylk]

[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long 
string columns

Add validate for long string columns

1. long string columns cannot be sort_columns
2. long string columns cannot be dictionary include
3. long string columns cannot be dictionary exclude
4. long string columns can only be string columns

This closes #2380


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/218a8deb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/218a8deb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/218a8deb

Branch: refs/heads/carbonstore
Commit: 218a8deb614e0dc160ab8c7e38c71d80711eb1a7
Parents: 091a28b
Author: xuchuanyin 
Authored: Wed Jun 20 18:55:04 2018 +0800
Committer: kumarvishal09 
Committed: Thu Jun 21 12:00:34 2018 +0530

--
 .../VarcharDataTypesBasicTestCase.scala | 57 
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +++-
 2 files changed, 67 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 419b306..9ea3f1f 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
 sql(s"drop table if exists $longStringTable")
   }
 
+  test("long string columns cannot be dictionary include") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 
'dictionary_include'='address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is 
unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be dictionay exclude") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 
'dictionary_exclude'='address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is 
unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be sort_columns") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 
'SORT_COLUMNS'='name, address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("sort_columns is unsupported 
for long string datatype column: name"))
+  }
+
+  test("long string columns can only be string columns") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("long_string_columns: id"))
+assert(exceptionCaught.getMessage.contains("its data type is not string"))
+  }
+
   private def prepareTable(): Unit = {
 sql(
   s"""

http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 

[36/50] [abbrv] carbondata git commit: Support adding local dictionary configuration in create table statement and show the configs in describe formatted table

[jackylk]

Support adding local dictionary configuration in create table statement and 
show the configs in describe formatted table

What changes were proposed in this pull request?
In this PR, in order to support local dictionary,

create table changes are made to support local dictionary configurations as 
table properties
show local dictionary properties in describe formatted command based on whether 
the local dictionary enabled or disabled.
Highlights:
basically we will have four properties

LOCAL_DICT_ENABLE => whether to enable or disable local dictionary
LOCAL_DICT_THRESHOLD => threshold property for the column to generate local 
dictionary
LOCAL_DICT_INCLUDE => columns for which local dictionary needs to be generated
LOCAL_DICT_EXCLUDE => columns for which local dictionary should not be generated

This closes#2375


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/be20fefb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/be20fefb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/be20fefb

Branch: refs/heads/carbonstore
Commit: be20fefbe0a01a501a567683ecc872e08f3fc001
Parents: ca466d9
Author: akashrn5 
Authored: Wed Jun 6 20:33:39 2018 +0530
Committer: kumarvishal09 
Committed: Tue Jun 19 21:08:27 2018 +0530

--
 .../core/constants/CarbonCommonConstants.java   |  31 +
 .../ThriftWrapperSchemaConverterImpl.java   |   5 +
 .../core/metadata/schema/table/CarbonTable.java |  68 +
 .../schema/table/column/ColumnSchema.java   |  19 +++
 .../apache/carbondata/core/util/CarbonUtil.java | 108 ++
 .../describeTable/TestDescribeTable.scala   |   4 +-
 .../carbondata/spark/util/CarbonScalaUtil.scala |   2 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 139 ++-
 .../command/carbonTableSchemaCommon.scala   |  13 +-
 .../table/CarbonDescribeFormattedCommand.scala  |  38 -
 10 files changed, 417 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 355bcb6..5f06d08 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -910,6 +910,37 @@ public final class CarbonCommonConstants {
   public static final String COLUMN_GROUPS = "column_groups";
   public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
   public static final String DICTIONARY_INCLUDE = "dictionary_include";
+
+  /**
+   * Table property to enable or disable local dictionary generation
+   */
+  public static final String LOCAL_DICTIONARY_ENABLE = 
"local_dictionary_enable";
+
+  /**
+   * default value for local dictionary generation
+   */
+  public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true";
+
+  /**
+   * Threshold value for local dictionary
+   */
+  public static final String LOCAL_DICTIONARY_THRESHOLD = 
"local_dictionary_threshold";
+
+  /**
+   * default value for local dictionary
+   */
+  public static final String LOCAL_DICTIONARY_THRESHOLD_DEFAULT = "1000";
+
+  /**
+   * Table property to specify the columns for which local dictionary needs to 
be generated.
+   */
+  public static final String LOCAL_DICTIONARY_INCLUDE = 
"local_dictionary_include";
+
+  /**
+   * Table property to specify the columns for which local dictionary should 
not be to be generated.
+   */
+  public static final String LOCAL_DICTIONARY_EXCLUDE = 
"local_dictionary_exclude";
+
   /**
* key for dictionary path
*/

http://git-wip-us.apache.org/repos/asf/carbondata/blob/be20fefb/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index f03b997..12f5fc3 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -37,6 +37,7 @@ import 
org.apache.carbondata.core.metadata.schema.table.TableInfo;
 import org.apache.carbondata.core.metadata.schema.table.TableSchema;
 import 

[24/50] [abbrv] carbondata git commit: [CARBONDATA-2604] Getting ArrayIndexOutOfBoundException during compaction after IUD in cluster is fixed

[jackylk]

[CARBONDATA-2604] Getting ArrayIndexOutOfBoundException during compaction after 
IUD in cluster is fixed

Issue: if some records are deleted then during filling the measure and 
dimension data no of valid rows count and actual rows may be different if
some records are deleted . and during filling of measure data it will iterrate 
over the scanned result. so it will cause ArrayIndexOutOfBoundException

Solution : Make a new temp list to collect the measure and dimension data 
during scan and fill inside RawBasedResultCollector and add it to final list

This closes #2369


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/efad40d5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/efad40d5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/efad40d5

Branch: refs/heads/carbonstore
Commit: efad40d5723849a351ec700e8e4e346cac8c3454
Parents: ff03645
Author: rahul 
Authored: Tue Jun 12 19:26:40 2018 +0530
Committer: manishgupta88 
Committed: Wed Jun 13 20:38:24 2018 +0530

--
 .../collector/impl/RawBasedResultCollector.java  | 12 +---
 .../sdv/generated/DataLoadingIUDTestCase.scala   | 19 +++
 2 files changed, 28 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/efad40d5/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
index d28df0a..7302b2c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -92,16 +92,22 @@ public class RawBasedResultCollector extends 
AbstractScannedResultCollector {
 // re initialized with left over value
 batchSize = 0;
   }
+  // for every iteration of available rows filling newly created list of 
Object[] and add it to
+  // the final list so there is no mismatch in the counter while filling 
dimension and
+  // measure data
+  List collectedData = new ArrayList<>(availableBatchRowCount);
   // fill dimension data
-  fillDimensionData(scannedResult, listBasedResult, queryMeasures, 
availableBatchRowCount);
-  fillMeasureData(scannedResult, listBasedResult);
+  fillDimensionData(scannedResult, collectedData, queryMeasures, 
availableBatchRowCount);
+  fillMeasureData(scannedResult, collectedData);
   // increment the number of rows scanned in scanned result statistics
   incrementScannedResultRowCounter(scannedResult, availableBatchRowCount);
   // assign the left over rows to batch size if the number of rows fetched 
are lesser
   // than batchSize
-  if (listBasedResult.size() < availableBatchRowCount) {
+  if (collectedData.size() < availableBatchRowCount) {
 batchSize += availableBatchRowCount - listBasedResult.size();
   }
+  // add the collected data to the final list
+  listBasedResult.addAll(collectedData);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/efad40d5/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
index 4c232be..79458f5 100644
--- 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/DataLoadingIUDTestCase.scala
@@ -3671,6 +3671,23 @@ test("HQ_Defect_TC_2016110901163", Include) {
sql(s"""drop table default.t_carbn01  """).collect
 }
 
+  test("[CARBONDATA-2604] ", Include){
+sql("drop table if exists brinjal").collect
+sql("create table brinjal (imei string,AMSize string,channelsId 
string,ActiveCountry string, Activecity string,gamePointId 
double,deviceInformationId double,productionDate Timestamp,deliveryDate 
timestamp,deliverycharge double) STORED BY 'org.apache.carbondata.format' 
TBLPROPERTIES('table_blocksize'='2000','sort_columns'='imei')").collect
+sql(s"""LOAD DATA INPATH 
'$resourcesPath/Data/InsertData/vardhandaterestruct.csv' INTO TABLE brinjal 

[47/50] [abbrv] carbondata git commit: [CARBONDATA-2615][32K] Support page size less than 32000 in CarbondataV3

[jackylk]

[CARBONDATA-2615][32K] Support page size less than 32000 in CarbondataV3

Since we support super long string, if it is long enough, a column page
with 32000 rows will exceed 2GB, so we support a page less than 32000
rows.

This closes #2383


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/091a28bf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/091a28bf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/091a28bf

Branch: refs/heads/carbonstore
Commit: 091a28bf833a5296dd3878ddb11b243f7f37a8fc
Parents: 2ea3b2d
Author: xuchuanyin 
Authored: Wed Jun 20 19:07:03 2018 +0800
Committer: kumarvishal09 
Committed: Thu Jun 21 11:00:02 2018 +0530

--
 .../testsuite/dataload/TestLoadDataGeneral.scala| 16 
 .../store/CarbonFactDataHandlerColumnar.java|  7 ++-
 2 files changed, 22 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/091a28bf/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 688928f..8b51090 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -259,6 +259,22 @@ class TestLoadDataGeneral extends QueryTest with 
BeforeAndAfterEach {
   CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_HDFS,
   originStatus)
   }
+
+  test("test data loading with page size less than 32000") {
+CarbonProperties.getInstance().addProperty(
+  CarbonCommonConstants.BLOCKLET_SIZE, "16000")
+
+val testData = s"$resourcesPath/sample.csv"
+sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
+checkAnswer(
+  sql("SELECT COUNT(*) FROM loadtest"),
+  Seq(Row(6))
+)
+
+
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+  CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+  }
+
   override def afterEach {
 sql("DROP TABLE if exists loadtest")
 sql("drop table if exists invalidMeasures")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/091a28bf/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
 
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index c0acadd..5fe3261 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -371,8 +371,13 @@ public class CarbonFactDataHandlerColumnar implements 
CarbonFactHandler {
 this.pageSize = Integer.parseInt(CarbonProperties.getInstance()
 .getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
 CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
+// support less than 32000 rows in one page, because we support super long 
string,
+// if it is long enough, a clomun page with 32000 rows will exceed 2GB
 if (version == ColumnarFormatVersion.V3) {
-  this.pageSize = 
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+  this.pageSize =
+  pageSize < 
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT ?
+  pageSize :
+  
CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
 }
 LOGGER.info("Number of rows per column blocklet " + pageSize);
 dataRows = new ArrayList<>(this.pageSize);

[30/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat folder for managed carbon table

[jackylk]

[CARBONDATA-2428] Support flat folder for managed carbon table

Currently carbondata writing happens in fixed path 
tablepath/Fact/Part0/Segment_NUM folder and it is not same as hive/parquet 
folder structure. This PR makes all files written will be inside tablepath, it 
does not maintain any segment folder structure. Only for partition it adds the 
folder.

This feature can be controlled through a table property flat_folder. The 
default value of it is false.

This closes #2207


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/60dfdd38
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/60dfdd38
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/60dfdd38

Branch: refs/heads/carbonstore
Commit: 60dfdd3857d037231844d9fb95967dcdb0071f40
Parents: 181f0ac
Author: ravipesala 
Authored: Fri Apr 20 07:34:00 2018 +0530
Committer: Jacky Li 
Committed: Sat Jun 16 10:56:10 2018 +0800

--
 .../core/constants/CarbonCommonConstants.java   |   8 ++
 .../apache/carbondata/core/datamap/Segment.java |  20 +++
 .../carbondata/core/datamap/TableDataMap.java   |   4 +-
 .../core/datamap/dev/DataMapWriter.java |   3 +-
 .../core/datastore/block/TableBlockInfo.java|  27 ++--
 .../blockletindex/BlockletDataMapFactory.java   |   4 +-
 .../core/metadata/SegmentFileStore.java |  91 ++
 .../core/metadata/schema/table/CarbonTable.java |  13 ++
 .../core/mutate/CarbonUpdateUtil.java   |  71 ++-
 .../executor/impl/AbstractQueryExecutor.java|  18 ++-
 .../core/scan/result/BlockletScannedResult.java |   3 +-
 .../scan/scanner/impl/BlockletFullScanner.java  |   6 +-
 .../SegmentUpdateStatusManager.java |  40 +++---
 .../apache/carbondata/core/util/CarbonUtil.java |  18 +--
 .../core/util/path/CarbonTablePath.java |  77 +---
 .../datastore/block/TableBlockInfoTest.java |   2 +-
 .../CarbonFormatDirectoryStructureTest.java |  22 +---
 .../bloom/BloomCoarseGrainDataMapFactory.java   |  10 +-
 .../lucene/LuceneDataMapFactoryBase.java|  17 ++-
 .../carbondata/hadoop/CarbonInputSplit.java |  35 --
 .../hadoop/api/CarbonInputFormat.java   |   2 +-
 .../hadoop/api/CarbonTableInputFormat.java  |  12 +-
 .../sdv/generated/DataLoadingIUDTestCase.scala  |   7 +-
 .../sdv/generated/MergeIndexTestCase.scala  |  55 
 .../lucene/LuceneFineGrainDataMapSuite.scala|  36 --
 .../dataload/TestLoadDataGeneral.scala  |   9 +-
 .../InsertIntoCarbonTableTestCase.scala |  33 ++---
 .../CarbonIndexFileMergeTestCase.scala  |  53 +---
 ...ompactionSupportGlobalSortFunctionTest.scala |  15 ++-
 ...mpactionSupportGlobalSortParameterTest.scala |  15 ++-
 .../dataload/TestBatchSortDataLoad.scala|  18 +--
 .../dataload/TestDataLoadWithFileName.scala |  26 ++--
 .../dataload/TestGlobalSortDataLoad.scala   |  23 +++-
 .../testsuite/datamap/CGDataMapTestCase.scala   |  10 +-
 .../testsuite/datamap/FGDataMapTestCase.scala   |   6 +-
 .../testsuite/datamap/TestDataMapCommand.scala  |  27 ++--
 .../FlatFolderTableLoadingTestCase.scala| 125 +++
 .../iud/UpdateCarbonTableTestCase.scala |  10 +-
 .../TestDataLoadingForPartitionTable.scala  |  27 ++--
 .../load/DataLoadProcessorStepOnSpark.scala |   2 +-
 .../spark/rdd/CarbonIUDMergerRDD.scala  |   4 +-
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |   7 +-
 .../carbondata/spark/util/CommonUtil.scala  |  17 +++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   2 +
 .../org/apache/spark/util/PartitionUtils.scala  |  73 +--
 .../datamap/IndexDataMapRebuildRDD.scala|   4 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|  89 ++---
 .../spark/rdd/CarbonTableCompactor.scala|   4 +-
 .../org/apache/spark/sql/CarbonSession.scala|   2 +-
 .../command/mutation/DeleteExecution.scala  |  23 ++--
 .../preaaggregate/PreAggregateTableHelper.scala |   3 +
 .../table/CarbonDescribeFormattedCommand.scala  |   5 +
 .../partition/TestAlterPartitionTable.scala |  35 --
 .../CarbonGetTableDetailComandTestCase.scala|   4 +-
 .../datamap/DataMapWriterListener.java  |   1 +
 .../loading/AbstractDataLoadProcessorStep.java  |   3 +-
 .../loading/TableProcessingOperations.java  |   6 +-
 .../loading/model/CarbonLoadModel.java  |  21 +++-
 .../merger/AbstractResultProcessor.java |   6 +-
 .../merger/CompactionResultSortProcessor.java   |   2 +-
 .../merger/RowResultMergerProcessor.java|   4 +-
 .../partition/spliter/RowResultProcessor.java   |   1 +
 .../store/CarbonFactDataHandlerModel.java   |   6 +-
 .../store/writer/AbstractFactDataWriter.java|   6 +-
 .../processing/util/CarbonLoaderUtil.java   |   4 +-
 .../store/worker/SearchRequestHandler.java  |  

[13/50] [abbrv] carbondata git commit: [CARBONDATA-2577] [CARBONDATA-2579] Fixed issue in Avro logical type for nested Array and document update

[jackylk]

[CARBONDATA-2577] [CARBONDATA-2579] Fixed issue in Avro logical type for nested 
Array and document update

Problem: Nested Array logical type of date, timestamp-millis, timestamp-micros 
is not working.

Root cause: During the preparation of carbon schema from avro schema. For array 
nested type logical types were not handled.

Solution: Handle the logical types for array nested type during carbon schema 
preparation.

This closes #2361


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/041603dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/041603dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/041603dc

Branch: refs/heads/carbonstore
Commit: 041603dccf1d98348db36c4bf8e2e60d50a5bcc8
Parents: d401e06
Author: ajantha-bhat 
Authored: Mon Jun 4 16:12:48 2018 +0530
Committer: kumarvishal09 
Committed: Tue Jun 5 19:21:07 2018 +0530

--
 docs/data-management-on-carbondata.md   |  7 -
 docs/sdk-guide.md   | 15 +++
 .../TestNonTransactionalCarbonTable.scala   | 28 +---
 .../carbondata/sdk/file/AvroCarbonWriter.java   | 26 --
 4 files changed, 58 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/docs/data-management-on-carbondata.md
--
diff --git a/docs/data-management-on-carbondata.md 
b/docs/data-management-on-carbondata.md
index 706209c..3326e9b 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -216,7 +216,12 @@ This tutorial is going to introduce all commands and data 
operations on CarbonDa
   This can be SDK output. Refer [SDK Writer 
Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-writer-guide.md).
 
   
   **Note:**
-  Dropping of the external table should not delete the files present in the 
location.
+  1. Dropping of the external table should not delete the files present in the 
location.
+  2. When external table is created on non-transactional table data, 
+  external table will be registered with the schema of carbondata files.
+  If multiple files with different schema is present, exception will be thrown.
+  So, If table registered with one schema and files are of different schema, 
+  suggest to drop the external table and create again to register table with 
new schema.  
 
 
 ## CREATE DATABASE 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/docs/sdk-guide.md
--
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 0f20dc3..e04698d 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -128,6 +128,21 @@ Each of SQL data types are mapped into data types of SDK. 
Following are the mapp
 | STRING | DataTypes.STRING |
 | DECIMAL | DataTypes.createDecimalType(precision, scale) |
 
+**NOTE:**
+ Carbon Supports below logical types of AVRO.
+ a. Date
+The date logical type represents a date within the calendar, with no 
reference to a particular time zone or time of day.
+A date logical type annotates an Avro int, where the int stores the number 
of days from the unix epoch, 1 January 1970 (ISO calendar). 
+ b. Timestamp (millisecond precision)
+The timestamp-millis logical type represents an instant on the global 
timeline, independent of a particular time zone or calendar, with a precision 
of one millisecond.
+A timestamp-millis logical type annotates an Avro long, where the long 
stores the number of milliseconds from the unix epoch, 1 January 1970 
00:00:00.000 UTC.
+ c. Timestamp (microsecond precision)
+The timestamp-micros logical type represents an instant on the global 
timeline, independent of a particular time zone or calendar, with a precision 
of one microsecond.
+A timestamp-micros logical type annotates an Avro long, where the long 
stores the number of microseconds from the unix epoch, 1 January 1970 
00:00:00.00 UTC.
+
+Currently the values of logical types are not validated by carbon. 
+Expect that avro record passed by the user is already validated by avro 
record generator tools.   
+
 ## Run SQL on files directly
 Instead of creating table and query it, you can also query that file directly 
with SQL.
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/041603dc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
 

[32/50] [abbrv] carbondata git commit: [CARBONDATA-2553] support ZSTD compression for sort temp file

[jackylk]

[CARBONDATA-2553] support ZSTD compression for sort temp file

This closes #2350


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ece06729
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ece06729
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ece06729

Branch: refs/heads/carbonstore
Commit: ece0672930b8bffba8e9bddad63560ff9d6cd582
Parents: 5593d16
Author: Manhua 
Authored: Tue May 29 09:21:52 2018 +0800
Committer: Jacky Li 
Committed: Mon Jun 18 21:31:02 2018 +0800

--
 core/pom.xml|  5 ++
 .../core/constants/CarbonCommonConstants.java   |  2 +-
 .../datastore/filesystem/LocalCarbonFile.java   |  8 +++
 .../carbondata/core/util/CarbonProperties.java  |  4 +-
 docs/useful-tips-on-carbondata.md   |  2 +-
 .../TestLoadWithSortTempCompressed.scala| 51 +++-
 6 files changed, 66 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index 7d87037..c145c3b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -68,6 +68,11 @@
   ${snappy.version}
 
 
+  com.github.luben
+  zstd-jni
+  1.3.2-2
+
+
   org.jmockit
   jmockit
   test

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 2fcf0f5..355bcb6 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1378,7 +1378,7 @@ public final class CarbonCommonConstants {
   public static final String CARBON_SORT_TEMP_COMPRESSOR = 
"carbon.sort.temp.compressor";
 
   /**
-   * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4'.
+   * The optional values are 'SNAPPY','GZIP','BZIP2','LZ4','ZSTD'.
* By default, empty means that Carbondata will not compress the sort temp 
files.
*/
   public static final String CARBON_SORT_TEMP_COMPRESSOR_DEFAULT = "";

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 60b7e17..5b6f657 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -42,6 +42,8 @@ import 
org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.util.CarbonUtil;
 
+import com.github.luben.zstd.ZstdInputStream;
+import com.github.luben.zstd.ZstdOutputStream;
 import net.jpountz.lz4.LZ4BlockInputStream;
 import net.jpountz.lz4.LZ4BlockOutputStream;
 import 
org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -290,6 +292,8 @@ public class LocalCarbonFile implements CarbonFile {
   inputStream = new SnappyInputStream(new FileInputStream(path));
 } else if ("LZ4".equalsIgnoreCase(compressor)) {
   inputStream = new LZ4BlockInputStream(new FileInputStream(path));
+} else if ("ZSTD".equalsIgnoreCase(compressor)) {
+  inputStream = new ZstdInputStream(new FileInputStream(path));
 } else {
   throw new IOException("Unsupported compressor: " + compressor);
 }
@@ -368,6 +372,10 @@ public class LocalCarbonFile implements CarbonFile {
   outputStream = new SnappyOutputStream(new FileOutputStream(path));
 } else if ("LZ4".equalsIgnoreCase(compressor)) {
   outputStream = new LZ4BlockOutputStream(new FileOutputStream(path));
+} else if ("ZSTD".equalsIgnoreCase(compressor)) {
+  // compression level 1 is cost-effective for sort temp file
+  // which is not used for storage
+  outputStream = new ZstdOutputStream(new FileOutputStream(path), 1);
 } else {
   throw new IOException("Unsupported compressor: " + compressor);
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ece06729/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java

[49/50] [abbrv] carbondata git commit: [CARBONDATA-2513][32K] Support write long string from dataframe

[jackylk]

[CARBONDATA-2513][32K] Support write long string from dataframe

support write long string from dataframe

This closes #2382


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/55f4bc6c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/55f4bc6c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/55f4bc6c

Branch: refs/heads/carbonstore
Commit: 55f4bc6c89f637b162b414033512901e9bd8a745
Parents: 218a8de
Author: xuchuanyin 
Authored: Wed Jun 20 19:01:24 2018 +0800
Committer: kumarvishal09 
Committed: Thu Jun 21 12:31:21 2018 +0530

--
 .../VarcharDataTypesBasicTestCase.scala | 32 +++-
 .../apache/carbondata/spark/CarbonOption.scala  |  2 ++
 .../spark/rdd/NewCarbonDataLoadRDD.scala| 15 +++--
 .../carbondata/spark/util/CarbonScalaUtil.scala |  3 +-
 .../spark/sql/CarbonDataFrameWriter.scala   |  1 +
 .../streaming/parser/FieldConverter.scala   | 11 +--
 6 files changed, 57 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 9ea3f1f..9798178 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -20,8 +20,9 @@ package org.apache.carbondata.spark.testsuite.longstring
 import java.io.{File, PrintWriter}
 
 import org.apache.commons.lang3.RandomStringUtils
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
 import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, 
StructType}
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -36,6 +37,7 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
   private val inputFile_2g_column_page = s"$inputDir$fileName_2g_column_page"
   private val lineNum = 1000
   private var content: Content = _
+  private var longStringDF: DataFrame = _
   private var originMemorySize = CarbonProperties.getInstance().getProperty(
 CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
 CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
@@ -257,6 +259,34 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
 // since after exception wrapper, we cannot get the root cause directly
   }
 
+  private def prepareDF(): Unit = {
+val schema = StructType(
+  StructField("id", IntegerType, nullable = true) ::
+  StructField("name", StringType, nullable = true) ::
+  StructField("description", StringType, nullable = true) ::
+  StructField("address", StringType, nullable = true) ::
+  StructField("note", StringType, nullable = true) :: Nil
+)
+longStringDF = sqlContext.sparkSession.read
+  .schema(schema)
+  .csv(inputFile)
+  }
+
+  test("write from dataframe with long string datatype") {
+prepareDF()
+// write spark dataframe to carbondata with `long_string_columns` property
+longStringDF.write
+  .format("carbondata")
+  .option("tableName", longStringTable)
+  .option("single_pass", "false")
+  .option("sort_columns", "name")
+  .option("long_string_columns", "description, note")
+  .mode(SaveMode.Overwrite)
+  .save()
+
+checkQuery()
+  }
+
   // will create 2 long string columns
   private def createFile(filePath: String, line: Int = 1, start: Int = 0,
   varcharLen: Int = Short.MaxValue + 1000): Content = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index a48e63d..5f23f77 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ 

[37/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test Cases for Local Dictionary Support for Create Table comand

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 65ff76d..9af8817 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -321,9 +321,8 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
 case scala.util.Failure(ex) =>
   LOGGER
 .debug(
-  "invalid value is configured for local_dictionary_threshold, 
considering the defaut" +
-  " " +
-  "value")
+  "invalid value is configured for local_dictionary_threshold, 
considering the " +
+  "default value")
   tableProperties.put(CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD,
 CarbonCommonConstants.LOCAL_DICTIONARY_THRESHOLD_DEFAULT)
   }
@@ -362,9 +361,11 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
   .distinct.length !=
 List(localDictIncludeCols, 
localDictExcludeCols).mkString(",").split(",")
   .length) {
+  val duplicateColumns = 
localDictIncludeCols.diff(localDictExcludeCols.distinct).distinct
   val errMsg =
-"Column ambiguity as duplicate columns present in 
LOCAL_DICTIONARY_INCLUDE and " +
-"LOCAL_DICTIONARY_INCLUDE.Duplicate columns are not allowed."
+"Column ambiguity as duplicate column(s):  " +
+duplicateColumns.mkString("(", ",", ")") + "are present in 
LOCAL_DICTIONARY_INCLUDE " +
+"and LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."
   throw new MalformedCarbonCommandException(errMsg)
 }
   }
@@ -411,10 +412,11 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
 
 // check if the duplicate columns are specified in table schema
 if (localDictColumns.distinct.lengthCompare(localDictColumns.size) != 0) {
-  val a = localDictColumns.diff(localDictColumns.distinct).distinct
-  val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains 
Duplicate Columns " +
-   a.mkString("(", ",", ")") +
-   ". Please check create table statement."
+  val duplicateColumns = 
localDictColumns.diff(localDictColumns.distinct).distinct
+  val errMsg =
+"LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE contains Duplicate 
Columns: " +
+duplicateColumns.mkString("(", ",", ")") +
+". Please check create table statement."
   throw new MalformedCarbonCommandException(errMsg)
 }
 
@@ -436,8 +438,9 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
  !x.dataType.get.equalsIgnoreCase("ARRAY"))) {
 val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE 
column: " +
dictColm.trim +
-   " is not a String datatype column. 
LOCAL_DICTIONARY_COLUMN should be no " +
-   "dictionary string datatype column.Please check create 
table statement."
+   " is not a String/complex datatype column. 
LOCAL_DICTIONARY_COLUMN should " +
+   "be no dictionary string/complex datatype column.Please 
check create table" +
+   " statement."
 throw new MalformedCarbonCommandException(errormsg)
   }
 }
@@ -448,10 +451,12 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
 
tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(",").map(_.trim)
   localDictColumns.foreach { distCol =>
 if (dictIncludeColumns.exists(x => x.equalsIgnoreCase(distCol.trim))) {
+  val duplicateColumns = 
dictIncludeColumns.diff(localDictColumns.distinct).distinct
   val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE 
column: " +
- distCol.trim +
+ duplicateColumns.mkString("(", ",", ")") +
  " specified in Dictionary include. Local Dictionary 
will not be " +
- "generated for Dictionary include. Please check 
create table statement."
+ "generated for Dictionary include columns. Please 
check create table " +
+ "statement."
   throw new MalformedCarbonCommandException(errormsg)
 }
   }

[18/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate carbonstore mv branch

[jackylk]

[CARBONDATA-2573] integrate carbonstore mv branch

Fixes bugs related to MV and added tests

This closes #2335


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0ef7e55c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0ef7e55c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0ef7e55c

Branch: refs/heads/carbonstore
Commit: 0ef7e55c46be9d3767539d1a51b780064cc7ad26
Parents: 83ee2c4
Author: ravipesala 
Authored: Wed May 30 09:11:13 2018 +0530
Committer: Jacky Li 
Committed: Mon Jun 11 21:25:31 2018 +0800

--
 .../carbondata/mv/datamap/MVAnalyzerRule.scala  |   2 +-
 .../apache/carbondata/mv/datamap/MVHelper.scala |  23 +
 .../apache/carbondata/mv/datamap/MVState.scala  |  55 --
 .../mv/rewrite/DefaultMatchMaker.scala  |  34 +-
 .../carbondata/mv/rewrite/Navigator.scala   |  50 +-
 .../carbondata/mv/rewrite/QueryRewrite.scala|  19 +-
 .../mv/rewrite/SummaryDatasetCatalog.scala  |  79 +-
 .../apache/carbondata/mv/rewrite/Utils.scala| 108 ++-
 .../carbondata/mv/session/MVSession.scala   |  84 ++
 .../mv/session/internal/SessionState.scala  |  56 ++
 .../mv/rewrite/MVCreateTestCase.scala   |  46 +-
 .../carbondata/mv/rewrite/MVTPCDSTestCase.scala |   2 +-
 .../SelectSelectExactChildrenSuite.scala|   5 +-
 .../carbondata/mv/rewrite/TestSQLSuite.scala|  99 +++
 .../carbondata/mv/rewrite/Tpcds_1_4_Suite.scala |  84 ++
 .../mv/rewrite/matching/TestSQLBatch.scala  |  23 +-
 .../rewrite/matching/TestTPCDS_1_4_Batch.scala  | 886 +--
 .../org/apache/carbondata/mv/dsl/package.scala  |   4 +-
 .../util/LogicalPlanSignatureGenerator.scala|  11 +-
 .../carbondata/mv/plans/util/SQLBuilder.scala   |  14 +-
 .../mv/testutil/Tpcds_1_4_Tables.scala  | 142 +--
 .../org/apache/carbondata/mv/TestSQLBatch.scala | 584 
 .../mv/plans/ExtractJoinConditionsSuite.scala   |   2 +-
 .../carbondata/mv/plans/IsSPJGHSuite.scala  |   3 +-
 .../mv/plans/LogicalToModularPlanSuite.scala|   3 +-
 .../carbondata/mv/plans/ModularToSQLSuite.scala | 232 +++--
 .../carbondata/mv/plans/SignatureSuite.scala|  95 +-
 .../mv/plans/Tpcds_1_4_BenchmarkSuite.scala |  86 ++
 .../carbondata/mv/testutil/TestSQLBatch.scala   | 584 
 .../carbondata/mv/testutil/TestSQLBatch2.scala  | 138 +++
 30 files changed, 2306 insertions(+), 1247 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
--
diff --git 
a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
 
b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
index 4e93f15..483780f 100644
--- 
a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
+++ 
b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVAnalyzerRule.scala
@@ -65,7 +65,7 @@ class MVAnalyzerRule(sparkSession: SparkSession) extends 
Rule[LogicalPlan] {
 val catalog = 
DataMapStoreManager.getInstance().getDataMapCatalog(dataMapProvider,
   DataMapClassProvider.MV.getShortName).asInstanceOf[SummaryDatasetCatalog]
 if (needAnalysis && catalog != null && isValidPlan(plan, catalog)) {
-  val modularPlan = catalog.mVState.rewritePlan(plan).withSummaryData
+  val modularPlan = 
catalog.mvSession.sessionState.rewritePlan(plan).withMVTable
   if (modularPlan.find (_.rewritten).isDefined) {
 val compactSQL = modularPlan.asCompactSQL
 LOGGER.audit(s"\n$compactSQL\n")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
--
diff --git 
a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
 
b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
index 0f9362f..a40fa2c 100644
--- 
a/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
+++ 
b/datamap/mv/core/src/main/scala/org/apache/carbondata/mv/datamap/MVHelper.scala
@@ -373,5 +373,28 @@ object MVHelper {
   case other => other
 }
   }
+
+  /**
+   * Rewrite the updated mv query with corresponding MV table.
+   */
+  def rewriteWithMVTable(rewrittenPlan: ModularPlan, rewrite: QueryRewrite): 
ModularPlan = {
+if (rewrittenPlan.find(_.rewritten).isDefined) {
+  val updatedDataMapTablePlan = rewrittenPlan transform {
+case s: Select =>
+  MVHelper.updateDataMap(s, rewrite)
+case g: GroupBy =>
+  MVHelper.updateDataMap(g, rewrite)
+  }
+  // 

[45/50] [abbrv] carbondata git commit: [CARBONDATA-2504][STREAM] Support StreamSQL for streaming job

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java 
b/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
index 9e338e7..daa1447 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/store/LocalCarbonStore.java
@@ -65,7 +65,7 @@ class LocalCarbonStore extends MetaCachedCarbonStore {
 Objects.requireNonNull(projectColumns);
 
 CarbonTable table = getTable(path);
-if (table.isStreamingTable() || table.isHivePartitionTable()) {
+if (table.isStreamingSink() || table.isHivePartitionTable()) {
   throw new UnsupportedOperationException("streaming and partition table 
is not supported");
 }
 // TODO: use InputFormat to prune data and read data

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
--
diff --git 
a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
 
b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
index 1696fdc..5a888ef 100644
--- 
a/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
+++ 
b/streaming/src/main/scala/org/apache/carbondata/streaming/parser/RowStreamParserImp.scala
@@ -17,7 +17,6 @@
 
 package org.apache.carbondata.streaming.parser
 
-import java.nio.charset.Charset
 import java.text.SimpleDateFormat
 
 import org.apache.hadoop.conf.Configuration

[50/50] [abbrv] carbondata git commit: [CARBONDATA-2623][DataMap] Add DataMap Pre and Pevent listener

[jackylk]

[CARBONDATA-2623][DataMap] Add DataMap Pre and Pevent listener

Added Pre and Post Execution Events for index datamap

This closes #2389


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b3f78206
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b3f78206
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b3f78206

Branch: refs/heads/carbonstore
Commit: b3f7820623d4bc9ab4408beb8ad708ba9b19b899
Parents: 55f4bc6
Author: mohammadshahidkhan 
Authored: Wed Jun 20 19:52:51 2018 +0530
Committer: manishgupta88 
Committed: Thu Jun 21 17:37:48 2018 +0530

--
 .../carbondata/events/DataMapEvents.scala   | 68 
 .../org/apache/carbondata/events/Events.scala   | 18 +-
 .../datamap/IndexDataMapRebuildRDD.scala| 11 +++-
 .../spark/rdd/CarbonTableCompactor.scala| 23 ++-
 .../datamap/CarbonCreateDataMapCommand.scala| 22 +++
 .../datamap/CarbonDataMapRebuildCommand.scala   | 12 
 .../datamap/CarbonDropDataMapCommand.scala  | 11 
 .../management/CarbonLoadDataCommand.scala  | 21 +-
 8 files changed, 181 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
new file mode 100644
index 000..8fb374f
--- /dev/null
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.events
+
+import org.apache.spark.sql.SparkSession
+
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+
+/**
+ * For handling operation's after finish of index creation over table with 
index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPostExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of update index datmap status over 
table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPreExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of  update index datmap status over 
table with index
+ * datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPostExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of index build over table with index 
datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPreExecutionEvent(sparkSession: SparkSession,
+identifier: AbsoluteTableIdentifier, dataMapNames: 
scala.collection.mutable.Seq[String])
+  extends Event with BuildDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of index build over table with index 
datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPostExecutionEvent(sparkSession: SparkSession,
+identifier: AbsoluteTableIdentifier)
+  extends Event with TableEventInfo
+
+/**
+ * For handling operation's before start of index creation over table with 
index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPreExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala
--
diff --git 

[33/50] [abbrv] carbondata git commit: [CARBONDATA-2610] Fix for datamap creation failed on table having loaded data with null value on string datatype

[jackylk]

[CARBONDATA-2610] Fix for datamap creation failed on table having loaded data 
with null value on string datatype

Problem: Datamap creation having null values already loaded in string datatype 
of table fails.
Solution: Check for null before converting data to the string.

This closes #2376


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e7fed361
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e7fed361
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e7fed361

Branch: refs/heads/carbonstore
Commit: e7fed361b93d7986392c469009f0f6633d71def5
Parents: ece0672
Author: Jatin 
Authored: Thu Jun 14 22:56:09 2018 +0530
Committer: Jacky Li 
Committed: Tue Jun 19 00:23:01 2018 +0800

--
 .../lucene/LuceneFineGrainDataMapSuite.scala| 31 
 .../datamap/IndexDataMapRebuildRDD.scala|  2 +-
 2 files changed, 32 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7fed361/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 6530ec0..6d774a2 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -829,6 +829,37 @@ class LuceneFineGrainDataMapSuite extends QueryTest with 
BeforeAndAfterAll {
   sql("select * from table_stop where 
text_match('suggestion:*is*')").collect().length == 1)
   }
 
+  test("test lucene data map on null values") {
+sql("DROP TABLE IF EXISTS datamap_test4")
+sql("DROP TABLE IF EXISTS datamap_copy")
+sql(
+  """
+| CREATE TABLE datamap_test4(id INT, name STRING, city STRING, age INT)
+| STORED BY 'carbondata'
+| TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT', 
'autorefreshdatamap' = 'false')
+  """.stripMargin)
+sql(
+  """
+| CREATE TABLE datamap_copy(id INT, name STRING, city STRING, age INT)
+| STORED BY 'carbondata'
+| TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT', 
'autorefreshdatamap' = 'false')
+  """.stripMargin)
+sql("insert into datamap_test4 select 1,'name','city',20")
+sql("insert into datamap_test4 select 2,'name1','city1',20")
+sql("insert into datamap_test4 select 25,cast(null as 
string),'city2',NULL")
+sql("insert into datamap_copy select * from datamap_test4")
+sql(
+  s"""
+ | CREATE DATAMAP dm4 ON TABLE datamap_test4
+ | USING 'lucene'
+ | DMProperties('INDEX_COLUMNS'='name , city')
+  """.stripMargin)
+checkAnswer(sql("SELECT * FROM datamap_test4 WHERE TEXT_MATCH('name:n*')"),
+  sql(s"select * from datamap_copy where name like '%n%'"))
+sql("drop table datamap_test4")
+sql("drop table datamap_copy")
+  }
+
   override protected def afterAll(): Unit = {
 LuceneFineGrainDataMapSuite.deleteFile(file2)
 sql("DROP TABLE IF EXISTS normal_test")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7fed361/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
 
b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
index f3f2650..cde6201 100644
--- 
a/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/carbondata/datamap/IndexDataMapRebuildRDD.scala
@@ -124,7 +124,7 @@ class OriginalReadSupport(dataTypes: Array[DataType]) 
extends CarbonReadSupport[
 
   override def readRow(data: Array[Object]): Array[Object] = {
 dataTypes.zipWithIndex.foreach { case (dataType, i) =>
-  if (dataType == DataTypes.STRING) {
+  if (dataType == DataTypes.STRING && data(i) != null) {
 data(i) = data(i).toString
   }
 }

[43/50] [abbrv] carbondata git commit: [HOTFIX] Added Performance Optimization for Presto by using MultiBlockSplit

[jackylk]

[HOTFIX] Added Performance Optimization for Presto by using MultiBlockSplit

Added Performance Optimization for Presto by using MultiBlockSplit

This closes #2265


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/01b48fc3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/01b48fc3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/01b48fc3

Branch: refs/heads/carbonstore
Commit: 01b48fc36a93b74827edff4d3330cbf5546a5d38
Parents: 0e1d550
Author: Bhavya 
Authored: Mon Apr 16 11:54:17 2018 +0530
Committer: chenliang613 
Committed: Wed Jun 20 22:53:38 2018 +0800

--
 integration/presto/README.md|   9 +-
 integration/presto/pom.xml  |  14 +-
 .../carbondata/presto/CarbonVectorBatch.java|   2 +-
 .../carbondata/presto/CarbondataMetadata.java   |  87 +++-
 .../carbondata/presto/CarbondataPageSource.java |  11 +-
 .../presto/CarbondataPageSourceProvider.java|  27 ++--
 .../carbondata/presto/CarbondataSplit.java  |  24 +++-
 .../presto/CarbondataSplitManager.java  |  34 -
 .../PrestoCarbonVectorizedRecordReader.java |  39 +-
 .../presto/impl/CarbonLocalInputSplit.java  |  19 ++-
 .../presto/impl/CarbonLocalMultiBlockSplit.java |  86 
 .../presto/impl/CarbonTableConfig.java  |  40 +-
 .../presto/impl/CarbonTableReader.java  | 133 ---
 .../readers/DecimalSliceStreamReader.java   | 105 ---
 .../presto/src/main/resources/log4j.properties  |  11 ++
 15 files changed, 462 insertions(+), 179 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/README.md
--
diff --git a/integration/presto/README.md b/integration/presto/README.md
index 53884a2..5a44f5c 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -77,6 +77,9 @@ Please follow the below steps to query carbondata in presto
   carbondata-store={schema-store-path}
   enable.unsafe.in.query.processing=false
   carbon.unsafe.working.memory.in.mb={value}
+  enable.unsafe.columnpage=false
+  enable.unsafe.sort=false
+
   ```
   Replace the schema-store-path with the absolute path of the parent directory 
of the schema.
   For example, if you have a schema named 'default' stored in 
hdfs://namenode:9000/test/carbondata/,
@@ -112,7 +115,11 @@ Please follow the below steps to query carbondata in presto
   Unsafe Properties
   enable.unsafe.in.query.processing property by default is true in CarbonData 
system, the carbon.unsafe.working.memory.in.mb 
   property defines the limit for Unsafe Memory usage in Mega Bytes, the 
default value is 512 MB.
-  If your tables are big you can increase the unsafe memory, or disable unsafe 
via setting enable.unsafe.in.query.processing=false.
+  Currently Presto does not support Unsafe Memory so we have to disable the 
unsafe feature by setting below properties to false.
+
+  enable.unsafe.in.query.processing=false.
+  enable.unsafe.columnpage=false
+  enable.unsafe.sort=false
 
   If you updated the jar balls or configuration files, make sure you have 
dispatched them
to all the presto nodes and restarted the presto servers on the nodes. The 
updates will not take effect before restarting.

http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index b91f070..c61023a 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -463,12 +463,6 @@
   provided
 
 
-  com.google.code.findbugs
-  jsr305
-  3.0.2
-  provided
-
-
   org.glassfish.hk2
   hk2-api
   2.5.0-b42
@@ -552,13 +546,7 @@
 src/test/scala
 
   
-src/resources
-  
-  
-.
-
-  CARBON_SPARK_INTERFACELogResource.properties
-
+src/main/resources
   
 
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/01b48fc3/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
index b230d6a..b6caaa3 100644
--- 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
+++ 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorBatch.java
@@ -25,7 +25,7 @@ import 

[46/50] [abbrv] carbondata git commit: [CARBONDATA-2504][STREAM] Support StreamSQL for streaming job

[jackylk]

[CARBONDATA-2504][STREAM] Support StreamSQL for streaming job

Currently, user need to write Spark Streaming APP to use carbon streaming 
ingest feature, which is not so easy for some users. By providing StreamSQL, 
user can manage the streaming job more easily.

This closes #2328


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2ea3b2dc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2ea3b2dc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2ea3b2dc

Branch: refs/heads/carbonstore
Commit: 2ea3b2dc5841c604c5fb44a6e7f18c7d2db8c543
Parents: 6eb360e
Author: Jacky Li 
Authored: Mon May 21 21:49:33 2018 +0800
Committer: ravipesala 
Committed: Thu Jun 21 09:44:22 2018 +0530

--
 .../exceptions/NoSuchStreamException.java   |  35 ++
 .../core/metadata/schema/table/CarbonTable.java |  15 +-
 .../spark/util/SparkDataTypeConverterImpl.java  |  45 +++
 .../apache/carbondata/spark/CarbonOption.scala  |   9 +-
 .../carbondata/spark/StreamingOption.scala  |  67 
 .../carbondata/spark/util/CarbonScalaUtil.scala |   5 +-
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   3 +
 .../spark/util/CarbonMetastoreTypes.scala   | 104 ++
 .../apache/spark/util/SparkTypeConverter.scala  | 134 +++
 .../carbondata/stream/StreamJobManager.scala| 198 ++
 .../org/apache/spark/sql/CarbonSource.scala |   2 +-
 .../spark/sql/CarbonSparkStreamingFactory.scala |   2 +-
 .../datamap/CarbonCreateDataMapCommand.scala|   2 +-
 .../stream/CarbonCreateStreamCommand.scala  | 130 +++
 .../stream/CarbonDropStreamCommand.scala|  36 ++
 .../stream/CarbonShowStreamsCommand.scala   |  76 
 .../command/table/CarbonDropTableCommand.scala  |   2 +-
 .../strategy/CarbonLateDecodeStrategy.scala |   2 +-
 .../sql/execution/strategy/DDLStrategy.scala|   2 +-
 .../strategy/StreamingTableStrategy.scala   |   2 +-
 .../sql/hive/CarbonPreAggregateRules.scala  |   6 +-
 .../apache/spark/sql/hive/CarbonRelation.scala  |  89 +
 .../sql/parser/CarbonSpark2SqlParser.scala  |  44 ++-
 .../spark/sql/parser/CarbonSparkSqlParser.scala |   4 +-
 .../apache/spark/util/SparkTypeConverter.scala  | 135 ---
 .../TestStreamingTableOperation.scala   | 359 ++-
 .../carbondata/store/LocalCarbonStore.java  |   2 +-
 .../streaming/parser/RowStreamParserImp.scala   |   1 -
 28 files changed, 1254 insertions(+), 257 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
--
diff --git 
a/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
 
b/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
new file mode 100644
index 000..77fa7fb
--- /dev/null
+++ 
b/common/src/main/java/org/apache/carbondata/common/exceptions/NoSuchStreamException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.common.exceptions;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Stable
+public class NoSuchStreamException extends Exception {
+  /**
+   * default serial version ID.
+   */
+  private static final long serialVersionUID = 133241232L;
+
+  public NoSuchStreamException(String streamName) {
+super("stream '" + streamName + "' not found");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2ea3b2dc/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
 

[27/50] [abbrv] carbondata git commit: [CARBONDATA-2593] Add an option 'carbon.insert.storage.level' to support configuring the storage level when insert into data with 'carbon.insert.persist.enable'=

[jackylk]

[CARBONDATA-2593] Add an option 'carbon.insert.storage.level' to support 
configuring the storage level when insert into data with 
'carbon.insert.persist.enable'='true'

When insert into data with 'carbon.insert.persist.enable'='true', the storage 
level of dataset is 'MEMORY_AND_DISK',
it should support configuring the storage level to correspond to different 
environment.

This closes #2373


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/181f0ac9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/181f0ac9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/181f0ac9

Branch: refs/heads/carbonstore
Commit: 181f0ac9bed6ff7d83268f6c058aee943b348ddc
Parents: f0c8834
Author: Zhang Zhichao <441586...@qq.com>
Authored: Thu Jun 14 14:48:47 2018 +0800
Committer: Jacky Li 
Committed: Sat Jun 16 03:32:36 2018 +0800

--
 .../core/constants/CarbonCommonConstants.java   | 25 
 .../carbondata/core/util/CarbonProperties.java  | 18 ++
 docs/configuration-parameters.md|  4 
 .../management/CarbonInsertIntoCommand.scala|  5 ++--
 4 files changed, 50 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c7281dd..19ff494 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -866,6 +866,7 @@ public final class CarbonCommonConstants {
* to run load and insert queries on source table concurrently then user can 
enable this flag
*/
   @CarbonProperty
+  @InterfaceStability.Evolving
   public static final String CARBON_INSERT_PERSIST_ENABLED = 
"carbon.insert.persist.enable";
 
   /**
@@ -875,6 +876,27 @@ public final class CarbonCommonConstants {
   public static final String CARBON_INSERT_PERSIST_ENABLED_DEFAULT = "false";
 
   /**
+   * Which storage level to persist dataset when insert into data
+   * with 'carbon.insert.persist.enable'='true'
+   */
+  @CarbonProperty
+  @InterfaceStability.Evolving
+  public static final String CARBON_INSERT_STORAGE_LEVEL =
+  "carbon.insert.storage.level";
+
+  /**
+   * The default value(MEMORY_AND_DISK) is the same as the default storage 
level of Dataset.
+   * Unlike `RDD.cache()`, the default storage level is set to be 
`MEMORY_AND_DISK` because
+   * recomputing the in-memory columnar representation of the underlying table 
is expensive.
+   *
+   * if user's executor has less memory, set the CARBON_INSERT_STORAGE_LEVEL
+   * to MEMORY_AND_DISK_SER or other storage level to correspond to different 
environment.
+   * You can get more recommendations about storage level in spark website:
+   * 
http://spark.apache.org/docs/latest/rdd-programming-guide.html#rdd-persistence.
+   */
+  public static final String CARBON_INSERT_STORAGE_LEVEL_DEFAULT = 
"MEMORY_AND_DISK";
+
+  /**
* default name of data base
*/
   public static final String DATABASE_DEFAULT_NAME = "default";
@@ -1094,6 +1116,7 @@ public final class CarbonCommonConstants {
* to determine to use the rdd persist or not.
*/
   @CarbonProperty
+  @InterfaceStability.Evolving
   public static final String isPersistEnabled = "carbon.update.persist.enable";
 
   /**
@@ -1117,6 +1140,7 @@ public final class CarbonCommonConstants {
* with 'carbon.update.persist.enable'='true'
*/
   @CarbonProperty
+  @InterfaceStability.Evolving
   public static final String CARBON_UPDATE_STORAGE_LEVEL =
   "carbon.update.storage.level";
 
@@ -1354,6 +1378,7 @@ public final class CarbonCommonConstants {
* Which storage level to persist rdd when sort_scope=global_sort
*/
   @CarbonProperty
+  @InterfaceStability.Evolving
   public static final String CARBON_GLOBAL_SORT_RDD_STORAGE_LEVEL =
   "carbon.global.sort.rdd.storage.level";
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/181f0ac9/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 6eb7de6..b134a7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1581,4 

[41/50] [abbrv] carbondata git commit: [CARBONDATA-2420][32K] Support string longer than 32000 characters

[jackylk]

[CARBONDATA-2420][32K] Support string longer than 32000 characters

Add a property in creating table 'long_string_columns' to support string 
columns that will contains more than 32000 characters.
Inside carbondata, it use an integer instead of short to store the length of 
bytes content.

Internally in Carbondata,

add a new datatype called varchar to represent the long string column
add a new encoding called DIRECT_COMPRESS_VARCHAR to the varcher column page 
meta
use an integer (previously short) to store the length of bytes content.
add 2GB constraint for one column page

This closes #2379


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc53dee2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc53dee2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc53dee2

Branch: refs/heads/carbonstore
Commit: dc53dee2448f366319764021d77c4be75d43b9e3
Parents: c5a4ec0
Author: xuchuanyin 
Authored: Sat Jun 2 15:17:04 2018 +0800
Committer: kumarvishal09 
Committed: Wed Jun 20 15:24:22 2018 +0530

--
 .../core/constants/CarbonCommonConstants.java   |   3 +
 .../impl/FixedLengthDimensionColumnPage.java|   2 +-
 .../impl/VariableLengthDimensionColumnPage.java |  11 +-
 ...mpressedDimensionChunkFileBasedReaderV1.java |   3 +-
 ...mpressedDimensionChunkFileBasedReaderV2.java |   3 +-
 ...mpressedDimensionChunkFileBasedReaderV3.java |   7 +-
 .../chunk/store/DimensionChunkStoreFactory.java |  22 +-
 ...ariableIntLengthDimensionDataChunkStore.java |  43 +++
 ...feVariableLengthDimensionDataChunkStore.java |  45 +--
 ...iableShortLengthDimensionDataChunkStore.java |  41 +++
 ...ariableIntLengthDimensionDataChunkStore.java |  44 +++
 ...feVariableLengthDimensionDataChunkStore.java |  54 ++--
 ...iableShortLengthDimensionDataChunkStore.java |  44 +++
 .../core/datastore/page/ColumnPage.java |  16 +-
 .../datastore/page/VarLengthColumnPageBase.java |   6 +
 .../page/encoding/DefaultEncodingFactory.java   |   1 +
 .../page/encoding/EncodingFactory.java  |   3 +-
 .../encoding/compress/DirectCompressCodec.java  |   6 +-
 .../legacy/HighCardDictDimensionIndexCodec.java |  13 +-
 .../statistics/LVLongStringStatsCollector.java  |  51 
 .../statistics/LVShortStringStatsCollector.java |  50 
 .../page/statistics/LVStringStatsCollector.java |  27 +-
 .../core/indexstore/UnsafeMemoryDMStore.java|  11 +-
 .../blockletindex/BlockletDataMap.java  |   8 +-
 .../core/indexstore/row/DataMapRow.java |   4 +-
 .../core/indexstore/row/UnsafeDataMapRow.java   |  60 ++--
 .../core/indexstore/schema/CarbonRowSchema.java |  10 +-
 .../core/metadata/blocklet/BlockletInfo.java|   2 +-
 .../ThriftWrapperSchemaConverterImpl.java   |   8 +
 .../core/metadata/datatype/DataType.java|   3 +
 .../core/metadata/datatype/DataTypes.java   |   5 +
 .../core/metadata/datatype/VarcharType.java |  34 +++
 .../core/metadata/encoder/Encoding.java |   5 +-
 .../schema/table/TableSchemaBuilder.java|   1 +
 .../util/AbstractDataFileFooterConverter.java   |   2 +
 .../apache/carbondata/core/util/CarbonUtil.java |   8 +-
 .../carbondata/core/util/DataTypeUtil.java  |   4 +
 .../ThriftWrapperSchemaConverterImplTest.java   |   2 +-
 format/src/main/thrift/schema.thrift|   3 +
 .../VarcharDataTypesBasicTestCase.scala | 279 +++
 .../carbondata/spark/util/CarbonScalaUtil.scala |   1 +
 .../spark/util/DataTypeConverterUtil.scala  |   1 +
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  36 ++-
 .../command/carbonTableSchemaCommon.scala   |  43 ++-
 .../apache/spark/sql/hive/CarbonRelation.scala  |   1 +
 .../impl/NonDictionaryFieldConverterImpl.java   |  12 +-
 .../loading/csvinput/CSVInputFormat.java|   4 +-
 .../loading/row/IntermediateSortTempRow.java|  19 +-
 .../loading/sort/SortStepRowHandler.java|  26 +-
 .../merger/CompactionResultSortProcessor.java   |  11 +-
 .../sort/sortdata/SortParameters.java   |  21 +-
 .../sort/sortdata/TableFieldStat.java   |  37 ++-
 .../carbondata/processing/store/TablePage.java  |  37 ++-
 .../util/CarbonDataProcessorUtil.java   |  20 ++
 54 files changed, 1049 insertions(+), 164 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc53dee2/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 5f06d08..118ff28 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 

[38/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test Cases for Local Dictionary Support for Create Table comand

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
new file mode 100644
index 000..5f9af69
--- /dev/null
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
@@ -0,0 +1,2102 @@
+package org.apache.carbondata.spark.testsuite.localdictionary
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import 
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
+class LocalDictionarySupportCreateTableTest extends QueryTest with 
BeforeAndAfterAll {
+
+  override protected def beforeAll(): Unit = {
+sql("DROP TABLE IF EXISTS LOCAL1")
+  }
+
+  test("test local dictionary default configuration") {
+sql("drop table if exists local1")
+sql(
+  """
+| CREATE TABLE local1(id int, name string, city string, age int)
+| STORED BY 'org.apache.carbondata.format'
+  """.stripMargin)
+
+val desc_result = sql("describe formatted local1")
+
+val descLoc = sql("describe formatted local1").collect
+descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match 
{
+  case Some(row) => assert(row.get(1).toString.contains("true"))
+}
+descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) 
match {
+  case Some(row) => assert(row.get(1).toString.contains("1000"))
+}
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_001") {
+sql("drop table if exists local1")
+sql(
+  """
+| CREATE TABLE local1(id int, name string, city string, age int)
+| STORED BY 'org.apache.carbondata.format'
+| tblproperties('local_dictionary_include'='name')
+  """.
+stripMargin)
+val descFormatted1 = sql("describe formatted local1").collect
+descFormatted1.find(_.get(0).toString.contains("Local Dictionary 
Enabled")) match {
+  case Some(row) => assert(row.get(1).toString.contains("true"))
+}
+descFormatted1.find(_.get(0).toString.contains("Local Dictionary 
Include")) match {
+  case Some(row) => assert(row.get(1).toString.contains("name"))
+}
+  }
+
+  test(
+"test local dictionary custom configurations for local dict columns _002")
+  {
+sql("drop table if exists local1")
+
+intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 'org.apache.carbondata.format'
+  | tblproperties('local_dictionary_include'='name,name')
+""".stripMargin)
+}
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_003") {
+sql("drop table if exists local1")
+val exception = intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 'org.apache.carbondata.format'
+  | tblproperties('local_dictionary_include'='')
+""".
+  stripMargin)
+}
+assert(exception.getMessage
+  .contains(
+"LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column:  does not 
exist in table. Please check " +
+"create table statement"))
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_004") {
+sql("drop table if exists local1")
+val exception1 = intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 'org.apache.carbondata.format'
+  | tblproperties('local_dictionary_include'='abc')
+""".
+  stripMargin)
+}
+assert(exception1.getMessage
+  .contains(
+"LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does 
not exist in table. Please check " +
+"create table " +
+"statement"))
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_005") {
+sql("drop table if exists local1")
+val exception = intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 'org.apache.carbondata.format'
+  | tblproperties('local_dictionary_include'='id')
+""".
+  stripMargin)
+}
+assert(exception.getMessage
+ 

[06/50] [abbrv] carbondata git commit: [CARBONDATA-2566] Optimize CarbonReaderExample

[jackylk]

[CARBONDATA-2566] Optimize CarbonReaderExample

Optimize CarbonReaderExample
1.Add different data type, including date and timestamp
2. update the doc
3.invoke the
Schema schema = CarbonSchemaReader
.readSchemaInSchemaFile(dataFiles[0].getAbsolutePath())
.asOriginOrder();

This closes #2356


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/56bf4e42
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/56bf4e42
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/56bf4e42

Branch: refs/heads/carbonstore
Commit: 56bf4e420747ddeb800fc7f004a6ec0d9f5e7d3f
Parents: 9469e6b
Author: xubo245 
Authored: Thu May 31 15:52:57 2018 +0800
Committer: kumarvishal09 
Committed: Fri Jun 1 16:33:28 2018 +0530

--
 docs/sdk-guide.md   | 15 ++--
 .../examples/sdk/CarbonReaderExample.java   | 92 +---
 2 files changed, 89 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/56bf4e42/docs/sdk-guide.md
--
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index ec70919..2371b33 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -415,17 +415,22 @@ External client can make use of this reader to read 
CarbonData files without Car
 String path = "./testWriteFiles";
 CarbonReader reader = CarbonReader
 .builder(path, "_temp")
-.projection(new String[]{"name", "age"})
+.projection(new String[]{"stringField", "shortField", "intField", 
"longField", 
+"doubleField", "boolField", "dateField", "timeField", 
"decimalField"})
 .build();
 
 // 2. Read data
+long day = 24L * 3600 * 1000;
 int i = 0;
 while (reader.hasNext()) {
-  Object[] row = (Object[]) reader.readNextRow();
-  System.out.println(row[0] + "\t" + row[1]);
-  i++;
+Object[] row = (Object[]) reader.readNextRow();
+
System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+i, row[0], row[1], row[2], row[3], row[4], row[5],
+new Date((day * ((int) row[6]))), new Timestamp((long) row[7] / 
1000), row[8]
+));
+i++;
 }
-
+
 // 3. Close this reader
 reader.close();
 ```

http://git-wip-us.apache.org/repos/asf/carbondata/blob/56bf4e42/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
--
diff --git 
a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
 
b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
index d7886c0..8d3ff0d 100644
--- 
a/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
+++ 
b/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/CarbonReaderExample.java
@@ -18,16 +18,19 @@
 package org.apache.carbondata.examples.sdk;
 
 import java.io.File;
+import java.io.FilenameFilter;
+import java.sql.Date;
+import java.sql.Timestamp;
 
 import org.apache.commons.io.FileUtils;
 
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.sdk.file.CarbonReader;
+import org.apache.carbondata.sdk.file.CarbonSchemaReader;
 import org.apache.carbondata.sdk.file.CarbonWriter;
 import org.apache.carbondata.sdk.file.Field;
 import org.apache.carbondata.sdk.file.Schema;
 
-
 /**
  * Example fo CarbonReader with close method
  * After readNextRow of CarbonReader, User should close the reader,
@@ -39,36 +42,99 @@ public class CarbonReaderExample {
 try {
 FileUtils.deleteDirectory(new File(path));
 
-Field[] fields = new Field[2];
-fields[0] = new Field("name", DataTypes.STRING);
-fields[1] = new Field("age", DataTypes.INT);
+Field[] fields = new Field[9];
+fields[0] = new Field("stringField", DataTypes.STRING);
+fields[1] = new Field("shortField", DataTypes.SHORT);
+fields[2] = new Field("intField", DataTypes.INT);
+fields[3] = new Field("longField", DataTypes.LONG);
+fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+fields[6] = new Field("dateField", DataTypes.DATE);
+fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+fields[8] = new Field("decimalField", 
DataTypes.createDecimalType(8, 2));
 
 CarbonWriter writer = CarbonWriter.builder()
-.outputPath(path)
-.persistSchemaFile(true)
-.buildWriterForCSVInput(new Schema(fields));
+

[01/50] [abbrv] carbondata git commit: [CARBONDATA-2521] Support create carbonReader without tableName

[jackylk]

Repository: carbondata
Updated Branches:
  refs/heads/carbonstore 638ed1fa7 -> b3f782062


[CARBONDATA-2521] Support create carbonReader without tableName

Add new method for creating carbonReader without tableName

1.add new interface: public static CarbonReaderBuilder builder(String tablePath)
2.Default value of table name is UnknownTable + time

This closes #2336


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5b2b9130
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5b2b9130
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5b2b9130

Branch: refs/heads/carbonstore
Commit: 5b2b9130411da7737a76b8901f61c59639113e5d
Parents: b338459
Author: xubo245 
Authored: Wed May 23 21:08:23 2018 +0800
Committer: Jacky Li 
Committed: Thu May 31 14:19:47 2018 +0800

--
 docs/sdk-guide.md   | 21 +-
 .../carbondata/sdk/file/CarbonReader.java   | 19 +
 .../carbondata/sdk/file/CarbonReaderTest.java   | 76 
 3 files changed, 113 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/docs/sdk-guide.md
--
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 1d225a9..360516a 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -429,14 +429,29 @@ Find example code at 
[CarbonReaderExample](https://github.com/apache/carbondata/
 
 ### Class org.apache.carbondata.sdk.file.CarbonReader
 ```
- /**
-  * Return a new CarbonReaderBuilder instance
-  */
+   /**
+* Return a new {@link CarbonReaderBuilder} instance
+*
+* @param tablePath table store path
+* @param tableName table name
+* @return CarbonReaderBuilder object
+*/
   public static CarbonReaderBuilder builder(String tablePath, String 
tableName);
 ```
 
 ```
   /**
+   * Return a new CarbonReaderBuilder instance
+   * Default value of table name is table + tablePath + time
+   *
+   * @param tablePath table path
+   * @return CarbonReaderBuilder object
+   */
+  public static CarbonReaderBuilder builder(String tablePath);
+```
+
+```
+  /**
* Return true if has next row
*/
   public boolean hasNext();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 60ead05..81db7b2 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -18,6 +18,8 @@
 package org.apache.carbondata.sdk.file;
 
 import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.List;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
@@ -83,12 +85,29 @@ public class CarbonReader {
 
   /**
* Return a new {@link CarbonReaderBuilder} instance
+   *
+   * @param tablePath table store path
+   * @param tableName table name
+   * @return CarbonReaderBuilder object
*/
   public static CarbonReaderBuilder builder(String tablePath, String 
tableName) {
 return new CarbonReaderBuilder(tablePath, tableName);
   }
 
   /**
+   * Return a new {@link CarbonReaderBuilder} instance
+   * Default value of table name is table + tablePath + time
+   *
+   * @param tablePath table path
+   * @return CarbonReaderBuilder object
+   */
+  public static CarbonReaderBuilder builder(String tablePath) {
+String time = new SimpleDateFormat("MMddHHmmssSSS").format(new Date());
+String tableName = "UnknownTable" + time;
+return builder(tablePath, tableName);
+  }
+
+  /**
* Close reader
*
* @throws IOException

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b2b9130/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
--
diff --git 
a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java 
b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index deb6d06..95c25f8 100644
--- 
a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ 
b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -225,6 +225,82 @@ public class CarbonReaderTest extends TestCase {
   }
 
   @Test
+  public void testWriteAndReadFilesWithoutTableName() throws IOException, 
InterruptedException {
+String path = "./testWriteFiles";
+FileUtils.deleteDirectory(new File(path));
+
+Field[] 

[19/50] [abbrv] carbondata git commit: [HOTFIX][CARBONDATA-2591] Fix SDK CarbonReader filter issue

[jackylk]

[HOTFIX][CARBONDATA-2591] Fix SDK CarbonReader filter issue

There are some issue in SDK CarbonReader filter function, please check the lira.

This closes #2363


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/290ef5a3
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/290ef5a3
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/290ef5a3

Branch: refs/heads/carbonstore
Commit: 290ef5a3a90081b3c95ea0dc418f643ea5ad694f
Parents: 0ef7e55
Author: xubo245 
Authored: Thu Jun 7 22:00:31 2018 +0800
Committer: ravipesala 
Committed: Tue Jun 12 10:54:04 2018 +0530

--
 .../core/metadata/schema/table/CarbonTable.java |  35 +++
 .../apache/carbondata/core/util/CarbonUtil.java |   1 +
 .../sdk/file/CarbonReaderBuilder.java   |   6 +-
 .../carbondata/sdk/file/CarbonReaderTest.java   | 251 ++-
 .../apache/carbondata/sdk/file/TestUtil.java|  14 ++
 5 files changed, 304 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 6949643..20bc7a1 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -17,6 +17,8 @@
 
 package org.apache.carbondata.core.metadata.schema.table;
 
+import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -57,6 +59,8 @@ import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 
+import static 
org.apache.carbondata.core.util.CarbonUtil.thriftColumnSchemaToWrapperColumnSchema;
+
 /**
  * Mapping class for Carbon actual table
  */
@@ -218,6 +222,37 @@ public class CarbonTable implements Serializable {
 }
   }
 
+  public static CarbonTable buildTable(
+  String tablePath,
+  String tableName) throws IOException {
+TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, 
"null", "null");
+File[] dataFiles = new File(tablePath).listFiles(new FilenameFilter() {
+  @Override
+  public boolean accept(File dir, String name) {
+if (name == null) {
+  return false;
+}
+return name.endsWith("carbonindex");
+  }
+});
+if (dataFiles == null || dataFiles.length < 1) {
+  throw new RuntimeException("Carbon index file not exists.");
+}
+org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil
+.inferSchemaFromIndexFile(dataFiles[0].toString(), tableName);
+List columnSchemaList = new ArrayList();
+for (org.apache.carbondata.format.ColumnSchema thriftColumnSchema : 
tableInfo
+.getFact_table().getTable_columns()) {
+  ColumnSchema columnSchema = 
thriftColumnSchemaToWrapperColumnSchema(thriftColumnSchema);
+  if (columnSchema.getColumnReferenceId() == null) {
+columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId());
+  }
+  columnSchemaList.add(columnSchema);
+}
+tableInfoInfer.getFactTable().setListOfColumns(columnSchemaList);
+return CarbonTable.buildFromTableInfo(tableInfoInfer);
+  }
+
   public static CarbonTable buildDummyTable(String tablePath) throws 
IOException {
 TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, 
"null", "null");
 return CarbonTable.buildFromTableInfo(tableInfoInfer);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/290ef5a3/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index e1e5e16..2aa4a05 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2209,6 +2209,7 @@ public final class CarbonUtil {
   org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
 ColumnSchema wrapperColumnSchema = new ColumnSchema();
 wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
+
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
 

[29/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat folder for managed carbon table

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index 99b536c..b4937e6 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -17,12 +17,15 @@
 
 package org.apache.carbondata.spark.testsuite.datacompaction
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
-import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, 
CarbonFileFilter}
+import org.apache.carbondata.core.datamap.Segment
 import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import 
org.apache.carbondata.core.indexstore.blockletindex.SegmentIndexFileStore
+import org.apache.carbondata.core.metadata.{CarbonMetadata, SegmentFileStore}
 import org.apache.carbondata.core.util.path.CarbonTablePath
 import org.apache.carbondata.core.writer.CarbonIndexFileMergeWriter
 
@@ -61,7 +64,7 @@ class CarbonIndexFileMergeTestCase
   """.stripMargin)
 sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE indexmerge 
OPTIONS('header'='false', " +
 s"'GLOBAL_SORT_PARTITIONS'='100')")
-val table = 
CarbonMetadata.getInstance().getCarbonTable("default","indexmerge")
+val table = CarbonMetadata.getInstance().getCarbonTable("default", 
"indexmerge")
 new CarbonIndexFileMergeWriter(table)
   .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
 assert(getIndexFileCount("default_indexmerge", "0") == 0)
@@ -84,7 +87,7 @@ class CarbonIndexFileMergeTestCase
 val rows = sql("""Select count(*) from nonindexmerge""").collect()
 assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
 assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
-val table = 
CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+val table = CarbonMetadata.getInstance().getCarbonTable("default", 
"nonindexmerge")
 new CarbonIndexFileMergeWriter(table)
   .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
 new CarbonIndexFileMergeWriter(table)
@@ -109,7 +112,7 @@ class CarbonIndexFileMergeTestCase
 val rows = sql("""Select count(*) from nonindexmerge""").collect()
 assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
 assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
-val table = 
CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+val table = CarbonMetadata.getInstance().getCarbonTable("default", 
"nonindexmerge")
 new CarbonIndexFileMergeWriter(table)
   .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
 new CarbonIndexFileMergeWriter(table)
@@ -138,7 +141,7 @@ class CarbonIndexFileMergeTestCase
 assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
 assert(getIndexFileCount("default_nonindexmerge", "1") == 100)
 sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
-val table = 
CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+val table = CarbonMetadata.getInstance().getCarbonTable("default", 
"nonindexmerge")
 new CarbonIndexFileMergeWriter(table)
   .mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
 assert(getIndexFileCount("default_nonindexmerge", "0.1") == 0)
@@ -167,7 +170,7 @@ class CarbonIndexFileMergeTestCase
 assert(getIndexFileCount("default_nonindexmerge", "2") == 100)
 assert(getIndexFileCount("default_nonindexmerge", "3") == 100)
 sql("ALTER TABLE nonindexmerge COMPACT 'minor'").collect()
-val table = 
CarbonMetadata.getInstance().getCarbonTable("default","nonindexmerge")
+val table = CarbonMetadata.getInstance().getCarbonTable("default", 
"nonindexmerge")
 new CarbonIndexFileMergeWriter(table)
   .mergeCarbonIndexFilesOfSegment("0.1", table.getTablePath, false)
 assert(getIndexFileCount("default_nonindexmerge", "0") == 100)
@@ -190,18 +193,32 @@ class CarbonIndexFileMergeTestCase
 sql("select * from mitable").show()
   }
 
-  private def getIndexFileCount(tableName: String, segment: String): Int = {
-val table = CarbonMetadata.getInstance().getCarbonTable(tableName)
-val path = CarbonTablePath
-  

[23/50] [abbrv] carbondata git commit: [HOTFIX] fix java style errors

[jackylk]

[HOTFIX] fix java style errors

This closes #2371


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ff036459
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ff036459
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ff036459

Branch: refs/heads/carbonstore
Commit: ff0364599c5bc1d403b4294237e6c88a254beaf4
Parents: 685087e
Author: Zhang Zhichao <441586...@qq.com>
Authored: Wed Jun 13 12:13:41 2018 +0800
Committer: Jacky Li 
Committed: Wed Jun 13 15:54:54 2018 +0800

--
 .../apache/spark/sql/catalyst/CarbonDDLSqlParser.scala|  2 +-
 .../command/management/CarbonLoadDataCommand.scala|  2 ++
 .../processing/loading/model/CarbonLoadModel.java |  2 +-
 .../processing/loading/model/CarbonLoadModelBuilder.java  |  3 ++-
 .../carbondata/processing/loading/model/LoadOption.java   |  5 +++--
 .../carbondata/processing/util/CarbonLoaderUtil.java  | 10 +-
 6 files changed, 14 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 1f04fa4..61a5b42 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ 
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -885,7 +885,7 @@ abstract class CarbonDDLSqlParser extends 
AbstractCarbonSparkSQLParser {
   "ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", 
"BAD_RECORD_PATH",
   "BATCH_SORT_SIZE_INMB", "GLOBAL_SORT_PARTITIONS", "SINGLE_PASS",
   "IS_EMPTY_DATA_BAD_RECORD", "HEADER", "TIMESTAMPFORMAT", 
"SKIP_EMPTY_LINE",
-  "SORT_COLUMN_BOUNDS","LOAD_MIN_SIZE_INMB"
+  "SORT_COLUMN_BOUNDS", "LOAD_MIN_SIZE_INMB"
 )
 var isSupported = true
 val invalidOptions = StringBuilder.newBuilder

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
index 4703b23..69db3ea 100644
--- 
a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
+++ 
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonLoadDataCommand.scala
@@ -24,6 +24,7 @@ import java.util.UUID
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
+
 import org.apache.commons.lang3.StringUtils
 import org.apache.hadoop.conf.Configuration
 import org.apache.spark.rdd.RDD
@@ -44,6 +45,7 @@ import org.apache.spark.sql.types._
 import org.apache.spark.storage.StorageLevel
 import org.apache.spark.unsafe.types.UTF8String
 import org.apache.spark.util.{CarbonReflectionUtils, CausedBy, FileUtils}
+
 import 
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
 import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
 import org.apache.carbondata.core.constants.{CarbonCommonConstants, 
CarbonLoadOptionConstants}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
 
b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
index f267fa7..90c297e 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModel.java
@@ -210,7 +210,7 @@ public class CarbonLoadModel implements Serializable {
* Flder path to where data should be written for this load.
*/
   private String dataWritePath;
-  
+
   /**
* sort columns bounds
*/

http://git-wip-us.apache.org/repos/asf/carbondata/blob/ff036459/processing/src/main/java/org/apache/carbondata/processing/loading/model/CarbonLoadModelBuilder.java

[12/50] [abbrv] carbondata git commit: [CARBONDATA-2575] Add document to explain DataMap Management

[jackylk]

[CARBONDATA-2575] Add document to explain DataMap Management

Add document to explain DataMap Management

This closes #2360


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d401e060
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d401e060
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d401e060

Branch: refs/heads/carbonstore
Commit: d401e060adcc531d66468dc61f4d468768cfea3f
Parents: 5f68a79
Author: Jacky Li 
Authored: Mon Jun 4 21:18:31 2018 +0800
Committer: Jacky Li 
Committed: Tue Jun 5 19:57:06 2018 +0800

--
 docs/datamap/datamap-management.md | 111 
 1 file changed, 111 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d401e060/docs/datamap/datamap-management.md
--
diff --git a/docs/datamap/datamap-management.md 
b/docs/datamap/datamap-management.md
new file mode 100644
index 000..01bb69f
--- /dev/null
+++ b/docs/datamap/datamap-management.md
@@ -0,0 +1,111 @@
+# CarbonData DataMap Management
+
+## Overview
+
+DataMap can be created using following DDL
+
+```
+  CREATE DATAMAP [IF NOT EXISTS] datamap_name
+  [ON TABLE main_table]
+  USING "datamap_provider"
+  [WITH DEFERRED REBUILD]
+  DMPROPERTIES ('key'='value', ...)
+  AS
+SELECT statement
+```
+
+Currently, there are 5 DataMap implementation in CarbonData.
+
+| DataMap Provider | Description  | DMPROPERTIES   
  | Management   |
+|  |  | 
 |  |
+| preaggregate | single table pre-aggregate table | No DMPROPERTY 
is required| Automatic|
+| timeseries   | time dimension rollup table. | event_time, 
xx_granularity, please refer to [Timeseries 
DataMap](https://github.com/apache/carbondata/blob/master/docs/datamap/timeseries-datamap-guide.md)
 | Automatic|
+| mv   | multi-table pre-aggregate table, | No DMPROPERTY 
is required| Manual   |
+| lucene   | lucene indexing for text column  | index_columns 
to specifying the index columns | Manual/Automatic |
+| bloom| bloom filter for high cardinality column, geospatial 
column | index_columns to specifying the index columns | Manual/Automatic |
+
+## DataMap Management
+
+There are two kinds of management semantic for DataMap.
+
+1. Autmatic Refresh: Create datamap without `WITH DEFERED REBUILD` in the 
statement
+2. Manual Refresh: Create datamap with `WITH DEFERED REBUILD` in the statement
+
+### Automatic Refresh
+
+When user creates a datamap on the main table without using `WITH DEFERED 
REBUILD` syntax, the datamap will be managed by system automatically.
+For every data load to the main table, system will immediately triger a load 
to the datamap automatically. These two data loading (to main table and 
datamap) is executed in a transactional manner, meaning that it will be either 
both success or neither success. 
+
+The data loading to datamap is incremental based on Segment concept, avoiding 
a expesive total rebuild.
+
+If user perform following command on the main table, system will return 
failure. (reject the operation)
+
+1. Data management command: `UPDATE/DELETE/DELETE SEGMENT`.
+2. Schema management command: `ALTER TABLE DROP COLUMN`, `ALTER TABLE CHANGE 
DATATYPE`,
+   `ALTER TABLE RENAME`. Note that adding a new column is supported, and for 
dropping columns and
+   change datatype command, CarbonData will check whether it will impact the 
pre-aggregate table, if
+not, the operation is allowed, otherwise operation will be rejected by 
throwing exception.
+3. Partition management command: `ALTER TABLE ADD/DROP PARTITION
+
+If user do want to perform above operations on the main table, user can first 
drop the datamap, perform the operation, and re-create the datamap again.
+
+If user drop the main table, the datamap will be dropped immediately too.
+
+### Manual Refresh
+
+When user creates a datamap specifying maunal refresh semantic, the datamap is 
created with status *disabled* and query will NOT use this datamap until user 
can issue REBUILD DATAMAP command to build the datamap. For every REBUILD 
DATAMAP command, system will trigger a full rebuild of the datamap. After 
rebuild is done, system will change datamap status to *enabled*, so that it can 
be used in query rewrite.
+
+For every new data loading, data update, delete, the related datamap will be 
made *disabled*.
+
+If the main table is dropped by user, the related datamap will be dropped 
immediately.
+
+*Note: If you are creating a datamap on external 

[08/50] [abbrv] carbondata git commit: [CARONDATA-2559]task id set for each carbonReader in threadlocal

[jackylk]

[CARONDATA-2559]task id set for each carbonReader in threadlocal

1. Task Id set for CarbonReader because for each CarbonReader object it should 
be separate Thread Local variable .
2. If sort-Column is not given to CarbonWriter Describe formatted showing 
default sort_cols is fixed
3. Issue : CarbonReader was being closed after one iteration. So when reader 
iterates over the next batch it gives NullPointerException because it is 
already closed.
Solution : reader is closed if any exception encountered. Else It will be 
closed explicitly by user.
4. CarbonProperties API for SDK moved to common API List because Property 
setting is common for both(carbonReader and CarbonWriter) .


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/92d9b925
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/92d9b925
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/92d9b925

Branch: refs/heads/carbonstore
Commit: 92d9b9256373763f05736e29d93b7e835e0da3dd
Parents: 4bb7e27
Author: rahulforallp 
Authored: Tue May 29 10:23:46 2018 +0530
Committer: kumarvishal09 
Committed: Mon Jun 4 17:49:05 2018 +0530

--
 docs/sdk-guide.md   | 95 ++--
 .../TestNonTransactionalCarbonTable.scala   | 13 +--
 .../carbondata/sdk/file/CarbonReader.java   |  5 ++
 .../sdk/file/CarbonReaderBuilder.java   | 10 ++-
 .../sdk/file/CarbonWriterBuilder.java   |  4 +-
 .../sdk/file/CSVCarbonWriterTest.java   |  2 +-
 .../carbondata/sdk/file/CarbonReaderTest.java   | 41 -
 .../apache/carbondata/sdk/file/TestUtil.java|  4 +-
 .../carbondata/store/LocalCarbonStoreTest.java  |  2 +-
 9 files changed, 96 insertions(+), 80 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/92d9b925/docs/sdk-guide.md
--
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 2371b33..5dbb5ac 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -350,52 +350,6 @@ public Schema(Field[] fields);
 public static Schema parseJson(String json);
 ```
 
-### Class org.apache.carbondata.core.util.CarbonProperties
-
-```
-/**
-* This method will be responsible to get the instance of CarbonProperties class
-*
-* @return carbon properties instance
-*/
-public static CarbonProperties getInstance();
-```
-
-```
-/**
-* This method will be used to add a new property
-*
-* @param key is a property name to set for carbon.
-* @param value is valid parameter corresponding to property.
-* @return CarbonProperties object
-*/
-public CarbonProperties addProperty(String key, String value);
-```
-
-```
-/**
-* This method will be used to get the property value. If property is not
-* present, then it will return the default value.
-*
-* @param key is a property name to get user specified value.
-* @return properties value for corresponding key. If not set, then returns 
null.
-*/
-public String getProperty(String key);
-```
-
-```
-/**
-* This method will be used to get the property value. If property is not
-* present, then it will return the default value.
-*
-* @param key is a property name to get user specified value..
-* @param defaultValue used to be returned by function if corrosponding key not 
set.
-* @return properties value for corresponding key. If not set, then returns 
specified defaultValue.
-*/
-public String getProperty(String key, String defaultValue);
-```
-Reference : [list of carbon 
properties](http://carbondata.apache.org/configuration-parameters.html)
-
 ### Class org.apache.carbondata.sdk.file.AvroCarbonWriter
 ```
 /**
@@ -705,3 +659,52 @@ Find example code at 
[CarbonReaderExample](https://github.com/apache/carbondata/
 ```
 
 Find S3 example code at 
[SDKS3Example](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/java/org/apache/carbondata/examples/sdk/SDKS3Example.java)
 in the CarbonData repo.
+
+
+# Common API List for CarbonReader and CarbonWriter
+
+### Class org.apache.carbondata.core.util.CarbonProperties
+
+```
+/**
+* This method will be responsible to get the instance of CarbonProperties class
+*
+* @return carbon properties instance
+*/
+public static CarbonProperties getInstance();
+```
+
+```
+/**
+* This method will be used to add a new property
+*
+* @param key is a property name to set for carbon.
+* @param value is valid parameter corresponding to property.
+* @return CarbonProperties object
+*/
+public CarbonProperties addProperty(String key, String value);
+```
+
+```
+/**
+* This method will be used to get the property value. If property is not
+* present, then it will return the default value.
+*
+* @param key is a property name to get user specified value.
+* @return properties value for corresponding key. If not set, 

[20/50] [abbrv] carbondata git commit: [CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure case for creation of multi reader for non-transactional table

[jackylk]

[CARBONDATA-2578] fixed memory leak inside CarbonReader and handled failure 
case for creation of
 multi reader for non-transactional table

Issue :
CarbonIterator inside CarbonRecordReader was keeping reference of RowBatch and 
it is not being
closed inside CarbonRecordReader. sort_column with measure was considering all 
the dimension
column along with given column. if creation of one CarbonReader for non 
transactional table
is failed then we are not able to create another CarbonReader.

Solution :
close() called inside hasNext() to clear the previous iterator before iterating 
over the next CarbonReader..
if sortcolumn is not empty and sortcolumnsList contains the fields check is 
added to finally in sortcolumn props.
Clear the datamap in catch block, if creation of CarbonReader is failed.

This closes #2362


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19312ab5
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19312ab5
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19312ab5

Branch: refs/heads/carbonstore
Commit: 19312ab5b35654e89f04ec79881d928c859f8a07
Parents: 290ef5a
Author: rahul 
Authored: Tue Jun 5 13:09:36 2018 +0530
Committer: kumarvishal09 
Committed: Tue Jun 12 17:01:53 2018 +0530

--
 .../createTable/TestNonTransactionalCarbonTable.scala | 7 ---
 .../java/org/apache/carbondata/sdk/file/CarbonReader.java | 2 ++
 .../org/apache/carbondata/sdk/file/CarbonWriterBuilder.java   | 5 -
 3 files changed, 6 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index b275bb8..805fc71 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -368,8 +368,9 @@ class TestNonTransactionalCarbonTable extends QueryTest 
with BeforeAndAfterAll {
  |'carbondata' LOCATION
  |'$writerPath' """.stripMargin)
 
-checkExistence(sql("describe formatted sdkOutputTable"), true, "age")
-
+checkExistence(sql("describe formatted sdkOutputTable"), true, 
"SORT_COLUMNSage")
+checkExistence(sql("describe formatted sdkOutputTable"), false, 
"SORT_COLUMNSname,age")
+checkExistence(sql("describe formatted sdkOutputTable"), false, 
"SORT_COLUMNSage,name")
 buildTestDataSingleFile()
 assert(new File(writerPath).exists())
 sql("DROP TABLE IF EXISTS sdkOutputTable")
@@ -402,7 +403,7 @@ class TestNonTransactionalCarbonTable extends QueryTest 
with BeforeAndAfterAll {
 intercept[RuntimeException] {
   buildTestDataWithSortColumns(List(""))
 }
-
+
 assert(!(new File(writerPath).exists()))
 cleanTestData()
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 9af710f..be809e6 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -74,6 +74,8 @@ public class CarbonReader {
 return false;
   } else {
 index++;
+// current reader is closed
+currentReader.close();
 currentReader = readers.get(index);
 return currentReader.nextKeyValue();
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/19312ab5/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index bd64568..0f670fe 100644
--- 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ 

[31/50] [abbrv] carbondata git commit: [CARBONDATA-2614] Fix the error when using FG in search mode and the prune result is none

[jackylk]

[CARBONDATA-2614] Fix the error when using FG in search mode and the prune 
result is none

the prune result is none, and can not set datamapWritePath, which will not 
generate bitSegGroup in 
org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl#applyFilter(org.apache.carbondata.core.scan.processor.RawBlockletColumnChunks,
 boolean), it mean the bitSetGroup is null. It will throw Fix the error when 
using FG in search mode and the prune result is none

This closes #2378


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5593d164
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5593d164
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5593d164

Branch: refs/heads/carbonstore
Commit: 5593d1646b075847a84ac2891fd25e78201451fe
Parents: 60dfdd3
Author: xubo245 
Authored: Fri Jun 15 18:50:26 2018 +0800
Committer: Jacky Li 
Committed: Mon Jun 18 20:29:59 2018 +0800

--
 .../apache/carbondata/core/scan/model/QueryModel.java  | 13 +
 .../apache/carbondata/hadoop/CarbonRecordReader.java   |  2 +-
 .../testsuite/detailquery/SearchModeTestCase.scala |  1 +
 .../carbondata/store/worker/SearchRequestHandler.java  |  1 +
 4 files changed, 16 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java 
b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index de11d11..55dafb9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -109,6 +109,11 @@ public class QueryModel {
*/
   private boolean requiredRowId;
 
+  /**
+   * whether it is FG with search mode
+   */
+  private boolean isFG;
+
   private QueryModel(CarbonTable carbonTable) {
 tableBlockInfos = new ArrayList();
 invalidSegmentIds = new ArrayList<>();
@@ -370,6 +375,14 @@ public class QueryModel {
 this.requiredRowId = requiredRowId;
   }
 
+  public boolean isFG() {
+return isFG;
+  }
+
+  public void setFG(boolean FG) {
+isFG = FG;
+  }
+
   @Override
   public String toString() {
 return String.format("scan on table %s.%s, %d projection columns with 
filter (%s)",

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
--
diff --git 
a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java 
b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 4911e41..6b56382 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -80,7 +80,7 @@ public class CarbonRecordReader extends 
AbstractRecordReader {
 }
 // It should use the exists tableBlockInfos if tableBlockInfos of 
queryModel is not empty
 // otherwise the prune is no use before this method
-if (queryModel.getTableBlockInfos().isEmpty()) {
+if (!queryModel.isFG()) {
   List tableBlockInfoList = 
CarbonInputSplit.createBlocks(splitList);
   queryModel.setTableBlockInfos(tableBlockInfoList);
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5593d164/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
index 3e6adaf..001f6c0 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
@@ -111,6 +111,7 @@ class SearchModeTestCase extends QueryTest with 
BeforeAndAfterAll {
   }
 
   test("test lucene datamap with search mode") {
+sql("set carbon.search.enabled = true")
 sql("DROP DATAMAP IF EXISTS dm ON TABLE main")
 sql("CREATE DATAMAP dm ON TABLE main USING 'lucene' 
DMProperties('INDEX_COLUMNS'='id') ")
 checkAnswer(sql("SELECT * FROM main WHERE TEXT_MATCH('id:10')"),

[21/50] [abbrv] carbondata git commit: [CARBONDATA-2603] Fix: error handling during reader build failure

[jackylk]

[CARBONDATA-2603] Fix: error handling during reader build failure

problem :
When the CarbonReaderBuilder.build() is failed due to some problems like invalid
projection that leads to query model creation failure. Blocklet datamap is not 
cleared for that table.So,
the next reader instance uses old blocklet datamap . That creates error.

Solution: Clear the blocklet datamap if the reader build is failed.

This closes #2368


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9b88a065
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9b88a065
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9b88a065

Branch: refs/heads/carbonstore
Commit: 9b88a065276c97e17ae6fa1c6f7a27b0d6d12a7c
Parents: 19312ab
Author: ajantha-bhat 
Authored: Mon Jun 11 19:17:33 2018 +0530
Committer: kumarvishal09 
Committed: Tue Jun 12 20:12:09 2018 +0530

--
 .../sdk/file/CarbonReaderBuilder.java   | 39 -
 .../carbondata/sdk/file/CarbonReaderTest.java   | 61 
 2 files changed, 84 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b88a065/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
--
diff --git 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index 83cb34e..ebee41a 100644
--- 
a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ 
b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -24,6 +24,7 @@ import java.util.Objects;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
 import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datamap.DataMapStoreManager;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.scan.expression.Expression;
@@ -200,23 +201,29 @@ public class CarbonReaderBuilder {
   format.setColumnProjection(job.getConfiguration(), projectionColumns);
 }
 
-final List splits =
-format.getSplits(new JobContextImpl(job.getConfiguration(), new 
JobID()));
-
-List> readers = new ArrayList<>(splits.size());
-for (InputSplit split : splits) {
-  TaskAttemptContextImpl attempt =
-  new TaskAttemptContextImpl(job.getConfiguration(), new 
TaskAttemptID());
-  RecordReader reader = format.createRecordReader(split, attempt);
-  try {
-reader.initialize(split, attempt);
-readers.add(reader);
-  } catch (Exception e) {
-reader.close();
-throw e;
+try {
+  final List splits =
+  format.getSplits(new JobContextImpl(job.getConfiguration(), new 
JobID()));
+
+  List> readers = new ArrayList<>(splits.size());
+  for (InputSplit split : splits) {
+TaskAttemptContextImpl attempt =
+new TaskAttemptContextImpl(job.getConfiguration(), new 
TaskAttemptID());
+RecordReader reader = format.createRecordReader(split, attempt);
+try {
+  reader.initialize(split, attempt);
+  readers.add(reader);
+} catch (Exception e) {
+  reader.close();
+  throw e;
+}
   }
+  return new CarbonReader<>(readers);
+} catch (Exception ex) {
+  // Clear the datamap cache as it can get added in getSplits() method
+  DataMapStoreManager.getInstance()
+  .clearDataMaps(table.getAbsoluteTableIdentifier());
+  throw ex;
 }
-
-return new CarbonReader<>(readers);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9b88a065/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
--
diff --git 
a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java 
b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index fb2e2bc..2bc4b1f 100644
--- 
a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ 
b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -353,6 +353,67 @@ public class CarbonReaderTest extends TestCase {
   }
 
   @Test
+  public void testWriteAndReadFilesWithReaderBuildFail() throws IOException, 
InterruptedException {
+String path1 = "./testWriteFiles";
+String path2 = "./testWriteFiles2";
+FileUtils.deleteDirectory(new File(path1));
+FileUtils.deleteDirectory(new File(path2));
+
+Field[] fields = new Field[] { new Field("c1", "string"),
+ new 

[04/50] [abbrv] carbondata git commit: [CARBONDATA-2529] Fixed S3 Issue for Hadoop 2.8.3

[jackylk]

[CARBONDATA-2529] Fixed S3 Issue for Hadoop 2.8.3

This issue fixes the issue while loading the data with S3 as backend

This closes #2340


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4d22ddc9
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4d22ddc9
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4d22ddc9

Branch: refs/heads/carbonstore
Commit: 4d22ddc9d932891af7d3f6557a423d65969f1fd3
Parents: 5ad7009
Author: Bhavya 
Authored: Thu May 24 21:17:58 2018 +0530
Committer: chenliang613 
Committed: Fri Jun 1 14:37:36 2018 +0800

--
 .../carbondata/core/datastore/filesystem/HDFSCarbonFile.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4d22ddc9/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
index 4663ac5..fc5420d 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -120,7 +120,8 @@ public class HDFSCarbonFile extends AbstractDFSCarbonFile {
 ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new 
Path(changetoName),
 org.apache.hadoop.fs.Options.Rename.OVERWRITE);
 return true;
-  } else if (fileStatus.getPath().toString().startsWith("s3n")) {
+  } else if (fileStatus.getPath().toString().startsWith("s3n")
+  || fileStatus.getPath().toString().startsWith("s3a")) {
 fs.delete(new Path(changetoName), true);
 return fs.rename(fileStatus.getPath(), new Path(changetoName));
   } else {

[14/50] [abbrv] carbondata git commit: [CARBONDATA-2569] Change the strategy of Search mode throw exception and run sparkSQL

[jackylk]

[CARBONDATA-2569] Change the strategy of Search mode throw exception and run 
sparkSQL

Search mode throw exception but test case pass, please check the jira.

This closes #2357


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83ee2c45
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83ee2c45
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83ee2c45

Branch: refs/heads/carbonstore
Commit: 83ee2c45fc10b220605916abe133b7a250007fdc
Parents: 041603d
Author: xubo245 
Authored: Fri Jun 1 11:50:52 2018 +0800
Committer: ravipesala 
Committed: Mon Jun 11 18:06:27 2018 +0530

--
 .../main/scala/org/apache/spark/sql/CarbonSession.scala | 12 
 1 file changed, 8 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/83ee2c45/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
--
diff --git 
a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala 
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 497f95a..93c0b4a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -37,9 +37,8 @@ import org.apache.spark.sql.profiler.{Profiler, SQLStart}
 import org.apache.spark.util.{CarbonReflectionUtils, Utils}
 
 import org.apache.carbondata.common.annotations.InterfaceAudience
-import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
 import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.scan.expression.LiteralExpression
 import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, 
ThreadLocalSessionInfo}
 import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
 import org.apache.carbondata.store.SparkCarbonStore
@@ -101,8 +100,8 @@ class CarbonSession(@transient val sc: SparkContext,
   } catch {
 case e: Exception =>
   logError(String.format(
-"Exception when executing search mode: %s, fallback to 
SparkSQL", e.getMessage))
-  new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
+"Exception when executing search mode: %s", e.getMessage))
+  throw e;
   }
 } else {
   new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
@@ -171,19 +170,24 @@ class CarbonSession(@transient val sc: SparkContext,
*/
   private def trySearchMode(qe: QueryExecution, sse: SQLStart): DataFrame = {
 val analyzed = qe.analyzed
+val LOG: LogService = 
LogServiceFactory.getLogService(this.getClass.getName)
 analyzed match {
   case _@Project(columns, _@Filter(expr, s: SubqueryAlias))
 if s.child.isInstanceOf[LogicalRelation] &&
s.child.asInstanceOf[LogicalRelation].relation
  .isInstanceOf[CarbonDatasourceHadoopRelation] =>
+LOG.info(s"Search service started and supports filter: ${sse.sqlText}")
 runSearch(analyzed, columns, expr, 
s.child.asInstanceOf[LogicalRelation])
   case gl@GlobalLimit(_, ll@LocalLimit(_, p@Project(columns, 
_@Filter(expr, s: SubqueryAlias
 if s.child.isInstanceOf[LogicalRelation] &&
s.child.asInstanceOf[LogicalRelation].relation
  .isInstanceOf[CarbonDatasourceHadoopRelation] =>
 val logicalRelation = s.child.asInstanceOf[LogicalRelation]
+LOG.info(s"Search service started and supports limit: ${sse.sqlText}")
 runSearch(analyzed, columns, expr, logicalRelation, gl.maxRows, 
ll.maxRows)
   case _ =>
+LOG.info(s"Search service started, but don't support: ${sse.sqlText}," 
+
+  s" and will run it with SparkSQL")
 new Dataset[Row](self, qe, RowEncoder(qe.analyzed.schema))
 }
   }

[17/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate carbonstore mv branch

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
--
diff --git 
a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
 
b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
index 074bf00..e564052 100644
--- 
a/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
+++ 
b/datamap/mv/core/src/test/scala/org/apache/carbondata/mv/rewrite/matching/TestTPCDS_1_4_Batch.scala
@@ -1,20 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
 package org.apache.carbondata.mv.rewrite.matching
 
 object TestTPCDS_1_4_Batch {
@@ -32,73 +15,67 @@ object TestTPCDS_1_4_Batch {
 |FROM item
 |WHERE i_item_sk IN (2, 3, 5, 7, 11, 13, 17, 19)
 """.stripMargin.trim,
-   """
-|SELECT
-|FROM 
-|WHERE 
-""".stripMargin.trim),
+   """ 
+   """.stripMargin.trim),
   // test case 2: test SELECT-SELECT-EXACT_MATCH with SELECT containing 
join (derive from q64)
-  ("case_2",
-   """
-|SELECT cs1.product_name, cs1.store_name, cs1.store_zip, 
cs1.b_street_number,
-|   cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, 
cs1.c_street_name,
-|   cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, 
cs1.s3, cs2.s1,
-|   cs2.s2, cs2.s3, cs2.syear, cs2.cnt
-|FROM cross_sales cs1,cross_sales cs2
-|WHERE cs1.item_sk=cs2.item_sk AND
-| cs1.syear = 1999 AND
-| cs2.syear = 1999 + 1 AND
-| cs2.cnt <= cs1.cnt AND
-| cs1.store_name = cs2.store_name AND
-| cs1.store_zip = cs2.store_zip
-""".stripMargin.trim,
-   """
-|SELECT cs1.product_name, cs1.store_name, cs1.store_zip, 
cs1.b_street_number,
-|   cs1.b_streen_name, cs1.b_city, cs1.b_zip, cs1.c_street_number, 
cs1.c_street_name,
-|   cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, 
cs1.s3, cs2.s1,
-|   cs2.s2, cs2.s3
-|FROM cross_sales cs1,cross_sales cs2
-|WHERE cs1.item_sk=cs2.item_sk AND
-| cs1.syear = 1999 AND
-| cs2.syear = 1999 + 1 AND
-| cs2.cnt <= cs1.cnt AND
-| cs1.store_name = cs2.store_name AND
-| cs1.store_zip = cs2.store_zip
-|ORDER BY cs1.product_name, cs1.store_name, cs2.cnt
-""".stripMargin.trim,
-   """
-|SELECT
-|FROM
-|WHERE
-""".stripMargin.trim),
+  // cross_sales not in Tpcds_1_4_Tables.scala
+//  ("case_2",
+//   """
+//|SELECT cs1.product_name, cs1.store_name, cs1.store_zip, 
cs1.b_street_number,
+//|   cs1.b_streen_name, cs1.b_city, cs1.b_zip, 
cs1.c_street_number, cs1.c_street_name,
+//|   cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, 
cs1.s3, cs2.s1,
+//|   cs2.s2, cs2.s3, cs2.syear, cs2.cnt
+//|FROM cross_sales cs1,cross_sales cs2
+//|WHERE cs1.item_sk=cs2.item_sk AND
+//| cs1.syear = 1999 AND
+//| cs2.syear = 1999 + 1 AND
+//| cs2.cnt <= cs1.cnt AND
+//| cs1.store_name = cs2.store_name AND
+//| cs1.store_zip = cs2.store_zip
+//""".stripMargin.trim,
+//   """
+//|SELECT cs1.product_name, cs1.store_name, cs1.store_zip, 
cs1.b_street_number,
+//|   cs1.b_streen_name, cs1.b_city, cs1.b_zip, 
cs1.c_street_number, cs1.c_street_name,
+//|   cs1.c_city, cs1.c_zip, cs1.syear, cs1.cnt, cs1.s1, cs1.s2, 
cs1.s3, cs2.s1,
+//|   cs2.s2, cs2.s3
+//|FROM cross_sales cs1,cross_sales cs2
+//|WHERE cs1.item_sk=cs2.item_sk AND
+//| cs1.syear = 1999 AND
+//| cs2.syear = 1999 + 1 AND
+//| cs2.cnt <= cs1.cnt AND
+//| cs1.store_name = cs2.store_name AND
+//| cs1.store_zip = cs2.store_zip
+//|ORDER BY cs1.product_name, cs1.store_name, 

[03/50] [abbrv] carbondata git commit: [Documentation] Editorial Review comment fixed

[jackylk]

[Documentation] Editorial Review comment fixed

Editorial Review comment fixed

This closes #2320


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5ad70095
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5ad70095
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5ad70095

Branch: refs/heads/carbonstore
Commit: 5ad7009573b7a95a181221d6a58df05e1fafbeb6
Parents: 6aadfe7
Author: sgururajshetty 
Authored: Thu May 31 17:36:26 2018 +0530
Committer: kunal642 
Committed: Thu May 31 17:40:30 2018 +0530

--
 docs/data-management-on-carbondata.md| 4 ++--
 docs/datamap/timeseries-datamap-guide.md | 8 
 2 files changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ad70095/docs/data-management-on-carbondata.md
--
diff --git a/docs/data-management-on-carbondata.md 
b/docs/data-management-on-carbondata.md
index 51e98ab..706209c 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -35,11 +35,11 @@ This tutorial is going to introduce all commands and data 
operations on CarbonDa
   
   ```
   CREATE TABLE [IF NOT EXISTS] [db_name.]table_name[(col_name data_type , ...)]
-  STORED BY 'carbondata'
+  STORED AS carbondata
   [TBLPROPERTIES (property_name=property_value, ...)]
   [LOCATION 'path']
   ```
-  **NOTE:** CarbonData also supports "STORED AS carbondata". Find example code 
at 
[CarbonSessionExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala)
 in the CarbonData repo.
+  **NOTE:** CarbonData also supports "STORED AS carbondata" and "USING 
carbondata". Find example code at 
[CarbonSessionExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala)
 in the CarbonData repo.
 ### Usage Guidelines
 
   Following are the guidelines for TBLPROPERTIES, CarbonData's additional 
table options can be set via carbon.properties.

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5ad70095/docs/datamap/timeseries-datamap-guide.md
--
diff --git a/docs/datamap/timeseries-datamap-guide.md 
b/docs/datamap/timeseries-datamap-guide.md
index 7847312..bea5286 100644
--- a/docs/datamap/timeseries-datamap-guide.md
+++ b/docs/datamap/timeseries-datamap-guide.md
@@ -1,12 +1,12 @@
 # CarbonData Timeseries DataMap
 
-* [Timeseries 
DataMap](#timeseries-datamap-intoduction-(alpha-feature-in-1.3.0))
+* [Timeseries DataMap Introduction](#timeseries-datamap-intoduction)
 * [Compaction](#compacting-pre-aggregate-tables)
 * [Data Management](#data-management-with-pre-aggregate-tables)
 
-## Timeseries DataMap Intoduction (Alpha feature in 1.3.0)
-Timeseries DataMap a pre-aggregate table implementation based on 
'preaggregate' DataMap. 
-Difference is that Timerseries DataMap has built-in understanding of time 
hierarchy and 
+## Timeseries DataMap Introduction (Alpha feature in 1.3.0)
+Timeseries DataMap a pre-aggregate table implementation based on 
'pre-aggregate' DataMap.
+Difference is that Timeseries DataMap has built-in understanding of time 
hierarchy and
 levels: year, month, day, hour, minute, so that it supports automatic roll-up 
in time dimension 
 for query.
 

[26/50] [abbrv] carbondata git commit: [CARBONDATA-2592][Integration] Getting NoSuchMethod error due to aws sdk multple version jar conflicts

[jackylk]

[CARBONDATA-2592][Integration] Getting NoSuchMethod error due to aws sdk 
multple version jar conflicts

## What changes were proposed in this pull request?
Currently in Carbon Spark2 project multiple dependency for the aws-sdk jar has 
been defined,this will create issue when we build carbon-assembly jars with
latest versions of hadoop/spark project. As part of hadoop-aws project, already 
aws-sdk 1.10.6 version jar will be fetched, since in the carbon-spark2 pom.xml
there is an explicit dependency defined/hardcoded for aws-sdk 1.7.4(old 
version)  this can lead to conflicts while loading the class files. because of 
this problem
when we run any carbon examples passing carbon-assembly jars as the argument 
using spark-submit  none of the testcases will work.
As a solution we can remove this dependency(aws-sdk 1.7.4) as already 
hadoop-aws dependency defined in the pom.xml of carbon-spark2 project
will fetch the latest aws-sdk jars.

## How was this patch tested?
After updating the pom, manually projects has been build and the use-case 
mentioned above  is manually tested.

This closes #2364


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f0c88348
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f0c88348
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f0c88348

Branch: refs/heads/carbonstore
Commit: f0c88348a88f46b1cdaaa1e9bc00a3ab6111e5ec
Parents: f116352
Author: sujith71955 
Authored: Thu Jun 7 12:38:56 2018 +0530
Committer: ravipesala 
Committed: Fri Jun 15 12:30:54 2018 +0530

--
 integration/spark2/pom.xml | 19 ---
 1 file changed, 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c88348/integration/spark2/pom.xml
--
diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml
index 2bce694..9b9e71d 100644
--- a/integration/spark2/pom.xml
+++ b/integration/spark2/pom.xml
@@ -103,25 +103,6 @@
   
 
 
-  com.amazonaws
-  aws-java-sdk
-  1.7.4
-  
-
-  com.fasterxml.jackson.core
-  jackson-core
-
-
-  com.fasterxml.jackson.core
-  jackson-annotations
-
-
-  com.fasterxml.jackson.core
-  jackson-databind
-
-  
-
-
   net.java.dev.jets3t
   jets3t
   0.9.0

[15/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate carbonstore mv branch

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
--
diff --git 
a/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
 
b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
new file mode 100644
index 000..a02cc89
--- /dev/null
+++ 
b/datamap/mv/plan/src/test/scala/org/apache/carbondata/mv/testutil/TestSQLBatch2.scala
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.mv.testutil
+
+object TestSQLBatch2 {
+
+  val testSQLBatch2 = Seq[String](
+  s"""
+ |SELECT f1.A,COUNT(*) AS B 
+ |FROM
+ |  fact f1
+ |  JOIN dim d1 ON (f1.K = d1.K)
+ |WHERE f1.E IS NULL AND (f1.C > d1.E OR d1.E = 3)
+ |GROUP BY f1.A
+  """.stripMargin.trim,
+  s"""
+ |SELECT f1.A,COUNT(*) AS B 
+ |FROM
+ |  fact f1
+ |  JOIN dim d1 ON (f1.K = d1.K)
+ |  JOIN dim1 d2 ON (f1.K = d2.K AND d2.G > 0)
+ |WHERE f1.E IS NULL AND f1.C > d1.E
+ |GROUP BY f1.A
+  """.stripMargin.trim,
+  s"""
+ |SELECT substr(item.i_item_desc,1,30) itemdesc, item.i_item_sk 
item_sk, date_dim.d_date solddate, count(*) cnt
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |  AND store_sales.ss_item_sk = item.i_item_sk
+ |  AND date_dim.d_year in (2000, 2000+1, 2000+2, 2000+3)
+ |GROUP BY substr(item.i_item_desc,1,30), 
item.i_item_sk,date_dim.d_date 
+  """.stripMargin.trim,
+  s"""
+ |SELECT item.i_item_desc, item.i_category, item.i_class, 
item.i_current_price, 
+ |   SUM(store_sales.ss_ext_sales_price) as itemrevenue,
+ |   
SUM(store_sales.ss_ext_sales_price)*100/sum(sum(store_sales.ss_ext_sales_price))
 over (partition by item.i_class) as revenueratio
+ |FROM date_dim, store_sales, item
+ |WHERE date_dim.d_date_sk = store_sales.ss_sold_date_sk
+ |  AND store_sales.ss_item_sk = item.i_item_sk
+ |  AND item.i_category in ('Sport', 'Books', 'Home')
+ |  AND date_dim.d_date between cast('1999-02-22' as date) AND 
(cast('1999-02-22' as date) + interval 30 days)
+ |GROUP BY item.i_item_id, item.i_item_desc, item.i_category, 
item.i_class, item.i_current_price 
+  """.stripMargin.trim,
+  s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, 
date_dim.d_year, date_dim.d_qoy, 
+ |   item.i_category, SUM(store_sales.ss_ext_sales_price) 
ext_sales_price 
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ |  AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ |  AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY channel, store_sales.ss_store_sk, date_dim.d_year, 
date_dim.d_qoy, item.i_category 
+  """.stripMargin.trim,
+  s"""
+ |SELECT 'store' channel, store_sales.ss_store_sk col_name, 
date_dim.d_year, date_dim.d_qoy, 
+ |   item.i_category, SUM(store_sales.ss_ext_sales_price) 
ext_sales_price 
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_store_sk IS NULL
+ |  AND store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ |  AND store_sales.ss_item_sk = item.i_item_sk
+ |GROUP BY store_sales.ss_store_sk, date_dim.d_year, date_dim.d_qoy, 
item.i_category 
+  """.stripMargin.trim,
+  s"""
+ |SELECT item.i_brand_id brand_id, item.i_brand brand, 
SUM(ss_ext_sales_price) ext_price 
+ |FROM date_dim, store_sales, item
+ |WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk
+ |  AND store_sales.ss_item_sk = item.i_item_sk
+ |  AND item.i_manager_id = 28
+ |  AND date_dim.d_year = 1999
+ |  AND date_dim.d_moy = 11
+ |GROUP BY item.i_brand_id, item.i_brand 
+  """.stripMargin.trim,
+  s"""
+ |SELECT item.i_brand_id 

[05/50] [abbrv] carbondata git commit: [CARBONDATA-2355] Support run SQL on carbondata files directly

[jackylk]

[CARBONDATA-2355] Support run SQL on carbondata files directly

Support run SQL on carbondata files directly

This closes #2181


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9469e6bd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9469e6bd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9469e6bd

Branch: refs/heads/carbonstore
Commit: 9469e6bd4da5c75ba836fb550112cec01f666544
Parents: 4d22ddc
Author: xubo245 <601450...@qq.com>
Authored: Wed Apr 18 17:34:12 2018 +0800
Committer: chenliang613 
Committed: Fri Jun 1 18:01:33 2018 +0800

--
 docs/sdk-guide.md   |   7 ++
 .../carbondata/examples/DirectSQLExample.scala  | 100 +++
 .../carbondata/examples/S3UsingSDkExample.scala |   2 +-
 ...FileInputFormatWithExternalCarbonTable.scala |   2 +-
 ...tCreateTableUsingSparkCarbonFileFormat.scala |  30 +-
 .../TestNonTransactionalCarbonTable.scala   |   2 +-
 ...ransactionalCarbonTableWithComplexType.scala |   2 +-
 ...tSparkCarbonFileFormatWithSparkSession.scala |   2 +-
 .../datasources/SparkCarbonFileFormat.scala |  26 -
 9 files changed, 164 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/docs/sdk-guide.md
--
diff --git a/docs/sdk-guide.md b/docs/sdk-guide.md
index 360516a..ec70919 100644
--- a/docs/sdk-guide.md
+++ b/docs/sdk-guide.md
@@ -128,7 +128,14 @@ Each of SQL data types are mapped into data types of SDK. 
Following are the mapp
 | STRING | DataTypes.STRING |
 | DECIMAL | DataTypes.createDecimalType(precision, scale) |
 
+## Run SQL on files directly
+Instead of creating table and query it, you can also query that file directly 
with SQL.
 
+### Example
+```
+SELECT * FROM carbonfile.`$Path`
+```
+Find example code at 
[DirectSQLExample](https://github.com/apache/carbondata/blob/master/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala)
 in the CarbonData repo.
 ## API List
 
 ### Class org.apache.carbondata.sdk.file.CarbonWriterBuilder

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9469e6bd/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
--
diff --git 
a/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
new file mode 100644
index 000..a011d80
--- /dev/null
+++ 
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/DirectSQLExample.scala
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.commons.io.FileUtils
+
+import org.apache.carbondata.core.metadata.datatype.DataTypes
+import org.apache.carbondata.examples.util.ExampleUtils
+import org.apache.carbondata.sdk.file.{CarbonWriter, Field, Schema}
+
+/**
+ * Running SQL on carbon files directly
+ * No need to create table first
+ * TODO: support more than one carbon file
+ */
+object DirectSQLExample {
+
+  // prepare SDK writer output
+  def buildTestData(
+  path: String,
+  num: Int = 3,
+  persistSchema: Boolean = false): Any = {
+
+// getCanonicalPath gives path with \, but the code expects /.
+val writerPath = path.replace("\\", "/");
+
+val fields: Array[Field] = new Array[Field](3)
+fields(0) = new Field("name", DataTypes.STRING)
+fields(1) = new Field("age", DataTypes.INT)
+fields(2) = new Field("height", DataTypes.DOUBLE)
+
+try {
+  val builder = CarbonWriter
+.builder()
+.outputPath(writerPath)
+.isTransactionalTable(true)
+.uniqueIdentifier(System.currentTimeMillis)
+.withBlockSize(2)
+  if (persistSchema) {
+builder.persistSchemaFile(true)
+  }
+  val writer = 

[09/50] [abbrv] carbondata git commit: [CARBONDATA-2571] Calculating the carbonindex and carbondata file size of a table is wrong

[jackylk]

[CARBONDATA-2571] Calculating the carbonindex and carbondata file size of a 
table is wrong

Problem:
While calculating the carbonindex files size, we are checking either index file 
or merge file. But in PR#2333, implementation is changed to fill both
the file name and the merge file name. So, we have to consider both fields.

Solution:
While calculating the carbonindex files size, we have to consider both the 
files and mergeFileName fields. We should get the list of index files from
these 2 fields and then calculate the size of the files.

This closes #2358


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27d70599
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27d70599
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27d70599

Branch: refs/heads/carbonstore
Commit: 27d7059984962b97bcaf576fed496653932ea743
Parents: 92d9b92
Author: dhatchayani 
Authored: Fri Jun 1 15:13:38 2018 +0530
Committer: manishgupta88 
Committed: Tue Jun 5 11:51:52 2018 +0530

--
 .../apache/carbondata/core/util/CarbonUtil.java | 37 +++-
 1 file changed, 20 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/27d70599/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
--
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1526047..5a7bce3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2688,27 +2688,30 @@ public final class CarbonUtil {
   throws IOException {
 long carbonDataSize = 0L;
 long carbonIndexSize = 0L;
-List listOfFilesRead = new ArrayList<>();
 HashMap dataAndIndexSize = new HashMap();
-if (fileStore.getLocationMap() != null) {
+Map locationMap = 
fileStore.getLocationMap();
+if (locationMap != null) {
   fileStore.readIndexFiles();
-  Map indexFiles = fileStore.getIndexFiles();
   Map> indexFilesMap = fileStore.getIndexFilesMap();
-  for (Map.Entry> entry : indexFilesMap.entrySet()) {
-// get the size of carbonindex file
-String indexFile = entry.getKey();
-String mergeIndexFile = indexFiles.get(indexFile);
-if (null != mergeIndexFile) {
-  String mergeIndexPath = indexFile
-  .substring(0, 
indexFile.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1)
-  + mergeIndexFile;
-  if (!listOfFilesRead.contains(mergeIndexPath)) {
-carbonIndexSize += 
FileFactory.getCarbonFile(mergeIndexPath).getSize();
-listOfFilesRead.add(mergeIndexPath);
-  }
-} else {
-  carbonIndexSize += FileFactory.getCarbonFile(indexFile).getSize();
+  // get the size of carbonindex file
+  for (Map.Entry entry : 
locationMap.entrySet()) {
+SegmentFileStore.FolderDetails folderDetails = entry.getValue();
+Set carbonindexFiles = folderDetails.getFiles();
+String mergeFileName = folderDetails.getMergeFileName();
+if (null != mergeFileName) {
+  String mergeIndexPath =
+  fileStore.getTablePath() + entry.getKey() + 
CarbonCommonConstants.FILE_SEPARATOR
+  + mergeFileName;
+  carbonIndexSize += 
FileFactory.getCarbonFile(mergeIndexPath).getSize();
 }
+for (String indexFile : carbonindexFiles) {
+  String indexPath =
+  fileStore.getTablePath() + entry.getKey() + 
CarbonCommonConstants.FILE_SEPARATOR
+  + indexFile;
+  carbonIndexSize += FileFactory.getCarbonFile(indexPath).getSize();
+}
+  }
+  for (Map.Entry> entry : indexFilesMap.entrySet()) {
 // get the size of carbondata files
 for (String blockFile : entry.getValue()) {
   carbonDataSize += FileFactory.getCarbonFile(blockFile).getSize();

[02/50] [abbrv] carbondata git commit: [CARBONDATA-2508] Fix the exception that can't get executorService when start search mode twice

[jackylk]

[CARBONDATA-2508] Fix the exception that can't get executorService when start 
search mode twice

This closes #2355


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6aadfe70
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6aadfe70
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6aadfe70

Branch: refs/heads/carbonstore
Commit: 6aadfe70a67bfd13ed5efedfaa368de57403a88f
Parents: 5b2b913
Author: xubo245 
Authored: Thu May 31 09:15:16 2018 +0800
Committer: Jacky Li 
Committed: Thu May 31 14:24:35 2018 +0800

--
 .../executor/impl/SearchModeDetailQueryExecutor.java   |  6 --
 .../impl/SearchModeVectorDetailQueryExecutor.java  |  6 --
 .../testsuite/detailquery/SearchModeTestCase.scala | 13 +
 3 files changed, 21 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
index aed472c..ae14327 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeDetailQueryExecutor.java
@@ -37,8 +37,10 @@ public class SearchModeDetailQueryExecutor extends 
AbstractQueryExecutor
   
LogServiceFactory.getLogService(SearchModeDetailQueryExecutor.class.getName());
   private static ExecutorService executorService = null;
 
-  static {
-initThreadPool();
+  public SearchModeDetailQueryExecutor() {
+if (executorService == null) {
+  initThreadPool();
+}
   }
 
   private static synchronized void initThreadPool() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
index 00fd511..705c451 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java
@@ -40,8 +40,10 @@ public class SearchModeVectorDetailQueryExecutor extends 
AbstractQueryExecutorhttp://git-wip-us.apache.org/repos/asf/carbondata/blob/6aadfe70/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
index d278fc5..3e6adaf 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SearchModeTestCase.scala
@@ -136,4 +136,17 @@ class SearchModeTestCase extends QueryTest with 
BeforeAndAfterAll {
 sql("DROP DATAMAP if exists dm3 ON TABLE main")
   }
 
+  test("start search mode twice") {
+sqlContext.sparkSession.asInstanceOf[CarbonSession].startSearchMode()
+
assert(sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+checkSearchAnswer("select id from main where id = '3' limit 10")
+sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode()
+
assert(!sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+
+// start twice
+sqlContext.sparkSession.asInstanceOf[CarbonSession].startSearchMode()
+
assert(sqlContext.sparkSession.asInstanceOf[CarbonSession].isSearchModeEnabled)
+checkSearchAnswer("select id from main where id = '3' limit 10")
+sqlContext.sparkSession.asInstanceOf[CarbonSession].stopSearchMode()
+  }
 }

[28/50] [abbrv] carbondata git commit: [CARBONDATA-2428] Support flat folder for managed carbon table

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
 
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 9d0c933..b76722b 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -279,7 +279,7 @@ public abstract class AbstractFactDataWriter implements 
CarbonFactDataWriter {
 this.carbonDataFileName = CarbonTablePath
 .getCarbonDataFileName(fileCount, 
model.getCarbonDataFileAttributes().getTaskId(),
 model.getBucketId(), model.getTaskExtension(),
-"" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+"" + model.getCarbonDataFileAttributes().getFactTimeStamp(), 
model.getSegmentId());
 this.carbonDataFileHdfsPath = model.getCarbonDataDirectoryPath() + 
File.separator
 + carbonDataFileName;
 try {
@@ -368,7 +368,7 @@ public abstract class AbstractFactDataWriter implements 
CarbonFactDataWriter {
   String rawFileName = model.getCarbonDataDirectoryPath() + File.separator 
+ CarbonTablePath
   
.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(),
   model.getBucketId(), model.getTaskExtension(),
-  "" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+  "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), 
model.getSegmentId());
   indexFileName = FileFactory.getUpdatedFilePath(rawFileName, 
FileFactory.FileType.HDFS);
 } else {
   // randomly choose a temp location for index file
@@ -378,7 +378,7 @@ public abstract class AbstractFactDataWriter implements 
CarbonFactDataWriter {
   indexFileName = chosenTempLocation + File.separator + CarbonTablePath
   
.getCarbonIndexFileName(model.getCarbonDataFileAttributes().getTaskId(),
   model.getBucketId(), model.getTaskExtension(),
-  "" + model.getCarbonDataFileAttributes().getFactTimeStamp());
+  "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), 
model.getSegmentId());
 }
 
 CarbonIndexFileWriter writer = new CarbonIndexFileWriter();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
--
diff --git 
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
 
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
index 0ea7223..da77cf6 100644
--- 
a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
+++ 
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonLoaderUtil.java
@@ -601,7 +601,9 @@ public final class CarbonLoaderUtil {
 long sizePerNode = 0;
 long totalFileSize = 0;
 if (BlockAssignmentStrategy.BLOCK_NUM_FIRST == blockAssignmentStrategy) {
-  sizePerNode = blockInfos.size() / noofNodes;
+  if (blockInfos.size() > 0) {
+sizePerNode = blockInfos.size() / noofNodes;
+  }
   sizePerNode = sizePerNode <= 0 ? 1 : sizePerNode;
 } else if (BlockAssignmentStrategy.BLOCK_SIZE_FIRST == 
blockAssignmentStrategy
 || BlockAssignmentStrategy.NODE_MIN_SIZE_FIRST == 
blockAssignmentStrategy) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/60dfdd38/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
--
diff --git 
a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
 
b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
index f6406c7..4bfadce 100644
--- 
a/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
+++ 
b/store/search/src/main/java/org/apache/carbondata/store/worker/SearchRequestHandler.java
@@ -37,15 +37,18 @@ import org.apache.carbondata.core.datastore.row.CarbonRow;
 import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.readcommitter.LatestFilesReadCommittedScope;
+import org.apache.carbondata.core.readcommitter.TableStatusReadCommittedScope;
 import 
org.apache.carbondata.core.scan.executor.impl.SearchModeDetailQueryExecutor;
 import 

[07/50] [abbrv] carbondata git commit: [Hoxfix] Upgrade dev version to 1.5.0-SNAPSHOT and fix some small issues

[jackylk]

[Hoxfix] Upgrade dev version to 1.5.0-SNAPSHOT and fix some small issues

1.Upgrade dev version to 1.5.0-SNAPSHOT
2.Fix carbon-spark-sql issue
3.Remove hadoop 2.2 profile

This closes #2359


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4bb7e278
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4bb7e278
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4bb7e278

Branch: refs/heads/carbonstore
Commit: 4bb7e2785f961b2697b7de2a3a2556c25a5bb6b3
Parents: 56bf4e4
Author: chenliang613 
Authored: Sat Jun 2 12:58:57 2018 +0800
Committer: ravipesala 
Committed: Sat Jun 2 12:39:10 2018 +0530

--
 assembly/pom.xml  | 2 +-
 bin/carbon-spark-sql  | 4 ++--
 common/pom.xml| 2 +-
 core/pom.xml  | 2 +-
 datamap/bloom/pom.xml | 2 +-
 datamap/examples/pom.xml  | 2 +-
 datamap/lucene/pom.xml| 2 +-
 datamap/mv/core/pom.xml   | 2 +-
 datamap/mv/plan/pom.xml   | 2 +-
 examples/flink/pom.xml| 2 +-
 examples/spark2/pom.xml   | 2 +-
 format/pom.xml| 2 +-
 hadoop/pom.xml| 2 +-
 integration/hive/pom.xml  | 2 +-
 integration/presto/pom.xml| 2 +-
 integration/spark-common-cluster-test/pom.xml | 2 +-
 integration/spark-common-test/pom.xml | 2 +-
 integration/spark-common/pom.xml  | 2 +-
 integration/spark2/pom.xml| 2 +-
 pom.xml   | 8 +---
 processing/pom.xml| 2 +-
 store/sdk/pom.xml | 2 +-
 store/search/pom.xml  | 2 +-
 streaming/pom.xml | 2 +-
 24 files changed, 25 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/assembly/pom.xml
--
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 56522d0..eb3d3a9 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.4.0-SNAPSHOT
+1.5.0-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/bin/carbon-spark-sql
--
diff --git a/bin/carbon-spark-sql b/bin/carbon-spark-sql
index 4b927d1..9916fef 100755
--- a/bin/carbon-spark-sql
+++ b/bin/carbon-spark-sql
@@ -36,13 +36,13 @@ export CARBON_SOURCE="$(cd "`dirname "$0"`"/..; pwd)"
 ASSEMBLY_DIR="$CARBON_SOURCE/assembly/target/scala-2.11"
 
 GREP_OPTIONS=
-num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" | wc 
-l)"
+num_jars="$(ls -1 "$ASSEMBLY_DIR" | grep "^apache-carbondata.*\.jar$" | wc -l)"
 if [ "$num_jars" -eq "0" -a -z "$ASSEMBLY_DIR" ]; then
   echo "Failed to find Carbondata assembly in $ASSEMBLY_DIR." 1>&2
   echo "You need to build Carbondata before running this program." 1>&2
   exit 1
 fi
-ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^carbondata.*hadoop.*\.jar$" || 
true)"
+ASSEMBLY_JARS="$(ls -1 "$ASSEMBLY_DIR" | grep "^apache-carbondata.*\.jar$" || 
true)"
 if [ "$num_jars" -gt "1" ]; then
   echo "Found multiple Carbondata assembly jars in $ASSEMBLY_DIR:" 1>&2
   echo "$ASSEMBLY_JARS" 1>&2

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/common/pom.xml
--
diff --git a/common/pom.xml b/common/pom.xml
index 433d575..1209388 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.4.0-SNAPSHOT
+1.5.0-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/core/pom.xml
--
diff --git a/core/pom.xml b/core/pom.xml
index d9c756e..7d87037 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -22,7 +22,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.4.0-SNAPSHOT
+1.5.0-SNAPSHOT
 ../pom.xml
   
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4bb7e278/datamap/bloom/pom.xml
--
diff --git a/datamap/bloom/pom.xml b/datamap/bloom/pom.xml
index b7db969..f13d477 100644
--- a/datamap/bloom/pom.xml
+++ b/datamap/bloom/pom.xml
@@ -6,7 +6,7 @@
   
 org.apache.carbondata
 carbondata-parent
-1.4.0-SNAPSHOT
+1.5.0-SNAPSHOT
 ../../pom.xml
   
 

[34/50] [abbrv] carbondata git commit: [CARBONDATA-2418] [Presto] [S3] Fixed Presto Can't Query CarbonData When CarbonStore is at S3

[jackylk]

[CARBONDATA-2418] [Presto] [S3] Fixed Presto Can't Query CarbonData When 
CarbonStore is at S3

This closes #2287


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc4f87ba
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc4f87ba
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc4f87ba

Branch: refs/heads/carbonstore
Commit: dc4f87ba5047568e3800c6ed873846b408131da0
Parents: e7fed36
Author: anubhav100 
Authored: Tue May 1 09:59:05 2018 +0530
Committer: Jacky Li 
Committed: Tue Jun 19 00:25:33 2018 +0800

--
 integration/presto/README.md| 28 +++
 integration/presto/pom.xml  | 49 +++-
 .../carbondata/presto/CarbondataMetadata.java   |  6 ++
 .../presto/impl/CarbonTableConfig.java  | 81 +++-
 .../presto/impl/CarbonTableReader.java  | 23 +-
 5 files changed, 183 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/README.md
--
diff --git a/integration/presto/README.md b/integration/presto/README.md
index 8da70d7..53884a2 100644
--- a/integration/presto/README.md
+++ b/integration/presto/README.md
@@ -82,6 +82,34 @@ Please follow the below steps to query carbondata in presto
   For example, if you have a schema named 'default' stored in 
hdfs://namenode:9000/test/carbondata/,
   Then set carbondata-store=hdfs://namenode:9000/test/carbondata
   
+ Connecting to carbondata store on s3
+ * In case you want to query carbonstore on S3 using S3A api put following 
additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties 
+   ```
+Required properties
+
+fs.s3a.access.key={value}
+fs.s3a.secret.key={value}
+
+Optional properties
+
+fs.s3a.endpoint={value}
+   ```
+ * In case you want to query carbonstore on s3 using S3 api put following 
additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties 
+```
+  fs.s3.awsAccessKeyId={value}
+  fs.s3.awsSecretAccessKey={value}
+```
+  * In case You want to query carbonstore on s3 using S3N api put following 
additional properties inside $PRESTO_HOME$/etc/catalog/carbondata.properties 
+```
+fs.s3n.awsAccessKeyId={value}
+fs.s3n.awsSecretAccessKey={value}
+ ```
+ 
+Replace the schema-store-path with the absolute path of the parent 
directory of the schema.
+For example, if you have a schema named 'default' stored in a bucket 
s3a://s3-carbon/store,
+Then set carbondata-store=s3a://s3-carbon/store
+
+  Unsafe Properties
   enable.unsafe.in.query.processing property by default is true in CarbonData 
system, the carbon.unsafe.working.memory.in.mb 
   property defines the limit for Unsafe Memory usage in Mega Bytes, the 
default value is 512 MB.
   If your tables are big you can increase the unsafe memory, or disable unsafe 
via setting enable.unsafe.in.query.processing=false.

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/pom.xml
--
diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml
index bfa05f9..b91f070 100644
--- a/integration/presto/pom.xml
+++ b/integration/presto/pom.xml
@@ -498,7 +498,54 @@
   lz4-java
   1.4.0
 
-
+
+  org.apache.hadoop
+  hadoop-aws
+  ${hadoop.version}
+  
+
+  com.fasterxml.jackson.core
+  jackson-core
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+  
+
+
+  com.amazonaws
+  aws-java-sdk
+  1.7.4
+  
+
+  com.fasterxml.jackson.core
+  jackson-core
+
+
+  com.fasterxml.jackson.core
+  jackson-annotations
+
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
+  
+
+
+  net.java.dev.jets3t
+  jets3t
+  0.9.0
+
+
+  org.apache.httpcomponents
+  httpcore
+  4.2
+
   
 
   

http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc4f87ba/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
--
diff --git 
a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
 
b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 718c628..8be7494 100755
--- 

[16/50] [abbrv] carbondata git commit: [CARBONDATA-2573] integrate carbonstore mv branch

[jackylk]

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
--
diff --git 
a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala 
b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
index 20b5e8a..241d6a8 100644
--- a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
+++ b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/dsl/package.scala
@@ -35,7 +35,7 @@ import org.apache.carbondata.mv.plans.util._
  */
 package object dsl {
 
-  // object plans {
+  object Plans {
 
 implicit class DslModularPlan(val modularPlan: ModularPlan) {
   def select(outputExprs: NamedExpression*)
@@ -96,6 +96,6 @@ package object dsl {
   def optimize: LogicalPlan = BirdcageOptimizer.execute(logicalPlan)
 }
 
-  // }
+   }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
--
diff --git 
a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
 
b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
index 0c5661e..2aff5c0 100644
--- 
a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
+++ 
b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/LogicalPlanSignatureGenerator.scala
@@ -30,8 +30,7 @@ object CheckSPJG {
   case a: Aggregate =>
 a.child.collect {
   case Join(_, _, _, _) | Project(_, _) | Filter(_, _) |
-//   CatalogRelation(_, _, _) |
-   LogicalRelation(_, _, _) | LocalRelation(_, _) => true
+   HiveTableRelation(_, _, _) | LogicalRelation(_, _, _) | 
LocalRelation(_, _) => true
   case _ => false
 }.forall(identity)
   case _ => false
@@ -59,10 +58,10 @@ object LogicalPlanRule extends SignatureRule[LogicalPlan] {
   case LogicalRelation(_, _, _) =>
 // TODO: implement this (link to BaseRelation)
 None
-//  case CatalogRelation(tableMeta, _, _) =>
-//Some(Signature(false,
-//  Set(Seq(tableMeta.database, 
tableMeta.identifier.table).mkString("."
-  case l: LocalRelation =>
+  case HiveTableRelation(tableMeta, _, _) =>
+Some(Signature(false,
+  Set(Seq(tableMeta.database, 
tableMeta.identifier.table).mkString("."
+  case l : LocalRelation =>
 // LocalRelation is for unit test cases
 Some(Signature(groupby = false, Set(l.toString(
   case Filter(_, _) =>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
--
diff --git 
a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
 
b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
index 4bc8b97..b6e62eb 100644
--- 
a/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
+++ 
b/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/plans/util/SQLBuilder.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
 
 import scala.collection.immutable
 
-import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, 
AttributeMap, AttributeReference, AttributeSet, Expression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, 
AttributeMap, AttributeReference, AttributeSet, Cast, Expression, 
NamedExpression}
 import org.apache.spark.sql.catalyst.rules.{Rule, RuleExecutor}
 
 import org.apache.carbondata.mv.expressions.modular._
@@ -71,6 +71,10 @@ class SQLBuilder private(
   CleanupQualifier,
   // Insert sub queries on top of operators that need to appear after 
FROM clause.
   AddSubquery
+  // Removes [[Cast Casts]] that are unnecessary when converting back 
to SQL
+  // Comment out for now, will add later by converting AttributMap to 
Map in SQLBuildDSL
+  // .scala
+  // RemoveCasts
 )
   )
 }
@@ -217,6 +221,14 @@ class SQLBuilder private(
 }
   }
 
+  object RemoveCasts extends Rule[ModularPlan] {
+def apply(tree: ModularPlan): ModularPlan = {
+  tree transformAllExpressions {
+case Cast(e, dataType, _) => e
+  }
+}
+  }
+
 }
 
 object SQLBuilder {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0ef7e55c/datamap/mv/plan/src/main/scala/org/apache/carbondata/mv/testutil/Tpcds_1_4_Tables.scala
--
diff --git 

[10/50] [abbrv] carbondata git commit: [CARBONDATA-2554] Added support for logical type

[jackylk]

[CARBONDATA-2554] Added support for logical type

Added support for date and timestamp logical types in AvroCarbonWriter.

This closes #2347


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2f234869
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2f234869
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2f234869

Branch: refs/heads/carbonstore
Commit: 2f2348690964ac87c2f38939280958f2469d212d
Parents: 27d7059
Author: kunal642 
Authored: Mon May 28 11:41:59 2018 +0530
Committer: kumarvishal09 
Committed: Tue Jun 5 11:52:09 2018 +0530

--
 .../DirectDictionaryGenerator.java  |   2 +
 .../DateDirectDictionaryGenerator.java  |   2 +-
 .../TimeStampDirectDictionaryGenerator.java |   2 +-
 .../TestNonTransactionalCarbonTable.scala   | 145 ++-
 .../processing/datatypes/PrimitiveDataType.java |  44 +-
 .../loading/dictionary/DirectDictionary.java|   4 +
 .../InputProcessorStepWithNoConverterImpl.java  |  24 ++-
 .../carbondata/sdk/file/AvroCarbonWriter.java   |  71 -
 8 files changed, 279 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
index 469fe1e..2139f31 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
@@ -40,6 +40,8 @@ public interface DirectDictionaryGenerator {
*/
   Object getValueFromSurrogate(int key);
 
+  int generateKey(long value);
+
   /**
* The method generate and returns the dictionary / surrogate key for direct 
dictionary column
* This Method is called while executing filter queries for getting direct 
surrogate members.

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index c49af9c..329e260 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -163,7 +163,7 @@ public class DateDirectDictionaryGenerator implements 
DirectDictionaryGenerator
 }
   }
 
-  private int generateKey(long timeValue) {
+  public int generateKey(long timeValue) {
 if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
   if (LOGGER.isDebugEnabled()) {
 LOGGER.debug("Value for date type column is not in valid range. Value 
considered as null.");

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2f234869/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index d218e99..c7a4194 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -206,7 +206,7 @@ public class TimeStampDirectDictionaryGenerator implements 
DirectDictionaryGener
 }
   }
 
-  private int generateKey(long timeValue) {
+  public int generateKey(long timeValue) {
 long time = (timeValue - cutOffTimeStamp) / granularityFactor;
 int keyValue = -1;
 if (time >= (long) Integer.MIN_VALUE && time <= (long) Integer.MAX_VALUE) {

[35/50] [abbrv] carbondata git commit: [CARBONDATA-1787] Updated data-management-on-carbondata.md for GLOBAL_SORT_PARTITIONS

[jackylk]

[CARBONDATA-1787] Updated data-management-on-carbondata.md for 
GLOBAL_SORT_PARTITIONS

This closes #1668


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ca466d9f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ca466d9f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ca466d9f

Branch: refs/heads/carbonstore
Commit: ca466d9f4b07db1a088dc62cebaf6b4733c28a9b
Parents: dc4f87b
Author: vandana 
Authored: Fri Dec 15 18:16:05 2017 +0530
Committer: Jacky Li 
Committed: Tue Jun 19 00:34:39 2018 +0800

--
 docs/data-management-on-carbondata.md | 10 ++
 1 file changed, 10 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/ca466d9f/docs/data-management-on-carbondata.md
--
diff --git a/docs/data-management-on-carbondata.md 
b/docs/data-management-on-carbondata.md
index 3326e9b..7e171aa 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -560,6 +560,16 @@ This tutorial is going to introduce all commands and data 
operations on CarbonDa
   'BAD_RECORDS_ACTION'='REDIRECT','IS_EMPTY_DATA_BAD_RECORD'='false')
   ```
 
+  - **GLOBAL_SORT_PARTITIONS:** If the SORT_SCOPE is defined as GLOBAL_SORT, 
then user can specify the number of partitions to use while shuffling data for 
sort using GLOBAL_SORT_PARTITIONS. If it is not configured, or configured less 
than 1, then it uses the number of map task as reduce task. It is recommended 
that each reduce task deal with 512MB-1GB data.
+
+  ```
+  OPTIONS('GLOBAL_SORT_PARTITIONS'='2')
+  ```
+
+   NOTE:
+   * GLOBAL_SORT_PARTITIONS should be Integer type, the range is 
[1,Integer.MaxValue].
+   * It is only used when the SORT_SCOPE is GLOBAL_SORT.
+
 ### INSERT DATA INTO CARBONDATA TABLE
 
   This command inserts data into a CarbonData table, it is defined as a 
combination of two queries Insert and Select query respectively. 

[11/50] [abbrv] carbondata git commit: [CARBONDATA-2557] [CARBONDATA-2472] [CARBONDATA-2570] Improve Carbon Reader performance on S3 and fixed datamap clear issue in reader

[jackylk]

[CARBONDATA-2557] [CARBONDATA-2472] [CARBONDATA-2570] Improve Carbon Reader 
performance on S3 and fixed datamap clear issue in reader

[CARBONDATA-2557] [CARBONDATA-2472] Problem : CarbonReaderBuilder.build() is 
slower in s3. It takes around 8 seconds to finish build()
Solution: S3 is slow in listFiles, open, FileExist, getCarbonFile operations. 
So, List down all the calls of those API in the reader flow and remove the 
redundant checks.

[CARBONDATA-2570] Problem : Carbon SDK Reader, second time reader instance have 
an issue in cluster test
Solution: Blocklet datamap's of first time reader is not cleared properly in 
the cluster. Need to change the API to clear the blocklet datamap.

so change
DataMapStoreManager.getInstance().getDefaultDataMap(queryModel.getTable()).clear();
to
DataMapStoreManager.getInstance().clearDataMaps(queryModel.getTable().getAbsoluteTableIdentifie());

This closes #2345


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f68a792
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f68a792
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f68a792

Branch: refs/heads/carbonstore
Commit: 5f68a792f2e83d15379740f715cf05d7ae9aaa05
Parents: 2f23486
Author: ajantha-bhat 
Authored: Sun May 27 22:49:23 2018 +0530
Committer: ravipesala 
Committed: Tue Jun 5 16:23:27 2018 +0530

--
 .../core/datamap/dev/CacheableDataMap.java  |   6 +-
 .../core/datastore/SegmentTaskIndexStore.java   |   2 +-
 .../indexstore/BlockletDataMapIndexStore.java   |  84 +++--
 .../TableBlockIndexUniqueIdentifierWrapper.java |  52 
 .../blockletindex/BlockletDataMapFactory.java   | 122 ---
 .../blockletindex/SegmentIndexFileStore.java|  15 +++
 .../core/metadata/schema/table/CarbonTable.java |  60 -
 .../LatestFilesReadCommittedScope.java  |  19 +--
 .../SegmentUpdateStatusManager.java |  15 ++-
 .../core/util/BlockletDataMapUtil.java  |  50 +++-
 .../apache/carbondata/core/util/CarbonUtil.java |  30 +
 .../TestBlockletDataMapFactory.java |  13 +-
 docs/sdk-guide.md   |  10 --
 .../examples/sdk/CarbonReaderExample.java   |   1 -
 .../carbondata/hadoop/CarbonRecordReader.java   |   3 +-
 .../hadoop/api/CarbonFileInputFormat.java   |  97 ---
 .../hadoop/api/CarbonInputFormat.java   |  24 
 ...FileInputFormatWithExternalCarbonTable.scala |   2 +-
 ...tCreateTableUsingSparkCarbonFileFormat.scala |   2 +-
 .../TestNonTransactionalCarbonTable.scala   |  11 +-
 .../sdk/file/CarbonReaderBuilder.java   |  51 ++--
 .../carbondata/sdk/file/CarbonReaderTest.java   |   4 +-
 22 files changed, 375 insertions(+), 298 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
 
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
index dba0840..e292c60 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/CacheableDataMap.java
@@ -22,7 +22,7 @@ import java.util.List;
 
 import org.apache.carbondata.core.datamap.DataMapDistributable;
 import org.apache.carbondata.core.indexstore.BlockletDataMapIndexWrapper;
-import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import 
org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper;
 import org.apache.carbondata.core.memory.MemoryException;
 
 /**
@@ -33,10 +33,10 @@ public interface CacheableDataMap {
   /**
* Add the blockletDataMapIndexWrapper to cache for key 
tableBlockIndexUniqueIdentifier
*
-   * @param tableBlockIndexUniqueIdentifier
+   * @param tableBlockIndexUniqueIdentifierWrapper
* @param blockletDataMapIndexWrapper
*/
-  void cache(TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier,
+  void cache(TableBlockIndexUniqueIdentifierWrapper 
tableBlockIndexUniqueIdentifierWrapper,
   BlockletDataMapIndexWrapper blockletDataMapIndexWrapper) throws 
IOException, MemoryException;
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f68a792/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
 
b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index d325f21..c642091 

[44/50] [abbrv] carbondata git commit: [CARBONDATA-2616][BloomDataMap] Fix bugs in querying bloom datamap with two index columns

[jackylk]

[CARBONDATA-2616][BloomDataMap] Fix bugs in querying bloom datamap with two 
index columns

During pruning in bloomfilter datamap, the same blocklets has been added
to result more than once, thus causing explaining and querying returning
incorrect result.

This closes #2386


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6eb360e1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6eb360e1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6eb360e1

Branch: refs/heads/carbonstore
Commit: 6eb360e1f5f577a576d185efb7dcbf1cc6a302e8
Parents: 01b48fc
Author: xuchuanyin 
Authored: Wed Jun 20 16:31:28 2018 +0800
Committer: Jacky Li 
Committed: Thu Jun 21 12:03:48 2018 +0800

--
 .../datamap/bloom/BloomCoarseGrainDataMap.java|  5 +++--
 .../bloom/BloomCoarseGrainDataMapSuite.scala  | 18 +-
 2 files changed, 20 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/6eb360e1/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
--
diff --git 
a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
 
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index e9af0ff..ed03256 100644
--- 
a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ 
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -21,6 +21,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -85,7 +86,7 @@ public class BloomCoarseGrainDataMap extends 
CoarseGrainDataMap {
   @Override
   public List prune(FilterResolverIntf filterExp, SegmentProperties 
segmentProperties,
   List partitions) {
-List hitBlocklets = new ArrayList();
+Set hitBlocklets = new HashSet<>();
 if (filterExp == null) {
   // null is different from empty here. Empty means after pruning, no 
blocklet need to scan.
   return null;
@@ -111,7 +112,7 @@ public class BloomCoarseGrainDataMap extends 
CoarseGrainDataMap {
 }
   }
 }
-return hitBlocklets;
+return new ArrayList<>(hitBlocklets);
   }
 
   private byte[] convertValueToBytes(DataType dataType, Object value) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/6eb360e1/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
--
diff --git 
a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
 
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 7df3901..c9a4097 100644
--- 
a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ 
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -71,7 +71,23 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with 
BeforeAndAfterAll with
 checkAnswer(
   checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 
'city_999'", dataMapName, shouldHit),
   sql(s"select * from $normalTable where city = 'city_999'"))
- checkAnswer(
+// query with two index_columns
+checkAnswer(
+  checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 1 and 
city='city_1'", dataMapName, shouldHit),
+  sql(s"select * from $normalTable where id = 1 and city='city_1'"))
+checkAnswer(
+  checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 999 
and city='city_999'", dataMapName, shouldHit),
+  sql(s"select * from $normalTable where id = 999 and city='city_999'"))
+checkAnswer(
+  checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 
'city_1' and id = 0", dataMapName, shouldHit),
+  sql(s"select * from $normalTable where city = 'city_1' and id = 0"))
+checkAnswer(
+  checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 
'city_999' and name='n999'", dataMapName, shouldHit),
+  sql(s"select * from $normalTable where city = 'city_999' and 
name='n999'"))
+checkAnswer(
+  checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 
'city_999' and name='n1'", dataMapName, shouldHit),
+  sql(s"select * from $normalTable where city = 'city_999' and name='n1'"))
+checkAnswer(
   sql(s"select min(id), max(id), min(name), max(name), min(city), 
max(city)" +

[42/50] [abbrv] carbondata git commit: [CARBONDATA-2617] Invalid tuple-id and block id getting formed for Non partition table

[jackylk]

[CARBONDATA-2617] Invalid tuple-id and block id getting formed for Non 
partition table

Problem
Invalid tuple and block id getting formed for non partition table

Analysis
While creating a partition table a segment file was written in the Metadata 
folder under table structure. This was introduced during
development of partition table feature. At that time segment file was written 
only for partition table and it was used to distinguish between
parition and non partition table in the code. But later the code was modified 
to write the segment file for both parititon and non partition
table and the code to distinguish partition and non partition table was not 
modified which is causing this incorrect formation of block and tuple id.

Fix
Modify the logic to distinguish partitioned and non partitioned table and the 
same has been handled in this PR.

This closes #2385


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0e1d550e
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0e1d550e
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0e1d550e

Branch: refs/heads/carbonstore
Commit: 0e1d550e8dacba798e9ffbdda25c4388e8933632
Parents: dc53dee
Author: rahul 
Authored: Tue Jun 19 19:23:26 2018 +0530
Committer: manishgupta88 
Committed: Wed Jun 20 16:37:23 2018 +0530

--
 .../core/mutate/CarbonUpdateUtil.java   |  4 +-
 .../executor/impl/AbstractQueryExecutor.java|  4 +-
 .../SegmentUpdateStatusManager.java | 20 ++---
 .../apache/carbondata/core/util/CarbonUtil.java |  4 +-
 .../iud/DeleteCarbonTableTestCase.scala | 83 
 .../command/mutation/DeleteExecution.scala  |  6 +-
 6 files changed, 100 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java 
b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 40d498c..8627bdb 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -81,10 +81,10 @@ public class CarbonUpdateUtil {
   /**
* Returns block path from tuple id
*/
-  public static String getTableBlockPath(String tid, String tablePath, boolean 
isSegmentFile) {
+  public static String getTableBlockPath(String tid, String tablePath, boolean 
isPartitionTable) {
 String partField = getRequiredFieldFromTID(tid, TupleIdEnum.PART_ID);
 // If it has segment file then partfield can be appended directly to table 
path
-if (isSegmentFile) {
+if (isPartitionTable) {
   return tablePath + CarbonCommonConstants.FILE_SEPARATOR + 
partField.replace("#", "/");
 }
 String part = CarbonTablePath.addPartPrefix(partField);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 2bbe75c..f365045 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -296,8 +296,8 @@ public abstract class AbstractQueryExecutor implements 
QueryExecutor {
 String blockId = CarbonUtil
 .getBlockId(queryModel.getAbsoluteTableIdentifier(), filePath, 
segment.getSegmentNo(),
 queryModel.getTable().getTableInfo().isTransactionalTable(),
-segment.getSegmentFileName() != null);
-if (segment.getSegmentFileName() != null) {
+queryModel.getTable().isHivePartitionTable());
+if (queryModel.getTable().isHivePartitionTable()) {
   
blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockIdForPartitionTable(blockId));
 } else {
   blockExecutionInfo.setBlockId(CarbonTablePath.getShortBlockId(blockId));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/0e1d550e/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
 
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 

[22/50] [abbrv] carbondata git commit: [CARBONDATA-2309][DataLoad] Add strategy to generate bigger carbondata files in case of small amout of data

[jackylk]

[CARBONDATA-2309][DataLoad] Add strategy to generate bigger carbondata files in 
case of small amout of data

This closes #2314


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/685087ed
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/685087ed
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/685087ed

Branch: refs/heads/carbonstore
Commit: 685087ed4de7ecc181de6ee43c9e5865eb26b650
Parents: 9b88a06
Author: ndwangsen 
Authored: Fri May 25 21:45:58 2018 +0800
Committer: Jacky Li 
Committed: Wed Jun 13 01:56:03 2018 +0800

--
 .../core/constants/CarbonCommonConstants.java   |  12 ++
 .../constants/CarbonLoadOptionConstants.java|  10 ++
 .../carbondata/core/util/CarbonProperties.java  |  12 ++
 docs/useful-tips-on-carbondata.md   |   3 +-
 .../dataload/TestTableLoadMinSize.scala | 149 +++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   2 +-
 .../spark/rdd/CarbonDataRDDFactory.scala|   9 +-
 .../management/CarbonLoadDataCommand.scala  |   4 +-
 .../loading/model/CarbonLoadModel.java  |  15 ++
 .../loading/model/CarbonLoadModelBuilder.java   |   1 +
 .../processing/loading/model/LoadOption.java|   2 +
 .../processing/util/CarbonLoaderUtil.java   | 105 +++--
 .../processing/util/CarbonLoaderUtilTest.java   |  51 ++-
 .../scala/org/apache/spark/rpc/Master.scala |   3 +-
 14 files changed, 355 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 08aa704..c7281dd 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1761,6 +1761,18 @@ public final class CarbonCommonConstants {
 
   public static final String CARBON_LUCENE_INDEX_STOP_WORDS_DEFAULT = "false";
 
+  /**
+   * The node loads the smallest amount of data
+   */
+  @CarbonProperty
+  public static final String CARBON_LOAD_MIN_SIZE_INMB = "load_min_size_inmb";
+  public static final String CARBON_LOAD_MIN_NODE_SIZE_INMB_DEFAULT = "256";
+
+  /**
+   *  the node minimum load data default value
+   */
+  public static final int CARBON_LOAD_MIN_SIZE_DEFAULT = 256;
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
index a2213d5..6b8281c 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonLoadOptionConstants.java
@@ -151,4 +151,14 @@ public final class CarbonLoadOptionConstants {
   public static final String CARBON_LOAD_SORT_MEMORY_SPILL_PERCENTAGE
   = "carbon.load.sortMemory.spill.percentage";
   public static final String CARBON_LOAD_SORT_MEMORY_SPILL_PERCENTAGE_DEFAULT 
= "0";
+
+  /**
+   *  if loading data is too small, the original loading method will produce 
many small files.
+   *  enable set the node load minimum amount of data,avoid producing many 
small files.
+   *  This option is especially useful when you encounter a lot of small 
amounts of data.
+   */
+  @CarbonProperty
+  public static final String ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE
+  = "carbon.load.min.size.enabled";
+  public static final String ENABLE_CARBON_LOAD_NODE_DATA_MIN_SIZE_DEFAULT = 
"false";
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/685087ed/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java 
b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 4ee5199..6eb7de6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -1310,6 +1310,18 @@ public final class CarbonProperties {
 
CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_SKEWED_DATA_OPTIMIZATION_DEFAULT);
 return 

[39/50] [abbrv] carbondata git commit: [CARBONDATA-2611] Added Test Cases for Local Dictionary Support for Create Table comand

[jackylk]

[CARBONDATA-2611] Added Test Cases for Local Dictionary Support for Create 
Table comand

What changes were proposed in this pull request?
In this PR, UTs and SDV test cases are added for local dictionary support for 
create table command and describe formatted command.

changed the error message for validation of local dictionary table properties 
and
fixed column name display without extra space in describe formatted command

This closes#2377


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c5a4ec07
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c5a4ec07
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c5a4ec07

Branch: refs/heads/carbonstore
Commit: c5a4ec07a8c03600741d6a2ae324a4fe4ab61659
Parents: be20fef
Author: praveenmeenakshi56 
Authored: Thu Jun 14 23:36:59 2018 +0530
Committer: kumarvishal09 
Committed: Wed Jun 20 12:13:09 2018 +0530

--
 ...CreateTableWithLocalDictionaryTestCase.scala | 2102 ++
 .../cluster/sdv/suite/SDVSuites.scala   |6 +-
 .../LocalDictionarySupportCreateTableTest.scala | 2102 ++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |   31 +-
 .../table/CarbonDescribeFormattedCommand.scala  |2 +-
 5 files changed, 4227 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c5a4ec07/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
--
diff --git 
a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
new file mode 100644
index 000..48a31b7
--- /dev/null
+++ 
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
@@ -0,0 +1,2102 @@
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import 
org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
+
+class CreateTableWithLocalDictionaryTestCase extends QueryTest with 
BeforeAndAfterAll {
+
+  override protected def beforeAll(): Unit = {
+sql("DROP TABLE IF EXISTS LOCAL1")
+  }
+
+  test("test local dictionary default configuration") {
+sql("drop table if exists local1")
+sql(
+  """
+| CREATE TABLE local1(id int, name string, city string, age int)
+| STORED BY 'org.apache.carbondata.format'
+  """.stripMargin)
+
+val desc_result = sql("describe formatted local1")
+
+val descLoc = sql("describe formatted local1").collect
+descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match 
{
+  case Some(row) => assert(row.get(1).toString.contains("true"))
+}
+descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) 
match {
+  case Some(row) => assert(row.get(1).toString.contains("1000"))
+}
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_001") {
+sql("drop table if exists local1")
+sql(
+  """
+| CREATE TABLE local1(id int, name string, city string, age int)
+| STORED BY 'org.apache.carbondata.format'
+| tblproperties('local_dictionary_include'='name')
+  """.
+stripMargin)
+val descFormatted1 = sql("describe formatted local1").collect
+descFormatted1.find(_.get(0).toString.contains("Local Dictionary 
Enabled")) match {
+  case Some(row) => assert(row.get(1).toString.contains("true"))
+}
+descFormatted1.find(_.get(0).toString.contains("Local Dictionary 
Include")) match {
+  case Some(row) => assert(row.get(1).toString.contains("name"))
+}
+  }
+
+  test(
+"test local dictionary custom configurations for local dict columns _002")
+  {
+sql("drop table if exists local1")
+
+intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 'org.apache.carbondata.format'
+  | tblproperties('local_dictionary_include'='name,name')
+""".stripMargin)
+}
+  }
+
+  test("test local dictionary custom configurations for local dict columns 
_003") {
+sql("drop table if exists local1")
+val exception = intercept[MalformedCarbonCommandException] {
+  sql(
+"""
+  | CREATE TABLE local1(id int, name string, city string, age int)
+  | STORED BY 

Jenkins build is back to stable : carbondata-master-spark-2.1 #2526

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2526

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Store SDK #567

[Apache Jenkins Server]

See 

Jenkins build is back to normal : carbondata-master-spark-2.2 #567

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #567

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark2 #567

[Apache Jenkins Server]

See 

Jenkins build is back to normal : carbondata-master-spark-2.2 » Apache CarbonData :: Parent #567

[Apache Jenkins Server]

See 

carbondata git commit: [CARBONDATA-2623][DataMap] Add DataMap Pre and Pevent listener

[manishgupta88]

Repository: carbondata
Updated Branches:
  refs/heads/master 55f4bc6c8 -> b3f782062


[CARBONDATA-2623][DataMap] Add DataMap Pre and Pevent listener

Added Pre and Post Execution Events for index datamap

This closes #2389


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b3f78206
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b3f78206
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b3f78206

Branch: refs/heads/master
Commit: b3f7820623d4bc9ab4408beb8ad708ba9b19b899
Parents: 55f4bc6
Author: mohammadshahidkhan 
Authored: Wed Jun 20 19:52:51 2018 +0530
Committer: manishgupta88 
Committed: Thu Jun 21 17:37:48 2018 +0530

--
 .../carbondata/events/DataMapEvents.scala   | 68 
 .../org/apache/carbondata/events/Events.scala   | 18 +-
 .../datamap/IndexDataMapRebuildRDD.scala| 11 +++-
 .../spark/rdd/CarbonTableCompactor.scala| 23 ++-
 .../datamap/CarbonCreateDataMapCommand.scala| 22 +++
 .../datamap/CarbonDataMapRebuildCommand.scala   | 12 
 .../datamap/CarbonDropDataMapCommand.scala  | 11 
 .../management/CarbonLoadDataCommand.scala  | 21 +-
 8 files changed, 181 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
new file mode 100644
index 000..8fb374f
--- /dev/null
+++ 
b/integration/spark-common/src/main/scala/org/apache/carbondata/events/DataMapEvents.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.events
+
+import org.apache.spark.sql.SparkSession
+
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+
+/**
+ * For handling operation's after finish of index creation over table with 
index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPostExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of update index datmap status over 
table with index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPreExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of  update index datmap status over 
table with index
+ * datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class UpdateDataMapPostExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+
+/**
+ * For handling operation's before start of index build over table with index 
datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPreExecutionEvent(sparkSession: SparkSession,
+identifier: AbsoluteTableIdentifier, dataMapNames: 
scala.collection.mutable.Seq[String])
+  extends Event with BuildDataMapEventsInfo
+
+/**
+ * For handling operation's after finish of index build over table with index 
datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class BuildDataMapPostExecutionEvent(sparkSession: SparkSession,
+identifier: AbsoluteTableIdentifier)
+  extends Event with TableEventInfo
+
+/**
+ * For handling operation's before start of index creation over table with 
index datamap
+ * example: bloom datamap, Lucene datamap
+ */
+case class CreateDataMapPreExecutionEvent(sparkSession: SparkSession,
+storePath: String) extends Event with CreateDataMapEventsInfo
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b3f78206/integration/spark-common/src/main/scala/org/apache/carbondata/events/Events.scala

Build failed in Jenkins: carbondata-master-spark-2.2 #566

[Apache Jenkins Server]

See 


Changes:

[kumarvishal09] [CARBONDATA-2513][32K] Support write long string from dataframe

--
[...truncated 62.39 MB...]
++

++
|count(1)|
++
|   8|
++

time for query on table with lucene datamap table:0.344
time for query on table without lucene datamap table:0.407
+-+-+
|   id| name|
+-+-+
|which test1 good5|who and name2|
|which test1 good5|who and name2|
|which test1 good9|who and name3|
|which test1 good9|who and name3|
+-+-+

+-+-+
|   id| name|
+-+-+
|which test1 good1|who and name1|
|which test1 good1|who and name1|
|which test1 good1|who and name3|
|which test1 good1|who and name3|
|which test1 good5|who and name2|
|which test1 good5|who and name2|
|which test1 good9|who and name3|
|which test1 good9|who and name3|
+-+-+

18/06/21 02:08:33 AUDIT CarbonDropTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Deleting table [persontable] under 
database [default]
18/06/21 02:08:33 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/21 02:08:33 AUDIT CarbonDropTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Deleted table [persontable] under 
database [default]
- LuceneDataMapExample
18/06/21 02:08:33 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [origin_table]
18/06/21 02:08:33 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/21 02:08:33 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [origin_table]
18/06/21 02:08:33 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 02:08:34 ERROR DataLoadExecutor: [Executor task launch worker for task 
2297][partitionID:table;queryID:19209353440077540] Data Load is partially 
success for table origin_table
18/06/21 02:08:34 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 02:08:34 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 02:08:34 ERROR DataLoadExecutor: [Executor task launch worker for task 
2298][partitionID:table;queryID:19209354003275850] Data Load is partially 
success for table origin_table
18/06/21 02:08:34 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 02:08:34 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 02:08:35 ERROR DataLoadExecutor: [Executor task launch worker for task 
2299][partitionID:table;queryID:19209354552143947] Data Load is partially 
success for table origin_table
18/06/21 02:08:35 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 02:08:35 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 02:08:35 ERROR DataLoadExecutor: [Executor task launch worker for task 
2300][partitionID:table;queryID:19209354941369470] Data Load is partially 
success for table origin_table
18/06/21 02:08:35 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
++
|count(1)|
++
|  40|
++

18/06/21 02:08:35 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [external_table]
18/06/21 02:08:35 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [external_table]
++
|count(1)|
++
|  40|
++

18/06/21 02:08:35 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 02:08:36 ERROR DataLoadExecutor: [Executor task launch worker for task 
2305][partitionID:table;queryID:19209355633280570] Data Load is partially 
success for table origin_table
18/06/21 02:08:36 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 02:08:36 AUDIT 

Jenkins build became unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Store SDK #566

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark2 #566

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #566

[Apache Jenkins Server]

See 

Build failed in Jenkins: carbondata-master-spark-2.2 » Apache CarbonData :: Parent #566

[Apache Jenkins Server]

See 


--
Established TCP socket on 35591
maven33-agent.jar already up to date
maven33-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
<===[JENKINS REMOTING CAPACITY]===>   channel started
Executing Maven:  -B -f 

 -Dmaven.repo.local=/home/jenkins/jenkins-slave/maven-repositories/1 -B -e 
clean install -U -Pspark-2.2 coveralls:report
[INFO] Error stacktraces are turned on.
[INFO] Scanning for projects...
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-presto:presto-plugin:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 621, column 15
[WARNING] 'build.plugins.plugin.version' for 
pl.project13.maven:git-commit-id-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 634, column 15
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-examples-spark2:jar:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-examples-spark2:[unknown-version], 

 line 163, column 15
[WARNING] 
[WARNING] It is highly recommended to fix these problems because they threaten 
the stability of your build.
[WARNING] 
[WARNING] For this reason, future Maven versions might no longer support 
building such malformed projects.
[WARNING] 
[INFO] 
[INFO] Reactor Build Order:
[INFO] 
[INFO] Apache CarbonData :: Parent
[INFO] Apache CarbonData :: Common
[INFO] Apache CarbonData :: Core
[INFO] Apache CarbonData :: Processing
[INFO] Apache CarbonData :: Hadoop
[INFO] Apache CarbonData :: Streaming
[INFO] Apache CarbonData :: Spark Common
[INFO] Apache CarbonData :: Store SDK
[INFO] Apache CarbonData :: Search
[INFO] Apache CarbonData :: Lucene Index DataMap
[INFO] Apache CarbonData :: Bloom Index DataMap
[INFO] Apache CarbonData :: Spark2
[INFO] Apache CarbonData :: Spark Common Test
[INFO] Apache CarbonData :: DataMap Examples
[INFO] Apache CarbonData :: Assembly
[INFO] Apache CarbonData :: Hive
[INFO] Apache CarbonData :: presto
[INFO] Apache CarbonData :: Spark2 Examples
[INFO] 
[INFO] 
[INFO] Building Apache CarbonData :: Parent 1.5.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ carbondata-parent ---
[INFO] Deleting 

[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent (default-prepare-agent) @ 
carbondata-parent ---
[INFO] argLine set to 
-javaagent:/home/jenkins/jenkins-slave/maven-repositories/1/org/jacoco/org.jacoco.agent/0.7.9/org.jacoco.agent-0.7.9-runtime.jar=destfile=
[INFO] 
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ 
carbondata-parent ---
[INFO] 
[INFO] >>> findbugs-maven-plugin:3.0.4:check (analyze-compile) > :findbugs @ 
carbondata-parent >>>
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:findbugs (findbugs) @ carbondata-parent 
---
[INFO] 
[INFO] <<< findbugs-maven-plugin:3.0.4:check (analyze-compile) < :findbugs @ 
carbondata-parent <<<
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:check (analyze-compile) @ 
carbondata-parent ---
[INFO] 
[INFO] --- maven-site-plugin:3.4:attach-descriptor (attach-descriptor) @ 
carbondata-parent ---
[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent-integration 
(default-prepare-agent-integration) @ carbondata-parent ---
[INFO] argLine set to 

Jenkins build is unstable: carbondata-master-spark-2.2 #565

[Apache Jenkins Server]

See 

Jenkins build is back to normal : carbondata-master-spark-2.2 » Apache CarbonData :: Parent #565

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #565

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2525

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.1 #2525

[Apache Jenkins Server]

See 

Jenkins build is unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2524

[Apache Jenkins Server]

See 

Jenkins build is unstable: carbondata-master-spark-2.1 #2524

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.1 » Apache CarbonData :: Store SDK #2523

[Apache Jenkins Server]

See 

Jenkins build is back to stable : carbondata-master-spark-2.2 » Apache CarbonData :: Spark Common Test #564

[Apache Jenkins Server]

See 

Build failed in Jenkins: carbondata-master-spark-2.2 #564

[Apache Jenkins Server]

See 


Changes:

[kumarvishal09] [CARBONDATA-2615][32K] Support page size less than 32000 in 
CarbondataV3

--
[...truncated 55.64 MB...]
|which test1 good5|who and name3|
|which test1 good8|who and name0|
|which test1 good8|who and name0|
|which test1 good9|who and name6|
|which test1 good9|who and name6|
+-+-+

+-+-+
|   id| name|
+-+-+
|which test1 good1|who and name4|
|which test1 good1|who and name4|
|which test1 good1|who and name7|
|which test1 good1|who and name7|
|which test1 good4|who and name5|
|which test1 good4|who and name5|
|which test1 good5|who and name3|
|which test1 good5|who and name3|
|which test1 good8|who and name0|
|which test1 good8|who and name0|
+-+-+

18/06/21 00:25:14 AUDIT CarbonDropTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Deleting table [persontable] under 
database [default]
18/06/21 00:25:15 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/21 00:25:15 AUDIT CarbonDropTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Deleted table [persontable] under 
database [default]
- LuceneDataMapExample
18/06/21 00:25:15 AUDIT CarbonCreateTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [origin_table]
18/06/21 00:25:15 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/21 00:25:15 AUDIT CarbonCreateTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [origin_table]
18/06/21 00:25:15 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:15 ERROR DataLoadExecutor: [Executor task launch worker for task 
2298][partitionID:table;queryID:52799987248250537] Data Load is partially 
success for table origin_table
18/06/21 00:25:15 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 00:25:15 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:15 ERROR DataLoadExecutor: [Executor task launch worker for task 
2299][partitionID:table;queryID:52799987704214472] Data Load is partially 
success for table origin_table
18/06/21 00:25:16 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 00:25:16 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:16 ERROR DataLoadExecutor: [Executor task launch worker for task 
2300][partitionID:table;queryID:52799988119687321] Data Load is partially 
success for table origin_table
18/06/21 00:25:16 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 00:25:16 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:16 ERROR DataLoadExecutor: [Executor task launch worker for task 
2301][partitionID:table;queryID:52799988531224311] Data Load is partially 
success for table origin_table
18/06/21 00:25:16 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
++
|count(1)|
++
|  40|
++

18/06/21 00:25:16 AUDIT CarbonCreateTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [external_table]
18/06/21 00:25:17 AUDIT CarbonCreateTableCommand: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [external_table]
++
|count(1)|
++
|  40|
++

18/06/21 00:25:17 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:17 ERROR DataLoadExecutor: [Executor task launch worker for task 
2306][partitionID:table;queryID:52799989249075746] Data Load is partially 
success for table origin_table
18/06/21 00:25:17 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/21 00:25:17 AUDIT CarbonDataRDDFactory$: 
[asf926.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/21 00:25:17 ERROR DataLoadExecutor: [Executor 

Build failed in Jenkins: carbondata-master-spark-2.2 » Apache CarbonData :: Parent #564

[Apache Jenkins Server]

See 


--
Established TCP socket on 40673
maven33-agent.jar already up to date
maven33-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
<===[JENKINS REMOTING CAPACITY]===>   channel started
Executing Maven:  -B -f 

 -Dmaven.repo.local=/home/jenkins/jenkins-slave/maven-repositories/0 -B -e 
clean install -U -Pspark-2.2 coveralls:report
[INFO] Error stacktraces are turned on.
[INFO] Scanning for projects...
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-presto:presto-plugin:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 621, column 15
[WARNING] 'build.plugins.plugin.version' for 
pl.project13.maven:git-commit-id-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 634, column 15
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-examples-spark2:jar:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-examples-spark2:[unknown-version], 

 line 163, column 15
[WARNING] 
[WARNING] It is highly recommended to fix these problems because they threaten 
the stability of your build.
[WARNING] 
[WARNING] For this reason, future Maven versions might no longer support 
building such malformed projects.
[WARNING] 
[INFO] 
[INFO] Reactor Build Order:
[INFO] 
[INFO] Apache CarbonData :: Parent
[INFO] Apache CarbonData :: Common
[INFO] Apache CarbonData :: Core
[INFO] Apache CarbonData :: Processing
[INFO] Apache CarbonData :: Hadoop
[INFO] Apache CarbonData :: Streaming
[INFO] Apache CarbonData :: Spark Common
[INFO] Apache CarbonData :: Store SDK
[INFO] Apache CarbonData :: Search
[INFO] Apache CarbonData :: Lucene Index DataMap
[INFO] Apache CarbonData :: Bloom Index DataMap
[INFO] Apache CarbonData :: Spark2
[INFO] Apache CarbonData :: Spark Common Test
[INFO] Apache CarbonData :: DataMap Examples
[INFO] Apache CarbonData :: Assembly
[INFO] Apache CarbonData :: Hive
[INFO] Apache CarbonData :: presto
[INFO] Apache CarbonData :: Spark2 Examples
[INFO] 
[INFO] 
[INFO] Building Apache CarbonData :: Parent 1.5.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ carbondata-parent ---
[INFO] Deleting 

[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent (default-prepare-agent) @ 
carbondata-parent ---
[INFO] argLine set to 
-javaagent:/home/jenkins/jenkins-slave/maven-repositories/0/org/jacoco/org.jacoco.agent/0.7.9/org.jacoco.agent-0.7.9-runtime.jar=destfile=
[INFO] 
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ 
carbondata-parent ---
[INFO] 
[INFO] >>> findbugs-maven-plugin:3.0.4:check (analyze-compile) > :findbugs @ 
carbondata-parent >>>
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:findbugs (findbugs) @ carbondata-parent 
---
[INFO] 
[INFO] <<< findbugs-maven-plugin:3.0.4:check (analyze-compile) < :findbugs @ 
carbondata-parent <<<
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:check (analyze-compile) @ 
carbondata-parent ---
[INFO] 
[INFO] --- maven-site-plugin:3.4:attach-descriptor (attach-descriptor) @ 
carbondata-parent ---
[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent-integration 
(default-prepare-agent-integration) @ carbondata-parent ---
[INFO] argLine set to 

carbondata git commit: [CARBONDATA-2513][32K] Support write long string from dataframe

[kumarvishal09]

Repository: carbondata
Updated Branches:
  refs/heads/master 218a8deb6 -> 55f4bc6c8


[CARBONDATA-2513][32K] Support write long string from dataframe

support write long string from dataframe

This closes #2382


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/55f4bc6c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/55f4bc6c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/55f4bc6c

Branch: refs/heads/master
Commit: 55f4bc6c89f637b162b414033512901e9bd8a745
Parents: 218a8de
Author: xuchuanyin 
Authored: Wed Jun 20 19:01:24 2018 +0800
Committer: kumarvishal09 
Committed: Thu Jun 21 12:31:21 2018 +0530

--
 .../VarcharDataTypesBasicTestCase.scala | 32 +++-
 .../apache/carbondata/spark/CarbonOption.scala  |  2 ++
 .../spark/rdd/NewCarbonDataLoadRDD.scala| 15 +++--
 .../carbondata/spark/util/CarbonScalaUtil.scala |  3 +-
 .../spark/sql/CarbonDataFrameWriter.scala   |  1 +
 .../streaming/parser/FieldConverter.scala   | 11 +--
 6 files changed, 57 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 9ea3f1f..9798178 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -20,8 +20,9 @@ package org.apache.carbondata.spark.testsuite.longstring
 import java.io.{File, PrintWriter}
 
 import org.apache.commons.lang3.RandomStringUtils
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, Row, SaveMode}
 import org.apache.spark.sql.test.util.QueryTest
+import org.apache.spark.sql.types.{IntegerType, StringType, StructField, 
StructType}
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -36,6 +37,7 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
   private val inputFile_2g_column_page = s"$inputDir$fileName_2g_column_page"
   private val lineNum = 1000
   private var content: Content = _
+  private var longStringDF: DataFrame = _
   private var originMemorySize = CarbonProperties.getInstance().getProperty(
 CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB,
 CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB_DEFAULT)
@@ -257,6 +259,34 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
 // since after exception wrapper, we cannot get the root cause directly
   }
 
+  private def prepareDF(): Unit = {
+val schema = StructType(
+  StructField("id", IntegerType, nullable = true) ::
+  StructField("name", StringType, nullable = true) ::
+  StructField("description", StringType, nullable = true) ::
+  StructField("address", StringType, nullable = true) ::
+  StructField("note", StringType, nullable = true) :: Nil
+)
+longStringDF = sqlContext.sparkSession.read
+  .schema(schema)
+  .csv(inputFile)
+  }
+
+  test("write from dataframe with long string datatype") {
+prepareDF()
+// write spark dataframe to carbondata with `long_string_columns` property
+longStringDF.write
+  .format("carbondata")
+  .option("tableName", longStringTable)
+  .option("single_pass", "false")
+  .option("sort_columns", "name")
+  .option("long_string_columns", "description, note")
+  .mode(SaveMode.Overwrite)
+  .save()
+
+checkQuery()
+  }
+
   // will create 2 long string columns
   private def createFile(filePath: String, line: Int = 1, start: Int = 0,
   varcharLen: Int = Short.MaxValue + 1000): Content = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/55f4bc6c/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
--
diff --git 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
 
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
index a48e63d..5f23f77 100644
--- 
a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonOption.scala
+++ 

carbondata git commit: [CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long string columns

[kumarvishal09]

Repository: carbondata
Updated Branches:
  refs/heads/master 091a28bf8 -> 218a8deb6


[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long 
string columns

Add validate for long string columns

1. long string columns cannot be sort_columns
2. long string columns cannot be dictionary include
3. long string columns cannot be dictionary exclude
4. long string columns can only be string columns

This closes #2380


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/218a8deb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/218a8deb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/218a8deb

Branch: refs/heads/master
Commit: 218a8deb614e0dc160ab8c7e38c71d80711eb1a7
Parents: 091a28b
Author: xuchuanyin 
Authored: Wed Jun 20 18:55:04 2018 +0800
Committer: kumarvishal09 
Committed: Thu Jun 21 12:00:34 2018 +0530

--
 .../VarcharDataTypesBasicTestCase.scala | 57 
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +++-
 2 files changed, 67 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
--
diff --git 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 419b306..9ea3f1f 100644
--- 
a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ 
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with 
BeforeAndAfterEach wi
 sql(s"drop table if exists $longStringTable")
   }
 
+  test("long string columns cannot be dictionary include") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 
'dictionary_include'='address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is 
unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be dictionay exclude") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 
'dictionary_exclude'='address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is 
unsupported for long string datatype column: address"))
+  }
+
+  test("long string columns cannot be sort_columns") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 
'SORT_COLUMNS'='name, address')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("sort_columns is unsupported 
for long string datatype column: name"))
+  }
+
+  test("long string columns can only be string columns") {
+val exceptionCaught = intercept[Exception] {
+  sql(
+s"""
+   | CREATE TABLE if not exists $longStringTable(
+   | id INT, name STRING, description STRING, address STRING, note 
STRING
+   | ) STORED BY 'carbondata'
+   | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note')
+   |""".
+  stripMargin)
+}
+assert(exceptionCaught.getMessage.contains("long_string_columns: id"))
+assert(exceptionCaught.getMessage.contains("its data type is not string"))
+  }
+
   private def prepareTable(): Unit = {
 sql(
   s"""

http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
--
diff --git 

Build failed in Jenkins: carbondata-master-spark-2.2 #562

[Apache Jenkins Server]

See 


Changes:

[jacky.likun] [CARBONDATA-2616][BloomDataMap] Fix bugs in querying bloom 
datamap with

--
[...truncated 63.07 MB...]
time for query on table without lucene datamap table:0.42
+-+-+
|   id| name|
+-+-+
|which test1 good7|who and name7|
|which test1 good7|who and name7|
|which test1 good8|who and name1|
|which test1 good8|who and name1|
+-+-+

+-+-+
|   id| name|
+-+-+
|which test1 good1|who and name6|
|which test1 good1|who and name6|
|which test1 good7|who and name7|
|which test1 good7|who and name7|
|which test1 good8|who and name1|
|which test1 good8|who and name1|
+-+-+

18/06/20 23:14:58 AUDIT CarbonDropTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Deleting table [persontable] under 
database [default]
18/06/20 23:14:58 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/20 23:14:58 AUDIT CarbonDropTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Deleted table [persontable] under 
database [default]
- LuceneDataMapExample
18/06/20 23:14:58 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [origin_table]
18/06/20 23:14:58 ERROR DataMapStoreManager: ScalaTest-main-running-RunExamples 
failed to get carbon table from table Path
18/06/20 23:14:58 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [origin_table]
18/06/20 23:14:58 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:14:59 ERROR DataLoadExecutor: [Executor task launch worker for task 
2297][partitionID:table;queryID:19198938320820255] Data Load is partially 
success for table origin_table
18/06/20 23:14:59 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/20 23:14:59 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:14:59 ERROR DataLoadExecutor: [Executor task launch worker for task 
2298][partitionID:table;queryID:19198938872558630] Data Load is partially 
success for table origin_table
18/06/20 23:14:59 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/20 23:14:59 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:15:00 ERROR DataLoadExecutor: [Executor task launch worker for task 
2299][partitionID:table;queryID:19198939601419967] Data Load is partially 
success for table origin_table
18/06/20 23:15:00 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/20 23:15:00 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:15:00 ERROR DataLoadExecutor: [Executor task launch worker for task 
2300][partitionID:table;queryID:19198940138227785] Data Load is partially 
success for table origin_table
18/06/20 23:15:00 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
++
|count(1)|
++
|  40|
++

18/06/20 23:15:00 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Creating Table with Database name 
[default] and Table name [external_table]
18/06/20 23:15:00 AUDIT CarbonCreateTableCommand: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Table created with Database name 
[default] and Table name [external_table]
++
|count(1)|
++
|  40|
++

18/06/20 23:15:01 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:15:01 ERROR DataLoadExecutor: [Executor task launch worker for task 
2305][partitionID:table;queryID:19198940911787014] Data Load is partially 
success for table origin_table
18/06/20 23:15:01 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load is successful for 
default.origin_table
18/06/20 23:15:01 AUDIT CarbonDataRDDFactory$: 
[asf933.gq1.ygridcore.net][jenkins][Thread-1]Data load request has been 
received for table default.origin_table
18/06/20 23:15:02 ERROR DataLoadExecutor: [Executor task launch 

Build failed in Jenkins: carbondata-master-spark-2.2 » Apache CarbonData :: Parent #562

[Apache Jenkins Server]

See 


--
Established TCP socket on 45958
maven33-agent.jar already up to date
maven33-interceptor.jar already up to date
maven3-interceptor-commons.jar already up to date
<===[JENKINS REMOTING CAPACITY]===>   channel started
Executing Maven:  -B -f 

 -Dmaven.repo.local=/home/jenkins/jenkins-slave/maven-repositories/0 -B -e 
clean install -U -Pspark-2.2 coveralls:report
[INFO] Error stacktraces are turned on.
[INFO] Scanning for projects...
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-presto:presto-plugin:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 621, column 15
[WARNING] 'build.plugins.plugin.version' for 
pl.project13.maven:git-commit-id-plugin is missing. @ 
org.apache.carbondata:carbondata-presto:[unknown-version], 

 line 634, column 15
[WARNING] 
[WARNING] Some problems were encountered while building the effective model for 
org.apache.carbondata:carbondata-examples-spark2:jar:1.5.0-SNAPSHOT
[WARNING] 'build.plugins.plugin.version' for 
com.ning.maven.plugins:maven-duplicate-finder-plugin is missing. @ 
org.apache.carbondata:carbondata-examples-spark2:[unknown-version], 

 line 163, column 15
[WARNING] 
[WARNING] It is highly recommended to fix these problems because they threaten 
the stability of your build.
[WARNING] 
[WARNING] For this reason, future Maven versions might no longer support 
building such malformed projects.
[WARNING] 
[INFO] 
[INFO] Reactor Build Order:
[INFO] 
[INFO] Apache CarbonData :: Parent
[INFO] Apache CarbonData :: Common
[INFO] Apache CarbonData :: Core
[INFO] Apache CarbonData :: Processing
[INFO] Apache CarbonData :: Hadoop
[INFO] Apache CarbonData :: Streaming
[INFO] Apache CarbonData :: Spark Common
[INFO] Apache CarbonData :: Store SDK
[INFO] Apache CarbonData :: Search
[INFO] Apache CarbonData :: Lucene Index DataMap
[INFO] Apache CarbonData :: Bloom Index DataMap
[INFO] Apache CarbonData :: Spark2
[INFO] Apache CarbonData :: Spark Common Test
[INFO] Apache CarbonData :: DataMap Examples
[INFO] Apache CarbonData :: Assembly
[INFO] Apache CarbonData :: Hive
[INFO] Apache CarbonData :: presto
[INFO] Apache CarbonData :: Spark2 Examples
[INFO] 
[INFO] 
[INFO] Building Apache CarbonData :: Parent 1.5.0-SNAPSHOT
[INFO] 
[INFO] 
[INFO] --- maven-clean-plugin:2.5:clean (default-clean) @ carbondata-parent ---
[INFO] Deleting 

[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent (default-prepare-agent) @ 
carbondata-parent ---
[INFO] argLine set to 
-javaagent:/home/jenkins/jenkins-slave/maven-repositories/0/org/jacoco/org.jacoco.agent/0.7.9/org.jacoco.agent-0.7.9-runtime.jar=destfile=
[INFO] 
[INFO] --- maven-remote-resources-plugin:1.5:process (default) @ 
carbondata-parent ---
[INFO] 
[INFO] >>> findbugs-maven-plugin:3.0.4:check (analyze-compile) > :findbugs @ 
carbondata-parent >>>
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:findbugs (findbugs) @ carbondata-parent 
---
[INFO] 
[INFO] <<< findbugs-maven-plugin:3.0.4:check (analyze-compile) < :findbugs @ 
carbondata-parent <<<
[INFO] 
[INFO] --- findbugs-maven-plugin:3.0.4:check (analyze-compile) @ 
carbondata-parent ---
[INFO] 
[INFO] --- maven-site-plugin:3.4:attach-descriptor (attach-descriptor) @ 
carbondata-parent ---
[INFO] 
[INFO] --- jacoco-maven-plugin:0.7.9:prepare-agent-integration 
(default-prepare-agent-integration) @ carbondata-parent ---
[INFO] argLine set to 

Jenkins build is still unstable: carbondata-master-spark-2.1 #2522

[Apache Jenkins Server]

See 

Jenkins build is still unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Spark Common Test #2522

[Apache Jenkins Server]

See 

Jenkins build became unstable: carbondata-master-spark-2.1 » Apache CarbonData :: Store SDK #2522

[Apache Jenkins Server]

See