date:20161115

[1/2] incubator-carbondata git commit: add no-kettle loading example

2016-11-15 Thread chenliang613

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master e60c9255f -> 24978ed7e


add no-kettle loading example

fix comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e84cd930
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e84cd930
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e84cd930

Branch: refs/heads/master
Commit: e84cd930d25961aff65588529316a7d93a21f34b
Parents: e60c925
Author: jackylk 
Authored: Thu Nov 10 23:20:52 2016 +0800
Committer: chenliang613 
Committed: Tue Nov 15 23:42:55 2016 +0800

--
 .../carbondata/examples/CarbonExample.scala | 21 
 .../carbondata/examples/util/ExampleUtils.scala |  1 +
 .../spark/CarbonDataFrameWriter.scala   | 10 +-
 3 files changed, 31 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
--
diff --git 
a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala 
b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
index df0a1e2..7c769ff 100644
--- a/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
+++ b/examples/src/main/scala/org/apache/carbondata/examples/CarbonExample.scala
@@ -32,6 +32,7 @@ object CarbonExample {
 
 cc.sql("DROP TABLE IF EXISTS t3")
 
+// Create table, 6 dimensions, 1 measure
 cc.sql("""
CREATE TABLE IF NOT EXISTS t3
(ID Int, date Timestamp, country String,
@@ -39,10 +40,15 @@ object CarbonExample {
STORED BY 'carbondata'
""")
 
+// Currently there are two data loading flows in CarbonData, one uses 
Kettle as ETL tool
+// in each node to do data loading, another uses a multi-thread framework 
without Kettleï¼See
+// AbstractDataLoadProcessorStep)
+// Load data with Kettle
 cc.sql(s"""
LOAD DATA LOCAL INPATH '$testData' into table t3
""")
 
+// Perform a query
 cc.sql("""
SELECT country, count(salary) AS amount
FROM t3
@@ -50,6 +56,21 @@ object CarbonExample {
GROUP BY country
""").show()
 
+// Load data without kettle
+cc.sql(s"""
+   LOAD DATA LOCAL INPATH '$testData' into table t3
+   OPTIONS('USE_KETTLE'='false')
+   """)
+
+// Perform a query
+cc.sql("""
+   SELECT country, count(salary) AS amount
+   FROM t3
+   WHERE country IN ('china','france')
+   GROUP BY country
+   """).show()
+
+// Drop table
 cc.sql("DROP TABLE IF EXISTS t3")
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
--
diff --git 
a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
 
b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
index cfcdde8..83010b6 100644
--- 
a/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ 
b/examples/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
@@ -84,6 +84,7 @@ object ExampleUtils {
 .format("carbondata")
 .option("tableName", tableName)
 .option("compress", "true")
+.option("useKettle", "false")
 .mode(mode)
 .save()
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e84cd930/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
--
diff --git 
a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
 
b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index c2b2190..a02751e 100644
--- 
a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ 
b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -161,11 +161,19 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) 
extends Logging {
   }
 
   private def makeLoadString(csvFolder: String, options: CarbonOption): String 
= {
-s"""
+if (options.useKettle) {
+  s"""
   LOAD DATA INPATH '$csvFolder'
   INTO TABLE ${options.dbName}.${options.tableName}
   OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
   """
+} else {
+  s"""
+

[1/2] incubator-carbondata git commit: implement test cases for core.load module

2016-11-15 Thread jackylk

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 6fe4f96e1 -> e60c9255f


implement test cases for core.load module

code formatted using formatter

replace assert with assertEquals

manage test case for timestamp


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/70537a6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/70537a6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/70537a6c

Branch: refs/heads/master
Commit: 70537a6cf8d04dcb4be216e4b54f72974e2f3ae1
Parents: 6fe4f96
Author: Anurag 
Authored: Fri Oct 28 10:34:51 2016 +0530
Committer: jackylk 
Committed: Tue Nov 15 21:02:31 2016 +0800

--
 .../core/load/LoadMetadataDetails.java  |   2 +-
 .../core/load/LoadMetadataDetailsUnitTest.java  | 134 +++
 2 files changed, 135 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70537a6c/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
--
diff --git 
a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java 
b/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
index 7c58ae7..d527ba4 100644
--- 
a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
+++ 
b/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
@@ -150,7 +150,7 @@ public class LoadMetadataDetails implements Serializable {
* return loadStartTime
* @return
*/
-  public long getLoadStartTimeAsLong() {
+  public Long getLoadStartTimeAsLong() {
 return getTimeStamp(loadStartTime);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70537a6c/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
--
diff --git 
a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
 
b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
new file mode 100644
index 000..4aed38f
--- /dev/null
+++ 
b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.load;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.TimeZone;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+import static junit.framework.Assert.assertNull;
+
+public class LoadMetadataDetailsUnitTest {
+
+  private LoadMetadataDetails loadMetadataDetails;
+  private static final LogService LOGGER =
+  LogServiceFactory.getLogService(LoadMetadataDetails.class.getName());
+
+  @Before public void setup() {
+loadMetadataDetails = new LoadMetadataDetails();
+  }
+
+  /**
+   * This method will test Hashcode which will return 31 if we don't set 
loadName.
+   *
+   * @throws Exception
+   */
+
+  @Test public void testHashCodeLoadNameNull() throws Exception {
+int expected_result = 31;
+int data = loadMetadataDetails.hashCode();
+assertEquals(expected_result, data);
+  }
+
+  @Test public void testHashCodeValueInLoadName() throws Exception {
+loadMetadataDetails.setLoadName("test");
+int data = loadMetadataDetails.hashCode();
+assertNotSame(31, data);
+  }
+
+  @Test public void testEqualsObjectIsNotLoadMetadataDetails() throws 
Exception {
+Object obj = new Object();
+boolean result = loadMetadataDetails.equals(obj);
+assertEquals(false, result);
+  }
+
+

[2/2] incubator-carbondata git commit: [CARBONDATA-340] implement test cases for core.load module This closes #267

2016-11-15 Thread jackylk

[CARBONDATA-340] implement test cases for core.load module This closes #267


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e60c9255
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e60c9255
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e60c9255

Branch: refs/heads/master
Commit: e60c9255f79b9dd67d02504c49ed348c2b42523f
Parents: 6fe4f96 70537a6
Author: jackylk 
Authored: Tue Nov 15 22:48:20 2016 +0800
Committer: jackylk 
Committed: Tue Nov 15 22:48:20 2016 +0800

--
 .../core/load/LoadMetadataDetails.java  |   2 +-
 .../core/load/LoadMetadataDetailsUnitTest.java  | 134 +++
 2 files changed, 135 insertions(+), 1 deletion(-)
--

[1/2] incubator-carbondata git commit: add no-kettle loading example

[1/2] incubator-carbondata git commit: implement test cases for core.load module

[2/2] incubator-carbondata git commit: [CARBONDATA-340] implement test cases for core.load module This closes #267

3 matches

Site Navigation

Mail list logo

Footer information