HIVE-17725: Fix misnamed tests which are not run during precommit runs. (Daniel Voros via Zoltan Haindrich)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3681b859 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3681b859 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3681b859 Branch: refs/heads/master Commit: 3681b8595d22568e2b5baaef8ab56d00a2005b2d Parents: 4258193 Author: Daniel Voros <daniel.vo...@gmail.com> Authored: Thu Oct 19 10:59:07 2017 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Thu Oct 19 10:59:07 2017 +0200 ---------------------------------------------------------------------- .../hadoop/hive/hbase/LazyHBaseCellMapTest.java | 72 ------- .../hadoop/hive/hbase/TestLazyHBaseCellMap.java | 73 +++++++ .../hive/hcatalog/cli/TestSemanticAnalysis.java | 18 +- .../hive/hcatalog/data/TestReaderWriter.java | 2 - .../hive/hcatalog/mapreduce/HCatBaseTest.java | 3 +- .../hcatalog/pig/TestHCatStorerWrapper.java | 2 +- .../thrift/TestThriftCliServiceMessageSize.java | 153 ++++++++++++++ .../thrift/ThriftCliServiceMessageSizeTest.java | 158 -------------- .../config/JdbcStorageConfigManagerTest.java | 89 -------- .../config/TestJdbcStorageConfigManager.java | 88 ++++++++ .../hive/storage/jdbc/JdbcInputFormatTest.java | 81 -------- .../storage/jdbc/QueryConditionBuilderTest.java | 151 -------------- .../hive/storage/jdbc/TestJdbcInputFormat.java | 80 +++++++ .../storage/jdbc/TestQueryConditionBuilder.java | 150 ++++++++++++++ .../org/apache/hive/storage/jdbc/TestSuite.java | 29 --- .../dao/GenericJdbcDatabaseAccessorTest.java | 206 ------------------- .../dao/TestGenericJdbcDatabaseAccessor.java | 205 ++++++++++++++++++ .../org/apache/hive/jdbc/HiveStatementTest.java | 58 ------ .../org/apache/hive/jdbc/TestHiveStatement.java | 58 ++++++ .../json/JSONMessageDeserializerTest.java | 106 ---------- .../json/TestJSONMessageDeserializer.java | 106 ++++++++++ .../repl/bootstrap/load/TaskTrackerTest.java | 46 ----- .../repl/bootstrap/load/TestTaskTracker.java | 46 +++++ .../ql/exec/tez/InputSplitComparatorTest.java | 39 ---- .../ql/exec/tez/TestInputSplitComparator.java | 39 ++++ .../hive/ql/parse/repl/CopyUtilsTest.java | 47 ----- .../hive/ql/parse/repl/TestCopyUtils.java | 47 +++++ .../ql/parse/repl/dump/HiveWrapperTest.java | 45 ---- .../ql/parse/repl/dump/TestHiveWrapper.java | 45 ++++ .../PrimaryToReplicaResourceFunctionTest.java | 105 ---------- .../TestPrimaryToReplicaResourceFunction.java | 104 ++++++++++ .../hive/service/cli/CLIServiceRestoreTest.java | 55 ----- .../hive/service/cli/TestCLIServiceRestore.java | 53 +++++ .../hadoop/hive/ql/util/JavaDataModelTest.java | 76 ------- .../hadoop/hive/ql/util/TestJavaDataModel.java | 76 +++++++ 35 files changed, 1334 insertions(+), 1377 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java deleted file mode 100644 index eb13e14..0000000 --- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/LazyHBaseCellMapTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.hbase; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hive.serde2.lazy.LazyFactory; -import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; -import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; -import org.apache.hadoop.io.Text; - -import junit.framework.TestCase; - -public class LazyHBaseCellMapTest extends TestCase { - public static final byte[] TEST_ROW = Bytes.toBytes("test-row"); - public static final byte[] COLUMN_FAMILY = Bytes.toBytes("a"); - public static final String QUAL_PREFIX = "col_"; - - - public void testInitColumnPrefix() throws Exception { - Text nullSequence = new Text("\\N"); - ObjectInspector oi = LazyFactory.createLazyObjectInspector( - TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0), - new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0); - - LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi); - - // Initialize a result - Cell[] cells = new KeyValue[2]; - - final String col1="1"; - final String col2="2"; - cells[0] = new KeyValue(TEST_ROW, COLUMN_FAMILY, - Bytes.toBytes(QUAL_PREFIX+col1), Bytes.toBytes("cfacol1")); - cells[1]=new KeyValue(TEST_ROW, COLUMN_FAMILY, - Bytes.toBytes(QUAL_PREFIX+col2), Bytes.toBytes("cfacol2")); - - Result r = Result.create(cells); - - List<Boolean> mapBinaryStorage = new ArrayList<Boolean>(); - mapBinaryStorage.add(false); - mapBinaryStorage.add(false); - - b.init(r, COLUMN_FAMILY, mapBinaryStorage, Bytes.toBytes(QUAL_PREFIX), true); - - assertNotNull(b.getMapValueElement(new Text(col1))); - assertNotNull(b.getMapValueElement(new Text(col2))); - - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java ---------------------------------------------------------------------- diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java new file mode 100644 index 0000000..a15f53c --- /dev/null +++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestLazyHBaseCellMap.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.hbase; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hive.serde2.lazy.LazyFactory; +import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertNotNull; + +public class TestLazyHBaseCellMap { + public static final byte[] TEST_ROW = Bytes.toBytes("test-row"); + public static final byte[] COLUMN_FAMILY = Bytes.toBytes("a"); + public static final String QUAL_PREFIX = "col_"; + + @Test + public void testInitColumnPrefix() throws Exception { + Text nullSequence = new Text("\\N"); + ObjectInspector oi = LazyFactory.createLazyObjectInspector( + TypeInfoUtils.getTypeInfosFromTypeString("map<string,string>").get(0), + new byte[]{(byte) 1, (byte) 2}, 0, nullSequence, false, (byte) 0); + + LazyHBaseCellMap b = new LazyHBaseCellMap((LazyMapObjectInspector) oi); + + // Initialize a result + Cell[] cells = new KeyValue[2]; + + final String col1 = "1"; + final String col2 = "2"; + cells[0] = new KeyValue(TEST_ROW, COLUMN_FAMILY, + Bytes.toBytes(QUAL_PREFIX + col1), Bytes.toBytes("cfacol1")); + cells[1] = new KeyValue(TEST_ROW, COLUMN_FAMILY, + Bytes.toBytes(QUAL_PREFIX + col2), Bytes.toBytes("cfacol2")); + + Result r = Result.create(cells); + + List<Boolean> mapBinaryStorage = new ArrayList<Boolean>(); + mapBinaryStorage.add(false); + mapBinaryStorage.add(false); + + b.init(r, COLUMN_FAMILY, mapBinaryStorage, Bytes.toBytes(QUAL_PREFIX), true); + + assertNotNull(b.getMapValueElement(new Text(col1))); + assertNotNull(b.getMapValueElement(new Text(col2))); + + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java index f259800..d249d43 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java @@ -18,15 +18,6 @@ */ package org.apache.hive.hcatalog.cli; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; @@ -52,6 +43,15 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + public class TestSemanticAnalysis extends HCatBaseTest { private static final Logger LOG = LoggerFactory.getLogger(TestSemanticAnalysis.class); http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java index 4f92b68..f38f136 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestReaderWriter.java @@ -32,10 +32,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.transfer.DataTransferFactory; import org.apache.hive.hcatalog.data.transfer.HCatReader; http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java index 6cc0a96..f78eb15 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.util.Shell; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.pig.ExecType; import org.apache.pig.PigServer; @@ -43,7 +42,7 @@ import java.util.Properties; /** * Simplify writing HCatalog tests that require a HiveMetaStore. */ -public class HCatBaseTest { +public abstract class HCatBaseTest { protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class); public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("user.dir") + "/build/test/data/" + HCatBaseTest.class.getCanonicalName() + "-" + System.currentTimeMillis()); http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java ---------------------------------------------------------------------- diff --git a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java index 2cf14aa..55c2e83 100644 --- a/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java +++ b/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java @@ -27,8 +27,8 @@ import java.util.UUID; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hive.hcatalog.HcatTestUtils; -import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; +import org.apache.hive.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.ExecType; import org.apache.pig.PigServer; http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCliServiceMessageSize.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCliServiceMessageSize.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCliServiceMessageSize.java new file mode 100644 index 0000000..0948efc --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCliServiceMessageSize.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.service.cli.thrift; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hive.service.auth.HiveAuthConstants; +import org.apache.hive.service.server.HiveServer2; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class TestThriftCliServiceMessageSize { + protected static int port; + protected static String host = "localhost"; + protected static HiveServer2 hiveServer2; + protected static ThriftCLIServiceClient client; + protected static HiveConf hiveConf; + protected static String USERNAME = "anonymous"; + protected static String PASSWORD = "anonymous"; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Find a free port + port = MetaStoreTestUtils.findFreePort(); + hiveServer2 = new HiveServer2(); + hiveConf = new HiveConf(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + protected static void startHiveServer2WithConf(HiveServer2 hiveServer2, HiveConf hiveConf) + throws Exception { + hiveServer2.init(hiveConf); + // Start HiveServer2 with given config + // Fail if server doesn't start + try { + hiveServer2.start(); + } catch (Throwable t) { + t.printStackTrace(); + fail(); + } + // Wait for startup to complete + Thread.sleep(2000); + System.out.println("HiveServer2 started on port " + port); + } + + protected static void stopHiveServer2(HiveServer2 hiveServer2) throws Exception { + if (hiveServer2 != null) { + hiveServer2.stop(); + } + } + + /** + * @throws java.lang.Exception + */ + @Before + public void setUp() throws Exception { + } + + /** + * @throws java.lang.Exception + */ + @After + public void tearDown() throws Exception { + + } + + @Test + public void testMessageSize() throws Exception { + String transportMode = "binary"; + + hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthConstants.AuthTypes.NONE.toString()); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + + HiveServer2 hiveServer2 = new HiveServer2(); + String url = "jdbc:hive2://localhost:" + port + "/default"; + Class.forName("org.apache.hive.jdbc.HiveDriver"); + + try { + // First start HS2 with high message size limit. This should allow connections + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 100*1024*1024); + startHiveServer2WithConf(hiveServer2, hiveConf); + + System.out.println("Started Thrift CLI service with message size limit " + + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE)); + + // With the high message size limit this connection should work + Connection connection = DriverManager.getConnection(url, "hiveuser", "hive"); + Statement stmt = connection.createStatement(); + assertNotNull("Statement is null", stmt); + stmt.execute("set hive.support.concurrency = false"); + connection.close(); + stopHiveServer2(hiveServer2); + + // Now start HS2 with low message size limit. This should prevent any connections + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 1); + hiveServer2 = new HiveServer2(); + startHiveServer2WithConf(hiveServer2, hiveConf); + System.out.println("Started Thrift CLI service with message size limit " + + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE)); + + Exception caughtException = null; + try { + // This should fail + connection = DriverManager.getConnection(url, "hiveuser", "hive"); + } catch (Exception err) { + caughtException = err; + } + // Verify we hit an error while connecting + assertNotNull(caughtException); + } finally { + stopHiveServer2(hiveServer2); + hiveServer2 = null; + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java deleted file mode 100644 index 926efa3..0000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.service.cli.thrift; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; -import org.apache.hive.service.Service; -import org.apache.hive.service.auth.HiveAuthConstants; -import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes; -import org.apache.hive.service.cli.SessionHandle; -import org.apache.hive.service.server.HiveServer2; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class ThriftCliServiceMessageSizeTest { - protected static int port; - protected static String host = "localhost"; - protected static HiveServer2 hiveServer2; - protected static ThriftCLIServiceClient client; - protected static HiveConf hiveConf; - protected static String USERNAME = "anonymous"; - protected static String PASSWORD = "anonymous"; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Find a free port - port = MetaStoreTestUtils.findFreePort(); - hiveServer2 = new HiveServer2(); - hiveConf = new HiveConf(); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - } - - protected static void startHiveServer2WithConf(HiveServer2 hiveServer2, HiveConf hiveConf) - throws Exception { - hiveServer2.init(hiveConf); - // Start HiveServer2 with given config - // Fail if server doesn't start - try { - hiveServer2.start(); - } catch (Throwable t) { - t.printStackTrace(); - fail(); - } - // Wait for startup to complete - Thread.sleep(2000); - System.out.println("HiveServer2 started on port " + port); - } - - protected static void stopHiveServer2(HiveServer2 hiveServer2) throws Exception { - if (hiveServer2 != null) { - hiveServer2.stop(); - } - } - - /** - * @throws java.lang.Exception - */ - @Before - public void setUp() throws Exception { - } - - /** - * @throws java.lang.Exception - */ - @After - public void tearDown() throws Exception { - - } - - @Test - public void testMessageSize() throws Exception { - String transportMode = "binary"; - - hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); - hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); - hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthConstants.AuthTypes.NONE.toString()); - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); - - HiveServer2 hiveServer2 = new HiveServer2(); - String url = "jdbc:hive2://localhost:" + port + "/default"; - Class.forName("org.apache.hive.jdbc.HiveDriver"); - - try { - // First start HS2 with high message size limit. This should allow connections - hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 100*1024*1024); - startHiveServer2WithConf(hiveServer2, hiveConf); - - System.out.println("Started Thrift CLI service with message size limit " - + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE)); - - // With the high message size limit this connection should work - Connection connection = DriverManager.getConnection(url, "hiveuser", "hive"); - Statement stmt = connection.createStatement(); - assertNotNull("Statement is null", stmt); - stmt.execute("set hive.support.concurrency = false"); - connection.close(); - stopHiveServer2(hiveServer2); - - // Now start HS2 with low message size limit. This should prevent any connections - hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE, 1); - hiveServer2 = new HiveServer2(); - startHiveServer2WithConf(hiveServer2, hiveConf); - System.out.println("Started Thrift CLI service with message size limit " - + hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_MESSAGE_SIZE)); - - Exception caughtException = null; - try { - // This should fail - connection = DriverManager.getConnection(url, "hiveuser", "hive"); - } catch (Exception err) { - caughtException = err; - } - // Verify we hit an error while connecting - assertNotNull(caughtException); - } finally { - stopHiveServer2(hiveServer2); - hiveServer2 = null; - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java b/jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java deleted file mode 100644 index 800172c..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/config/JdbcStorageConfigManagerTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.config; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; - -import org.junit.Test; -import org.junit.Ignore; - -import org.apache.hive.storage.jdbc.conf.DatabaseType; -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class JdbcStorageConfigManagerTest { - - @Test - public void testWithAllRequiredSettingsDefined() throws Exception { - Properties props = new Properties(); - props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); - props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); - props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); - props.put(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "com.mysql.jdbc.Driver"); - - Map<String, String> jobMap = new HashMap<String, String>(); - JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); - - assertThat(jobMap, is(notNullValue())); - assertThat(jobMap.size(), is(equalTo(4))); - assertThat(jobMap.get(JdbcStorageConfig.DATABASE_TYPE.getPropertyName()), is(equalTo("MYSQL"))); - assertThat(jobMap.get(JdbcStorageConfig.JDBC_URL.getPropertyName()), is(equalTo("jdbc://localhost:3306/hive"))); - assertThat(jobMap.get(JdbcStorageConfig.QUERY.getPropertyName()), - is(equalTo("SELECT col1,col2,col3 FROM sometable"))); - } - - - // since metastore connections don't require the url, this is allowable. - @Ignore @Test(expected = IllegalArgumentException.class) - public void testWithJdbcUrlMissing() throws Exception { - Properties props = new Properties(); - props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); - props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); - - Map<String, String> jobMap = new HashMap<String, String>(); - JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); - } - - - @Test(expected = IllegalArgumentException.class) - public void testWithDatabaseTypeMissing() throws Exception { - Properties props = new Properties(); - props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); - props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); - - Map<String, String> jobMap = new HashMap<String, String>(); - JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); - } - - - @Test(expected = IllegalArgumentException.class) - public void testWithUnknownDatabaseType() throws Exception { - Properties props = new Properties(); - props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "Postgres"); - props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); - props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); - - Map<String, String> jobMap = new HashMap<String, String>(); - JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/config/TestJdbcStorageConfigManager.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/config/TestJdbcStorageConfigManager.java b/jdbc-handler/src/test/java/org/apache/hive/config/TestJdbcStorageConfigManager.java new file mode 100644 index 0000000..3db15b7 --- /dev/null +++ b/jdbc-handler/src/test/java/org/apache/hive/config/TestJdbcStorageConfigManager.java @@ -0,0 +1,88 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.config; + +import org.apache.hive.storage.jdbc.conf.DatabaseType; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfigManager; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +public class TestJdbcStorageConfigManager { + + @Test + public void testWithAllRequiredSettingsDefined() throws Exception { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + props.put(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "com.mysql.jdbc.Driver"); + + Map<String, String> jobMap = new HashMap<String, String>(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + + assertThat(jobMap, is(notNullValue())); + assertThat(jobMap.size(), is(equalTo(4))); + assertThat(jobMap.get(JdbcStorageConfig.DATABASE_TYPE.getPropertyName()), is(equalTo("MYSQL"))); + assertThat(jobMap.get(JdbcStorageConfig.JDBC_URL.getPropertyName()), is(equalTo("jdbc://localhost:3306/hive"))); + assertThat(jobMap.get(JdbcStorageConfig.QUERY.getPropertyName()), + is(equalTo("SELECT col1,col2,col3 FROM sometable"))); + } + + + // since metastore connections don't require the url, this is allowable. + @Ignore @Test(expected = IllegalArgumentException.class) + public void testWithJdbcUrlMissing() throws Exception { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), DatabaseType.MYSQL.toString()); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map<String, String> jobMap = new HashMap<String, String>(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + + + @Test(expected = IllegalArgumentException.class) + public void testWithDatabaseTypeMissing() throws Exception { + Properties props = new Properties(); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map<String, String> jobMap = new HashMap<String, String>(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + + + @Test(expected = IllegalArgumentException.class) + public void testWithUnknownDatabaseType() throws Exception { + Properties props = new Properties(); + props.put(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "Postgres"); + props.put(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc://localhost:3306/hive"); + props.put(JdbcStorageConfig.QUERY.getPropertyName(), "SELECT col1,col2,col3 FROM sometable"); + + Map<String, String> jobMap = new HashMap<String, String>(); + JdbcStorageConfigManager.copyConfigurationToJob(props, jobMap); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java deleted file mode 100644 index cc6acf1..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/JdbcInputFormatTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc; - -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.when; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.InputSplit; -import org.apache.hadoop.mapred.JobConf; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; -import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; - -import java.io.IOException; - -@RunWith(MockitoJUnitRunner.class) -public class JdbcInputFormatTest { - - @Mock - private DatabaseAccessor mockDatabaseAccessor; - - - @Test - public void testSplitLogic_noSpillOver() throws HiveJdbcDatabaseAccessException, IOException { - JdbcInputFormat f = new JdbcInputFormat(); - when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); - f.setDbAccessor(mockDatabaseAccessor); - - JobConf conf = new JobConf(); - conf.set("mapred.input.dir", "/temp"); - InputSplit[] splits = f.getSplits(conf, 3); - - assertThat(splits, is(notNullValue())); - assertThat(splits.length, is(3)); - - assertThat(splits[0].getLength(), is(5L)); - } - - - @Test - public void testSplitLogic_withSpillOver() throws HiveJdbcDatabaseAccessException, IOException { - JdbcInputFormat f = new JdbcInputFormat(); - when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); - f.setDbAccessor(mockDatabaseAccessor); - - JobConf conf = new JobConf(); - conf.set("mapred.input.dir", "/temp"); - InputSplit[] splits = f.getSplits(conf, 6); - - assertThat(splits, is(notNullValue())); - assertThat(splits.length, is(6)); - - for (int i = 0; i < 3; i++) { - assertThat(splits[i].getLength(), is(3L)); - } - - for (int i = 3; i < 6; i++) { - assertThat(splits[i].getLength(), is(2L)); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java deleted file mode 100644 index 5cdae47..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/QueryConditionBuilderTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc; - -import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.ql.plan.TableScanDesc; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.junit.BeforeClass; -import org.junit.Test; - -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; - -import java.io.IOException; -import java.util.Scanner; - -public class QueryConditionBuilderTest { - - private static String condition1; - private static String condition2; - - - @BeforeClass - public static void setup() throws IOException { - condition1 = readFileContents("condition1.xml"); - condition2 = readFileContents("condition2.xml"); - } - - - private static String readFileContents(String name) throws IOException { - try (Scanner s = new Scanner(QueryConditionBuilderTest.class.getClassLoader().getResourceAsStream(name))) { - return s.useDelimiter("\\Z").next(); - } - } - - - @Test - public void testSimpleCondition_noTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("(visitor_id = 'x')"))); - } - - - @Test - public void testSimpleCondition_withTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=vid, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("(vid = 'x')"))); - } - - - @Test - public void testSimpleCondition_withDateType() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=vid:date, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); - } - - - @Test - public void testSimpleCondition_withVariedCaseMappings() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_ID,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), - "visitor_id=VID:date, sentiment=sentiment, tracking_id=tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); - } - - - @Test - public void testMultipleConditions_noTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("((visitor_id = 'x') and (sentiment = 'y'))"))); - } - - - @Test - public void testMultipleConditions_withTranslation() { - Configuration conf = new Configuration(); - conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition, is(equalToIgnoringWhiteSpace("((v = 'x') and (s = 'y'))"))); - } - - - @Test - public void testWithNullConf() { - String condition = QueryConditionBuilder.getInstance().buildCondition(null); - assertThat(condition, is(notNullValue())); - assertThat(condition.trim().isEmpty(), is(true)); - } - - - @Test - public void testWithUndefinedFilterExpr() { - Configuration conf = new Configuration(); - conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); - conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); - String condition = QueryConditionBuilder.getInstance().buildCondition(conf); - - assertThat(condition, is(notNullValue())); - assertThat(condition.trim().isEmpty(), is(true)); - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestJdbcInputFormat.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestJdbcInputFormat.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestJdbcInputFormat.java new file mode 100644 index 0000000..e904774 --- /dev/null +++ b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestJdbcInputFormat.java @@ -0,0 +1,80 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.InputSplit; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hive.storage.jdbc.dao.DatabaseAccessor; +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; + +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TestJdbcInputFormat { + + @Mock + private DatabaseAccessor mockDatabaseAccessor; + + + @Test + public void testSplitLogic_noSpillOver() throws HiveJdbcDatabaseAccessException, IOException { + JdbcInputFormat f = new JdbcInputFormat(); + when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); + f.setDbAccessor(mockDatabaseAccessor); + + JobConf conf = new JobConf(); + conf.set("mapred.input.dir", "/temp"); + InputSplit[] splits = f.getSplits(conf, 3); + + assertThat(splits, is(notNullValue())); + assertThat(splits.length, is(3)); + + assertThat(splits[0].getLength(), is(5L)); + } + + + @Test + public void testSplitLogic_withSpillOver() throws HiveJdbcDatabaseAccessException, IOException { + JdbcInputFormat f = new JdbcInputFormat(); + when(mockDatabaseAccessor.getTotalNumberOfRecords(any(Configuration.class))).thenReturn(15); + f.setDbAccessor(mockDatabaseAccessor); + + JobConf conf = new JobConf(); + conf.set("mapred.input.dir", "/temp"); + InputSplit[] splits = f.getSplits(conf, 6); + + assertThat(splits, is(notNullValue())); + assertThat(splits.length, is(6)); + + for (int i = 0; i < 3; i++) { + assertThat(splits[i].getLength(), is(3L)); + } + + for (int i = 3; i < 6; i++) { + assertThat(splits[i].getLength(), is(2L)); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java new file mode 100644 index 0000000..a59645d --- /dev/null +++ b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestQueryConditionBuilder.java @@ -0,0 +1,150 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.ql.plan.TableScanDesc; +import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Scanner; + +import static org.hamcrest.Matchers.equalToIgnoringWhiteSpace; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +public class TestQueryConditionBuilder { + + private static String condition1; + private static String condition2; + + + @BeforeClass + public static void setup() throws IOException { + condition1 = readFileContents("condition1.xml"); + condition2 = readFileContents("condition2.xml"); + } + + + private static String readFileContents(String name) throws IOException { + try (Scanner s = new Scanner(TestQueryConditionBuilder.class.getClassLoader().getResourceAsStream(name))) { + return s.useDelimiter("\\Z").next(); + } + } + + + @Test + public void testSimpleCondition_noTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("(visitor_id = 'x')"))); + } + + + @Test + public void testSimpleCondition_withTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=vid, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("(vid = 'x')"))); + } + + + @Test + public void testSimpleCondition_withDateType() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=vid:date, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); + } + + + @Test + public void testSimpleCondition_withVariedCaseMappings() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition1); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_ID,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), + "visitor_id=VID:date, sentiment=sentiment, tracking_id=tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("({d vid} = 'x')"))); + } + + + @Test + public void testMultipleConditions_noTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("((visitor_id = 'x') and (sentiment = 'y'))"))); + } + + + @Test + public void testMultipleConditions_withTranslation() { + Configuration conf = new Configuration(); + conf.set(TableScanDesc.FILTER_EXPR_CONF_STR, condition2); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition, is(equalToIgnoringWhiteSpace("((v = 'x') and (s = 'y'))"))); + } + + + @Test + public void testWithNullConf() { + String condition = QueryConditionBuilder.getInstance().buildCondition(null); + assertThat(condition, is(notNullValue())); + assertThat(condition.trim().isEmpty(), is(true)); + } + + + @Test + public void testWithUndefinedFilterExpr() { + Configuration conf = new Configuration(); + conf.set(serdeConstants.LIST_COLUMNS, "visitor_id,sentiment,tracking_id"); + conf.set(JdbcStorageConfig.COLUMN_MAPPING.getPropertyName(), "visitor_id=v,sentiment=s,tracking_id=t"); + String condition = QueryConditionBuilder.getInstance().buildCondition(conf); + + assertThat(condition, is(notNullValue())); + assertThat(condition.trim().isEmpty(), is(true)); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestSuite.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestSuite.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestSuite.java deleted file mode 100644 index df8eab7..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/TestSuite.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; -import org.junit.runners.Suite.SuiteClasses; - -import org.apache.hive.config.JdbcStorageConfigManagerTest; -import org.apache.hive.storage.jdbc.QueryConditionBuilderTest; -import org.apache.hive.storage.jdbc.dao.GenericJdbcDatabaseAccessorTest; - -@RunWith(Suite.class) -@SuiteClasses({ JdbcStorageConfigManagerTest.class, GenericJdbcDatabaseAccessorTest.class, - QueryConditionBuilderTest.class }) -public class TestSuite { -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java deleted file mode 100644 index b2442e8a..0000000 --- a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/GenericJdbcDatabaseAccessorTest.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.storage.jdbc.dao; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.equalToIgnoringCase; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.junit.Assert.assertThat; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; -import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; - -import java.util.List; -import java.util.Map; - -public class GenericJdbcDatabaseAccessorTest { - - @Test - public void testGetColumnNames_starQuery() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - List<String> columnNames = accessor.getColumnNames(conf); - - assertThat(columnNames, is(notNullValue())); - assertThat(columnNames.size(), is(equalTo(7))); - assertThat(columnNames.get(0), is(equalToIgnoringCase("strategy_id"))); - } - - - @Test - public void testGetColumnNames_fieldListQuery() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select name,referrer from test_strategy"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - List<String> columnNames = accessor.getColumnNames(conf); - - assertThat(columnNames, is(notNullValue())); - assertThat(columnNames.size(), is(equalTo(2))); - assertThat(columnNames.get(0), is(equalToIgnoringCase("name"))); - } - - - @Test(expected = HiveJdbcDatabaseAccessException.class) - public void testGetColumnNames_invalidQuery() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from invalid_strategy"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - @SuppressWarnings("unused") - List<String> columnNames = accessor.getColumnNames(conf); - } - - - @Test - public void testGetTotalNumberOfRecords() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - int numRecords = accessor.getTotalNumberOfRecords(conf); - - assertThat(numRecords, is(equalTo(5))); - } - - - @Test - public void testGetTotalNumberOfRecords_whereClause() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '5'"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - int numRecords = accessor.getTotalNumberOfRecords(conf); - - assertThat(numRecords, is(equalTo(1))); - } - - - @Test - public void testGetTotalNumberOfRecords_noRecords() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - int numRecords = accessor.getTotalNumberOfRecords(conf); - - assertThat(numRecords, is(equalTo(0))); - } - - - @Test(expected = HiveJdbcDatabaseAccessException.class) - public void testGetTotalNumberOfRecords_invalidQuery() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx where strategy_id = '5'"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - @SuppressWarnings("unused") - int numRecords = accessor.getTotalNumberOfRecords(conf); - } - - - @Test - public void testGetRecordIterator() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 0); - - assertThat(iterator, is(notNullValue())); - - int count = 0; - while (iterator.hasNext()) { - Map<String, Object> record = iterator.next(); - count++; - - assertThat(record, is(notNullValue())); - assertThat(record.size(), is(equalTo(7))); - assertThat(record.get("STRATEGY_ID"), is(equalTo(count))); - } - - assertThat(count, is(equalTo(2))); - iterator.close(); - } - - - @Test - public void testGetRecordIterator_offsets() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 2); - - assertThat(iterator, is(notNullValue())); - - int count = 0; - while (iterator.hasNext()) { - Map<String, Object> record = iterator.next(); - count++; - - assertThat(record, is(notNullValue())); - assertThat(record.size(), is(equalTo(7))); - assertThat(record.get("STRATEGY_ID"), is(equalTo(count + 2))); - } - - assertThat(count, is(equalTo(2))); - iterator.close(); - } - - - @Test - public void testGetRecordIterator_emptyResultSet() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); - - assertThat(iterator, is(notNullValue())); - assertThat(iterator.hasNext(), is(false)); - iterator.close(); - } - - - @Test - public void testGetRecordIterator_largeOffset() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 10, 25); - - assertThat(iterator, is(notNullValue())); - assertThat(iterator.hasNext(), is(false)); - iterator.close(); - } - - - @Test(expected = HiveJdbcDatabaseAccessException.class) - public void testGetRecordIterator_invalidQuery() throws HiveJdbcDatabaseAccessException { - Configuration conf = buildConfiguration(); - conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx"); - DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); - @SuppressWarnings("unused") - JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); - } - - - private Configuration buildConfiguration() { - String scriptPath = - GenericJdbcDatabaseAccessorTest.class.getClassLoader().getResource("test_script.sql") - .getPath(); - Configuration config = new Configuration(); - config.set(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "H2"); - config.set(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "org.h2.Driver"); - config.set(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc:h2:mem:test;MODE=MySQL;INIT=runscript from '" - + scriptPath + "'"); - config.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy"); - - return config; - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/TestGenericJdbcDatabaseAccessor.java ---------------------------------------------------------------------- diff --git a/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/TestGenericJdbcDatabaseAccessor.java b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/TestGenericJdbcDatabaseAccessor.java new file mode 100644 index 0000000..34f061e --- /dev/null +++ b/jdbc-handler/src/test/java/org/apache/hive/storage/jdbc/dao/TestGenericJdbcDatabaseAccessor.java @@ -0,0 +1,205 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.storage.jdbc.dao; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hive.storage.jdbc.conf.JdbcStorageConfig; +import org.apache.hive.storage.jdbc.exception.HiveJdbcDatabaseAccessException; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.equalToIgnoringCase; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +public class TestGenericJdbcDatabaseAccessor { + + @Test + public void testGetColumnNames_starQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + List<String> columnNames = accessor.getColumnNames(conf); + + assertThat(columnNames, is(notNullValue())); + assertThat(columnNames.size(), is(equalTo(7))); + assertThat(columnNames.get(0), is(equalToIgnoringCase("strategy_id"))); + } + + + @Test + public void testGetColumnNames_fieldListQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select name,referrer from test_strategy"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + List<String> columnNames = accessor.getColumnNames(conf); + + assertThat(columnNames, is(notNullValue())); + assertThat(columnNames.size(), is(equalTo(2))); + assertThat(columnNames.get(0), is(equalToIgnoringCase("name"))); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetColumnNames_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from invalid_strategy"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + List<String> columnNames = accessor.getColumnNames(conf); + } + + + @Test + public void testGetTotalNumberOfRecords() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(5))); + } + + + @Test + public void testGetTotalNumberOfRecords_whereClause() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '5'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(1))); + } + + + @Test + public void testGetTotalNumberOfRecords_noRecords() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + int numRecords = accessor.getTotalNumberOfRecords(conf); + + assertThat(numRecords, is(equalTo(0))); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetTotalNumberOfRecords_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx where strategy_id = '5'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + int numRecords = accessor.getTotalNumberOfRecords(conf); + } + + + @Test + public void testGetRecordIterator() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 0); + + assertThat(iterator, is(notNullValue())); + + int count = 0; + while (iterator.hasNext()) { + Map<String, Object> record = iterator.next(); + count++; + + assertThat(record, is(notNullValue())); + assertThat(record.size(), is(equalTo(7))); + assertThat(record.get("STRATEGY_ID"), is(equalTo(count))); + } + + assertThat(count, is(equalTo(2))); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_offsets() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 2, 2); + + assertThat(iterator, is(notNullValue())); + + int count = 0; + while (iterator.hasNext()) { + Map<String, Object> record = iterator.next(); + count++; + + assertThat(record, is(notNullValue())); + assertThat(record.size(), is(equalTo(7))); + assertThat(record.get("STRATEGY_ID"), is(equalTo(count + 2))); + } + + assertThat(count, is(equalTo(2))); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_emptyResultSet() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy where strategy_id = '25'"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); + + assertThat(iterator, is(notNullValue())); + assertThat(iterator.hasNext(), is(false)); + iterator.close(); + } + + + @Test + public void testGetRecordIterator_largeOffset() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 10, 25); + + assertThat(iterator, is(notNullValue())); + assertThat(iterator.hasNext(), is(false)); + iterator.close(); + } + + + @Test(expected = HiveJdbcDatabaseAccessException.class) + public void testGetRecordIterator_invalidQuery() throws HiveJdbcDatabaseAccessException { + Configuration conf = buildConfiguration(); + conf.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from strategyx"); + DatabaseAccessor accessor = DatabaseAccessorFactory.getAccessor(conf); + @SuppressWarnings("unused") + JdbcRecordIterator iterator = accessor.getRecordIterator(conf, 0, 2); + } + + + private Configuration buildConfiguration() { + String scriptPath = + TestGenericJdbcDatabaseAccessor.class.getClassLoader().getResource("test_script.sql") + .getPath(); + Configuration config = new Configuration(); + config.set(JdbcStorageConfig.DATABASE_TYPE.getPropertyName(), "H2"); + config.set(JdbcStorageConfig.JDBC_DRIVER_CLASS.getPropertyName(), "org.h2.Driver"); + config.set(JdbcStorageConfig.JDBC_URL.getPropertyName(), "jdbc:h2:mem:test;MODE=MySQL;INIT=runscript from '" + + scriptPath + "'"); + config.set(JdbcStorageConfig.QUERY.getPropertyName(), "select * from test_strategy"); + + return config; + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc/src/test/org/apache/hive/jdbc/HiveStatementTest.java ---------------------------------------------------------------------- diff --git a/jdbc/src/test/org/apache/hive/jdbc/HiveStatementTest.java b/jdbc/src/test/org/apache/hive/jdbc/HiveStatementTest.java deleted file mode 100644 index b06703d..0000000 --- a/jdbc/src/test/org/apache/hive/jdbc/HiveStatementTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.jdbc; - -import org.junit.Test; - -import java.sql.SQLException; - -import static org.junit.Assert.assertEquals; - -public class HiveStatementTest { - - @Test - public void testSetFetchSize1() throws SQLException { - HiveStatement stmt = new HiveStatement(null, null, null); - stmt.setFetchSize(123); - assertEquals(123, stmt.getFetchSize()); - } - - @Test - public void testSetFetchSize2() throws SQLException { - HiveStatement stmt = new HiveStatement(null, null, null); - int initial = stmt.getFetchSize(); - stmt.setFetchSize(0); - assertEquals(initial, stmt.getFetchSize()); - } - - @Test(expected = SQLException.class) - public void testSetFetchSize3() throws SQLException { - HiveStatement stmt = new HiveStatement(null, null, null); - stmt.setFetchSize(-1); - } - - @Test - public void testaddBatch() throws SQLException { - HiveStatement stmt = new HiveStatement(null, null, null); - try { - stmt.addBatch(null); - } catch (SQLException e) { - assertEquals("java.sql.SQLFeatureNotSupportedException: Method not supported", e.toString()); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java ---------------------------------------------------------------------- diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java b/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java new file mode 100644 index 0000000..eeb6b5d --- /dev/null +++ b/jdbc/src/test/org/apache/hive/jdbc/TestHiveStatement.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.jdbc; + +import org.junit.Test; + +import java.sql.SQLException; + +import static org.junit.Assert.assertEquals; + +public class TestHiveStatement { + + @Test + public void testSetFetchSize1() throws SQLException { + HiveStatement stmt = new HiveStatement(null, null, null); + stmt.setFetchSize(123); + assertEquals(123, stmt.getFetchSize()); + } + + @Test + public void testSetFetchSize2() throws SQLException { + HiveStatement stmt = new HiveStatement(null, null, null); + int initial = stmt.getFetchSize(); + stmt.setFetchSize(0); + assertEquals(initial, stmt.getFetchSize()); + } + + @Test(expected = SQLException.class) + public void testSetFetchSize3() throws SQLException { + HiveStatement stmt = new HiveStatement(null, null, null); + stmt.setFetchSize(-1); + } + + @Test + public void testaddBatch() throws SQLException { + HiveStatement stmt = new HiveStatement(null, null, null); + try { + stmt.addBatch(null); + } catch (SQLException e) { + assertEquals("java.sql.SQLFeatureNotSupportedException: Method not supported", e.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java b/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java deleted file mode 100644 index c278338..0000000 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializerTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.codehaus.jackson.annotate.JsonProperty; -import org.json.JSONException; -import org.junit.Test; -import org.skyscreamer.jsonassert.JSONAssert; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.*; - -public class JSONMessageDeserializerTest { - - public static class MyClass { - @JsonProperty - private int a; - @JsonProperty - private Map<String, String> map; - private long l; - private String shouldNotSerialize = "shouldNotSerialize"; - - //for jackson to instantiate - MyClass() { - } - - MyClass(int a, Map<String, String> map, long l) { - this.a = a; - this.map = map; - this.l = l; - } - - @JsonProperty - long getL() { - return l; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - MyClass myClass = (MyClass) o; - - if (a != myClass.a) - return false; - if (l != myClass.l) - return false; - if (!map.equals(myClass.map)) - return false; - return shouldNotSerialize.equals(myClass.shouldNotSerialize); - } - - @Override - public int hashCode() { - int result = a; - result = 31 * result + map.hashCode(); - result = 31 * result + (int) (l ^ (l >>> 32)); - result = 31 * result + shouldNotSerialize.hashCode(); - return result; - } - } - - @Test - public void shouldNotSerializePropertiesNotAnnotated() throws IOException, JSONException { - MyClass obj = new MyClass(Integer.MAX_VALUE, new HashMap<String, String>() {{ - put("a", "a"); - put("b", "b"); - }}, Long.MAX_VALUE); - String json = JSONMessageDeserializer.mapper.writeValueAsString(obj); - JSONAssert.assertEquals( - "{\"a\":2147483647,\"map\":{\"b\":\"b\",\"a\":\"a\"},\"l\":9223372036854775807}", json, - false); - } - - @Test - public void shouldDeserializeJsonStringToObject() throws IOException { - String json = "{\"a\":47,\"map\":{\"a\":\"a\",\"b\":\"a value for b\"},\"l\":98}"; - MyClass actual = JSONMessageDeserializer.mapper.readValue(json, MyClass.class); - MyClass expected = new MyClass(47, new HashMap<String, String>() {{ - put("a", "a"); - put("b", "a value for b"); - }}, 98L); - assertEquals(expected, actual); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java b/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java new file mode 100644 index 0000000..9e22d8f --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/messaging/json/TestJSONMessageDeserializer.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.messaging.json; + +import org.codehaus.jackson.annotate.JsonProperty; +import org.json.JSONException; +import org.junit.Test; +import org.skyscreamer.jsonassert.JSONAssert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TestJSONMessageDeserializer { + + public static class MyClass { + @JsonProperty + private int a; + @JsonProperty + private Map<String, String> map; + private long l; + private String shouldNotSerialize = "shouldNotSerialize"; + + //for jackson to instantiate + MyClass() { + } + + MyClass(int a, Map<String, String> map, long l) { + this.a = a; + this.map = map; + this.l = l; + } + + @JsonProperty + long getL() { + return l; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + MyClass myClass = (MyClass) o; + + if (a != myClass.a) + return false; + if (l != myClass.l) + return false; + if (!map.equals(myClass.map)) + return false; + return shouldNotSerialize.equals(myClass.shouldNotSerialize); + } + + @Override + public int hashCode() { + int result = a; + result = 31 * result + map.hashCode(); + result = 31 * result + (int) (l ^ (l >>> 32)); + result = 31 * result + shouldNotSerialize.hashCode(); + return result; + } + } + + @Test + public void shouldNotSerializePropertiesNotAnnotated() throws IOException, JSONException { + MyClass obj = new MyClass(Integer.MAX_VALUE, new HashMap<String, String>() {{ + put("a", "a"); + put("b", "b"); + }}, Long.MAX_VALUE); + String json = JSONMessageDeserializer.mapper.writeValueAsString(obj); + JSONAssert.assertEquals( + "{\"a\":2147483647,\"map\":{\"b\":\"b\",\"a\":\"a\"},\"l\":9223372036854775807}", json, + false); + } + + @Test + public void shouldDeserializeJsonStringToObject() throws IOException { + String json = "{\"a\":47,\"map\":{\"a\":\"a\",\"b\":\"a value for b\"},\"l\":98}"; + MyClass actual = JSONMessageDeserializer.mapper.readValue(json, MyClass.class); + MyClass expected = new MyClass(47, new HashMap<String, String>() {{ + put("a", "a"); + put("b", "a value for b"); + }}, 98L); + assertEquals(expected, actual); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/3681b859/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTrackerTest.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTrackerTest.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTrackerTest.java deleted file mode 100644 index ed0ebef..0000000 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/TaskTrackerTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.ql.exec.repl.bootstrap.load; - -import org.apache.hadoop.hive.ql.exec.Task; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.powermock.modules.junit4.PowerMockRunner; - -import java.io.Serializable; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -@RunWith(PowerMockRunner.class) - public class TaskTrackerTest { - @Mock - private Task<? extends Serializable> task; - - @Test - public void taskTrackerCompositionInitializesTheMaxTasksCorrectly() { - TaskTracker taskTracker = new TaskTracker(1); - assertTrue(taskTracker.canAddMoreTasks()); - taskTracker.addTask(task); - assertFalse(taskTracker.canAddMoreTasks()); - - TaskTracker taskTracker2 = new TaskTracker(taskTracker); - assertFalse(taskTracker2.canAddMoreTasks()); - } -} \ No newline at end of file
