Repository: lens Updated Branches: refs/heads/master 87049563a -> b71be2dc9
LENS-869 : Fix DefaultEstimatedQueryCollection.remove method Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/b71be2dc Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/b71be2dc Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/b71be2dc Branch: refs/heads/master Commit: b71be2dc906688b6f283bb16afa778d19fc1c0bf Parents: 8704956 Author: Rajat Khandelwal <pro...@apache.org> Authored: Wed Nov 25 18:39:07 2015 +0530 Committer: Amareshwari Sriramadasu <amareshw...@apache.org> Committed: Wed Nov 25 18:39:07 2015 +0530 ---------------------------------------------------------------------- .../lens/cli/TestLensNativeTableCommands.java | 6 +- .../org/apache/lens/driver/hive/HiveDriver.java | 6 +- .../lens/server/api/LensServerAPITestUtil.java | 47 +++ .../api/query/TestAbstractQueryContext.java | 2 +- .../lens/server/api/util/TestLensUtil.java | 24 -- .../org/apache/lens/server/LensServerConf.java | 1 - .../server/query/QueryExecutionServiceImpl.java | 18 +- .../DefaultEstimatedQueryCollection.java | 4 +- .../query/collect/DefaultQueryCollection.java | 2 + .../ThreadSafeEstimatedQueryCollection.java | 13 +- .../org/apache/lens/server/LensJerseyTest.java | 8 +- .../apache/lens/server/LensServerTestUtil.java | 298 +++++++++++++++++++ .../org/apache/lens/server/LensTestUtil.java | 298 ------------------- .../org/apache/lens/server/TestServerMode.java | 2 +- .../apache/lens/server/TestServerRestart.java | 14 +- .../server/metastore/TestMetastoreService.java | 4 +- .../metrics/TestResourceMethodMetrics.java | 8 +- .../lens/server/query/TestQueryConstraints.java | 285 ++++++++++++++++++ .../server/query/TestQueryEndEmailNotifier.java | 10 +- .../lens/server/query/TestQueryService.java | 53 ++-- .../lens/server/query/TestResultFormatting.java | 38 ++- .../session/TestDatabaseResourceService.java | 4 +- .../server/session/TestSessionClassLoaders.java | 4 +- 23 files changed, 746 insertions(+), 403 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java ---------------------------------------------------------------------- diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java index 40a473a..d453803 100644 --- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java +++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensNativeTableCommands.java @@ -20,7 +20,7 @@ package org.apache.lens.cli; import org.apache.lens.cli.commands.LensNativeTableCommands; import org.apache.lens.client.LensClient; -import org.apache.lens.server.LensTestUtil; +import org.apache.lens.server.LensServerTestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +50,7 @@ public class TestLensNativeTableCommands extends LensCliApplicationTest { LOG.debug("Starting to test nativetable commands"); String tblList = command.showNativeTables(); Assert.assertFalse(tblList.contains("test_native_table_command")); - LensTestUtil.createHiveTable("test_native_table_command"); + LensServerTestUtil.createHiveTable("test_native_table_command"); tblList = command.showNativeTables(); Assert.assertTrue(tblList.contains("test_native_table_command")); @@ -61,7 +61,7 @@ public class TestLensNativeTableCommands extends LensCliApplicationTest { Assert.assertTrue(desc.contains("MANAGED_TABLE")); Assert.assertTrue(desc.contains("test.hive.table.prop")); } finally { - LensTestUtil.dropHiveTable("test_native_table_command"); + LensServerTestUtil.dropHiveTable("test_native_table_command"); } } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java index c96ef20..19c4793 100644 --- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java +++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java @@ -95,7 +95,7 @@ public class HiveDriver implements LensDriver { public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW"; public static final String SESSION_KEY_DELIMITER = "."; - private static final String QUERY_LAUNCHIG_CONSTRAINT_FACTORIES_KEY + public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = "lens.driver.hive.query.launching.constraint.factories"; private static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY @@ -143,7 +143,7 @@ public class HiveDriver implements LensDriver { private DriverQueryHook queryHook; @Getter - private ImmutableSet<QueryLaunchingConstraint> queryConstraints; + protected ImmutableSet<QueryLaunchingConstraint> queryConstraints; private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies; private String sessionDbKey(String sessionHandle, String database) { @@ -364,7 +364,7 @@ public class HiveDriver implements LensDriver { } catch (InstantiationException | IllegalAccessException e) { throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e); } - queryConstraints = getImplementations(QUERY_LAUNCHIG_CONSTRAINT_FACTORIES_KEY, driverConf); + queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf); selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf); } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server-api/src/test/java/org/apache/lens/server/api/LensServerAPITestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/LensServerAPITestUtil.java b/lens-server-api/src/test/java/org/apache/lens/server/api/LensServerAPITestUtil.java new file mode 100644 index 0000000..84139a4 --- /dev/null +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/LensServerAPITestUtil.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api; + +import org.apache.lens.api.LensConf; + +import org.apache.hadoop.conf.Configuration; + +public class LensServerAPITestUtil { + private LensServerAPITestUtil() { + + } + + public static Configuration getConfiguration(Object... args) { + Configuration conf = new Configuration(false); + assert (args.length % 2 == 0); + for (int i = 0; i < args.length; i += 2) { + conf.set(args[i].toString(), args[i + 1].toString()); + } + return conf; + } + + public static LensConf getLensConf(Object... args) { + assert (args.length % 2 == 0); + LensConf conf = new LensConf(); + for (int i = 0; i < args.length; i += 2) { + conf.addProperty(args[i], args[i + 1]); + } + return conf; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java index e41f2f4..02b652e 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestAbstractQueryContext.java @@ -20,7 +20,7 @@ package org.apache.lens.server.api.query; import static org.apache.lens.api.Priority.HIGH; import static org.apache.lens.server.api.LensConfConstants.*; -import static org.apache.lens.server.api.util.TestLensUtil.getConfiguration; +import static org.apache.lens.server.api.LensServerAPITestUtil.getConfiguration; import static org.testng.Assert.*; http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server-api/src/test/java/org/apache/lens/server/api/util/TestLensUtil.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/util/TestLensUtil.java b/lens-server-api/src/test/java/org/apache/lens/server/api/util/TestLensUtil.java index a6acb7d..caee12f 100644 --- a/lens-server-api/src/test/java/org/apache/lens/server/api/util/TestLensUtil.java +++ b/lens-server-api/src/test/java/org/apache/lens/server/api/util/TestLensUtil.java @@ -18,14 +18,8 @@ */ package org.apache.lens.server.api.util; -import static org.testng.Assert.assertEquals; - import java.io.IOException; -import org.apache.lens.api.LensConf; - -import org.apache.hadoop.conf.Configuration; - import org.testng.Assert; import org.testng.annotations.Test; @@ -61,22 +55,4 @@ public class TestLensUtil { } Assert.assertEquals(LensUtil.getCauseMessage(th), "run time exception"); } - - public static Configuration getConfiguration(Object... args) { - Configuration conf = new Configuration(); - Assert.assertEquals(args.length % 2, 0, "Odd number of arguments not supported"); - for (int i = 0; i < args.length; i += 2) { - conf.set(args[i].toString(), args[i + 1].toString()); - } - return conf; - } - - public static LensConf getLensConf(Object... args) { - assertEquals(args.length % 2, 0); - LensConf conf = new LensConf(); - for (int i = 0; i < args.length; i += 2) { - conf.addProperty(args[i], args[i + 1]); - } - return conf; - } } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java index 6db720d..e977ebd 100644 --- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java +++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java @@ -52,7 +52,6 @@ public final class LensServerConf { OVERRIDING_CONF_FOR_DRIVER.set(prop.getKey(), prop.getValue()); } } - } } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java index fdc8bfd..941b1e5 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java @@ -86,9 +86,7 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; -import lombok.Getter; -import lombok.NonNull; -import lombok.ToString; +import lombok.*; import lombok.extern.slf4j.Slf4j; /** @@ -141,6 +139,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE /** * The launched queries. */ + @Getter private EstimatedQueryCollection launchedQueries; /** @@ -2569,6 +2568,19 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE return finishedQueries.size(); } + @Data + @AllArgsConstructor + public static class QueryCount { + long running, queued, waiting; + } + + public QueryCount getQueryCountSnapshot() { + removalFromLaunchedQueriesLock.lock(); + QueryCount count = new QueryCount(getRunningQueriesCount(), getQueuedQueriesCount(), getWaitingQueriesCount()); + removalFromLaunchedQueriesLock.unlock(); + return count; + } + /** * Handle driver session start. * http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java index 908b86f..4e5aa68 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java @@ -37,6 +37,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import lombok.NonNull; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; /** @@ -48,6 +49,7 @@ import lombok.extern.slf4j.Slf4j; * */ @Slf4j +@ToString public class DefaultEstimatedQueryCollection implements EstimatedQueryCollection { private final QueryCollection queries; @@ -109,7 +111,7 @@ public class DefaultEstimatedQueryCollection implements EstimatedQueryCollection */ @Override public boolean remove(QueryContext query) { - this.queriesByDriver.remove(query.getSelectedDriver()); + this.queriesByDriver.remove(query.getSelectedDriver(), query); return this.queries.remove(query); } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java index 844237a..6f4230b 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java @@ -27,6 +27,7 @@ import org.apache.commons.collections.map.MultiValueMap; import com.google.common.collect.Sets; import lombok.NonNull; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; /** @@ -36,6 +37,7 @@ import lombok.extern.slf4j.Slf4j; * @see QueryCollection */ @Slf4j +@ToString public class DefaultQueryCollection implements QueryCollection { private final Set<QueryContext> queries; http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java ---------------------------------------------------------------------- diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java index ca24a8b..0e68395 100644 --- a/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java +++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java @@ -50,22 +50,22 @@ public class ThreadSafeEstimatedQueryCollection implements EstimatedQueryCollect } @Override - public synchronized QueryCost getTotalQueryCost(String user) { + public synchronized QueryCost getTotalQueryCost(String user) { return this.estimatedQueries.getTotalQueryCost(user); } @Override - public synchronized Set<QueryContext> getQueries() { + public synchronized Set<QueryContext> getQueries() { return this.estimatedQueries.getQueries(); } @Override - public synchronized Set<QueryContext> getQueries(String user) { + public synchronized Set<QueryContext> getQueries(String user) { return this.estimatedQueries.getQueries(user); } @Override - public synchronized int getQueriesCount() { + public synchronized int getQueriesCount() { return this.estimatedQueries.getQueriesCount(); } @@ -93,4 +93,9 @@ public class ThreadSafeEstimatedQueryCollection implements EstimatedQueryCollect public synchronized boolean removeAll(Set<QueryContext> queries) { return this.estimatedQueries.removeAll(queries); } + + @Override + public synchronized String toString() { + return getClass().getSimpleName() + "(estimatedQueries=" + this.estimatedQueries + ")"; + } } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index 3dad050..8ba9353 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -18,6 +18,10 @@ */ package org.apache.lens.server; +import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS; +import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS_2; +import static org.apache.lens.server.LensServerTestUtil.createTestDatabaseResources; + import static org.testng.Assert.*; import java.io.IOException; @@ -123,10 +127,10 @@ public abstract class LensJerseyTest extends JerseyTest { hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3); - LensTestUtil.createTestDatabaseResources(new String[]{LensTestUtil.DB_WITH_JARS, LensTestUtil.DB_WITH_JARS_2}, + createTestDatabaseResources(new String[]{DB_WITH_JARS, DB_WITH_JARS_2}, hiveConf); - LensServices.get().init(LensServerConf.getHiveConf()); + LensServices.get().init(getServerConf()); LensServices.get().start(); // Check if mock service is started http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java new file mode 100644 index 0000000..57bedee --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -0,0 +1,298 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.*; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.GenericType; +import javax.ws.rs.core.MediaType; + +import org.apache.lens.api.LensConf; +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.query.LensQuery; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.api.query.QueryStatus; +import org.apache.lens.api.result.LensAPIResult; +import org.apache.lens.server.api.LensConfConstants; + +import org.apache.commons.io.FileUtils; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; + +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; + +import lombok.extern.slf4j.Slf4j; + +/** + * The Class LensServerTestUtil. + */ +@Slf4j +public final class LensServerTestUtil { + + public static final String DB_WITH_JARS = "test_db_static_jars"; + public static final String DB_WITH_JARS_2 = "test_db_static_jars_2"; + private LensServerTestUtil() { + + } + + /** + * Creates the table. + * + * @param tblName the tbl name + * @param parent the parent + * @param lensSessionId the lens session id + * @param schemaStr the schema string, with surrounding parenthesis. + * @throws InterruptedException the interrupted exception + */ + public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, String schemaStr) + throws InterruptedException { + LensConf conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + final WebTarget target = parent.path("queryapi/queries"); + + final FormDataMultiPart mp = new FormDataMultiPart(); + String createTable = "CREATE TABLE IF NOT EXISTS " + tblName + schemaStr; + + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, + MediaType.APPLICATION_XML_TYPE)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, + MediaType.APPLICATION_XML_TYPE)); + + final QueryHandle handle = target.request() + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), + new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); + // wait till the query finishes + LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() + .get(LensQuery.class); + QueryStatus stat = ctx.getStatus(); + while (!stat.finished()) { + ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); + stat = ctx.getStatus(); + Thread.sleep(1000); + } + final String debugHelpMsg = "Query Handle:"+ctx.getQueryHandleString(); + assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg); + assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg); + assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg); + assertTrue(ctx.getDriverStartTime() > 0, debugHelpMsg); + assertTrue(ctx.getDriverFinishTime() > 0, debugHelpMsg); + assertTrue(ctx.getFinishTime() > 0, debugHelpMsg); + } + + public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId) + throws InterruptedException { + createTable(tblName, parent, lensSessionId, "(ID INT, IDSTR STRING)"); + } + + public static void loadData(String tblName, final String testDataFile, WebTarget parent, + LensSessionHandle lensSessionId) throws InterruptedException { + LensConf conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + final WebTarget target = parent.path("queryapi/queries"); + + final FormDataMultiPart mp = new FormDataMultiPart(); + String dataLoad = "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE " + tblName; + + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, + MediaType.APPLICATION_XML_TYPE)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), dataLoad)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, + MediaType.APPLICATION_XML_TYPE)); + + final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), + new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); + + // wait till the query finishes + LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() + .get(LensQuery.class); + QueryStatus stat = ctx.getStatus(); + while (!stat.finished()) { + ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); + stat = ctx.getStatus(); + Thread.sleep(1000); + } + assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + /** + * Load data. + * + * @param tblName the tbl name + * @param testDataFile the test data file + * @param parent the parent + * @param lensSessionId the lens session id + * @throws InterruptedException the interrupted exception + */ + public static void loadDataFromClasspath(String tblName, final String testDataFile, WebTarget parent, + LensSessionHandle lensSessionId) throws InterruptedException { + + String absolutePath = LensServerTestUtil.class.getClassLoader().getResource(testDataFile).getPath(); + loadData(tblName, absolutePath, parent, lensSessionId); + } + + /** + * Drop table. + * + * @param tblName the tbl name + * @param parent the parent + * @param lensSessionId the lens session id + * @throws InterruptedException the interrupted exception + */ + public static void dropTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId) + throws InterruptedException { + LensConf conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); + dropTableWithConf(tblName, parent, lensSessionId, conf); + } + + /** + * Drop table with conf passed. + * + * @param tblName the tbl name + * @param parent the parent + * @param lensSessionId the lens session id + * @param conf the query conf + * + * @throws InterruptedException + */ + public static void dropTableWithConf(String tblName, WebTarget parent, LensSessionHandle lensSessionId, + LensConf conf) throws InterruptedException { + final WebTarget target = parent.path("queryapi/queries"); + + final FormDataMultiPart mp = new FormDataMultiPart(); + String createTable = "DROP TABLE IF EXISTS " + tblName; + + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, + MediaType.APPLICATION_XML_TYPE)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, + MediaType.APPLICATION_XML_TYPE)); + + final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), + new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); + + // wait till the query finishes + LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() + .get(LensQuery.class); + QueryStatus stat = ctx.getStatus(); + while (!stat.finished()) { + ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); + stat = ctx.getStatus(); + Thread.sleep(1000); + } + assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); + } + + /** + * Creates the hive table. + * + * @param tableName the table name + * @throws HiveException the hive exception + */ + public static void createHiveTable(String tableName) throws HiveException { + List<FieldSchema> columns = new ArrayList<FieldSchema>(); + columns.add(new FieldSchema("col1", "string", "")); + List<FieldSchema> partCols = new ArrayList<FieldSchema>(); + partCols.add(new FieldSchema("pcol1", "string", "")); + Map<String, String> params = new HashMap<String, String>(); + params.put("test.hive.table.prop", "tvalue"); + Table tbl = Hive.get().newTable(tableName); + tbl.setTableType(TableType.MANAGED_TABLE); + tbl.getTTable().getSd().setCols(columns); + tbl.setPartCols(partCols); + tbl.getTTable().getParameters().putAll(params); + Hive.get().createTable(tbl); + } + + /** + * Drop hive table. + * + * @param tableName the table name + * @throws HiveException the hive exception + */ + public static void dropHiveTable(String tableName) throws HiveException { + Hive.get().dropTable(tableName); + } + + public static void createTestDatabaseResources(String[] testDatabases, HiveConf conf) throws Exception { + File srcJarDir = new File("target/testjars/"); + if (!srcJarDir.exists()) { + // nothing to setup + return; + } + File resDir = new File("target/resources"); + if (!resDir.exists()) { + resDir.mkdir(); + } + + // Create databases and resource dirs + Hive hive = Hive.get(conf); + File testJarFile = new File("target/testjars/test.jar"); + File serdeJarFile = new File("target/testjars/serde.jar"); + for (String db : testDatabases) { + Database database = new Database(); + database.setName(db); + hive.createDatabase(database, true); + File dbDir = new File(resDir, db); + if (!dbDir.exists()) { + dbDir.mkdir(); + } + // Add a jar in the directory + try { + + String[] jarOrder = { + "x_" + db + ".jar", + "y_" + db + ".jar", + "z_" + db + ".jar", + "serde.jar", + }; + + // Jar order is -> z, y, x, File listing order is x, y, z + // We are explicitly specifying jar order + FileUtils.writeLines(new File(dbDir, "jar_order"), Arrays.asList(jarOrder[2], jarOrder[1], + jarOrder[0], jarOrder[3])); + + FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[0])); + FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[1])); + FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[2])); + FileUtils.copyFile(serdeJarFile, new File(dbDir, jarOrder[3])); + } catch (FileNotFoundException fnf) { + log.error("File not found.", fnf); + } + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java deleted file mode 100644 index 62e9954..0000000 --- a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java +++ /dev/null @@ -1,298 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.lens.server; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.*; - -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.WebTarget; -import javax.ws.rs.core.GenericType; -import javax.ws.rs.core.MediaType; - -import org.apache.lens.api.LensConf; -import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.query.LensQuery; -import org.apache.lens.api.query.QueryHandle; -import org.apache.lens.api.query.QueryStatus; -import org.apache.lens.api.result.LensAPIResult; -import org.apache.lens.server.api.LensConfConstants; - -import org.apache.commons.io.FileUtils; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.TableType; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; - -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; - -import lombok.extern.slf4j.Slf4j; - -/** - * The Class LensTestUtil. - */ -@Slf4j -public final class LensTestUtil { - - public static final String DB_WITH_JARS = "test_db_static_jars"; - public static final String DB_WITH_JARS_2 = "test_db_static_jars_2"; - private LensTestUtil() { - - } - - /** - * Creates the table. - * - * @param tblName the tbl name - * @param parent the parent - * @param lensSessionId the lens session id - * @param schemaStr the schema string, with surrounding parenthesis. - * @throws InterruptedException the interrupted exception - */ - public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId, String schemaStr) - throws InterruptedException { - LensConf conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - final WebTarget target = parent.path("queryapi/queries"); - - final FormDataMultiPart mp = new FormDataMultiPart(); - String createTable = "CREATE TABLE IF NOT EXISTS " + tblName + schemaStr; - - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - - final QueryHandle handle = target.request() - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - // wait till the query finishes - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - final String debugHelpMsg = "Query Handle:"+ctx.getQueryHandleString(); - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg); - assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg); - assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg); - assertTrue(ctx.getDriverStartTime() > 0, debugHelpMsg); - assertTrue(ctx.getDriverFinishTime() > 0, debugHelpMsg); - assertTrue(ctx.getFinishTime() > 0, debugHelpMsg); - } - - public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId) - throws InterruptedException { - createTable(tblName, parent, lensSessionId, "(ID INT, IDSTR STRING)"); - } - - public static void loadData(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId) throws InterruptedException { - LensConf conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - final WebTarget target = parent.path("queryapi/queries"); - - final FormDataMultiPart mp = new FormDataMultiPart(); - String dataLoad = "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE " + tblName; - - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), dataLoad)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - - final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - - // wait till the query finishes - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - } - /** - * Load data. - * - * @param tblName the tbl name - * @param testDataFile the test data file - * @param parent the parent - * @param lensSessionId the lens session id - * @throws InterruptedException the interrupted exception - */ - public static void loadDataFromClasspath(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId) throws InterruptedException { - - String absolutePath = LensTestUtil.class.getClassLoader().getResource(testDataFile).getPath(); - loadData(tblName, absolutePath, parent, lensSessionId); - } - - /** - * Drop table. - * - * @param tblName the tbl name - * @param parent the parent - * @param lensSessionId the lens session id - * @throws InterruptedException the interrupted exception - */ - public static void dropTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId) - throws InterruptedException { - LensConf conf = new LensConf(); - conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - dropTableWithConf(tblName, parent, lensSessionId, conf); - } - - /** - * Drop table with conf passed. - * - * @param tblName the tbl name - * @param parent the parent - * @param lensSessionId the lens session id - * @param conf the query conf - * - * @throws InterruptedException - */ - public static void dropTableWithConf(String tblName, WebTarget parent, LensSessionHandle lensSessionId, - LensConf conf) throws InterruptedException { - final WebTarget target = parent.path("queryapi/queries"); - - final FormDataMultiPart mp = new FormDataMultiPart(); - String createTable = "DROP TABLE IF EXISTS " + tblName; - - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - MediaType.APPLICATION_XML_TYPE)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), createTable)); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - MediaType.APPLICATION_XML_TYPE)); - - final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - - // wait till the query finishes - LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() - .get(LensQuery.class); - QueryStatus stat = ctx.getStatus(); - while (!stat.finished()) { - ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class); - stat = ctx.getStatus(); - Thread.sleep(1000); - } - assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); - } - - /** - * Creates the hive table. - * - * @param tableName the table name - * @throws HiveException the hive exception - */ - public static void createHiveTable(String tableName) throws HiveException { - List<FieldSchema> columns = new ArrayList<FieldSchema>(); - columns.add(new FieldSchema("col1", "string", "")); - List<FieldSchema> partCols = new ArrayList<FieldSchema>(); - partCols.add(new FieldSchema("pcol1", "string", "")); - Map<String, String> params = new HashMap<String, String>(); - params.put("test.hive.table.prop", "tvalue"); - Table tbl = Hive.get().newTable(tableName); - tbl.setTableType(TableType.MANAGED_TABLE); - tbl.getTTable().getSd().setCols(columns); - tbl.setPartCols(partCols); - tbl.getTTable().getParameters().putAll(params); - Hive.get().createTable(tbl); - } - - /** - * Drop hive table. - * - * @param tableName the table name - * @throws HiveException the hive exception - */ - public static void dropHiveTable(String tableName) throws HiveException { - Hive.get().dropTable(tableName); - } - - public static void createTestDatabaseResources(String[] testDatabases, HiveConf conf) throws Exception { - File srcJarDir = new File("target/testjars/"); - if (!srcJarDir.exists()) { - // nothing to setup - return; - } - File resDir = new File("target/resources"); - if (!resDir.exists()) { - resDir.mkdir(); - } - - // Create databases and resource dirs - Hive hive = Hive.get(conf); - File testJarFile = new File("target/testjars/test.jar"); - File serdeJarFile = new File("target/testjars/serde.jar"); - for (String db : testDatabases) { - Database database = new Database(); - database.setName(db); - hive.createDatabase(database, true); - File dbDir = new File(resDir, db); - if (!dbDir.exists()) { - dbDir.mkdir(); - } - // Add a jar in the directory - try { - - String[] jarOrder = { - "x_" + db + ".jar", - "y_" + db + ".jar", - "z_" + db + ".jar", - "serde.jar", - }; - - // Jar order is -> z, y, x, File listing order is x, y, z - // We are explicitly specifying jar order - FileUtils.writeLines(new File(dbDir, "jar_order"), Arrays.asList(jarOrder[2], jarOrder[1], - jarOrder[0], jarOrder[3])); - - FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[0])); - FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[1])); - FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[2])); - FileUtils.copyFile(serdeJarFile, new File(dbDir, jarOrder[3])); - } catch (FileNotFoundException fnf) { - log.error("File not found.", fnf); - } - } - } -} http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java index def5683..75f21e1 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java @@ -61,7 +61,7 @@ public class TestServerMode extends LensAllApplicationJerseyTest { @BeforeTest public void setUp() throws Exception { super.setUp(); - LensTestUtil.createTable("test_table", target(), RestAPITestUtil.openFooBarSession(target())); + LensServerTestUtil.createTable("test_table", target(), RestAPITestUtil.openFooBarSession(target())); } /* http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java index 49d62f9..7b6c560 100644 --- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java +++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java @@ -18,6 +18,8 @@ */ package org.apache.lens.server; +import static org.apache.lens.server.LensServerTestUtil.createTable; +import static org.apache.lens.server.LensServerTestUtil.loadData; import static org.apache.lens.server.common.RestAPITestUtil.execute; import static org.testng.Assert.assertEquals; @@ -135,8 +137,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { createRestartTestDataFile(); // Create a test table - LensTestUtil.createTable("test_server_restart", target(), lensSessionId); - LensTestUtil.loadData("test_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), lensSessionId); + createTable("test_server_restart", target(), lensSessionId); + loadData("test_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), lensSessionId); log.info("Loaded data"); // test post execute op @@ -211,7 +213,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { } } log.info("End server restart test"); - LensTestUtil.dropTable("test_server_restart", target(), lensSessionId); + LensServerTestUtil.dropTable("test_server_restart", target(), lensSessionId); queryService.closeSession(lensSessionId); } @@ -242,8 +244,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { log.info("@@ Added resource {}", dataFile.toURI()); // Create a test table - LensTestUtil.createTable("test_hive_server_restart", target(), lensSessionId); - LensTestUtil.loadData("test_hive_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), + createTable("test_hive_server_restart", target(), lensSessionId); + loadData("test_hive_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), lensSessionId); log.info("Loaded data"); @@ -354,7 +356,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest { // "Expected to be successful " + handle); log.info("End hive server restart test"); - LensTestUtil.dropTable("test_hive_server_restart", target(), lensSessionId); + LensServerTestUtil.dropTable("test_hive_server_restart", target(), lensSessionId); queryService.closeSession(lensSessionId); } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java index 3bc692e..e0c0923 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java @@ -48,8 +48,8 @@ import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.cube.metadata.*; import org.apache.lens.cube.metadata.ExprColumn.ExprSpec; import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; -import org.apache.lens.server.LensTestUtil; import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.api.util.LensUtil; @@ -2418,7 +2418,7 @@ public class TestMetastoreService extends LensJerseyTest { // create hive table String tableName = "test_simple_table"; SessionState.get().setCurrentDatabase(DB); - LensTestUtil.createHiveTable(tableName); + LensServerTestUtil.createHiveTable(tableName); WebTarget target = target().path("metastore").path("nativetables"); // get all native tables http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java index 94e8069..515ac13 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java +++ b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java @@ -41,8 +41,8 @@ import org.apache.lens.api.query.QueryHandle; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.server.LensAllApplicationJerseyTest; import org.apache.lens.server.LensApplication; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; -import org.apache.lens.server.LensTestUtil; import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.api.metrics.MethodMetrics; import org.apache.lens.server.api.metrics.MetricsService; @@ -87,16 +87,16 @@ public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest { } private void createTable(String tblName) throws InterruptedException { - LensTestUtil.createTable(tblName, target(), lensSessionId); + LensServerTestUtil.createTable(tblName, target(), lensSessionId); } private void loadData(String tblName, final String testDataFile) throws InterruptedException { - LensTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); + LensServerTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); } @AfterTest public void tearDown() throws Exception { - LensTestUtil.dropTable(TestQueryService.TEST_TABLE, target(), lensSessionId); + LensServerTestUtil.dropTable(TestQueryService.TEST_TABLE, target(), lensSessionId); metastoreService.closeSession(lensSessionId); super.tearDown(); } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java new file mode 100644 index 0000000..eb94c89 --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryConstraints.java @@ -0,0 +1,285 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.query; + +import static org.apache.lens.server.api.LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY; +import static org.apache.lens.server.api.util.LensUtil.getImplementations; + +import static org.testng.Assert.*; + +import java.util.*; + +import javax.ws.rs.core.Application; + +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.jaxb.LensJAXBContextResolver; +import org.apache.lens.api.query.QueryHandle; +import org.apache.lens.driver.hive.HiveDriver; +import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServerConf; +import org.apache.lens.server.LensServerTestUtil; +import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.LensServerAPITestUtil; +import org.apache.lens.server.api.driver.DriverSelector; +import org.apache.lens.server.api.driver.LensDriver; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.metrics.MetricsService; +import org.apache.lens.server.api.query.AbstractQueryContext; +import org.apache.lens.server.api.query.QueryExecutionService; +import org.apache.lens.server.common.RestAPITestUtil; +import org.apache.lens.server.common.TestResourceFile; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; + +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.test.TestProperties; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; + +import com.beust.jcommander.internal.Lists; +import com.google.common.base.Optional; +import lombok.extern.slf4j.Slf4j; + +/** + * The Class TestQueryService. + */ +@Slf4j +@Test(groups = "two-working-drivers", dependsOnGroups = "filter-test") +public class TestQueryConstraints extends LensJerseyTest { + private HiveConf serverConf; + + public static class MockHiveDriverBase extends HiveDriver { + + private final Configuration customConf; + + /** + * Instantiates a new hive driver. + * + * @throws LensException the lens exception + */ + public MockHiveDriverBase() throws LensException { + customConf = new Configuration(); + customConf.setInt("driver.max.concurrent.launched.queries", 2); + customConf.set(HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, + "org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory"); + } + + @Override + public void configure(Configuration conf) throws LensException { + super.configure(conf); + queryConstraints = getImplementations(HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, customConf); + } + } + + public static class HiveDriver1 extends MockHiveDriverBase { + + /** + * Instantiates a new hive driver. + * + * @throws LensException the lens exception + */ + public HiveDriver1() throws LensException { + + } + } + + public static class HiveDriver2 extends MockHiveDriverBase { + + /** + * Instantiates a new hive driver. + * + * @throws LensException the lens exception + */ + public HiveDriver2() throws LensException { + } + } + + public static class RoundRobinSelector implements DriverSelector { + int counter = 0; + + @Override + public LensDriver select(AbstractQueryContext ctx, Configuration conf) { + final Collection<LensDriver> drivers = ctx.getDriverContext().getDriversWithValidQueryCost(); + LensDriver driver = drivers.toArray(new LensDriver[drivers.size()])[counter]; + counter = (counter + 1) % 2; + return driver; + } + } + + /** The query service. */ + QueryExecutionServiceImpl queryService; + + /** The metrics svc. */ + MetricsService metricsSvc; + + /** The lens session id. */ + LensSessionHandle lensSessionId; + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#setUp() + */ + @BeforeTest + public void setUp() throws Exception { + super.setUp(); + queryService = LensServices.get().getService(QueryExecutionService.NAME); + metricsSvc = LensServices.get().getService(MetricsService.NAME); + Map<String, String> sessionConf = new HashMap<>(); + sessionConf.put("test.session.key", "svalue"); + lensSessionId = queryService.openSession("foo@localhost", "bar", sessionConf); // @localhost should be removed + // automatically + createTable(TEST_TABLE); + loadData(TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue()); + } + + @Override + public HiveConf getServerConf() { + if (serverConf == null) { + serverConf = new HiveConf(super.getServerConf()); + serverConf.set(LensConfConstants.DRIVER_CLASSES, + HiveDriver1.class.getName() + "," + HiveDriver2.class.getName()); + serverConf.set("lens.server.driver.selector.class", RoundRobinSelector.class.getName()); + LensServerConf.getConfForDrivers().addResource(serverConf); + } + return serverConf; + } + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#tearDown() + */ + @AfterTest + public void tearDown() throws Exception { + dropTable(TEST_TABLE); + queryService.closeSession(lensSessionId); + for (LensDriver driver : queryService.getDrivers()) { + if (driver instanceof HiveDriver) { + assertFalse(((HiveDriver) driver).hasLensSession(lensSessionId)); + } + } + super.tearDown(); + } + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#configure() + */ + @Override + protected Application configure() { + enable(TestProperties.LOG_TRAFFIC); + enable(TestProperties.DUMP_ENTITY); + return new TestQueryService.QueryServiceTestApp(); + } + + /* + * (non-Javadoc) + * + * @see org.glassfish.jersey.test.JerseyTest#configureClient(org.glassfish.jersey.client.ClientConfig) + */ + @Override + protected void configureClient(ClientConfig config) { + config.register(MultiPartFeature.class); + config.register(LensJAXBContextResolver.class); + } + + /** The test table. */ + public static final String TEST_TABLE = "TEST_TABLE"; + + /** + * Creates the table. + * + * @param tblName the tbl name + * @throws InterruptedException the interrupted exception + */ + private void createTable(String tblName) throws InterruptedException { + LensServerTestUtil.createTable(tblName, target(), lensSessionId); + } + + /** + * Load data. + * + * @param tblName the tbl name + * @param testDataFile the test data file + * @throws InterruptedException the interrupted exception + */ + private void loadData(String tblName, final String testDataFile) throws InterruptedException { + LensServerTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); + } + + /** + * Drop table. + * + * @param tblName the tbl name + * @throws InterruptedException the interrupted exception + */ + private void dropTable(String tblName) throws InterruptedException { + LensServerTestUtil.dropTable(tblName, target(), lensSessionId); + } + + @Test + public void testThrottling() throws InterruptedException { + List<QueryHandle> handles = Lists.newArrayList(); + for (int j = 0; j < 5; j++) { + for (int i = 0; i < 10; i++) { + handles.add(launchQuery()); + assertValidity(); + } + // No harm in sleeping, the queries will anyway take time. + Thread.sleep(1000); + } + for (QueryHandle handle : handles) { + RestAPITestUtil.waitForQueryToFinish(target(), lensSessionId, handle); + assertValidity(); + } + for (QueryHandle handle : handles) { + RestAPITestUtil.getLensQueryResult(target(), lensSessionId, handle); + assertValidity(); + } + } + + private void assertValidity() { + QueryExecutionServiceImpl.QueryCount count = queryService.getQueryCountSnapshot(); + assertTrue(count.running <= 4, System.currentTimeMillis() + " " + count.running + " running queries: " + + queryService.getLaunchedQueries()); + if (count.running == 4) { + assertEquals(count.queued, 0); + } else { + assertEquals(count.waiting, 0); + } + } + + private QueryHandle launchQuery() { + return RestAPITestUtil.executeAndGetHandle(target(), Optional.of(lensSessionId), + Optional.of("select ID from " + TEST_TABLE), + Optional.of(LensServerAPITestUtil.getLensConf(QUERY_METRIC_UNIQUE_ID_CONF_KEY, UUID.randomUUID()))); + } + + @AfterMethod + private void waitForPurge() throws InterruptedException { + waitForPurge(0, queryService.finishedQueries); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java index ec6fd86..4ac42b2 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java @@ -18,7 +18,7 @@ */ package org.apache.lens.server.query; -import static org.apache.lens.server.api.util.TestLensUtil.getLensConf; +import static org.apache.lens.server.api.LensServerAPITestUtil.getLensConf; import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.assertEquals; @@ -41,8 +41,8 @@ import org.apache.lens.api.query.QueryResult; import org.apache.lens.api.query.QueryStatus.Status; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; -import org.apache.lens.server.LensTestUtil; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.common.TestResourceFile; @@ -144,7 +144,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void createTable(String tblName) throws InterruptedException { - LensTestUtil.createTable(tblName, target(), lensSessionId); + LensServerTestUtil.createTable(tblName, target(), lensSessionId); } /** @@ -155,7 +155,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void loadData(String tblName, final String testDataFile) throws InterruptedException { - LensTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); + LensServerTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); } /** @@ -165,7 +165,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void dropTable(String tblName) throws InterruptedException { - LensTestUtil.dropTable(tblName, target(), lensSessionId); + LensServerTestUtil.dropTable(tblName, target(), lensSessionId); } private QueryHandle launchAndWaitForQuery(LensConf conf, String query, Status expectedStatus) http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index c37b0ed..c8a1cc6 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -20,7 +20,9 @@ package org.apache.lens.server.query; import static javax.ws.rs.core.Response.Status.*; -import static org.apache.lens.server.api.util.TestLensUtil.getLensConf; +import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS; +import static org.apache.lens.server.LensServerTestUtil.DB_WITH_JARS_2; +import static org.apache.lens.server.api.LensServerAPITestUtil.getLensConf; import static org.apache.lens.server.common.RestAPITestUtil.*; import static org.testng.Assert.*; @@ -48,8 +50,8 @@ import org.apache.lens.api.result.QueryCostTO; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.driver.hive.HiveDriver; import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.LensServices; -import org.apache.lens.server.LensTestUtil; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.driver.LensDriver; import org.apache.lens.server.api.error.LensException; @@ -183,7 +185,7 @@ public class TestQueryService extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void createTable(String tblName) throws InterruptedException { - LensTestUtil.createTable(tblName, target(), lensSessionId); + LensServerTestUtil.createTable(tblName, target(), lensSessionId); } /** @@ -194,7 +196,7 @@ public class TestQueryService extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void loadData(String tblName, final String testDataFile) throws InterruptedException { - LensTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); + LensServerTestUtil.loadDataFromClasspath(tblName, testDataFile, target(), lensSessionId); } /** @@ -204,7 +206,7 @@ public class TestQueryService extends LensJerseyTest { * @throws InterruptedException the interrupted exception */ private void dropTable(String tblName) throws InterruptedException { - LensTestUtil.dropTable(tblName, target(), lensSessionId); + LensServerTestUtil.dropTable(tblName, target(), lensSessionId); } // test get a random query, should return 400 @@ -1248,23 +1250,23 @@ public class TestQueryService extends LensJerseyTest { // Open session with a DB which has static jars LensSessionHandle sessionHandle = - sessionService.openSession("foo@localhost", "bar", LensTestUtil.DB_WITH_JARS, new HashMap<String, String>()); + sessionService.openSession("foo@localhost", "bar", DB_WITH_JARS, new HashMap<String, String>()); // Add a jar in the session File testJarFile = new File("target/testjars/test2.jar"); sessionService.addResourceToAllServices(sessionHandle, "jar", "file://" + testJarFile.getAbsolutePath()); - log.info("@@@ Opened session " + sessionHandle.getPublicId() + " with database " + LensTestUtil.DB_WITH_JARS); + log.info("@@@ Opened session " + sessionHandle.getPublicId() + " with database " + DB_WITH_JARS); LensSessionImpl session = sessionService.getSession(sessionHandle); // Jars should be pending until query is run - assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).size(), 1); - assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).size(), 1); + assertEquals(session.getPendingSessionResourcesForDatabase(DB_WITH_JARS).size(), 1); + assertEquals(session.getPendingSessionResourcesForDatabase(DB_WITH_JARS_2).size(), 1); final String tableInDBWithJars = "testHiveDriverGetsDBJars"; try { // First execute query on the session with db should load jars from DB - LensTestUtil.createTable(tableInDBWithJars, target(), sessionHandle, "(ID INT, IDSTR STRING) " + LensServerTestUtil.createTable(tableInDBWithJars, target(), sessionHandle, "(ID INT, IDSTR STRING) " + "ROW FORMAT SERDE \"DatabaseJarSerde\""); boolean addedToHiveDriver = false; @@ -1272,37 +1274,36 @@ public class TestQueryService extends LensJerseyTest { for (LensDriver driver : queryService.getDrivers()) { if (driver instanceof HiveDriver) { addedToHiveDriver = - ((HiveDriver) driver).areDBResourcesAddedForSession(sessionHandle.getPublicId().toString(), - LensTestUtil.DB_WITH_JARS); + ((HiveDriver) driver).areDBResourcesAddedForSession(sessionHandle.getPublicId().toString(), DB_WITH_JARS); } } assertTrue(addedToHiveDriver); // Switch database log.info("@@@# database switch test"); - session.setCurrentDatabase(LensTestUtil.DB_WITH_JARS_2); - LensTestUtil.createTable(tableInDBWithJars + "_2", target(), sessionHandle, "(ID INT, IDSTR STRING) " + session.setCurrentDatabase(DB_WITH_JARS_2); + LensServerTestUtil.createTable(tableInDBWithJars + "_2", target(), sessionHandle, "(ID INT, IDSTR STRING) " + "ROW FORMAT SERDE \"DatabaseJarSerde\""); // All db jars should have been added - assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS_2).isEmpty()); - assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS).isEmpty()); + assertTrue(session.getDBResources(DB_WITH_JARS_2).isEmpty()); + assertTrue(session.getDBResources(DB_WITH_JARS).isEmpty()); // All session resources must have been added to both DBs assertFalse(session.getLensSessionPersistInfo().getResources().isEmpty()); for (LensSessionImpl.ResourceEntry resource : session.getLensSessionPersistInfo().getResources()) { - assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS_2)); - assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS)); + assertTrue(resource.isAddedToDatabase(DB_WITH_JARS_2)); + assertTrue(resource.isAddedToDatabase(DB_WITH_JARS)); } - assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).isEmpty()); - assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).isEmpty()); + assertTrue(session.getPendingSessionResourcesForDatabase(DB_WITH_JARS).isEmpty()); + assertTrue(session.getPendingSessionResourcesForDatabase(DB_WITH_JARS_2).isEmpty()); } finally { log.info("@@@ TEST_OVER"); try { - LensTestUtil.dropTable(tableInDBWithJars, target(), sessionHandle); - LensTestUtil.dropTable(tableInDBWithJars + "_2", target(), sessionHandle); + LensServerTestUtil.dropTable(tableInDBWithJars, target(), sessionHandle); + LensServerTestUtil.dropTable(tableInDBWithJars + "_2", target(), sessionHandle); } catch (Throwable th) { log.error("Exception while dropping table.", th); } @@ -1361,13 +1362,13 @@ public class TestQueryService extends LensJerseyTest { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); String tblName = "testNonSelectQueriesWithPersistResult"; - LensTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); + LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true"); - LensTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); + LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); - LensTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); + LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "false"); - LensTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); + LensServerTestUtil.dropTableWithConf(tblName, target(), lensSessionId, conf); } @Test http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java index 987129c..30d1e34 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java @@ -18,7 +18,9 @@ */ package org.apache.lens.server.query; -import static org.testng.Assert.assertTrue; +import static org.apache.lens.server.LensServerTestUtil.*; + +import static org.testng.Assert.*; import java.io.IOException; import java.util.HashMap; @@ -38,17 +40,23 @@ import org.apache.lens.api.query.QueryStatus.Status; import org.apache.lens.api.result.LensAPIResult; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServices; -import org.apache.lens.server.LensTestUtil; import org.apache.lens.server.api.LensConfConstants; -import org.apache.lens.server.api.query.*; +import org.apache.lens.server.api.query.InMemoryOutputFormatter; +import org.apache.lens.server.api.query.PersistedOutputFormatter; +import org.apache.lens.server.api.query.QueryContext; +import org.apache.lens.server.api.query.QueryExecutionService; import org.apache.lens.server.common.TestResourceFile; import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe; import org.glassfish.jersey.client.ClientConfig; -import org.glassfish.jersey.media.multipart.*; -import org.testng.Assert; -import org.testng.annotations.*; +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.testng.annotations.AfterTest; +import org.testng.annotations.BeforeTest; +import org.testng.annotations.Test; import lombok.extern.slf4j.Slf4j; @@ -75,9 +83,9 @@ public class TestResultFormatting extends LensJerseyTest { super.setUp(); queryService = LensServices.get().getService(QueryExecutionService.NAME); lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>()); - LensTestUtil.createTable(testTable, target(), lensSessionId, + createTable(testTable, target(), lensSessionId, "(ID INT, IDSTR STRING, IDARR ARRAY<INT>, IDSTRARR ARRAY<STRING>)"); - LensTestUtil.loadDataFromClasspath(testTable, TestResourceFile.TEST_DATA2_FILE.getValue(), target(), lensSessionId); + loadDataFromClasspath(testTable, TestResourceFile.TEST_DATA2_FILE.getValue(), target(), lensSessionId); } /* @@ -87,7 +95,7 @@ public class TestResultFormatting extends LensJerseyTest { */ @AfterTest public void tearDown() throws Exception { - LensTestUtil.dropTable(testTable, target(), lensSessionId); + dropTable(testTable, target(), lensSessionId); queryService.closeSession(lensSessionId); super.tearDown(); } @@ -212,7 +220,7 @@ public class TestResultFormatting extends LensJerseyTest { .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), new GenericType<LensAPIResult<QueryHandle>>() {}).getData(); - Assert.assertNotNull(handle); + assertNotNull(handle); // Get query LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request() @@ -225,7 +233,7 @@ public class TestResultFormatting extends LensJerseyTest { Thread.sleep(1000); } - Assert.assertEquals(ctx.getStatus().getStatus(), status); + assertEquals(ctx.getStatus().getStatus(), status); if (status.equals(QueryStatus.Status.SUCCESSFUL)) { QueryContext qctx = queryService.getQueryContext(handle); @@ -239,12 +247,12 @@ public class TestResultFormatting extends LensJerseyTest { } else if (!isDir) { // isDir is true if the formatter is skipped due to result being the max size allowed if (qctx.isDriverPersistent()) { - Assert.assertTrue(qctx.getQueryOutputFormatter() instanceof PersistedOutputFormatter); + assertTrue(qctx.getQueryOutputFormatter() instanceof PersistedOutputFormatter); } else { - Assert.assertTrue(qctx.getQueryOutputFormatter() instanceof InMemoryOutputFormatter); + assertTrue(qctx.getQueryOutputFormatter() instanceof InMemoryOutputFormatter); } } else { - Assert.assertNull(qctx.getQueryOutputFormatter()); + assertNull(qctx.getQueryOutputFormatter()); } // fetch results TestQueryService.validatePersistedResult(handle, target(), lensSessionId, new String[][]{ @@ -259,7 +267,7 @@ public class TestResultFormatting extends LensJerseyTest { assertTrue(ctx.getDriverStartTime() > 0); assertTrue(ctx.getDriverFinishTime() > 0); assertTrue(ctx.getFinishTime() > 0); - Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED); + assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED); } } http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 28bed27..1008faf 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -24,7 +24,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import org.apache.lens.server.LensTestUtil; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.api.LensConfConstants; import org.apache.hadoop.hive.conf.HiveConf; @@ -53,7 +53,7 @@ public class TestDatabaseResourceService { @BeforeClass public void setup() throws Exception { - LensTestUtil.createTestDatabaseResources(testDatabases, conf); + LensServerTestUtil.createTestDatabaseResources(testDatabases, conf); // Start resource service. conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources"); dbResService = new DatabaseResourceService(DatabaseResourceService.NAME); http://git-wip-us.apache.org/repos/asf/lens/blob/b71be2dc/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java index 464533d..5feab71 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java @@ -25,7 +25,7 @@ import java.util.HashMap; import org.apache.lens.api.LensSessionHandle; import org.apache.lens.server.LensServerConf; -import org.apache.lens.server.LensTestUtil; +import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.user.UserConfigLoaderFactory; @@ -57,7 +57,7 @@ public class TestSessionClassLoaders { * test2.jar containing ClassLoaderTestClass2.class added to session via addResource */ // Create test databases and tables - LensTestUtil.createTestDatabaseResources(new String[]{DB1}, conf); + LensServerTestUtil.createTestDatabaseResources(new String[]{DB1}, conf); conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName()); conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources");