LENS-1449 : lens.query.result.parent.dir to be set at driver level.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/41051ea4 Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/41051ea4 Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/41051ea4 Branch: refs/heads/current-release-line Commit: 41051ea47f164f004f5fccaa7ea91765007811fc Parents: 503a46d Author: Raghavendra Singh <raghavsi...@apache.org> Authored: Thu Jul 6 14:03:59 2017 +0530 Committer: Rajat Khandelwal <rajatgupt...@gmail.com> Committed: Thu Jul 13 14:42:59 2017 +0530 ---------------------------------------------------------------------- .../api/query/DriverSelectorQueryContext.java | 4 +++- .../lens/server/api/query/QueryContext.java | 5 +++++ .../lens/server/query/TestQueryService.java | 6 ++++-- .../lens/server/query/TestResultFormatting.java | 20 +++++++++++++++++++- .../drivers/hive/hive1/hivedriver-site.xml | 5 +++++ .../drivers/hive/hive2/hivedriver-site.xml | 6 ++++++ .../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 5 +++++ lens-server/src/test/resources/lens-site.xml | 5 ----- 8 files changed, 47 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java index 8e431d1..41e533e 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java @@ -155,7 +155,9 @@ public class DriverSelectorQueryContext { private Configuration mergeConf(LensDriver driver, Configuration queryConf) { Configuration conf = new Configuration(driver.getConf()); for (Map.Entry<String, String> entry : queryConf) { - conf.set(entry.getKey(), entry.getValue()); + if (!conf.getFinalParameters().contains(entry.getKey())) { + conf.set(entry.getKey(), entry.getValue()); + } } conf.setClassLoader(queryConf.getClassLoader()); return conf; http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java index 9923589..ccdef87 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java @@ -328,6 +328,11 @@ public class QueryContext extends AbstractQueryContext implements FailureContext } public String getResultSetParentDir() { + if (getSelectedDriver() != null && getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) != null) { + log.info("Fetching Parent Dir from driver conf:- " + + getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR)); + return getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR); + } return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT); } http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java index 1149696..b7cdb88 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java @@ -848,9 +848,11 @@ public class TestQueryService extends LensJerseyTest { LensResultSet rs = queryService.getResultset(handle3); //check persisted result path String expectedPath = - ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString() + ctx3.getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString() + ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN); - assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath)); + assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath) + , "Result Path " + ((PersistentResultSet) rs).getOutputPath() + + " does not contain expected path: " + expectedPath); validateHttpEndPoint(target(), null, handle3, null); } http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java ---------------------------------------------------------------------- diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java index 4f8b8ff..76d3720 100644 --- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java +++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java @@ -157,6 +157,22 @@ public class TestResultFormatting extends LensJerseyTest { } /** + * Test result formatter hdfs persistent result location override from driver conf. + * + * @throws InterruptedException the interrupted exception + * @throws IOException Signals that an I/O exception has occurred. + */ + @Test + public void testResultDirOverrideFromDriverConf() throws InterruptedException, IOException { + LensConf conf = new LensConf(); + conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true"); + LensQuery lensQuery = testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false, + null, MediaType.APPLICATION_XML_TYPE); + assertEquals(lensQuery.getResultSetPath(), "file:"+System.getProperty("user.dir")+"/target/hive-lens-results/" + +lensQuery.getQueryHandle().getHandleIdString()+".csv"); + } + + /** * Test persistent result with max size. * * @throws InterruptedException the interrupted exception @@ -196,7 +212,7 @@ public class TestResultFormatting extends LensJerseyTest { * @throws InterruptedException the interrupted exception * @throws IOException Signals that an I/O exception has occurred. */ - private void testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt) + private LensQuery testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt) throws InterruptedException, IOException { // test post execute op final WebTarget target = target().path("queryapi/queries"); @@ -266,6 +282,8 @@ public class TestResultFormatting extends LensJerseyTest { || ctx.getStatus().getStatusMessage().equals(ResultFormatter.ERROR_MESSAGE)); assertEquals(ctx.getStatus().getErrorMessage(), "Class NonexistentSerde.class not found"); } + + return ctx; } @AfterTest http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml index 1f0ff43..83143e7 100644 --- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml @@ -82,4 +82,9 @@ </description> </property> + <property> + <name>lens.query.result.parent.dir</name> + <value>target/hive-lens-results</value> + <final>true</final> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml index eb902e0..4170e9b 100644 --- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml +++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml @@ -82,4 +82,10 @@ </description> </property> + <property> + <name>lens.query.result.parent.dir</name> + <value>target/hive-lens-results</value> + <final>true</final> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml index 5b8b43f..ad068f8 100644 --- a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml +++ b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml @@ -55,4 +55,9 @@ <name>lens.driver.test.drivername</name> <value>jdbc/jdbc1</value> </property> + <property> + <name>lens.query.result.parent.dir</name> + <value>target/jdbc-lens-results</value> + <final>true</final> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/lens-site.xml ---------------------------------------------------------------------- diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml index 0060fa7..7e5f522 100644 --- a/lens-server/src/test/resources/lens-site.xml +++ b/lens-server/src/test/resources/lens-site.xml @@ -49,11 +49,6 @@ <name>lens.server.persist.location</name> <value>target/persist-dir</value> </property> - - <property> - <name>lens.query.result.parent.dir</name> - <value>target/lens-results</value> - </property> <property> <name>hive.server2.authentication</name>