FALCON-445 Propagate hive credentials defined in cluster entity to hive-site.xml. Contributed by Sowmya Ramesh
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/3d3bde55 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/3d3bde55 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/3d3bde55 Branch: refs/heads/master Commit: 3d3bde558027af1f6ceb914b396cf6a41682625e Parents: b7afe36 Author: Venkatesh Seetharam <[email protected]> Authored: Fri May 23 11:31:03 2014 -0700 Committer: Venkatesh Seetharam <[email protected]> Committed: Fri May 23 11:31:03 2014 -0700 ---------------------------------------------------------------------- CHANGES.txt | 5 ++++- .../org/apache/falcon/entity/ClusterHelper.java | 20 ++++++++++++++++++++ .../falcon/workflow/OozieWorkflowBuilder.java | 6 ++++++ .../OozieProcessWorkflowBuilderTest.java | 3 ++- .../resources/config/cluster/cluster-0.1.xml | 1 + 5 files changed, 33 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0e26c7d..e06fc80 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,7 +7,10 @@ Trunk (Unreleased) NEW FEATURES IMPROVEMENTS - FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré + FALCON-445 Propagate hive credentials defined in cluster entity to + hive-site.xml (Sowmya Ramesh via Venkatesh Seetharam) + + FALCON-133 Upgrade to slf4j 1.7.5 and use SLF4J logger. (Jean-Baptiste Onofré via Shwetha GS) OPTIMIZATIONS http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java index 3bf9d95..5284d68 100644 --- a/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/ClusterHelper.java @@ -27,6 +27,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + /** * Helper to get end points relating to the cluster. */ @@ -135,4 +139,20 @@ public final class ClusterHelper { } return null; } + + public static Map<String, String> geHiveProperties(Cluster cluster) { + if (cluster.getProperties() != null) { + List<Property> properties = cluster.getProperties().getProperties(); + if (properties != null && !properties.isEmpty()) { + Map<String, String> hiveProperties = new HashMap<String, String>(); + for (Property prop : properties) { + if (prop.getName().startsWith("hive.")) { + hiveProperties.put(prop.getName(), prop.getValue()); + } + } + return hiveProperties; + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java ---------------------------------------------------------------------- diff --git a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java index f5ff27a..7d84938 100644 --- a/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/workflow/OozieWorkflowBuilder.java @@ -483,6 +483,12 @@ public abstract class OozieWorkflowBuilder<T extends Entity> extends WorkflowBui "Registry interface is not defined in cluster: " + cluster.getName()); } + // Propagate the hive properties from cluster entity + Map<String, String> hiveProperties = ClusterHelper.geHiveProperties(cluster); + if (hiveProperties != null && !hiveProperties.isEmpty()) { + hiveCredentials.putAll(hiveProperties); + } + hiveCredentials.put(METASTOREURIS, metaStoreUrl); hiveCredentials.put("hive.metastore.execute.setugi", "true"); hiveCredentials.put("hcatNode", metaStoreUrl.replace("thrift", "hcat")); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java ---------------------------------------------------------------------- diff --git a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java index 8cfa9fc..2522ca3 100644 --- a/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java +++ b/process/src/test/java/org/apache/falcon/converter/OozieProcessWorkflowBuilderTest.java @@ -289,8 +289,9 @@ public class OozieProcessWorkflowBuilderTest extends AbstractTestBase { props.put(prop.getName(), prop.getValue()); } - // verify table props + // verify table and hive props Map<String, String> expected = getExpectedProperties(inFeed, outFeed, process); + expected.putAll(ClusterHelper.geHiveProperties(cluster)); for (Map.Entry<String, String> entry : props.entrySet()) { if (expected.containsKey(entry.getKey())) { Assert.assertEquals(entry.getValue(), expected.get(entry.getKey())); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/3d3bde55/process/src/test/resources/config/cluster/cluster-0.1.xml ---------------------------------------------------------------------- diff --git a/process/src/test/resources/config/cluster/cluster-0.1.xml b/process/src/test/resources/config/cluster/cluster-0.1.xml index 3780b3f..032cc77 100644 --- a/process/src/test/resources/config/cluster/cluster-0.1.xml +++ b/process/src/test/resources/config/cluster/cluster-0.1.xml @@ -39,5 +39,6 @@ <properties> <property name="field1" value="value1"/> <property name="field2" value="value2"/> + <property name="hive.metastore.client.socket.timeout" value="20"/> </properties> </cluster>
