Repository: oozie Updated Branches: refs/heads/master 74bf03270 -> eed805ec3
OOZIE-1799 Document hcatalog integration steps for Oozie in a secure cluster (venkatnrangan via bzhang) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/eed805ec Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/eed805ec Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/eed805ec Branch: refs/heads/master Commit: eed805ec3c704252472dd945fe02d067df42e1a3 Parents: 74bf032 Author: Bowen Zhang <[email protected]> Authored: Thu Apr 24 09:43:37 2014 -0700 Committer: Bowen Zhang <[email protected]> Committed: Thu Apr 24 09:43:37 2014 -0700 ---------------------------------------------------------------------- examples/src/main/apps/hcatalog/README | 10 ++++ examples/src/main/apps/hcatalog/job.properties | 2 + .../main/apps/hcatalog/job.properties.security | 38 ++++++++++++ examples/src/main/apps/hcatalog/workflow.xml | 2 +- .../main/apps/hcatalog/workflow.xml.security | 62 ++++++++++++++++++++ 5 files changed, 113 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/eed805ec/examples/src/main/apps/hcatalog/README ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hcatalog/README b/examples/src/main/apps/hcatalog/README index f5ba0a1..e9c5462 100644 --- a/examples/src/main/apps/hcatalog/README +++ b/examples/src/main/apps/hcatalog/README @@ -20,3 +20,13 @@ Running the coordinator example with HCatalog (End-to-end test) 7. Run Oozie job using the job.properties. Coordinator actions will be in WAITING 8. Make input dependencies available throught HCat client by "alter table invites add partition (ds='2010-01-01', region='usa')". This event will start the workflows with pig action 9. First workflow will SUCCEED as expected, however second one will fail due to 'partition already exists' error. Disregard this. The example demonstrates working as expected. +10. When running in a secure cluster + a. The files job.properties.security and workflow.xml.security can be + used and modified as needed. Please refer to those two files and follow + the instructions. + b. Make sure oozie-site.xml under /etc/oozie/conf has the following property + specified. + <property> + <name>oozie.credentials.credentialclasses</name> + <value>hcat=org.apache.oozie.action.hadoop.HCatCredentials</value> + </property> http://git-wip-us.apache.org/repos/asf/oozie/blob/eed805ec/examples/src/main/apps/hcatalog/job.properties ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hcatalog/job.properties b/examples/src/main/apps/hcatalog/job.properties index f8f92ca..e8d1f1b 100644 --- a/examples/src/main/apps/hcatalog/job.properties +++ b/examples/src/main/apps/hcatalog/job.properties @@ -20,6 +20,8 @@ nameNode=hdfs://localhost:8020 jobTracker=localhost:8021 queueName=default examplesRoot=examples +oozie.use.system.libpath=true + oozie.coord.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hcatalog hcatNode=hcat://localhost:11002 http://git-wip-us.apache.org/repos/asf/oozie/blob/eed805ec/examples/src/main/apps/hcatalog/job.properties.security ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hcatalog/job.properties.security b/examples/src/main/apps/hcatalog/job.properties.security new file mode 100644 index 0000000..2fbcc6e --- /dev/null +++ b/examples/src/main/apps/hcatalog/job.properties.security @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +nameNode=hdfs://localhost:8020 +jobTracker=localhost:8021 +queueName=default +examplesRoot=examples +oozie.use.system.libpath=true + +hive.metastore.sasl.enabled=true +hive.metastore.kerberos.principal=hive/<host>@<realm> + + +oozie.coord.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/hcatalog +hcatNode=hcat://localhost:11002 +db=default +table=invites +start=2010-01-01T01:00Z +end=2010-01-01T01:20Z +workflowAppUri=${nameNode}/user/${user.name}/${examplesRoot}/apps/hcatalog +dataOut=2011-01-01 +outputtable=oozie +region=usa http://git-wip-us.apache.org/repos/asf/oozie/blob/eed805ec/examples/src/main/apps/hcatalog/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hcatalog/workflow.xml b/examples/src/main/apps/hcatalog/workflow.xml index 09adf5f..d6593df 100644 --- a/examples/src/main/apps/hcatalog/workflow.xml +++ b/examples/src/main/apps/hcatalog/workflow.xml @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -<workflow-app xmlns="uri:oozie:workflow:0.2" name="pig-wf"> +<workflow-app xmlns="uri:oozie:workflow:0.5" name="pig-wf"> <start to="pig-node"/> <action name="pig-node"> <pig> http://git-wip-us.apache.org/repos/asf/oozie/blob/eed805ec/examples/src/main/apps/hcatalog/workflow.xml.security ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/hcatalog/workflow.xml.security b/examples/src/main/apps/hcatalog/workflow.xml.security new file mode 100644 index 0000000..aa3522a --- /dev/null +++ b/examples/src/main/apps/hcatalog/workflow.xml.security @@ -0,0 +1,62 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<workflow-app xmlns="uri:oozie:workflow:0.5" name="pig-wf"> + <credentials> + <credential name='hcatauth' type='hcat'> + <property> + <name>hcat.metastore.uri</name> + <value>thrift://<host>:<port></value> + </property> + <property> + <name>hcat.metastore.principal</name> + <value>hive/<host>@<realm></value> + </property> + </credential> + </credentials> + <start to="pig-node"/> + <action name="pig-node" cred="hcatauth"> + <pig> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + <property> + <name>mapred.compress.map.output</name> + <value>true</value> + </property> + </configuration> + <script>id.pig</script> + <param>DB=${DB}</param> + <param>TABLE=${TABLE}</param> + <param>FILTER=${FILTER}</param> + <param>OUTPUT_DB=${OUTPUT_DB}</param> + <param>OUTPUT_TABLE=${OUTPUT_TABLE}</param> + <param>OUTPUT_PARTITION=${OUTPUT_PARTITION}</param> + <file>lib/hive-site.xml</file> + </pig> + <ok to="end"/> + <error to="fail"/> + </action> + <kill name="fail"> + <message>Pig failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> + <end name="end"/> +</workflow-app>
