Temp: addressing CodeReview comments (JobConf mostly) Change-Id: I5c9c173f894d62aa68452bbe56bbced1b9b6e8fa
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/74f4afb2 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/74f4afb2 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/74f4afb2 Branch: refs/heads/oya Commit: 74f4afb223da3ce367304f6e115b4f5c9455a8b9 Parents: 4c237f9 Author: Peter Bacsko <pbac...@cloudera.com> Authored: Tue Jan 17 18:21:05 2017 +0100 Committer: Peter Bacsko <pbac...@cloudera.com> Committed: Tue Jan 17 18:21:05 2017 +0100 ---------------------------------------------------------------------- .../org/apache/oozie/client/XOozieClient.java | 2 +- .../java/org/apache/oozie/WorkflowJobBean.java | 2 +- .../apache/oozie/action/hadoop/Credentials.java | 38 ------ .../action/hadoop/CredentialsProvider.java | 92 +++---------- .../hadoop/CredentialsProviderFactory.java | 102 +++++++++++++++ .../action/hadoop/HCatCredentialHelper.java | 19 +-- .../oozie/action/hadoop/HCatCredentials.java | 14 +- .../oozie/action/hadoop/HbaseCredentials.java | 22 ++-- .../oozie/action/hadoop/Hive2Credentials.java | 13 +- .../oozie/action/hadoop/HiveActionExecutor.java | 5 +- .../oozie/action/hadoop/JavaActionExecutor.java | 128 +++++++------------ .../action/hadoop/LauncherMapperHelper.java | 14 +- .../action/hadoop/MapReduceActionExecutor.java | 15 ++- .../oozie/action/hadoop/PigActionExecutor.java | 5 +- .../action/hadoop/SparkActionExecutor.java | 5 +- .../action/hadoop/SqoopActionExecutor.java | 4 +- .../oozie/service/HadoopAccessorService.java | 6 +- .../oozie/action/hadoop/CredentialForTest.java | 15 +-- .../oozie/action/hadoop/InsertTestToken.java | 14 +- .../action/hadoop/TestHCatCredentials.java | 34 ++--- .../action/hadoop/TestJavaActionExecutor.java | 27 ++-- .../oozie/action/hadoop/TestOozieJobInfo.java | 3 +- .../oozie/client/TestWorkflowXClient.java | 2 +- .../command/wf/TestActionCheckXCommand.java | 7 +- .../command/wf/TestActionStartXCommand.java | 4 +- .../oozie/service/TestConfigurationService.java | 4 +- .../oozie/service/TestRecoveryService.java | 2 +- .../hadoop/TestMapReduceActionExecutor.java | 28 ++-- 28 files changed, 300 insertions(+), 326 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/client/src/main/java/org/apache/oozie/client/XOozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/XOozieClient.java b/client/src/main/java/org/apache/oozie/client/XOozieClient.java index a76f544..379819b 100644 --- a/client/src/main/java/org/apache/oozie/client/XOozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/XOozieClient.java @@ -116,7 +116,7 @@ public class XOozieClient extends OozieClient { private void validateHttpSubmitConf(Properties conf) { String RM = conf.getProperty(XOozieClient.RM); if (RM == null) { - throw new RuntimeException("jobtracker is not specified in conf"); + throw new RuntimeException("Resource manager is not specified in conf"); } String NN = conf.getProperty(XOozieClient.NN); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/WorkflowJobBean.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java index f2f79dc..2042063 100644 --- a/core/src/main/java/org/apache/oozie/WorkflowJobBean.java +++ b/core/src/main/java/org/apache/oozie/WorkflowJobBean.java @@ -604,7 +604,7 @@ public class WorkflowJobBean implements Writable, WorkflowJob, JsonBean { @SuppressWarnings("unchecked") public List<WorkflowAction> getActions() { - return (List<WorkflowAction>)(List<?>) actions; + return (List) actions; } public void setActions(List<WorkflowActionBean> nodes) { http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java deleted file mode 100644 index 728d626..0000000 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Credentials.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.mapred.JobConf; -import org.apache.oozie.action.ActionExecutor.Context; - -public abstract class Credentials { - - /** - * This is the interface for all the Credentials implementation. Any new credential implementaion must implement - * this function. This function should modify the jobconf which will be used further to pass the credentials to the - * tasks while running it. Credentials properties and context is also provided by that user can get all the - * necessary configuration. - * - * @param jobconf job conf where the token will insert into - * @param props properties for getting credential token or certificate - * @param context workflow context - * @throws Exception thrown if failed - */ - public abstract void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception; -} http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java index 9cc1c28..1801b2d 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java @@ -18,85 +18,25 @@ package org.apache.oozie.action.hadoop; -import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.Credentials; +import org.apache.oozie.action.ActionExecutor.Context; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.util.XLog; +public interface CredentialsProvider { -public class CredentialsProvider { - Credentials cred; - String type; - public static final String CRED_KEY = "oozie.credentials.credentialclasses"; - private static final XLog LOG = XLog.getLog(CredentialsProvider.class); - - public CredentialsProvider(String type) { - this.type = type; - this.cred = null; - LOG.debug("Credentials Provider is created for Type: " + type); - } - - /** - * Create Credential object - * - * @return Credential object - * @throws Exception - */ - public Credentials createCredentialObject() throws Exception { - String type; - String classname; - for (String function : ConfigurationService.getStrings(CRED_KEY)) { - function = Trim(function); - LOG.debug("Creating Credential class for : " + function); - String[] str = function.split("="); - if (str.length > 0) { - type = str[0]; - classname = str[1]; - if (classname != null) { - LOG.debug("Creating Credential type : '" + type + "', class Name : '" + classname + "'"); - if (this.type.equalsIgnoreCase(str[0])) { - Class<?> klass = null; - try { - klass = Thread.currentThread().getContextClassLoader().loadClass(classname); - } - catch (ClassNotFoundException ex) { - LOG.warn("Exception while loading the class", ex); - throw ex; - } - - cred = (Credentials) ReflectionUtils.newInstance(klass, null); - } - } - } - } - return cred; - } - - /** - * Relogs into Kerberos using the Keytab for the Oozie server user. This should be called before attempting to get delegation - * tokens via {@link Credentials} implementations to ensure that the Kerberos credentials are current and won't expire too soon. + /** + * This is the interface for all the Credentials implementation. Any new credential implementation must implement + * this function. This function should modify the jobconf which will be used further to pass the credentials to the + * tasks while running it. Credentials properties and context is also provided by that user can get all the + * necessary configuration. * - * @throws IOException - */ - public static void ensureKerberosLogin() throws IOException { - LOG.debug("About to relogin from keytab"); - UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab(); - LOG.debug("Relogin from keytab successful"); - } - - /** - * To trim string + * @param credentials the credentials object which is updated + * @param config launcher AM configuration + * @param props properties for getting credential token or certificate + * @param context workflow context + * @throws Exception thrown if failed * - * @param str - * @return trim string */ - public String Trim(String str) { - if (str != null) { - str = str.replaceAll("\\n", ""); - str = str.replaceAll("\\t", ""); - str = str.trim(); - } - return str; - } + public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, Context context) + throws Exception; } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java new file mode 100644 index 0000000..fc907c4 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.io.IOException; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.util.XLog; + +public class CredentialsProviderFactory { + CredentialsProvider cred; + String type; + public static final String CRED_KEY = "oozie.credentials.credentialclasses"; + private static final XLog LOG = XLog.getLog(CredentialsProviderFactory.class); + + public CredentialsProviderFactory(String type) { + this.type = type; + this.cred = null; + LOG.debug("Credentials Provider is created for Type: " + type); + } + + /** + * Create Credential object + * + * @return Credential object + * @throws Exception + */ + public CredentialsProvider createCredentialsProvider() throws Exception { + String type; + String classname; + for (String function : ConfigurationService.getStrings(CRED_KEY)) { + function = Trim(function); + LOG.debug("Creating Credential class for : " + function); + String[] str = function.split("="); + if (str.length > 0) { + type = str[0]; + classname = str[1]; + if (classname != null) { + LOG.debug("Creating Credential type : '" + type + "', class Name : '" + classname + "'"); + if (this.type.equalsIgnoreCase(str[0])) { + Class<?> klass = null; + try { + klass = Thread.currentThread().getContextClassLoader().loadClass(classname); + } + catch (ClassNotFoundException ex) { + LOG.warn("Exception while loading the class", ex); + throw ex; + } + + cred = (CredentialsProvider) ReflectionUtils.newInstance(klass, null); + } + } + } + } + return cred; + } + + /** + * Relogs into Kerberos using the Keytab for the Oozie server user. This should be called before attempting to get delegation + * tokens via {@link CredentialsProvider} implementations to ensure that the Kerberos credentials are current and won't expire too soon. + * + * @throws IOException + */ + public static void ensureKerberosLogin() throws IOException { + LOG.debug("About to relogin from keytab"); + UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab(); + LOG.debug("Relogin from keytab successful"); + } + + /** + * To trim string + * + * @param str + * @return trim string + */ + public String Trim(String str) { + if (str != null) { + str = str.replaceAll("\\n", ""); + str = str.replaceAll("\\t", ""); + str = str.trim(); + } + return str; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java index 4f12152..4d9d3fa 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java @@ -18,10 +18,11 @@ package org.apache.oozie.action.hadoop; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -50,18 +51,18 @@ public class HCatCredentialHelper { * @param server - Serevr URI for HCat server * @throws Exception */ - public void set(JobConf launcherJobConf, String principal, String server) throws Exception { + public void set(Credentials credentials, Configuration launcherConfig, String principal, String server) throws Exception { HCatClient client = null; try { - client = getHCatClient(launcherJobConf, principal, server); + client = getHCatClient(launcherConfig, principal, server); XLog.getLog(getClass()).debug( "HCatCredentialHelper: set: User name for which token will be asked from HCat: " - + launcherJobConf.get(USER_NAME)); - String tokenStrForm = client.getDelegationToken(launcherJobConf.get(USER_NAME), UserGroupInformation + + launcherConfig.get(USER_NAME)); + String tokenStrForm = client.getDelegationToken(launcherConfig.get(USER_NAME), UserGroupInformation .getLoginUser().getShortUserName()); Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>(); hcatToken.decodeFromUrlString(tokenStrForm); - launcherJobConf.getCredentials().addToken(new Text("HCat Token"), hcatToken); + credentials.addToken(new Text("HCat Token"), hcatToken); XLog.getLog(getClass()).debug("Added the HCat token in job conf"); } catch (Exception ex) { @@ -77,13 +78,13 @@ public class HCatCredentialHelper { /** * Getting the HCat client. - * @param launcherJobConf + * @param launcherConfig * @param principal * @param server * @return HCatClient * @throws HCatException */ - public HCatClient getHCatClient(JobConf launcherJobConf, + public HCatClient getHCatClient(Configuration launcherConfig, String principal, String server) throws HCatException { HiveConf hiveConf = null; HCatClient hiveclient = null; @@ -95,7 +96,7 @@ public class HCatCredentialHelper { hiveConf.set(HIVE_METASTORE_KERBEROS_PRINCIPAL, principal); hiveConf.set(HIVE_METASTORE_LOCAL, "false"); hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, server); - String protection = launcherJobConf.get(HADOOP_RPC_PROTECTION, + String protection = launcherConfig.get(HADOOP_RPC_PROTECTION, SaslRpcServer.QualityOfProtection.AUTHENTICATION.name() .toLowerCase()); XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to " + protection); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java index d7689a9..52abbf1 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentials.java @@ -21,16 +21,13 @@ package org.apache.oozie.action.hadoop; import java.util.HashMap; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.Credentials; import org.apache.oozie.ErrorCode; import org.apache.oozie.action.ActionExecutor.Context; import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.Services; import org.apache.oozie.util.XLog; -import com.google.common.annotations.VisibleForTesting; - /** * Credentials implementation to store in jobConf, HCat-specific properties such as Principal and Uri * User specifies these credential properties along with the action configuration @@ -39,7 +36,7 @@ import com.google.common.annotations.VisibleForTesting; * User can extend the parent class to implement own class as well * for handling custom token-based credentials and add to the above server property */ -public class HCatCredentials extends Credentials { +public class HCatCredentials implements CredentialsProvider { private static final String HCAT_METASTORE_PRINCIPAL = "hcat.metastore.principal"; private static final String HCAT_METASTORE_URI = "hcat.metastore.uri"; @@ -54,7 +51,8 @@ public class HCatCredentials extends Credentials { * @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context) */ @Override - public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception { + public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, + Context context) throws Exception { try { String principal = getProperty(props.getProperties(), HCAT_METASTORE_PRINCIPAL, HIVE_METASTORE_PRINCIPAL); @@ -69,7 +67,7 @@ public class HCatCredentials extends Credentials { HCAT_METASTORE_URI + " is required to get hcat credential"); } HCatCredentialHelper hcch = new HCatCredentialHelper(); - hcch.set(jobconf, principal, server); + hcch.set(credentials, config, principal, server); } catch (Exception e) { XLog.getLog(getClass()).warn("Exception in addtoJobConf", e); @@ -102,4 +100,6 @@ public class HCatCredentials extends Credentials { } return value; } + + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java index 3a99b6a..327ba83 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.token.AuthenticationTokenIdentifier; import org.apache.hadoop.hbase.security.token.TokenUtil; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.oozie.action.ActionExecutor.Context; @@ -40,17 +40,16 @@ import org.apache.oozie.util.XLog; * Oozie server should be configured to use this Credentials class by including it via property 'oozie.credentials.credentialclasses' * */ -public class HbaseCredentials extends Credentials { - - +public class HbaseCredentials implements CredentialsProvider { /* (non-Javadoc) * @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context) */ @Override - public void addtoJobConf(JobConf jobConf, CredentialsProperties props, Context context) throws Exception { + public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, + Context context) throws Exception { try { - copyHbaseConfToJobConf(jobConf, props); - obtainToken(jobConf, context); + copyHbaseConfToJobConf(config, props); + obtainToken(credentials, config, context); } catch (Exception e) { XLog.getLog(getClass()).warn("Exception in receiving hbase credentials", e); @@ -58,7 +57,7 @@ public class HbaseCredentials extends Credentials { } } - void copyHbaseConfToJobConf(JobConf jobConf, CredentialsProperties props) { + void copyHbaseConfToJobConf(Configuration jobConf, CredentialsProperties props) { // Create configuration using hbase-site.xml/hbase-default.xml Configuration hbaseConf = new Configuration(false); HBaseConfiguration.addHbaseResources(hbaseConf); @@ -71,7 +70,8 @@ public class HbaseCredentials extends Credentials { injectConf(hbaseConf, jobConf); } - private void obtainToken(final JobConf jobConf, Context context) throws IOException, InterruptedException { + private void obtainToken(Credentials credentials, final Configuration jobConf, Context context) + throws IOException, InterruptedException { String user = context.getWorkflow().getUser(); UserGroupInformation ugi = UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser()); User u = User.create(ugi); @@ -84,7 +84,7 @@ public class HbaseCredentials extends Credentials { } } ); - jobConf.getCredentials().addToken(token.getService(), token); + credentials.addToken(token.getService(), token); } private void addPropsConf(CredentialsProperties props, Configuration destConf) { @@ -102,4 +102,6 @@ public class HbaseCredentials extends Credentials { } } } + + } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java index bd2f1f7..0b495f7 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2Credentials.java @@ -20,9 +20,11 @@ package org.apache.oozie.action.hadoop; import java.sql.Connection; import java.sql.DriverManager; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hive.jdbc.HiveConnection; import org.apache.oozie.ErrorCode; @@ -37,14 +39,15 @@ import org.apache.oozie.util.XLog; * User can extend the parent class to implement own class as well * for handling custom token-based credentials and add to the above server property */ -public class Hive2Credentials extends Credentials { +public class Hive2Credentials implements CredentialsProvider { private static final String USER_NAME = "user.name"; private static final String HIVE2_JDBC_URL = "hive2.jdbc.url"; private static final String HIVE2_SERVER_PRINCIPAL = "hive2.server.principal"; @Override - public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception { + public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, + Context context) throws Exception { try { // load the driver Class.forName("org.apache.hive.jdbc.HiveDriver"); @@ -66,7 +69,7 @@ public class Hive2Credentials extends Credentials { con = DriverManager.getConnection(url); XLog.getLog(getClass()).debug("Connected successfully to " + url); // get delegation token for the given proxy user - tokenStr = ((HiveConnection)con).getDelegationToken(jobconf.get(USER_NAME), principal); + tokenStr = ((HiveConnection)con).getDelegationToken(config.get(USER_NAME), principal); } finally { if (con != null) { con.close(); @@ -76,7 +79,7 @@ public class Hive2Credentials extends Credentials { Token<DelegationTokenIdentifier> hive2Token = new Token<DelegationTokenIdentifier>(); hive2Token.decodeFromUrlString(tokenStr); - jobconf.getCredentials().addToken(new Text("hive.server2.delegation.token"), hive2Token); + credentials.addToken(new Text("hive.server2.delegation.token"), hive2Token); XLog.getLog(getClass()).debug("Added the Hive Server 2 token in job conf"); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java index a2d4ca0..012d1ec 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java @@ -20,7 +20,6 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.service.ConfigurationService; @@ -138,10 +137,10 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { boolean loadDefaultResources = ConfigurationService .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); - JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); + Configuration conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); return conf; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 0f3f816..eb59ff4 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -26,7 +26,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; import java.nio.ByteBuffer; -import java.security.PrivilegedExceptionAction; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@ -56,10 +55,7 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager; import org.apache.hadoop.mapreduce.v2.util.MRApps; -import org.apache.hadoop.mapred.JobID; -import org.apache.hadoop.mapred.RunningJob; -import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker; @@ -237,21 +233,20 @@ public class JavaActionExecutor extends ActionExecutor { } } - public JobConf createBaseHadoopConf(Context context, Element actionXml) { + public Configuration createBaseHadoopConf(Context context, Element actionXml) { return createBaseHadoopConf(context, actionXml, true); } - protected JobConf createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) { + protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) { Namespace ns = actionXml.getNamespace(); String jobTracker = actionXml.getChild("job-tracker", ns).getTextTrim(); String nameNode = actionXml.getChild("name-node", ns).getTextTrim(); - JobConf conf = null; + Configuration conf = null; if (loadResources) { conf = Services.get().get(HadoopAccessorService.class).createJobConf(jobTracker); } else { - conf = new JobConf(false); - // conf.set(HadoopAccessorService.OOZIE_HADOOP_ACCESSOR_SERVICE_CREATED, "true"); + conf = new Configuration(false); } conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER)); @@ -268,7 +263,7 @@ public class JavaActionExecutor extends ActionExecutor { return conf; } - protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { return createBaseHadoopConf(context, actionXml); } @@ -777,7 +772,7 @@ public class JavaActionExecutor extends ActionExecutor { } @SuppressWarnings("unchecked") - JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf) + Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf) throws ActionExecutorException { try { @@ -788,7 +783,7 @@ public class JavaActionExecutor extends ActionExecutor { } // launcher job configuration - JobConf launcherJobConf = createBaseHadoopConf(context, actionXml); + Configuration launcherJobConf = createBaseHadoopConf(context, actionXml); // cancel delegation token on a launcher job which stays alive till child job(s) finishes // otherwise (in mapred action), doesn't cancel not to disturb running child job launcherJobConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); @@ -823,15 +818,6 @@ public class JavaActionExecutor extends ActionExecutor { } setLibFilesArchives(context, actionXml, appPathRoot, launcherJobConf); - String jobName = launcherJobConf.get(HADOOP_JOB_NAME); - if (jobName == null || jobName.isEmpty()) { - jobName = XLog.format( - "oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(), - context.getWorkflow().getAppName(), action.getName(), - context.getWorkflow().getId()); - launcherJobConf.setJobName(jobName); - } - // Inject Oozie job information if enabled. injectJobInfo(launcherJobConf, actionConf, context, action); @@ -912,7 +898,7 @@ public class JavaActionExecutor extends ActionExecutor { injectCallback(context, launcherConf); } - private void actionConfToLauncherConf(Configuration actionConf, JobConf launcherConf) { + private void actionConfToLauncherConf(Configuration actionConf, Configuration launcherConf) { for (String name : SPECIAL_PROPERTIES) { if (actionConf.get(name) != null && launcherConf.get("oozie.launcher." + name) == null) { launcherConf.set(name, actionConf.get(name)); @@ -938,14 +924,6 @@ public class JavaActionExecutor extends ActionExecutor { LOG.debug("Setting LibFilesArchives "); setLibFilesArchives(context, actionXml, appPathRoot, actionConf); - String jobName = actionConf.get(HADOOP_JOB_NAME); - if (jobName == null || jobName.isEmpty()) { - jobName = XLog.format("oozie:action:T={0}:W={1}:A={2}:ID={3}", - getType(), context.getWorkflow().getAppName(), - action.getName(), context.getWorkflow().getId()); - actionConf.set(HADOOP_JOB_NAME, jobName); - } - injectActionCallback(context, actionConf); if(actionConf.get(ACL_MODIFY_JOB) == null || actionConf.get(ACL_MODIFY_JOB).trim().equals("")) { @@ -959,15 +937,17 @@ public class JavaActionExecutor extends ActionExecutor { } // Setting the credential properties in launcher conf - JobConf credentialsConf = null; + Configuration credentialsConf = null; + HashMap<String, CredentialsProperties> credentialsProperties = setCredentialPropertyToActionConf(context, action, actionConf); + Credentials credentials = null; if (credentialsProperties != null) { - + credentials = new Credentials(); // Adding if action need to set more credential tokens - credentialsConf = new JobConf(false); + credentialsConf = new Configuration(false); XConfiguration.copy(actionConf, credentialsConf); - setCredentialTokens(credentialsConf, context, action, credentialsProperties); + setCredentialTokens(credentials, credentialsConf, context, action, credentialsProperties); // insert conf to action conf from credentialsConf for (Entry<String, String> entry : credentialsConf) { @@ -976,7 +956,7 @@ public class JavaActionExecutor extends ActionExecutor { } } } - JobConf launcherJobConf = createLauncherConf(actionFs, context, action, actionXml, actionConf); + Configuration launcherJobConf = createLauncherConf(actionFs, context, action, actionXml, actionConf); String consoleUrl; String launcherId = LauncherMapperHelper.getRecoveryId(launcherJobConf, context.getActionDir(), context @@ -1010,21 +990,21 @@ public class JavaActionExecutor extends ActionExecutor { // launcherJobConf.getCredentials().addToken(HadoopAccessorService.MR_TOKEN_ALIAS, mrdt); // insert credentials tokens to launcher job conf if needed - if (needInjectCredentials() && credentialsConf != null) { - for (Token<? extends TokenIdentifier> tk : credentialsConf.getCredentials().getAllTokens()) { + if (credentialsConf != null) { + for (Token<? extends TokenIdentifier> tk :credentials.getAllTokens()) { Text fauxAlias = new Text(tk.getKind() + "_" + tk.getService()); LOG.debug("ADDING TOKEN: " + fauxAlias); - launcherJobConf.getCredentials().addToken(fauxAlias, tk); + credentials.addToken(fauxAlias, tk); } - if (credentialsConf.getCredentials().numberOfSecretKeys() > 0) { + if (credentials.numberOfSecretKeys() > 0) { for (Entry<String, CredentialsProperties> entry : credentialsProperties.entrySet()) { CredentialsProperties credProps = entry.getValue(); if (credProps != null) { Text credName = new Text(credProps.getName()); - byte[] secKey = credentialsConf.getCredentials().getSecretKey(credName); + byte[] secKey = credentials.getSecretKey(credName); if (secKey != null) { LOG.debug("ADDING CREDENTIAL: " + credProps.getName()); - launcherJobConf.getCredentials().addSecretKey(credName, secKey); + credentials.addSecretKey(credName, secKey); } } } @@ -1039,7 +1019,7 @@ public class JavaActionExecutor extends ActionExecutor { YarnClientApplication newApp = yarnClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); ApplicationSubmissionContext appContext = - createAppSubmissionContext(appId, launcherJobConf, user, context, actionConf); + createAppSubmissionContext(appId, launcherJobConf, user, context, actionConf, action.getName(), credentials); yarnClient.submitApplication(appContext); launcherId = appId.toString(); @@ -1061,20 +1041,26 @@ public class JavaActionExecutor extends ActionExecutor { } } - private ApplicationSubmissionContext createAppSubmissionContext(ApplicationId appId, JobConf launcherJobConf, String user, - Context context, Configuration actionConf) + private ApplicationSubmissionContext createAppSubmissionContext(ApplicationId appId, Configuration launcherJobConf, + String user, Context context, Configuration actionConf, String actionName, + Credentials credentials) throws IOException, HadoopAccessorException, URISyntaxException { ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); + String jobName = XLog.format( + "oozie:launcher:T={0}:W={1}:A={2}:ID={3}", getType(), + context.getWorkflow().getAppName(), actionName, + context.getWorkflow().getId()); + appContext.setApplicationId(appId); - appContext.setApplicationName(launcherJobConf.getJobName()); + appContext.setApplicationName(jobName); appContext.setApplicationType("Oozie Launcher"); Priority pri = Records.newRecord(Priority.class); int priority = 0; // TODO: OYA: Add a constant or a config pri.setPriority(priority); appContext.setPriority(pri); - appContext.setQueue(launcherJobConf.getQueueName()); + appContext.setQueue("default"); // TODO: will be possible to set in <launcher> ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); // Set the resources to localize @@ -1096,7 +1082,6 @@ public class JavaActionExecutor extends ActionExecutor { // This adds the Hadoop jars to the classpath in the Launcher JVM ClasspathUtils.setupClasspath(env, launcherJobConf); - // FIXME: move this to specific places where it's actually needed - keeping it here for now if (needToAddMRJars()) { ClasspathUtils.addMapReduceToClasspath(env, launcherJobConf); } @@ -1132,9 +1117,11 @@ public class JavaActionExecutor extends ActionExecutor { appContext.setAMContainerSpec(amContainer); // Set tokens - DataOutputBuffer dob = new DataOutputBuffer(); - launcherJobConf.getCredentials().writeTokenStorageToStream(dob); - amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); + if (credentials != null) { + DataOutputBuffer dob = new DataOutputBuffer(); + credentials.writeTokenStorageToStream(dob); + amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); + } // Set Resources // TODO: OYA: make resources allocated for the AM configurable and choose good defaults (memory MB, vcores) @@ -1144,24 +1131,6 @@ public class JavaActionExecutor extends ActionExecutor { return appContext; } - private boolean needInjectCredentials() { - boolean methodExists = true; - - Class<?> klass; - try { - klass = Class.forName("org.apache.hadoop.mapred.JobConf"); - klass.getMethod("getCredentials"); - } - catch (ClassNotFoundException ex) { - methodExists = false; - } - catch (NoSuchMethodException ex) { - methodExists = false; - } - - return methodExists; - } - protected HashMap<String, CredentialsProperties> setCredentialPropertyToActionConf(Context context, WorkflowAction action, Configuration actionConf) throws Exception { HashMap<String, CredentialsProperties> credPropertiesMap = null; @@ -1199,20 +1168,20 @@ public class JavaActionExecutor extends ActionExecutor { return credPropertiesMap; } - protected void setCredentialTokens(JobConf jobconf, Context context, WorkflowAction action, + protected void setCredentialTokens(Credentials credentials, Configuration jobconf, Context context, WorkflowAction action, HashMap<String, CredentialsProperties> credPropertiesMap) throws Exception { if (context != null && action != null && credPropertiesMap != null) { // Make sure we're logged into Kerberos; if not, or near expiration, it will relogin - CredentialsProvider.ensureKerberosLogin(); + CredentialsProviderFactory.ensureKerberosLogin(); for (Entry<String, CredentialsProperties> entry : credPropertiesMap.entrySet()) { String credName = entry.getKey(); CredentialsProperties credProps = entry.getValue(); if (credProps != null) { - CredentialsProvider credProvider = new CredentialsProvider(credProps.getType()); - Credentials credentialObject = credProvider.createCredentialObject(); - if (credentialObject != null) { - credentialObject.addtoJobConf(jobconf, credProps, context); + CredentialsProviderFactory tokenProviderFactory = new CredentialsProviderFactory(credProps.getType()); + CredentialsProvider tokenProvider = tokenProviderFactory.createCredentialsProvider(); + if (tokenProvider != null) { + tokenProvider.updateCredentials(credentials, jobconf, credProps, context); LOG.debug("Retrieved Credential '" + credName + "' for action " + action.getId()); } else { @@ -1224,7 +1193,6 @@ public class JavaActionExecutor extends ActionExecutor { } } } - } protected HashMap<String, CredentialsProperties> getActionCredentialsProperties(Context context, @@ -1351,7 +1319,7 @@ public class JavaActionExecutor extends ActionExecutor { * @return YarnClient * @throws HadoopAccessorException */ - protected YarnClient createYarnClient(Context context, JobConf jobConf) throws HadoopAccessorException { + protected YarnClient createYarnClient(Context context, Configuration jobConf) throws HadoopAccessorException { String user = context.getWorkflow().getUser(); return Services.get().get(HadoopAccessorService.class).createYarnClient(user, jobConf); } @@ -1391,7 +1359,7 @@ public class JavaActionExecutor extends ActionExecutor { YarnClient yarnClient = null; try { Element actionXml = XmlUtils.parseXml(action.getConf()); - JobConf jobConf = createBaseHadoopConf(context, actionXml); + Configuration jobConf = createBaseHadoopConf(context, actionXml); FileSystem actionFs = context.getAppFileSystem(); yarnClient = createYarnClient(context, jobConf); FinalApplicationStatus appStatus = null; @@ -1528,7 +1496,7 @@ public class JavaActionExecutor extends ActionExecutor { try { Element actionXml = XmlUtils.parseXml(action.getConf()); - JobConf jobConf = createBaseHadoopConf(context, actionXml); + Configuration jobConf = createBaseHadoopConf(context, actionXml); yarnClient = createYarnClient(context, jobConf); yarnClient.killApplication(ConverterUtils.toApplicationId(action.getExternalId())); @@ -1641,7 +1609,7 @@ public class JavaActionExecutor extends ActionExecutor { HadoopAccessorException, URISyntaxException { } - private void injectJobInfo(JobConf launcherJobConf, Configuration actionConf, Context context, WorkflowAction action) { + private void injectJobInfo(Configuration launcherJobConf, Configuration actionConf, Context context, WorkflowAction action) { if (OozieJobInfo.isJobInfoEnabled()) { try { OozieJobInfo jobInfo = new OozieJobInfo(actionConf, context, action); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java index e2a667e..8bd476e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java @@ -39,7 +39,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Counters; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RunningJob; import org.apache.hadoop.security.UserGroupInformation; import org.apache.oozie.client.OozieClient; @@ -119,14 +118,9 @@ public class LauncherMapperHelper { launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax); } - public static void setupLauncherInfo(JobConf launcherConf, String jobId, String actionId, Path actionDir, + public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir, String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException { - launcherConf.setMapperClass(LauncherMapper.class); - launcherConf.setSpeculativeExecution(false); - launcherConf.setNumMapTasks(1); - launcherConf.setNumReduceTasks(0); - launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId); launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId); launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString()); @@ -147,13 +141,9 @@ public class LauncherMapperHelper { actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()])); launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true); } - - launcherConf.setInputFormat(OozieLauncherInputFormat.class); - launcherConf.setOutputFormat(OozieLauncherOutputFormat.class); - launcherConf.setOutputCommitter(OozieLauncherOutputCommitter.class); } - public static void setupYarnRestartHandling(JobConf launcherJobConf, Configuration actionConf, String launcherTag, + public static void setupYarnRestartHandling(Configuration launcherJobConf, Configuration actionConf, String launcherTag, long launcherTime) throws NoSuchAlgorithmException { launcherJobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, launcherTime); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java index ff89e48..918a269 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java @@ -209,8 +209,8 @@ public class MapReduceActionExecutor extends JavaActionExecutor { try { if (action.getStatus() == WorkflowAction.Status.OK) { Element actionXml = XmlUtils.parseXml(action.getConf()); - JobConf jobConf = createBaseHadoopConf(context, actionXml); - jobClient = createJobClient(context, jobConf); + Configuration jobConf = createBaseHadoopConf(context, actionXml); + jobClient = createJobClient(context, new JobConf(jobConf)); RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalChildIDs())); if (runningJob == null) { throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002", @@ -297,19 +297,20 @@ public class MapReduceActionExecutor extends JavaActionExecutor { } @Override - JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, + Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf) throws ActionExecutorException { // If the user is using a regular MapReduce job and specified an uber jar, we need to also set it for the launcher; // so we override createLauncherConf to call super and then to set the uber jar if specified. At this point, checking that // uber jars are enabled and resolving the uber jar path is already done by setupActionConf() when it parsed the actionConf // argument and we can just look up the uber jar in the actionConf argument. - JobConf launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf); + Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf); Namespace ns = actionXml.getNamespace(); if (actionXml.getChild("streaming", ns) == null && actionXml.getChild("pipes", ns) == null) { // Set for uber jar String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR); if (uberJar != null && uberJar.trim().length() > 0) { - launcherJobConf.setJar(uberJar); + // TODO + // launcherJobConf.setJar(uberJar); } } return launcherJobConf; @@ -350,7 +351,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { @Override public void check(Context context, WorkflowAction action) throws ActionExecutorException { Map<String, String> actionData = Collections.emptyMap(); - JobConf jobConf = null; + Configuration jobConf = null; try { FileSystem actionFs = context.getAppFileSystem(); @@ -372,7 +373,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor { boolean exception = false; try { - jobClient = createJobClient(context, jobConf); + jobClient = createJobClient(context, new JobConf(jobConf)); RunningJob runningJob = jobClient.getJob(JobID.forName(newId)); if (runningJob == null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java index 347b034..17508eb 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java @@ -23,7 +23,6 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.XOozieClient; @@ -129,10 +128,10 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor { } @Override - protected JobConf loadHadoopDefaultResources(Context context, Element actionXml) { + protected Configuration loadHadoopDefaultResources(Context context, Element actionXml) { boolean loadDefaultResources = ConfigurationService .getBoolean(HadoopAccessorService.ACTION_CONFS_LOAD_DEFAULT_RESOURCES); - JobConf conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); + Configuration conf = super.createBaseHadoopConf(context, actionXml, loadDefaultResources); return conf; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index c373758..b411c4c 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -21,7 +21,6 @@ package org.apache.oozie.action.hadoop; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.action.ActionExecutorException; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.service.ConfigurationService; @@ -99,10 +98,10 @@ public class SparkActionExecutor extends JavaActionExecutor { } @Override - JobConf createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, + Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml, Configuration actionConf) throws ActionExecutorException { - JobConf launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf); + Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf); if (launcherJobConf.get("oozie.launcher." + TASK_USER_PRECEDENCE) == null) { launcherJobConf.set(TASK_USER_PRECEDENCE, "true"); } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java index 54a860e..7e2561d 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java @@ -130,8 +130,8 @@ public class SqoopActionExecutor extends JavaActionExecutor { try { if (action.getStatus() == WorkflowAction.Status.OK) { Element actionXml = XmlUtils.parseXml(action.getConf()); - JobConf jobConf = createBaseHadoopConf(context, actionXml); - jobClient = createJobClient(context, jobConf); + Configuration jobConf = createBaseHadoopConf(context, actionXml); + jobClient = createJobClient(context, new JobConf(jobConf)); // Cumulative counters for all Sqoop mapreduce jobs Counters counters = null; http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java index 514f01e..3a36588 100644 --- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java +++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.oozie.ErrorCode; -import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.util.IOUtils; import org.apache.oozie.util.ParamChecker; @@ -52,7 +51,6 @@ import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.InetAddress; import java.net.URI; @@ -534,6 +532,10 @@ public class HadoopAccessorService implements Service { } } + public JobClient createJobClient(String user, Configuration conf) throws HadoopAccessorException { + return createJobClient(user, new JobConf(conf)); + } + /** * Return a YarnClient created with the provided user and configuration. * http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java b/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java index 0629891..23dd78a 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/CredentialForTest.java @@ -20,20 +20,17 @@ package org.apache.oozie.action.hadoop; import java.util.Map.Entry; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.Credentials; import org.apache.oozie.ErrorCode; import org.apache.oozie.action.ActionExecutor.Context; -import org.apache.oozie.action.hadoop.Credentials; import org.apache.oozie.action.hadoop.CredentialsProperties; - - -@SuppressWarnings("deprecation") -public class CredentialForTest extends Credentials { +public class CredentialForTest implements CredentialsProvider { @Override - public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception { - + public void updateCredentials(Credentials credentials, Configuration config, + CredentialsProperties props, Context context) throws Exception { String paramA = null; String paramB = null; for (Entry<String, String> parameter : props.getProperties().entrySet()) { @@ -50,7 +47,7 @@ public class CredentialForTest extends Credentials { throw new CredentialException(ErrorCode.E0510, "required parameters is null."); } - jobconf.set(props.getName(), "testcert"); + config.set(props.getName(), "testcert"); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java b/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java index 9da8fbe..9558aa5 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/InsertTestToken.java @@ -18,15 +18,15 @@ package org.apache.oozie.action.hadoop; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.oozie.action.ActionExecutor.Context; import org.apache.oozie.util.XLog; - -public class InsertTestToken extends Credentials{ +public class InsertTestToken implements CredentialsProvider { public static String DUMMY_SECRET_KEY = "DummySecretKey"; public InsertTestToken() { } @@ -34,14 +34,16 @@ public class InsertTestToken extends Credentials{ /* (non-Javadoc) * @see org.apache.oozie.action.hadoop.Credentials#addtoJobConf(org.apache.hadoop.mapred.JobConf, org.apache.oozie.action.hadoop.CredentialsProperties, org.apache.oozie.action.ActionExecutor.Context) */ + @Override - public void addtoJobConf(JobConf jobconf, CredentialsProperties props, Context context) throws Exception { + public void updateCredentials(Credentials credentials, Configuration config, CredentialsProperties props, Context context) + throws Exception { try { Token<DelegationTokenIdentifier> abctoken = new Token<DelegationTokenIdentifier>(); - jobconf.getCredentials().addToken(new Text("ABC Token"), abctoken); + credentials.addToken(new Text("ABC Token"), abctoken); XLog.getLog(getClass()).debug("Added the ABC token in job conf"); - jobconf.getCredentials().addSecretKey(new Text(DUMMY_SECRET_KEY), DUMMY_SECRET_KEY.getBytes("UTF-8")); + credentials.addSecretKey(new Text(DUMMY_SECRET_KEY), DUMMY_SECRET_KEY.getBytes("UTF-8")); XLog.getLog(getClass()).debug("Added the " + DUMMY_SECRET_KEY + " in job conf"); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java index e6d43ca..204245c 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestHCatCredentials.java @@ -34,6 +34,7 @@ import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.Credentials; import org.apache.oozie.service.HCatAccessorService; import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; @@ -130,17 +131,18 @@ public class TestHCatCredentials { credProps.setProperties(new HashMap<String, String>()); HCatCredentials hcatCred = new HCatCredentials(); final JobConf jobConf = new JobConf(false); + Credentials credentials = new Credentials(); PowerMockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - JobConf jConf = (JobConf) args[0]; - jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[1]); - jConf.set(HCAT_METASTORE_URI, (String) args[2]); + Configuration jConf = (Configuration) args[1]; + jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]); + jConf.set(HCAT_METASTORE_URI, (String) args[3]); return null; } - }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2); - hcatCred.addtoJobConf(jobConf, credProps, null); + }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2); + hcatCred.updateCredentials(credentials, jobConf, credProps, null); assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL)); assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI)); assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL)); @@ -155,19 +157,20 @@ public class TestHCatCredentials { credProps.setProperties(new HashMap<String, String>()); HCatCredentials hcatCred = new HCatCredentials(); final JobConf jobConf = new JobConf(false); + Credentials credentials = new Credentials(); HCatCredentialHelper hcatCredHelper = Mockito.mock(HCatCredentialHelper.class); PowerMockito.whenNew(HCatCredentialHelper.class).withNoArguments().thenReturn(hcatCredHelper); PowerMockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - JobConf jConf = (JobConf) args[0]; - jConf.set(HIVE_METASTORE_PRINCIPAL, (String) args[1]); - jConf.set(HIVE_METASTORE_URI, (String) args[2]); + Configuration jConf = (Configuration) args[1]; + jConf.set(HIVE_METASTORE_PRINCIPAL, (String) args[2]); + jConf.set(HIVE_METASTORE_URI, (String) args[3]); return null; } - }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL, TEST_HIVE_METASTORE_URI); - hcatCred.addtoJobConf(jobConf, credProps, null); + }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL, TEST_HIVE_METASTORE_URI); + hcatCred.updateCredentials(credentials, jobConf, credProps, null); assertEquals(TEST_HIVE_METASTORE_PRINCIPAL, jobConf.get(HIVE_METASTORE_PRINCIPAL)); assertEquals(TEST_HIVE_METASTORE_URI, jobConf.get(HIVE_METASTORE_URI)); assertNull(jobConf.get(HCAT_METASTORE_PRINCIPAL)); @@ -186,17 +189,18 @@ public class TestHCatCredentials { credProps.setProperties(prop); HCatCredentials hcatCred = new HCatCredentials(); final JobConf jobConf = new JobConf(false); + Credentials credentials = new Credentials(); PowerMockito.doAnswer(new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Object[] args = invocation.getArguments(); - JobConf jConf = (JobConf) args[0]; - jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[1]); - jConf.set(HCAT_METASTORE_URI, (String) args[2]); + JobConf jConf = (JobConf) args[1]; + jConf.set(HCAT_METASTORE_PRINCIPAL, (String) args[2]); + jConf.set(HCAT_METASTORE_URI, (String) args[3]); return null; } - }).when(hcatCredHelper).set(jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2); - hcatCred.addtoJobConf(jobConf, credProps, null); + }).when(hcatCredHelper).set(credentials, jobConf, TEST_HIVE_METASTORE_PRINCIPAL2, TEST_HIVE_METASTORE_URI2); + hcatCred.updateCredentials(credentials, jobConf, credProps, null); assertEquals(TEST_HIVE_METASTORE_PRINCIPAL2, jobConf.get(HCAT_METASTORE_PRINCIPAL)); assertEquals(TEST_HIVE_METASTORE_URI2, jobConf.get(HCAT_METASTORE_URI)); assertNull(jobConf.get(HIVE_METASTORE_PRINCIPAL)); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java index aa485ef..5996083 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.api.records.YarnApplicationState; @@ -879,10 +880,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // Try to load the token without it being defined in oozie-site; should get an exception JobConf credentialsConf = new JobConf(); + Credentials credentials = new Credentials(); Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf); XConfiguration.copy(launcherConf, credentialsConf); try { - ae.setCredentialTokens(credentialsConf, context, action, credProperties); + ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties); fail("Should have gotten an exception but did not"); } catch (ActionExecutorException aee) { @@ -896,13 +898,14 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // Try to load the token after being defined in oozie-site; should work correctly credentialsConf = new JobConf(); + credentials = new Credentials(); launcherConf = ae.createBaseHadoopConf(context, actionXmlconf); XConfiguration.copy(launcherConf, credentialsConf); - ae.setCredentialTokens(credentialsConf, context, action, credProperties); - Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token")); + ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties); + Token<? extends TokenIdentifier> tk = credentials.getToken(new Text("ABC Token")); assertNotNull(tk); - byte[] secKey = credentialsConf.getCredentials().getSecretKey(new Text(InsertTestToken.DUMMY_SECRET_KEY)); + byte[] secKey = credentials.getSecretKey(new Text(InsertTestToken.DUMMY_SECRET_KEY)); assertNotNull(secKey); assertEquals(InsertTestToken.DUMMY_SECRET_KEY, new String(secKey, "UTF-8")); } @@ -938,8 +941,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { try { // Setting the credential properties in launcher conf should fail - HashMap<String, CredentialsProperties> credProperties = ae.setCredentialPropertyToActionConf(context, action, - actionConf); + ae.setCredentialPropertyToActionConf(context, action, actionConf); } catch (ActionExecutorException e) { assertEquals(e.getErrorCode(), "JA021"); @@ -1083,10 +1085,11 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { // Try to load the token without it being defined in oozie-site; should get an exception JobConf credentialsConf = new JobConf(); + Credentials credentials = new Credentials(); Configuration launcherConf = ae.createBaseHadoopConf(context, actionXmlconf); XConfiguration.copy(launcherConf, credentialsConf); - ae.setCredentialTokens(credentialsConf, context, action, credProperties); - Token<? extends TokenIdentifier> tk = credentialsConf.getCredentials().getToken(new Text("ABC Token")); + ae.setCredentialTokens(credentials, credentialsConf, context, action, credProperties); + Token<? extends TokenIdentifier> tk = credentials.getToken(new Text("ABC Token")); if (expectingTokens) { assertNotNull(tk); } else { @@ -1523,7 +1526,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { getFileSystem().mkdirs(javaShareLibPath); Services.get().setService(ShareLibService.class); - JobConf conf = ae.createBaseHadoopConf(context, eActionXml); + Configuration conf = ae.createBaseHadoopConf(context, eActionXml); // Despite systemLibPath is not fully qualified and the action refers to the // second namenode the next line won't throw exception because default fs is used ae.addShareLib(conf, new String[] { "java-action-executor" }); @@ -1547,7 +1550,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { Context context = createContext(actionXml, null); Path appPath = new Path("localfs://namenode:port/mydir"); JavaActionExecutor ae = new JavaActionExecutor(); - JobConf conf = ae.createBaseHadoopConf(context, eActionXml); + Configuration conf = ae.createBaseHadoopConf(context, eActionXml); Services.get().destroy(); setSystemProperty(HadoopAccessorService.SUPPORTED_FILESYSTEMS, "hdfs,viewfs"); new Services().init(); @@ -1868,10 +1871,10 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase { WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); action.setType(ae.getType()); Context context = new Context(wf, action); - JobConf actionConf = new JobConf(); + Configuration actionConf = new JobConf(); // Test when server side setting is not enabled - JobConf launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); + Configuration launcherConf = ae.createLauncherConf(getFileSystem(), context, action, actionXml, actionConf); assertEquals("false", launcherConf.get(JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED)); // disabled by default ConfigurationService.set("oozie.action.launcher." + JavaActionExecutor.HADOOP_YARN_TIMELINE_SERVICE_ENABLED, "true"); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java index 0925398..551adff 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestOozieJobInfo.java @@ -33,7 +33,6 @@ import java.util.regex.Matcher; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapred.JobConf; import org.apache.oozie.BundleActionBean; import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorJobBean; @@ -154,7 +153,7 @@ public class TestOozieJobInfo extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfbean, actionList.get(1), false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(actionList.get(1).getConf())); String user = conf.get("user.name"); FileSystem fs = getFileSystem(); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java index 633ef37..b4bce60 100644 --- a/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java +++ b/core/src/test/java/org/apache/oozie/client/TestWorkflowXClient.java @@ -152,7 +152,7 @@ public class TestWorkflowXClient extends DagServletTestCase { fail("submit client without JT should throw exception"); } catch (RuntimeException exception) { - assertEquals("java.lang.RuntimeException: jobtracker is not specified in conf", exception.toString()); + assertEquals("java.lang.RuntimeException: Resource manager is not specified in conf", exception.toString()); } conf.setProperty(XOozieClient.RM, "localhost:9001"); try { http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java index 135c34a..4069a95 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionCheckXCommand.java @@ -23,6 +23,7 @@ import java.io.Writer; import java.util.Date; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobClient; @@ -257,7 +258,7 @@ public class TestActionCheckXCommand extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); @@ -408,7 +409,7 @@ public class TestActionCheckXCommand extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job2, action1, false, false); WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action2.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action2.getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); @@ -480,7 +481,7 @@ public class TestActionCheckXCommand extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job0, action1, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf())); String user = conf.get("user.name"); JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java index 59cfdd2..6c3051b 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestActionStartXCommand.java @@ -160,7 +160,7 @@ public class TestActionStartXCommand extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); String launcherId = action.getExternalId(); @@ -227,7 +227,7 @@ public class TestActionStartXCommand extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job, action, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action.getConf())); String user = conf.get("user.name"); String launcherId = action.getExternalId(); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java index 8d0e081..35a846a 100644 --- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java +++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java @@ -19,7 +19,7 @@ package org.apache.oozie.service; import org.apache.hadoop.conf.Configuration; -import org.apache.oozie.action.hadoop.CredentialsProvider; +import org.apache.oozie.action.hadoop.CredentialsProviderFactory; import org.apache.oozie.action.hadoop.DistcpActionExecutor; import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.action.hadoop.LauncherMapper; @@ -212,7 +212,7 @@ public class TestConfigurationService extends XTestCase { assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED)); assertEquals(0, ConfigurationService.getStrings("no.defined").length); - assertEquals(0, ConfigurationService.getStrings(CredentialsProvider.CRED_KEY).length); + assertEquals(0, ConfigurationService.getStrings(CredentialsProviderFactory.CRED_KEY).length); assertEquals(1, ConfigurationService.getStrings(DistcpActionExecutor.CLASS_NAMES).length); assertEquals("distcp=org.apache.hadoop.tools.DistCp", ConfigurationService.getStrings(DistcpActionExecutor.CLASS_NAMES)[0]); http://git-wip-us.apache.org/repos/asf/oozie/blob/74f4afb2/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java index a3270e9..c19d62a 100644 --- a/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java +++ b/core/src/test/java/org/apache/oozie/service/TestRecoveryService.java @@ -246,7 +246,7 @@ public class TestRecoveryService extends XDataTestCase { ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job1, action1, false, false); MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor(); - JobConf conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf())); + Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf())); String launcherId = action1.getExternalId(); waitUntilYarnAppDoneAndAssertSuccess(launcherId);