Author: gates Date: Mon Jul 22 23:47:03 2013 New Revision: 1505854 URL: http://svn.apache.org/r1505854 Log: HIVE-4677 WebHCat e2e tests fail on Hadoop 2 (daijy via gates)
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original) +++ hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java Mon Jul 22 23:47:03 2013 @@ -101,4 +101,6 @@ public interface HCatHadoopShims { */ public boolean isFileInHDFS(FileSystem fs, Path path) throws IOException; + public InetSocketAddress getAddress(Configuration conf); + } Modified: hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original) +++ hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 22 23:47:03 2013 @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.shims.HCatHadoopShims; /* * Communicate with the JobTracker as a specific user. @@ -35,14 +36,13 @@ public class TempletonJobTracker { /** * Create a connection to the Job Tracker. */ - public TempletonJobTracker(InetSocketAddress addr, - Configuration conf) + public TempletonJobTracker(Configuration conf) throws IOException { UserGroupInformation ugi = UserGroupInformation.getLoginUser(); cnx = (JobSubmissionProtocol) RPC.getProxy(JobSubmissionProtocol.class, JobSubmissionProtocol.versionID, - addr, + HCatHadoopShims.Instance.get().getAddress(conf), ugi, conf, NetUtils.getSocketFactory(conf, Modified: hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (original) +++ hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Mon Jul 22 23:47:03 2013 @@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Outpu import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.Progressable; public class HCatHadoopShims20S implements HCatHadoopShims { @@ -149,4 +150,11 @@ public class HCatHadoopShims20S implemen // In hadoop 1.x.x the file system URI is sufficient to determine the uri of the file return "hdfs".equals(fs.getUri().getScheme()); } + + @Override + public InetSocketAddress getAddress(Configuration conf) { + String jobTrackerStr = + conf.get("mapred.job.tracker", "localhost:8012"); + return NetUtils.createSocketAddr(jobTrackerStr); + } } Modified: hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original) +++ hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 22 23:47:03 2013 @@ -32,8 +32,7 @@ public class TempletonJobTracker { /** * Create a connection to the Job Tracker. */ - public TempletonJobTracker(InetSocketAddress addr, - Configuration conf) + public TempletonJobTracker(Configuration conf) throws IOException { jc = new JobClient(conf); Modified: hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original) +++ hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Mon Jul 22 23:47:03 2013 @@ -127,4 +127,9 @@ public class HCatHadoopShims23 implement // resolvePath is a sure shot way of knowing which file system the file is. return "hdfs".equals(fs.resolvePath(path).toUri().getScheme()); } + + @Override + public InetSocketAddress getAddress(Configuration conf) { + return null; + } } Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Mon Jul 22 23:47:03 2013 @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.TempletonJobTracker; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.shims.HCatHadoopShims; import org.apache.hcatalog.templeton.tool.JobState; /** @@ -39,8 +40,7 @@ public class DeleteDelegator extends Tem TempletonJobTracker tracker = null; JobState state = null; try { - tracker = new TempletonJobTracker(getAddress(appConf), - appConf); + tracker = new TempletonJobTracker(appConf); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) throw new BadParam("Invalid jobid: " + id); Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java Mon Jul 22 23:47:03 2013 @@ -25,6 +25,7 @@ import java.util.ArrayList; import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.mapred.TempletonJobTracker; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hcatalog.shims.HCatHadoopShims; import org.apache.hcatalog.templeton.tool.JobState; /** @@ -41,8 +42,7 @@ public class ListDelegator extends Templ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); TempletonJobTracker tracker = null; try { - tracker = new TempletonJobTracker(getAddress(appConf), - appConf); + tracker = new TempletonJobTracker(appConf); ArrayList<String> ids = new ArrayList<String>(); Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java Mon Jul 22 23:47:03 2013 @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.mapred.JobID; import org.apache.hadoop.mapred.JobProfile; import org.apache.hadoop.mapred.JobStatus; +import org.apache.hcatalog.shims.HCatHadoopShims; import org.apache.hadoop.mapred.TempletonJobTracker; import org.apache.hcatalog.templeton.tool.JobState; @@ -44,8 +45,7 @@ public class StatusDelegator extends Tem TempletonJobTracker tracker = null; JobState state = null; try { - tracker = new TempletonJobTracker(getAddress(appConf), - appConf); + tracker = new TempletonJobTracker(appConf); JobID jobid = StatusDelegator.StringToJobID(id); if (jobid == null) throw new BadParam("Invalid jobid: " + id); Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff ============================================================================== --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Mon Jul 22 23:47:03 2013 @@ -21,7 +21,6 @@ package org.apache.hcatalog.templeton; import java.net.InetSocketAddress; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; /** * The helper class for all the Templeton delegator classes. A @@ -34,10 +33,4 @@ public class TempletonDelegator { public TempletonDelegator(AppConfig appConf) { this.appConf = appConf; } - - public static InetSocketAddress getAddress(Configuration conf) { - String jobTrackerStr = - conf.get("mapred.job.tracker", "localhost:8012"); - return NetUtils.createSocketAddr(jobTrackerStr); - } }