svn commit: r1353580 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Author: tgraves Date: Mon Jun 25 14:39:01 2012 New Revision: 1353580 URL: http://svn.apache.org/viewvc?rev=1353580&view=rev Log: MAPREDUCE-4361. Fix detailed metrics for protobuf-based RPC on 0.23 (Jason Lowe via tgraves) Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1353580&r1=1353579&r2=1353580&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Mon Jun 25 14:39:01 2012 @@ -271,6 +271,14 @@ public abstract class Server { } /** + * Returns a handle to the rpcDetailedMetrics (required in tests) + * @return rpc detailed metrics + */ + public RpcDetailedMetrics getRpcDetailedMetrics() { +return rpcDetailedMetrics; + } + + /** * Refresh the service authorization ACL for the service handled by this server. */ public void refreshServiceAcl(Configuration conf, PolicyProvider provider) {
svn commit: r1353627 - /hadoop/common/trunk/hadoop-tools/hadoop-pipes/
Author: tucu Date: Mon Jun 25 16:02:39 2012 New Revision: 1353627 URL: http://svn.apache.org/viewvc?rev=1353627&view=rev Log: Adding svn-ignore for hdfs-raid and pipe target dirs Modified: hadoop/common/trunk/hadoop-tools/hadoop-pipes/ (props changed) Propchange: hadoop/common/trunk/hadoop-tools/hadoop-pipes/ -- --- svn:ignore (added) +++ svn:ignore Mon Jun 25 16:02:39 2012 @@ -0,0 +1 @@ +target
svn commit: r1353629 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/
Author: tucu Date: Mon Jun 25 16:04:28 2012 New Revision: 1353629 URL: http://svn.apache.org/viewvc?rev=1353629&view=rev Log: Adding svn-ignore for pipes target dir Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/ (props changed) Propchange: hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/ -- --- svn:ignore (added) +++ svn:ignore Mon Jun 25 16:04:28 2012 @@ -0,0 +1 @@ +target
svn commit: r1353691 - in /hadoop/common/branches/branch-1: CHANGES.txt src/native/acinclude.m4 src/native/configure.ac
Author: eli Date: Mon Jun 25 18:22:11 2012 New Revision: 1353691 URL: http://svn.apache.org/viewvc?rev=1353691&view=rev Log: HADOOP-7868. Hadoop native fails to compile when default linker option is -Wl,--as-needed. Contributed by Trevor Robinson Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/native/acinclude.m4 hadoop/common/branches/branch-1/src/native/configure.ac Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353691&r1=1353690&r2=1353691&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 18:22:11 2012 @@ -34,6 +34,9 @@ Release 1.2.0 - unreleased HADOOP-8512. AuthenticatedURL should reset the Token when the server returns other than OK on authentication (tucu) +HADOOP-7868. Hadoop native fails to compile when default linker +option is -Wl,--as-needed. (Trevor Robinson via eli) + Release 1.1.0 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/native/acinclude.m4 URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/native/acinclude.m4?rev=1353691&r1=1353690&r2=1353691&view=diff == --- hadoop/common/branches/branch-1/src/native/acinclude.m4 (original) +++ hadoop/common/branches/branch-1/src/native/acinclude.m4 Mon Jun 25 18:22:11 2012 @@ -1,4 +1,4 @@ -# AC_COMPUTE_NEEDED_DSO(LIBRARY, PREPROC_SYMBOL) +# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL) # -- # Compute the 'actual' dynamic-library used # for LIBRARY and set it to PREPROC_SYMBOL @@ -6,7 +6,7 @@ AC_DEFUN([AC_COMPUTE_NEEDED_DSO], [ AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'], ac_cv_libname_$1, [ - echo 'int main(int argc, char **argv){return 0;}' > conftest.c + echo '$2' > conftest.c if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then dnl Try objdump and ldd in that order to get the dynamic library if test ! -z "`which objdump | grep -v 'no objdump'`"; then @@ -22,5 +22,5 @@ AC_CACHE_CHECK([Checking for the 'actual rm -f conftest* ] ) -AC_DEFINE_UNQUOTED($2, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) +AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for '-l$1']) ])# AC_COMPUTE_NEEDED_DSO Modified: hadoop/common/branches/branch-1/src/native/configure.ac URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/native/configure.ac?rev=1353691&r1=1353690&r2=1353691&view=diff == --- hadoop/common/branches/branch-1/src/native/configure.ac (original) +++ hadoop/common/branches/branch-1/src/native/configure.ac Mon Jun 25 18:22:11 2012 @@ -87,10 +87,20 @@ CPPFLAGS=$cppflags_bak AC_SUBST([JNI_CPPFLAGS]) dnl Check for zlib headers -AC_CHECK_HEADERS([zlib.h zconf.h], AC_COMPUTE_NEEDED_DSO(z,HADOOP_ZLIB_LIBRARY), AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) +AC_CHECK_HEADERS([zlib.h zconf.h], + AC_COMPUTE_NEEDED_DSO(z, +[#include "zlib.h" +int main(int argc, char **argv){zlibVersion();return 0;}], +HADOOP_ZLIB_LIBRARY), + AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib to build. Please install the requisite zlib development package.)) dnl Check for snappy headers -AC_CHECK_HEADERS([snappy-c.h], AC_COMPUTE_NEEDED_DSO(snappy,HADOOP_SNAPPY_LIBRARY), AC_MSG_WARN(Snappy headers were not found... building without snappy.)) +AC_CHECK_HEADERS([snappy-c.h], + AC_COMPUTE_NEEDED_DSO(snappy, +[#include "snappy-c.h" +int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}], +HADOOP_SNAPPY_LIBRARY), + AC_MSG_WARN(Snappy headers were not found... building without snappy.)) dnl Check for headers needed by the native Group resolution implementation AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some system headers not found... please ensure their presence on your platform.))
svn commit: r1353753 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
Author: tucu Date: Mon Jun 25 21:39:35 2012 New Revision: 1353753 URL: http://svn.apache.org/viewvc?rev=1353753&view=rev Log: MAPREDUCE-2289. Permissions race can make getStagingDir fail on local filesystem (ahmed via tucu) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353753&r1=1353752&r2=1353753&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 21:39:35 2012 @@ -37,6 +37,9 @@ Release 1.2.0 - unreleased HADOOP-7868. Hadoop native fails to compile when default linker option is -Wl,--as-needed. (Trevor Robinson via eli) +MAPREDUCE-2289. Permissions race can make getStagingDir fail on local filesystem +(ahmed via tucu) + Release 1.1.0 - unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java?rev=1353753&r1=1353752&r2=1353753&view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java Mon Jun 25 21:39:35 2012 @@ -27,6 +27,10 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.conf.Configuration; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + /** * A utility to manage job submission files. * Note that this class is for framework internal usage only and is @@ -34,6 +38,8 @@ import org.apache.hadoop.conf.Configurat */ public class JobSubmissionFiles { + private final static Log LOG = LogFactory.getLog(JobSubmissionFiles.class); + // job submission directory is private! final public static FsPermission JOB_DIR_PERMISSION = FsPermission.createImmutable((short) 0700); // rwx @@ -103,14 +109,18 @@ public class JobSubmissionFiles { if (fs.exists(stagingArea)) { FileStatus fsStatus = fs.getFileStatus(stagingArea); String owner = fsStatus.getOwner(); - if (!(owner.equals(currentUser) || owner.equals(realUser)) || - !fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) { - throw new IOException("The ownership/permissions on the staging " + - "directory " + stagingArea + " is not as expected. " + - "It is owned by " + owner + " and permissions are "+ - fsStatus.getPermission() + ". The directory must " + + if (!(owner.equals(currentUser) || owner.equals(realUser))) { + throw new IOException("The ownership on the staging directory " + + stagingArea + " is not as expected. " + + "It is owned by " + owner + ". The directory must " + "be owned by the submitter " + currentUser + " or " + - "by " + realUser + " and permissions must be rwx--"); + "by " + realUser); + } + if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) { +LOG.info("Permissions on staging directory " + stagingArea + " are " + + "incorrect: " + fsStatus.getPermission() + ". Fixing permissions " + + "to correct value " + JOB_DIR_PERMISSION); +fs.setPermission(stagingArea, JOB_DIR_PERMISSION); } } else { fs.mkdirs(stagingArea,
svn commit: r1353759 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobClient.java src/test/org/apache/hadoop/mapred/TestJobClient.java
Author: tucu Date: Mon Jun 25 22:00:45 2012 New Revision: 1353759 URL: http://svn.apache.org/viewvc?rev=1353759&view=rev Log: MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via tucu) Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353759&r1=1353758&r2=1353759&view=diff == --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 22:00:45 2012 @@ -8,6 +8,8 @@ Release 1.2.0 - unreleased HADOOP-8023. Add unset() method to Configuration (tucu) +MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via tucu) + IMPROVEMENTS HDFS-3515. Port HDFS-1457 to branch-1. (eli) Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1353759&r1=1353758&r2=1353759&view=diff == --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java Mon Jun 25 22:00:45 2012 @@ -1063,13 +1063,28 @@ public class JobClient extends Configure } return false; } - + + /** + * Get {@link JobStatus} of a job. Returns null if the id does not correspond + * to any known job. + * + * @param jobid + * the jobid of the job. + * @return the {@link JobStatus} object to retrieve the job stats, null if the + * jobid doesn't correspond to any known job. + * @throws IOException + */ + public JobStatus getJobStatus(final JobID jobid) throws IOException { +return jobSubmitClient.getJobStatus(jobid); + } + /** - * Get an {@link RunningJob} object to track an ongoing job. Returns - * null if the id does not correspond to any known job. + * Get an {@link RunningJob} object to track an ongoing job. Returns null if + * the id does not correspond to any known job. * - * @param jobid the jobid of the job. - * @return the {@link RunningJob} handle to track the job, null if the + * @param jobid + * the jobid of the job. + * @return the {@link RunningJob} handle to track the job, null if the * jobid doesn't correspond to any known job. * @throws IOException */ Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java?rev=1353759&r1=1353758&r2=1353759&view=diff == --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java Mon Jun 25 22:00:45 2012 @@ -19,7 +19,6 @@ package org.apache.hadoop.mapred; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; @@ -41,6 +40,8 @@ public class TestJobClient extends Clust private static final Log LOG = LogFactory.getLog(TestJobClient.class); + private JobConf conf; + private String runJob() throws Exception { OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt")); Writer wr = new OutputStreamWriter(os); @@ -49,7 +50,7 @@ public class TestJobClient extends Clust wr.write("hello3\n"); wr.close(); -JobConf conf = createJobConf(); +conf = createJobConf(); conf.setJobName("mr"); conf.setJobPriority(JobPriority.HIGH); @@ -127,4 +128,15 @@ public class TestJobClient extends Clust assertEquals("Exit code", 0, exitCode); verifyJobPriority(jobId, "VERY_LOW"); } + + public void testGetJobStatus() throws Exception { +String jobIdString = runJob(); +JobID jobid = JobID.forName(jobIdString); + +JobClient client = new JobClient(conf); +assertEquals("JobClient intialization", conf, client.getConf()); +JobStatus jobStatus = client.getJobStatus(jobid); +assertNotNull("Fetch JobStatus", jobStatus); +assertEquals("JobId check", jobid, jobStatus.getJobID()); + } }
[Hadoop Wiki] Trivial Update of "ContributorsGroup" by QwertyManiac
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "ContributorsGroup" page has been changed by QwertyManiac: http://wiki.apache.org/hadoop/ContributorsGroup?action=diff&rev1=28&rev2=29 Comment: Add Robert Vandehey * RickFarnell * RidaBenjelloun * RobertEvans + * RobertVandehey * RyanRawson * Ryu Kobayashi * SanjayRadia