svn commit: r1353580 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Author: tgraves
Date: Mon Jun 25 14:39:01 2012
New Revision: 1353580
URL: http://svn.apache.org/viewvc?rev=1353580&view=rev
Log:
MAPREDUCE-4361. Fix detailed metrics for protobuf-based RPC on 0.23 (Jason Lowe
via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1353580&r1=1353579&r2=1353580&view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
Mon Jun 25 14:39:01 2012
@@ -271,6 +271,14 @@ public abstract class Server {
}
/**
+ * Returns a handle to the rpcDetailedMetrics (required in tests)
+ * @return rpc detailed metrics
+ */
+ public RpcDetailedMetrics getRpcDetailedMetrics() {
+return rpcDetailedMetrics;
+ }
+
+ /**
* Refresh the service authorization ACL for the service handled by this
server.
*/
public void refreshServiceAcl(Configuration conf, PolicyProvider provider) {
svn commit: r1353627 - /hadoop/common/trunk/hadoop-tools/hadoop-pipes/
Author: tucu Date: Mon Jun 25 16:02:39 2012 New Revision: 1353627 URL: http://svn.apache.org/viewvc?rev=1353627&view=rev Log: Adding svn-ignore for hdfs-raid and pipe target dirs Modified: hadoop/common/trunk/hadoop-tools/hadoop-pipes/ (props changed) Propchange: hadoop/common/trunk/hadoop-tools/hadoop-pipes/ -- --- svn:ignore (added) +++ svn:ignore Mon Jun 25 16:02:39 2012 @@ -0,0 +1 @@ +target
svn commit: r1353629 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/
Author: tucu Date: Mon Jun 25 16:04:28 2012 New Revision: 1353629 URL: http://svn.apache.org/viewvc?rev=1353629&view=rev Log: Adding svn-ignore for pipes target dir Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/ (props changed) Propchange: hadoop/common/branches/branch-2/hadoop-tools/hadoop-pipes/ -- --- svn:ignore (added) +++ svn:ignore Mon Jun 25 16:04:28 2012 @@ -0,0 +1 @@ +target
svn commit: r1353691 - in /hadoop/common/branches/branch-1: CHANGES.txt src/native/acinclude.m4 src/native/configure.ac
Author: eli
Date: Mon Jun 25 18:22:11 2012
New Revision: 1353691
URL: http://svn.apache.org/viewvc?rev=1353691&view=rev
Log:
HADOOP-7868. Hadoop native fails to compile when default linker option is
-Wl,--as-needed. Contributed by Trevor Robinson
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/native/acinclude.m4
hadoop/common/branches/branch-1/src/native/configure.ac
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353691&r1=1353690&r2=1353691&view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 18:22:11 2012
@@ -34,6 +34,9 @@ Release 1.2.0 - unreleased
HADOOP-8512. AuthenticatedURL should reset the Token when the server
returns other than OK on authentication (tucu)
+HADOOP-7868. Hadoop native fails to compile when default linker
+option is -Wl,--as-needed. (Trevor Robinson via eli)
+
Release 1.1.0 - unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1/src/native/acinclude.m4
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/native/acinclude.m4?rev=1353691&r1=1353690&r2=1353691&view=diff
==
--- hadoop/common/branches/branch-1/src/native/acinclude.m4 (original)
+++ hadoop/common/branches/branch-1/src/native/acinclude.m4 Mon Jun 25 18:22:11
2012
@@ -1,4 +1,4 @@
-# AC_COMPUTE_NEEDED_DSO(LIBRARY, PREPROC_SYMBOL)
+# AC_COMPUTE_NEEDED_DSO(LIBRARY, TEST_PROGRAM, PREPROC_SYMBOL)
# --
# Compute the 'actual' dynamic-library used
# for LIBRARY and set it to PREPROC_SYMBOL
@@ -6,7 +6,7 @@ AC_DEFUN([AC_COMPUTE_NEEDED_DSO],
[
AC_CACHE_CHECK([Checking for the 'actual' dynamic-library for '-l$1'],
ac_cv_libname_$1,
[
- echo 'int main(int argc, char **argv){return 0;}' > conftest.c
+ echo '$2' > conftest.c
if test -z "`${CC} ${LDFLAGS} -o conftest conftest.c -l$1 2>&1`"; then
dnl Try objdump and ldd in that order to get the dynamic library
if test ! -z "`which objdump | grep -v 'no objdump'`"; then
@@ -22,5 +22,5 @@ AC_CACHE_CHECK([Checking for the 'actual
rm -f conftest*
]
)
-AC_DEFINE_UNQUOTED($2, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for
'-l$1'])
+AC_DEFINE_UNQUOTED($3, ${ac_cv_libname_$1}, [The 'actual' dynamic-library for
'-l$1'])
])# AC_COMPUTE_NEEDED_DSO
Modified: hadoop/common/branches/branch-1/src/native/configure.ac
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/native/configure.ac?rev=1353691&r1=1353690&r2=1353691&view=diff
==
--- hadoop/common/branches/branch-1/src/native/configure.ac (original)
+++ hadoop/common/branches/branch-1/src/native/configure.ac Mon Jun 25 18:22:11
2012
@@ -87,10 +87,20 @@ CPPFLAGS=$cppflags_bak
AC_SUBST([JNI_CPPFLAGS])
dnl Check for zlib headers
-AC_CHECK_HEADERS([zlib.h zconf.h],
AC_COMPUTE_NEEDED_DSO(z,HADOOP_ZLIB_LIBRARY), AC_MSG_ERROR(Zlib headers were
not found... native-hadoop library needs zlib to build. Please install the
requisite zlib development package.))
+AC_CHECK_HEADERS([zlib.h zconf.h],
+ AC_COMPUTE_NEEDED_DSO(z,
+[#include "zlib.h"
+int main(int argc, char **argv){zlibVersion();return 0;}],
+HADOOP_ZLIB_LIBRARY),
+ AC_MSG_ERROR(Zlib headers were not found... native-hadoop library needs zlib
to build. Please install the requisite zlib development package.))
dnl Check for snappy headers
-AC_CHECK_HEADERS([snappy-c.h],
AC_COMPUTE_NEEDED_DSO(snappy,HADOOP_SNAPPY_LIBRARY), AC_MSG_WARN(Snappy headers
were not found... building without snappy.))
+AC_CHECK_HEADERS([snappy-c.h],
+ AC_COMPUTE_NEEDED_DSO(snappy,
+[#include "snappy-c.h"
+int main(int argc, char **argv){snappy_compress(0,0,0,0);return 0;}],
+HADOOP_SNAPPY_LIBRARY),
+ AC_MSG_WARN(Snappy headers were not found... building without snappy.))
dnl Check for headers needed by the native Group resolution implementation
AC_CHECK_HEADERS([fcntl.h stdlib.h string.h unistd.h], [], AC_MSG_ERROR(Some
system headers not found... please ensure their presence on your platform.))
svn commit: r1353753 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
Author: tucu
Date: Mon Jun 25 21:39:35 2012
New Revision: 1353753
URL: http://svn.apache.org/viewvc?rev=1353753&view=rev
Log:
MAPREDUCE-2289. Permissions race can make getStagingDir fail on local
filesystem (ahmed via tucu)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353753&r1=1353752&r2=1353753&view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 21:39:35 2012
@@ -37,6 +37,9 @@ Release 1.2.0 - unreleased
HADOOP-7868. Hadoop native fails to compile when default linker
option is -Wl,--as-needed. (Trevor Robinson via eli)
+MAPREDUCE-2289. Permissions race can make getStagingDir fail on local
filesystem
+(ahmed via tucu)
+
Release 1.1.0 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java?rev=1353753&r1=1353752&r2=1353753&view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapreduce/JobSubmissionFiles.java
Mon Jun 25 21:39:35 2012
@@ -27,6 +27,10 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.conf.Configuration;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* A utility to manage job submission files.
* Note that this class is for framework internal usage only and is
@@ -34,6 +38,8 @@ import org.apache.hadoop.conf.Configurat
*/
public class JobSubmissionFiles {
+ private final static Log LOG = LogFactory.getLog(JobSubmissionFiles.class);
+
// job submission directory is private!
final public static FsPermission JOB_DIR_PERMISSION =
FsPermission.createImmutable((short) 0700); // rwx
@@ -103,14 +109,18 @@ public class JobSubmissionFiles {
if (fs.exists(stagingArea)) {
FileStatus fsStatus = fs.getFileStatus(stagingArea);
String owner = fsStatus.getOwner();
- if (!(owner.equals(currentUser) || owner.equals(realUser)) ||
- !fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) {
- throw new IOException("The ownership/permissions on the staging " +
- "directory " + stagingArea + " is not as expected. " +
- "It is owned by " + owner + " and permissions are "+
- fsStatus.getPermission() + ". The directory must " +
+ if (!(owner.equals(currentUser) || owner.equals(realUser))) {
+ throw new IOException("The ownership on the staging directory " +
+ stagingArea + " is not as expected. " +
+ "It is owned by " + owner + ". The directory must " +
"be owned by the submitter " + currentUser + " or " +
- "by " + realUser + " and permissions must be rwx--");
+ "by " + realUser);
+ }
+ if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) {
+LOG.info("Permissions on staging directory " + stagingArea + " are " +
+ "incorrect: " + fsStatus.getPermission() + ". Fixing permissions " +
+ "to correct value " + JOB_DIR_PERMISSION);
+fs.setPermission(stagingArea, JOB_DIR_PERMISSION);
}
} else {
fs.mkdirs(stagingArea,
svn commit: r1353759 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobClient.java src/test/org/apache/hadoop/mapred/TestJobClient.java
Author: tucu
Date: Mon Jun 25 22:00:45 2012
New Revision: 1353759
URL: http://svn.apache.org/viewvc?rev=1353759&view=rev
Log:
MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl via
tucu)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1353759&r1=1353758&r2=1353759&view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Mon Jun 25 22:00:45 2012
@@ -8,6 +8,8 @@ Release 1.2.0 - unreleased
HADOOP-8023. Add unset() method to Configuration (tucu)
+MAPREDUCE-4355. Add JobStatus getJobStatus(JobID) to JobClient. (kkambatl
via tucu)
+
IMPROVEMENTS
HDFS-3515. Port HDFS-1457 to branch-1. (eli)
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1353759&r1=1353758&r2=1353759&view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobClient.java
Mon Jun 25 22:00:45 2012
@@ -1063,13 +1063,28 @@ public class JobClient extends Configure
}
return false;
}
-
+
+ /**
+ * Get {@link JobStatus} of a job. Returns null if the id does not correspond
+ * to any known job.
+ *
+ * @param jobid
+ * the jobid of the job.
+ * @return the {@link JobStatus} object to retrieve the job stats, null if
the
+ * jobid doesn't correspond to any known job.
+ * @throws IOException
+ */
+ public JobStatus getJobStatus(final JobID jobid) throws IOException {
+return jobSubmitClient.getJobStatus(jobid);
+ }
+
/**
- * Get an {@link RunningJob} object to track an ongoing job. Returns
- * null if the id does not correspond to any known job.
+ * Get an {@link RunningJob} object to track an ongoing job. Returns null if
+ * the id does not correspond to any known job.
*
- * @param jobid the jobid of the job.
- * @return the {@link RunningJob} handle to track the job, null if the
+ * @param jobid
+ * the jobid of the job.
+ * @return the {@link RunningJob} handle to track the job, null if the
* jobid doesn't correspond to any known job.
* @throws IOException
*/
Modified:
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java?rev=1353759&r1=1353758&r2=1353759&view=diff
==
---
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java
(original)
+++
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestJobClient.java
Mon Jun 25 22:00:45 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.mapred;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
@@ -41,6 +40,8 @@ public class TestJobClient extends Clust
private static final Log LOG = LogFactory.getLog(TestJobClient.class);
+ private JobConf conf;
+
private String runJob() throws Exception {
OutputStream os = getFileSystem().create(new Path(getInputDir(),
"text.txt"));
Writer wr = new OutputStreamWriter(os);
@@ -49,7 +50,7 @@ public class TestJobClient extends Clust
wr.write("hello3\n");
wr.close();
-JobConf conf = createJobConf();
+conf = createJobConf();
conf.setJobName("mr");
conf.setJobPriority(JobPriority.HIGH);
@@ -127,4 +128,15 @@ public class TestJobClient extends Clust
assertEquals("Exit code", 0, exitCode);
verifyJobPriority(jobId, "VERY_LOW");
}
+
+ public void testGetJobStatus() throws Exception {
+String jobIdString = runJob();
+JobID jobid = JobID.forName(jobIdString);
+
+JobClient client = new JobClient(conf);
+assertEquals("JobClient intialization", conf, client.getConf());
+JobStatus jobStatus = client.getJobStatus(jobid);
+assertNotNull("Fetch JobStatus", jobStatus);
+assertEquals("JobId check", jobid, jobStatus.getJobID());
+ }
}
[Hadoop Wiki] Trivial Update of "ContributorsGroup" by QwertyManiac
Dear Wiki user, You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change notification. The "ContributorsGroup" page has been changed by QwertyManiac: http://wiki.apache.org/hadoop/ContributorsGroup?action=diff&rev1=28&rev2=29 Comment: Add Robert Vandehey * RickFarnell * RidaBenjelloun * RobertEvans + * RobertVandehey * RyanRawson * Ryu Kobayashi * SanjayRadia
