Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java Tue Jan 22 19:33:02 2013 @@ -1,3 +1,20 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.hadoop.mapreduce.v2.hs; import java.io.IOException; @@ -6,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapred.TaskCompletionEvent; import org.apache.hadoop.mapreduce.JobACL; import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; @@ -126,7 +144,7 @@ public class MockHistoryJobs extends Moc } @Override - public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + public TaskCompletionEvent[] getMapAttemptCompletionEvents( int startIndex, int maxEvents) { return job.getMapAttemptCompletionEvents(startIndex, maxEvents); }
Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEntities.java Tue Jan 22 19:33:02 2013 @@ -1,3 +1,20 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.hadoop.mapreduce.v2.hs; import static junit.framework.Assert.assertEquals; Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml Tue Jan 22 19:33:02 2013 @@ -148,6 +148,15 @@ </additionalClasspathElements> </configuration> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/test/java/org/apache/hadoop/cli/data60bytes</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> </project> Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Tue Jan 22 19:33:02 2013 @@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.http.HttpConfig; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -69,14 +70,16 @@ import org.apache.hadoop.security.token. import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ClientToken; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.ProtoUtils; public class ClientServiceDelegate { private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class); @@ -176,13 +179,10 @@ public class ClientServiceDelegate { serviceAddr = NetUtils.createSocketAddrForHost( application.getHost(), application.getRpcPort()); if (UserGroupInformation.isSecurityEnabled()) { - String clientTokenEncoded = application.getClientToken(); - Token<ApplicationTokenIdentifier> clientToken = - new Token<ApplicationTokenIdentifier>(); - clientToken.decodeFromUrlString(clientTokenEncoded); - // RPC layer client expects ip:port as service for tokens - SecurityUtil.setTokenService(clientToken, serviceAddr); - newUgi.addToken(clientToken); + ClientToken clientToken = application.getClientToken(); + Token<ClientTokenIdentifier> token = + ProtoUtils.convertFromProtoFormat(clientToken, serviceAddr); + newUgi.addToken(token); } LOG.debug("Connecting to " + serviceAddr); final InetSocketAddress finalServiceAddr = serviceAddr; Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java Tue Jan 22 19:33:02 2013 @@ -24,6 +24,8 @@ import java.util.HashMap; import org.apache.commons.lang.NotImplementedException; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest; @@ -48,6 +50,8 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptResponse; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskResponse; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup; import org.apache.hadoop.mapreduce.v2.api.records.Counters; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; @@ -58,8 +62,8 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -82,10 +86,9 @@ public class NotRunningJob implements MR .newRecordInstance(ApplicationAttemptId.class); // Setting AppState to NEW and finalStatus to UNDEFINED as they are never - // used - // for a non running job + // used for a non running job return BuilderUtils.newApplicationReport(unknownAppId, unknownAttemptId, - "N/A", "N/A", "N/A", "N/A", 0, "", YarnApplicationState.NEW, "N/A", + "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A", "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A"); } @@ -215,7 +218,21 @@ public class NotRunningJob implements MR /* Should not be invoked by anyone. */ throw new NotImplementedException(); } - + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + /* Should not be invoked by anyone. */ + throw new NotImplementedException(); + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + /* Should not be invoked by anyone. */ + throw new NotImplementedException(); + } + @Override public InetSocketAddress getConnectAddress() { /* Should not be invoked by anyone. Normally used to set token service */ Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java Tue Jan 22 19:33:02 2013 @@ -33,7 +33,6 @@ import org.apache.hadoop.mapreduce.Queue import org.apache.hadoop.mapreduce.QueueInfo; import org.apache.hadoop.mapreduce.TaskTrackerInfo; import org.apache.hadoop.mapreduce.TypeConverter; -import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; @@ -41,9 +40,9 @@ import org.apache.hadoop.yarn.api.Client import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.client.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ProtoUtils; -import org.apache.hadoop.yarn.client.YarnClientImpl; public class ResourceMgrDelegate extends YarnClientImpl { private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); @@ -62,11 +61,6 @@ public class ResourceMgrDelegate extends init(conf); start(); } - - public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0) - throws IOException, InterruptedException { - return; - } public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException { @@ -168,13 +162,6 @@ public class ResourceMgrDelegate extends return 0; } - public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0) - throws IOException, InterruptedException { - // TODO: Implement renewDelegationToken - LOG.warn("renewDelegationToken - Not implemented"); - return 0; - } - public ApplicationId getApplicationId() { return applicationId; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Tue Jan 22 19:33:02 2013 @@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.util.Build import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ProtoUtils; +import com.google.common.annotations.VisibleForTesting; /** * This class enables the current JobClient (0.22 hadoop) to run on YARN. @@ -100,7 +101,7 @@ public class YARNRunner implements Clien private Configuration conf; private final FileContext defaultFileContext; - /* usually is false unless the jobclient getdelegation token is + /* usually is false unless the jobclient get delegation token is * called. This is a hack wherein we do return a token from RM * on getDelegationtoken but due to the restricted api on jobclient * we just add a job history DT token when submitting a job. @@ -158,18 +159,18 @@ public class YARNRunner implements Clien @Override public void cancelDelegationToken(Token<DelegationTokenIdentifier> arg0) throws IOException, InterruptedException { - resMgrDelegate.cancelDelegationToken(arg0); + throw new UnsupportedOperationException("Use Token.renew instead"); } @Override public TaskTrackerInfo[] getActiveTrackers() throws IOException, InterruptedException { - return resMgrDelegate.getActiveTrackers(); + return resMgrDelegate.getActiveTrackers(); } @Override public JobStatus[] getAllJobs() throws IOException, InterruptedException { - return resMgrDelegate.getAllJobs(); + return resMgrDelegate.getAllJobs(); } @Override @@ -184,18 +185,18 @@ public class YARNRunner implements Clien return resMgrDelegate.getClusterMetrics(); } - private Token<?> getDelegationTokenFromHS( - MRClientProtocol hsProxy, Text renewer) throws IOException, - InterruptedException { + @VisibleForTesting + Token<?> getDelegationTokenFromHS(MRClientProtocol hsProxy) + throws IOException, InterruptedException { GetDelegationTokenRequest request = recordFactory .newRecordInstance(GetDelegationTokenRequest.class); - request.setRenewer(renewer.toString()); + request.setRenewer(Master.getMasterPrincipal(conf)); DelegationToken mrDelegationToken = hsProxy.getDelegationToken(request) .getDelegationToken(); return ProtoUtils.convertFromProtoFormat(mrDelegationToken, hsProxy.getConnectAddress()); } - + @Override public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer) throws IOException, InterruptedException { @@ -269,8 +270,7 @@ public class YARNRunner implements Clien // the delegation tokens for the HistoryServer also. if (conf.getBoolean(JobClient.HS_DELEGATION_TOKEN_REQUIRED, DEFAULT_HS_DELEGATION_TOKEN_REQUIRED)) { - Token hsDT = getDelegationTokenFromHS(hsProxy, new Text( - conf.get(JobClient.HS_DELEGATION_TOKEN_RENEWER))); + Token hsDT = getDelegationTokenFromHS(hsProxy); ts.addToken(hsDT.getService(), hsDT); } } @@ -324,8 +324,16 @@ public class YARNRunner implements Clien // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); - capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB, - MRJobConfig.DEFAULT_MR_AM_VMEM_MB)); + capability.setMemory( + conf.getInt( + MRJobConfig.MR_AM_VMEM_MB, MRJobConfig.DEFAULT_MR_AM_VMEM_MB + ) + ); + capability.setVirtualCores( + conf.getInt( + MRJobConfig.MR_AM_CPU_VCORES, MRJobConfig.DEFAULT_MR_AM_CPU_VCORES + ) + ); LOG.debug("AppMaster capability = " + capability); // Setup LocalResources @@ -386,9 +394,31 @@ public class YARNRunner implements Clien MRJobConfig.MR_AM_LOG_LEVEL, MRJobConfig.DEFAULT_MR_AM_LOG_LEVEL); MRApps.addLog4jSystemProperties(logLevel, logSize, vargs); - vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, - MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS)); - + // Check for Java Lib Path usage in MAP and REDUCE configs + warnForJavaLibPath(conf.get(MRJobConfig.MAP_JAVA_OPTS,""), "map", + MRJobConfig.MAP_JAVA_OPTS, MRJobConfig.MAP_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS,""), "map", + MRJobConfig.MAPRED_MAP_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.REDUCE_JAVA_OPTS,""), "reduce", + MRJobConfig.REDUCE_JAVA_OPTS, MRJobConfig.REDUCE_ENV); + warnForJavaLibPath(conf.get(MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS,""), "reduce", + MRJobConfig.MAPRED_REDUCE_ADMIN_JAVA_OPTS, MRJobConfig.MAPRED_ADMIN_USER_ENV); + + // Add AM admin command opts before user command opts + // so that it can be overridden by user + String mrAppMasterAdminOptions = conf.get(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, + MRJobConfig.DEFAULT_MR_AM_ADMIN_COMMAND_OPTS); + warnForJavaLibPath(mrAppMasterAdminOptions, "app master", + MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, MRJobConfig.MR_AM_ADMIN_USER_ENV); + vargs.add(mrAppMasterAdminOptions); + + // Add AM user command opts + String mrAppMasterUserOptions = conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, + MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS); + warnForJavaLibPath(mrAppMasterUserOptions, "app master", + MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.MR_AM_ENV); + vargs.add(mrAppMasterUserOptions); + vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT); @@ -412,6 +442,9 @@ public class YARNRunner implements Clien Map<String, String> environment = new HashMap<String, String>(); MRApps.setClasspath(environment, conf); + // Setup the environment variables for Admin first + MRApps.setEnvFromInputString(environment, + conf.get(MRJobConfig.MR_AM_ADMIN_USER_ENV)); // Setup the environment variables (LD_LIBRARY_PATH, etc) MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MR_AM_ENV)); @@ -466,7 +499,7 @@ public class YARNRunner implements Clien @Override public long renewDelegationToken(Token<DelegationTokenIdentifier> arg0) throws IOException, InterruptedException { - return resMgrDelegate.renewDelegationToken(arg0); + throw new UnsupportedOperationException("Use Token.renew instead"); } @@ -569,4 +602,15 @@ public class YARNRunner implements Clien throws IOException { return clientCache.getClient(jobID).getLogFilePath(jobID, taskAttemptID); } + + private static void warnForJavaLibPath(String opts, String component, + String javaConf, String envConf) { + if (opts != null && opts.contains("-Djava.library.path")) { + LOG.warn("Usage of -Djava.library.path in " + javaConf + " can cause " + + "programs to no longer function if hadoop native libraries " + + "are used. These values should be set as part of the " + + "LD_LIBRARY_PATH in the " + component + " JVM env using " + + envConf + " config settings."); + } + } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo Tue Jan 22 19:33:02 2013 @@ -1 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# org.apache.hadoop.mapreduce.v2.security.client.ClientHSSecurityInfo Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java Tue Jan 22 19:33:02 2013 @@ -90,8 +90,8 @@ public abstract class NotificationTestCa } public static class NotificationServlet extends HttpServlet { - public static int counter = 0; - public static int failureCounter = 0; + public static volatile int counter = 0; + public static volatile int failureCounter = 0; private static final long serialVersionUID = 1L; protected void doGet(HttpServletRequest req, HttpServletResponse res) @@ -155,7 +155,11 @@ public abstract class NotificationTestCa System.out.println(launchWordCount(this.createJobConf(), "a b c d e f g h", 1, 1)); - Thread.sleep(2000); + boolean keepTrying = true; + for (int tries = 0; tries < 30 && keepTrying; tries++) { + Thread.sleep(50); + keepTrying = !(NotificationServlet.counter == 2); + } assertEquals(2, NotificationServlet.counter); assertEquals(0, NotificationServlet.failureCounter); @@ -173,14 +177,22 @@ public abstract class NotificationTestCa // run a job with KILLED status System.out.println(UtilsForTests.runJobKill(this.createJobConf(), inDir, outDir).getID()); - Thread.sleep(2000); + keepTrying = true; + for (int tries = 0; tries < 30 && keepTrying; tries++) { + Thread.sleep(50); + keepTrying = !(NotificationServlet.counter == 4); + } assertEquals(4, NotificationServlet.counter); assertEquals(0, NotificationServlet.failureCounter); // run a job with FAILED status System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir, outDir).getID()); - Thread.sleep(2000); + keepTrying = true; + for (int tries = 0; tries < 30 && keepTrying; tries++) { + Thread.sleep(50); + keepTrying = !(NotificationServlet.counter == 6); + } assertEquals(6, NotificationServlet.counter); assertEquals(0, NotificationServlet.failureCounter); } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Tue Jan 22 19:33:02 2013 @@ -25,10 +25,10 @@ import java.util.Iterator; import junit.framework.Assert; -import org.apache.hadoop.ipc.Server; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRConfig; @@ -69,6 +69,8 @@ import org.apache.hadoop.metrics2.lib.De import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ClientRMProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; @@ -87,6 +89,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -339,6 +343,18 @@ public class TestClientRedirect { GetDelegationTokenRequest request) throws YarnRemoteException { return null; } + + @Override + public RenewDelegationTokenResponse renewDelegationToken( + RenewDelegationTokenRequest request) throws YarnRemoteException { + return null; + } + + @Override + public CancelDelegationTokenResponse cancelDelegationToken( + CancelDelegationTokenRequest request) throws YarnRemoteException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { @@ -504,6 +520,20 @@ public class TestClientRedirect { throws YarnRemoteException { return null; } + + @Override + public org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse renewDelegationToken( + org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest request) + throws YarnRemoteException { + return null; + } + + @Override + public org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse cancelDelegationToken( + org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest request) + throws YarnRemoteException { + return null; + } } static Counters getMyCounters() { Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestJHSSecurity.java Tue Jan 22 19:33:02 2013 @@ -18,19 +18,29 @@ package org.apache.hadoop.mapreduce.security; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; +import java.net.InetSocketAddress; import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import junit.framework.Assert; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.v2.api.HSClientProtocol; import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest; +import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest; +import org.apache.hadoop.mapreduce.v2.hs.JHSDelegationTokenSecretManager; import org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer; import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; @@ -49,6 +59,8 @@ import org.junit.Test; public class TestJHSSecurity { + private static final Log LOG = LogFactory.getLog(TestJHSSecurity.class); + @Test public void testDelegationToken() throws IOException, InterruptedException { @@ -63,55 +75,208 @@ public class TestJHSSecurity { conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); - - final JobHistoryServer jobHistoryServer = new JobHistoryServer() { - protected void doSecureLogin(Configuration conf) throws IOException { - // no keytab based login + + final long initialInterval = 10000l; + final long maxLifetime= 20000l; + final long renewInterval = 10000l; + + JobHistoryServer jobHistoryServer = null; + MRClientProtocol clientUsingDT = null; + long tokenFetchTime; + try { + jobHistoryServer = new JobHistoryServer() { + protected void doSecureLogin(Configuration conf) throws IOException { + // no keytab based login + }; + + protected JHSDelegationTokenSecretManager createJHSSecretManager( + Configuration conf) { + return new JHSDelegationTokenSecretManager(initialInterval, + maxLifetime, renewInterval, 3600000); + } }; - }; - jobHistoryServer.init(conf); - jobHistoryServer.start(); - - // Fake the authentication-method - UserGroupInformation loggedInUser = UserGroupInformation.getCurrentUser(); - loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); +// final JobHistoryServer jobHistoryServer = jhServer; + jobHistoryServer.init(conf); + jobHistoryServer.start(); + final MRClientProtocol hsService = jobHistoryServer.getClientService() + .getClientHandler(); + + // Fake the authentication-method + UserGroupInformation loggedInUser = UserGroupInformation + .createRemoteUser("testrene...@apache.org"); + Assert.assertEquals("testrenewer", loggedInUser.getShortUserName()); + // Default realm is APACHE.ORG + loggedInUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS); + + + DelegationToken token = getDelegationToken(loggedInUser, hsService, + loggedInUser.getShortUserName()); + tokenFetchTime = System.currentTimeMillis(); + LOG.info("Got delegation token at: " + tokenFetchTime); + + // Now try talking to JHS using the delegation token + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "TheDarkLord", conf); + + GetJobReportRequest jobReportRequest = + Records.newRecord(GetJobReportRequest.class); + jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1)); + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (YarnRemoteException e) { + Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); + } + + // Renew after 50% of token age. + while(System.currentTimeMillis() < tokenFetchTime + initialInterval / 2) { + Thread.sleep(500l); + } + long nextExpTime = renewDelegationToken(loggedInUser, hsService, token); + long renewalTime = System.currentTimeMillis(); + LOG.info("Renewed token at: " + renewalTime + ", NextExpiryTime: " + + nextExpTime); + + // Wait for first expiry, but before renewed expiry. + while (System.currentTimeMillis() > tokenFetchTime + initialInterval + && System.currentTimeMillis() < nextExpTime) { + Thread.sleep(500l); + } + Thread.sleep(50l); + + // Valid token because of renewal. + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (UndeclaredThrowableException e) { + Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); + } + + // Wait for expiry. + while(System.currentTimeMillis() < renewalTime + renewInterval) { + Thread.sleep(500l); + } + Thread.sleep(50l); + LOG.info("At time: " + System.currentTimeMillis() + ", token should be invalid"); + // Token should have expired. + try { + clientUsingDT.getJobReport(jobReportRequest); + fail("Should not have succeeded with an expired token"); + } catch (UndeclaredThrowableException e) { + assertTrue(e.getCause().getMessage().contains("is expired")); + } + + // Test cancellation + // Stop the existing proxy, start another. + if (clientUsingDT != null) { +// RPC.stopProxy(clientUsingDT); + clientUsingDT = null; + } + token = getDelegationToken(loggedInUser, hsService, + loggedInUser.getShortUserName()); + tokenFetchTime = System.currentTimeMillis(); + LOG.info("Got delegation token at: " + tokenFetchTime); + + // Now try talking to HSService using the delegation token + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "loginuser2", conf); + + + try { + clientUsingDT.getJobReport(jobReportRequest); + } catch (UndeclaredThrowableException e) { + fail("Unexpected exception" + e); + } + cancelDelegationToken(loggedInUser, hsService, token); + if (clientUsingDT != null) { +// RPC.stopProxy(clientUsingDT); + clientUsingDT = null; + } + + // Creating a new connection. + clientUsingDT = getMRClientProtocol(token, jobHistoryServer + .getClientService().getBindAddress(), "loginuser2", conf); + LOG.info("Cancelled delegation token at: " + System.currentTimeMillis()); + // Verify cancellation worked. + try { + clientUsingDT.getJobReport(jobReportRequest); + fail("Should not have succeeded with a cancelled delegation token"); + } catch (UndeclaredThrowableException e) { + } + + + + } finally { + jobHistoryServer.stop(); + } + } + private DelegationToken getDelegationToken( + final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final String renewerString) + throws IOException, InterruptedException { // Get the delegation token directly as it is a little difficult to setup // the kerberos based rpc. - DelegationToken token = - loggedInUser.doAs(new PrivilegedExceptionAction<DelegationToken>() { + DelegationToken token = loggedInUser + .doAs(new PrivilegedExceptionAction<DelegationToken>() { @Override public DelegationToken run() throws YarnRemoteException { - GetDelegationTokenRequest request = - Records.newRecord(GetDelegationTokenRequest.class); - request.setRenewer("OneRenewerToRuleThemAll"); - return jobHistoryServer.getClientService().getClientHandler() - .getDelegationToken(request).getDelegationToken(); + GetDelegationTokenRequest request = Records + .newRecord(GetDelegationTokenRequest.class); + request.setRenewer(renewerString); + return hsService.getDelegationToken(request).getDelegationToken(); } + }); + return token; + } + + private long renewDelegationToken(final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final DelegationToken dToken) + throws IOException, InterruptedException { + long nextExpTime = loggedInUser.doAs(new PrivilegedExceptionAction<Long>() { + + @Override + public Long run() throws YarnRemoteException { + RenewDelegationTokenRequest request = Records + .newRecord(RenewDelegationTokenRequest.class); + request.setDelegationToken(dToken); + return hsService.renewDelegationToken(request).getNextExpirationTime(); + } + }); + return nextExpTime; + } + + private void cancelDelegationToken(final UserGroupInformation loggedInUser, + final MRClientProtocol hsService, final DelegationToken dToken) + throws IOException, InterruptedException { + + loggedInUser.doAs(new PrivilegedExceptionAction<Void>() { + @Override + public Void run() throws YarnRemoteException { + CancelDelegationTokenRequest request = Records + .newRecord(CancelDelegationTokenRequest.class); + request.setDelegationToken(dToken); + hsService.cancelDelegationToken(request); + return null; + } + }); + } + + private MRClientProtocol getMRClientProtocol(DelegationToken token, + final InetSocketAddress hsAddress, String user, final Configuration conf) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); + ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress)); - // Now try talking to JHS using the delegation token - UserGroupInformation ugi = - UserGroupInformation.createRemoteUser("TheDarkLord"); - ugi.addToken(ProtoUtils.convertFromProtoFormat( - token, jobHistoryServer.getClientService().getBindAddress())); final YarnRPC rpc = YarnRPC.create(conf); - MRClientProtocol userUsingDT = - ugi.doAs(new PrivilegedAction<MRClientProtocol>() { + MRClientProtocol hsWithDT = ugi + .doAs(new PrivilegedAction<MRClientProtocol>() { + @Override public MRClientProtocol run() { return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, - jobHistoryServer.getClientService().getBindAddress(), conf); + hsAddress, conf); } }); - GetJobReportRequest jobReportRequest = - Records.newRecord(GetJobReportRequest.class); - jobReportRequest.setJobId(MRBuilderUtils.newJobId(123456, 1, 1)); - try { - userUsingDT.getJobReport(jobReportRequest); - } catch (YarnRemoteException e) { - Assert.assertEquals("Unknown job job_123456_0001", e.getMessage()); - } + return hsWithDT; } } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml Tue Jan 22 19:33:02 2013 @@ -134,7 +134,15 @@ <effort>Max</effort> </configuration> </plugin> - + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes> + <exclude>src/main/java/org/apache/hadoop/examples/dancing/puzzle1.dta</exclude> + </excludes> + </configuration> + </plugin> </plugins> </build> </project> Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java Tue Jan 22 19:33:02 2013 @@ -21,6 +21,7 @@ package org.apache.hadoop.examples; import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; +import java.util.Random; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -77,8 +78,7 @@ public class QuasiMonteCarlo extends Con static final String DESCRIPTION = "A map/reduce program that estimates Pi using a quasi-Monte Carlo method."; /** tmp directory for input/output */ - static private final Path TMP_DIR = new Path( - QuasiMonteCarlo.class.getSimpleName() + "_TMP_3_141592654"); + static private final String TMP_DIR_PREFIX = QuasiMonteCarlo.class.getSimpleName(); /** 2-dimensional Halton sequence {H(i)}, * where H(i) is a 2-dimensional point and i >= 1 is the index. @@ -228,9 +228,9 @@ public class QuasiMonteCarlo extends Con @Override public void cleanup(Context context) throws IOException { //write output to a file - Path outDir = new Path(TMP_DIR, "out"); - Path outFile = new Path(outDir, "reduce-out"); Configuration conf = context.getConfiguration(); + Path outDir = new Path(conf.get(FileOutputFormat.OUTDIR)); + Path outFile = new Path(outDir, "reduce-out"); FileSystem fileSys = FileSystem.get(conf); SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf, outFile, LongWritable.class, LongWritable.class, @@ -246,7 +246,7 @@ public class QuasiMonteCarlo extends Con * @return the estimated value of Pi */ public static BigDecimal estimatePi(int numMaps, long numPoints, - Configuration conf + Path tmpDir, Configuration conf ) throws IOException, ClassNotFoundException, InterruptedException { Job job = new Job(conf); //setup job conf @@ -269,14 +269,14 @@ public class QuasiMonteCarlo extends Con job.setSpeculativeExecution(false); //setup input/output directories - final Path inDir = new Path(TMP_DIR, "in"); - final Path outDir = new Path(TMP_DIR, "out"); + final Path inDir = new Path(tmpDir, "in"); + final Path outDir = new Path(tmpDir, "out"); FileInputFormat.setInputPaths(job, inDir); FileOutputFormat.setOutputPath(job, outDir); final FileSystem fs = FileSystem.get(conf); - if (fs.exists(TMP_DIR)) { - throw new IOException("Tmp directory " + fs.makeQualified(TMP_DIR) + if (fs.exists(tmpDir)) { + throw new IOException("Tmp directory " + fs.makeQualified(tmpDir) + " already exists. Please remove it first."); } if (!fs.mkdirs(inDir)) { @@ -325,7 +325,7 @@ public class QuasiMonteCarlo extends Con .multiply(BigDecimal.valueOf(numInside.get())) .divide(numTotal, RoundingMode.HALF_UP); } finally { - fs.delete(TMP_DIR, true); + fs.delete(tmpDir, true); } } @@ -344,12 +344,15 @@ public class QuasiMonteCarlo extends Con final int nMaps = Integer.parseInt(args[0]); final long nSamples = Long.parseLong(args[1]); + long now = System.currentTimeMillis(); + int rand = new Random().nextInt(Integer.MAX_VALUE); + final Path tmpDir = new Path(TMP_DIR_PREFIX + "_" + now + "_" + rand); System.out.println("Number of Maps = " + nMaps); System.out.println("Samples per Map = " + nSamples); System.out.println("Estimated value of Pi is " - + estimatePi(nMaps, nSamples, getConf())); + + estimatePi(nMaps, nSamples, tmpDir, getConf())); return 0; } Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java Tue Jan 22 19:33:02 2013 @@ -174,16 +174,16 @@ public class DistributedPentomino extend return 2; } // check for passed parameters, otherwise use defaults - int width = PENT_WIDTH; - int height = PENT_HEIGHT; - int depth = PENT_DEPTH; + int width = conf.getInt(Pentomino.WIDTH, PENT_WIDTH); + int height = conf.getInt(Pentomino.HEIGHT, PENT_HEIGHT); + int depth = conf.getInt(Pentomino.DEPTH, PENT_DEPTH); for (int i = 0; i < args.length; i++) { if (args[i].equalsIgnoreCase("-depth")) { - depth = Integer.parseInt(args[i++].trim()); + depth = Integer.parseInt(args[++i].trim()); } else if (args[i].equalsIgnoreCase("-height")) { - height = Integer.parseInt(args[i++].trim()); + height = Integer.parseInt(args[++i].trim()); } else if (args[i].equalsIgnoreCase("-width") ) { - width = Integer.parseInt(args[i++].trim()); + width = Integer.parseInt(args[++i].trim()); } } // now set the values within conf for M/R tasks to read, this Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/test/java/org/apache/hadoop/examples/TestWordStats.java Tue Jan 22 19:33:02 2013 @@ -1,3 +1,20 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.hadoop.examples; import static org.junit.Assert.assertEquals; Modified: hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml?rev=1437113&r1=1437112&r2=1437113&view=diff ============================================================================== --- hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml (original) +++ hadoop/common/branches/branch-trunk-win/hadoop-mapreduce-project/pom.xml Tue Jan 22 19:33:02 2013 @@ -214,9 +214,11 @@ with tarfile.open(tar_name, "w:gz") as t <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> <configuration> - <includes> - <include>pom.xml</include> - </includes> + <excludes> + <exclude>.eclipse.templates/</exclude> + <exclude>CHANGES.txt</exclude> + <exclude>lib/jdiff/**</exclude> + </excludes> </configuration> </plugin> <plugin>
