Author: shv
Date: Mon Sep 24 20:54:20 2012
New Revision: 1389568
URL: http://svn.apache.org/viewvc?rev=1389568&view=rev
Log:
MAPREDUCE-4645. Provide a random seed to Slive to make the sequence of file
names deterministic. Contributed by Ravi Prakash.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1389568&r1=1389567&r2=1389568&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Sep 24
20:54:20 2012
@@ -126,6 +126,9 @@ Trunk (Unreleased)
MAPREDUCE-3868. Make Raid Compile. (Weiyan Wang via schen)
+ MAPREDUCE-4645. Provide a random seed to Slive to make the sequence
+ of file names deterministic. (Ravi Prakash via shv)
+
Release 2.0.3-alpha - Unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java?rev=1389568&r1=1389567&r2=1389568&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/Operation.java
Mon Sep 24 20:54:20 2012
@@ -41,7 +41,9 @@ abstract class Operation {
this.config = cfg;
this.type = type;
this.rnd = rnd;
- this.finder = new PathFinder(cfg, rnd);
+ // Use a new Random instance so that the sequence of file names produced is
+ // the same even in case of unsuccessful operations
+ this.finder = new PathFinder(cfg, new Random(rnd.nextInt()));
}
/**
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java?rev=1389568&r1=1389567&r2=1389568&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/slive/SliveMapper.java
Mon Sep 24 20:54:20 2012
@@ -32,6 +32,8 @@ import org.apache.hadoop.mapred.MapReduc
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.util.StringUtils;
/**
@@ -50,8 +52,7 @@ public class SliveMapper extends MapRedu
private FileSystem filesystem;
private ConfigExtractor config;
- private WeightSelector selector;
- private Random rnd;
+ private int taskId;
/*
* (non-Javadoc)
@@ -70,19 +71,19 @@ public class SliveMapper extends MapRedu
}
try {
config = new ConfigExtractor(conf);
- Long rndSeed = config.getRandomSeed();
- if (rndSeed != null) {
- rnd = new Random(rndSeed);
- } else {
- rnd = new Random();
- }
- selector = new WeightSelector(config, rnd);
ConfigExtractor.dumpOptions(config);
} catch (Exception e) {
LOG.error("Unable to setup slive " + StringUtils.stringifyException(e));
throw new RuntimeException("Unable to setup slive configuration", e);
}
-
+ if(conf.get(MRJobConfig.TASK_ATTEMPT_ID) != null ) {
+ this.taskId =
TaskAttemptID.forName(conf.get(MRJobConfig.TASK_ATTEMPT_ID))
+ .getTaskID().getId();
+ } else {
+ // So that branch-1/0.20 can run this same code as well
+ this.taskId = TaskAttemptID.forName(conf.get("mapred.task.id"))
+ .getTaskID().getId();
+ }
}
/**
@@ -95,15 +96,6 @@ public class SliveMapper extends MapRedu
}
/**
- * Gets the operation selector to use for this object
- *
- * @return WeightSelector
- */
- private WeightSelector getSelector() {
- return selector;
- }
-
- /**
* Logs to the given reporter and logs to the internal logger at info level
*
* @param r
@@ -154,6 +146,10 @@ public class SliveMapper extends MapRedu
Reporter reporter) throws IOException {
logAndSetStatus(reporter, "Running slive mapper for dummy key " + key
+ " and dummy value " + value);
+ //Add taskID to randomSeed to deterministically seed rnd.
+ Random rnd = config.getRandomSeed() != null ?
+ new Random(this.taskId + config.getRandomSeed()) : new Random();
+ WeightSelector selector = new WeightSelector(config, rnd);
long startTime = Timer.now();
long opAm = 0;
long sleepOps = 0;
@@ -163,7 +159,6 @@ public class SliveMapper extends MapRedu
if (sleepRange != null) {
sleeper = new SleepOp(getConfig(), rnd);
}
- WeightSelector selector = getSelector();
while (Timer.elapsed(startTime) < duration) {
try {
logAndSetStatus(reporter, "Attempting to select operation #"
