HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Contributed by Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b61ee333 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b61ee333 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b61ee333 Branch: refs/heads/HDFS-EC Commit: b61ee333a52455b9dc02768ab2e235d33fdbfe8a Parents: 2e213fc Author: Arpit Agarwal <[email protected]> Authored: Mon Jan 12 21:22:58 2015 -0800 Committer: Zhe Zhang <[email protected]> Committed: Tue Jan 13 07:33:17 2015 -0800 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++ .../dev-support/findbugs-exclude.xml | 7 ++++ .../gridmix/CompressionEmulationUtil.java | 8 +++-- .../gridmix/DistributedCacheEmulator.java | 10 ++++-- .../mapred/gridmix/GenerateDistCacheData.java | 6 +++- .../apache/hadoop/mapred/gridmix/Gridmix.java | 36 ++++++++++---------- .../hadoop/mapred/gridmix/PseudoLocalFs.java | 2 +- .../hadoop/mapred/gridmix/Statistics.java | 26 +++++++++++--- 8 files changed, 68 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a9300c2..59e6d80 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job (Akshay Radia via brandonli) + HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via + Arpit Agarwal) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml index 92458d4..cb043c4 100644 --- a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml +++ b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml @@ -30,4 +30,11 @@ <Bug pattern="REC_CATCH_EXCEPTION"/> <Bug code="REC"/> </Match> + + <!-- This has been done knowingly and meant to fool JVM so that it doesn't optimize code --> + <Match> + <Class name="org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin$DefaultCpuUsageEmulator"/> + <Field name="returnValue"/> + <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD"/> + </Match> </FindBugsFilter> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java index 526878c..bd7e878 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java @@ -21,6 +21,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -97,7 +98,9 @@ class CompressionEmulationUtil { private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE = new CompressionRatioLookupTable(); - + + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + /** * This is a {@link Mapper} implementation for generating random text data. * It uses {@link RandomTextDataGenerator} for generating text data and the @@ -133,7 +136,8 @@ class CompressionEmulationUtil { String randomKey = rtg.getRandomWord(); String randomValue = rtg.getRandomWord(); context.write(new Text(randomKey), new Text(randomValue)); - bytes -= (randomValue.getBytes().length + randomKey.getBytes().length); + bytes -= (randomValue.getBytes(charsetUTF8).length + + randomKey.getBytes(charsetUTF8).length); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java index ad3b113..d30a51d 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java @@ -41,6 +41,7 @@ import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -112,6 +113,8 @@ class DistributedCacheEmulator { Configuration conf; // gridmix configuration + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + // Pseudo local file system where local FS based distributed cache files are // created by gridmix. FileSystem pseudoLocalFs = null; @@ -436,9 +439,10 @@ class DistributedCacheEmulator { for (Iterator it = dcFiles.iterator(); it.hasNext();) { Map.Entry entry = (Map.Entry)it.next(); LongWritable fileSize = - new LongWritable(Long.valueOf(entry.getValue().toString())); + new LongWritable(Long.parseLong(entry.getValue().toString())); BytesWritable filePath = - new BytesWritable(entry.getKey().toString().getBytes()); + new BytesWritable( + entry.getKey().toString().getBytes(charsetUTF8)); byteCount += fileSize.get(); bytesSync += fileSize.get(); @@ -515,7 +519,7 @@ class DistributedCacheEmulator { // local FS based distributed cache file. // Create this file on the pseudo local FS. String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString(); - long fileSize = Long.valueOf(fileSizes[i]); + long fileSize = Long.parseLong(fileSizes[i]); Path mappedLocalFilePath = PseudoLocalFs.generateFilePath(fileId, fileSize) .makeQualified(pseudoLocalFs.getUri(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java index 0512746..4a75cde 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred.gridmix; import java.io.IOException; +import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; @@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob { */ static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644; + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + public GenerateDistCacheData(Configuration conf) throws IOException { super(conf, 0L, JOB_NAME); } @@ -152,7 +155,8 @@ class GenerateDistCacheData extends GridmixJob { public void map(LongWritable key, BytesWritable value, Context context) throws IOException, InterruptedException { - String fileName = new String(value.getBytes(), 0, value.getLength()); + String fileName = new String(value.getBytes(), 0, + value.getLength(), charsetUTF8); Path path = new Path(fileName); FSDataOutputStream dos = http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java index 8ac590b..4386bc1 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java @@ -729,40 +729,40 @@ public class Gridmix extends Configured implements Tool { out.println(" -users <usersResourceURI> : URI that contains the users list."); out.println("Configuration parameters:"); out.println(" General parameters:"); - out.printf(" %-48s : Output directory\n", GRIDMIX_OUT_DIR); - out.printf(" %-48s : Submitting threads\n", GRIDMIX_SUB_THR); - out.printf(" %-48s : Queued job desc\n", GRIDMIX_QUE_DEP); - out.printf(" %-48s : User resolution class\n", GRIDMIX_USR_RSV); - out.printf(" %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes()); + out.printf(" %-48s : Output directory%n", GRIDMIX_OUT_DIR); + out.printf(" %-48s : Submitting threads%n", GRIDMIX_SUB_THR); + out.printf(" %-48s : Queued job desc%n", GRIDMIX_QUE_DEP); + out.printf(" %-48s : User resolution class%n", GRIDMIX_USR_RSV); + out.printf(" %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes()); out.println(" Parameters related to job submission:"); - out.printf(" %-48s : Default queue\n", + out.printf(" %-48s : Default queue%n", GridmixJob.GRIDMIX_DEFAULT_QUEUE); - out.printf(" %-48s : Enable/disable using queues in trace\n", + out.printf(" %-48s : Enable/disable using queues in trace%n", GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE); - out.printf(" %-48s : Job submission policy (%s)\n", + out.printf(" %-48s : Job submission policy (%s)%n", GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies()); out.println(" Parameters specific for LOADJOB:"); - out.printf(" %-48s : Key fraction of rec\n", + out.printf(" %-48s : Key fraction of rec%n", AvgRecordFactory.GRIDMIX_KEY_FRC); out.println(" Parameters specific for SLEEPJOB:"); - out.printf(" %-48s : Whether to ignore reduce tasks\n", + out.printf(" %-48s : Whether to ignore reduce tasks%n", SleepJob.SLEEPJOB_MAPTASK_ONLY); - out.printf(" %-48s : Number of fake locations for map tasks\n", + out.printf(" %-48s : Number of fake locations for map tasks%n", JobCreator.SLEEPJOB_RANDOM_LOCATIONS); - out.printf(" %-48s : Maximum map task runtime in mili-sec\n", + out.printf(" %-48s : Maximum map task runtime in mili-sec%n", SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME); - out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n", + out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n", SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME); out.println(" Parameters specific for STRESS submission throttling policy:"); - out.printf(" %-48s : jobs vs task-tracker ratio\n", + out.printf(" %-48s : jobs vs task-tracker ratio%n", StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO); - out.printf(" %-48s : maps vs map-slot ratio\n", + out.printf(" %-48s : maps vs map-slot ratio%n", StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO); - out.printf(" %-48s : reduces vs reduce-slot ratio\n", + out.printf(" %-48s : reduces vs reduce-slot ratio%n", StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO); - out.printf(" %-48s : map-slot share per job\n", + out.printf(" %-48s : map-slot share per job%n", StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB); - out.printf(" %-48s : reduce-slot share per job\n", + out.printf(" %-48s : reduce-slot share per job%n", StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java index 497108a..d7ef563 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java @@ -124,7 +124,7 @@ class PseudoLocalFs extends FileSystem { } else { String[] parts = path.toUri().getPath().split("\\."); try { - fileSize = Long.valueOf(parts[parts.length - 1]); + fileSize = Long.parseLong(parts[parts.length - 1]); valid = (fileSize >= 0); } catch (NumberFormatException e) { valid = false; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b61ee333/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java index 322d755..915788b 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java @@ -119,7 +119,23 @@ public class Statistics implements Component<Statistics.JobStats> { } return new JobStats(maps, reds, job); } - + + private static void addToNumMapsSubmitted(int numMaps) { + numMapsSubmitted += numMaps; + } + + private static void addToNumReducesSubmitted(int numReduces) { + numReducesSubmitted += numReduces; + } + + private static void subtractFromNumMapsSubmitted(int numMaps) { + numMapsSubmitted -= numMaps; + } + + private static void subtractFromNumReducesSubmitted(int numReduces) { + numReducesSubmitted -= numReduces; + } + /** * Add a submitted job for monitoring. */ @@ -131,8 +147,8 @@ public class Statistics implements Component<Statistics.JobStats> { return; } submittedJobsMap.put(seq, stats); - numMapsSubmitted += stats.getNoOfMaps(); - numReducesSubmitted += stats.getNoOfReds(); + addToNumMapsSubmitted(stats.getNoOfMaps()); + addToNumReducesSubmitted(stats.getNoOfReds()); } /** @@ -156,8 +172,8 @@ public class Statistics implements Component<Statistics.JobStats> { } // update the total number of submitted map/reduce task count - numMapsSubmitted -= stat.getNoOfMaps(); - numReducesSubmitted -= stat.getNoOfReds(); + subtractFromNumMapsSubmitted(stat.getNoOfMaps()); + subtractFromNumReducesSubmitted(stat.getNoOfReds()); completedJobsInCurrentInterval++; //check if we have reached the maximum level of job completions.
