Author: wheat9 Date: Mon Mar 3 19:40:45 2014 New Revision: 1573694 URL: http://svn.apache.org/r1573694 Log: HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer. Contributed by Akira Ajisaka.
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1573694&r1=1573693&r2=1573694&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar 3 19:40:45 2014 @@ -517,6 +517,9 @@ Release 2.4.0 - UNRELEASED HDFS-5956. A file size is multiplied by the replication factor in 'hdfs oiv -p FileDistribution' option. (Akira Ajisaka via wheat9) + HDFS-5866. '-maxSize' and '-step' option fail in OfflineImageViewer. + (Akira Ajisaka via wheat9) + BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java?rev=1573694&r1=1573693&r2=1573694&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java Mon Mar 3 19:40:45 2014 @@ -62,6 +62,7 @@ import com.google.common.io.LimitInputSt final class FileDistributionCalculator { private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37 private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21 + private final static int MAX_INTERVALS = 0x8000000; // 128 M = 2^27 private final Configuration conf; private final long maxSize; @@ -82,9 +83,11 @@ final class FileDistributionCalculator { this.steps = steps == 0 ? INTERVAL_DEFAULT : steps; this.out = out; long numIntervals = this.maxSize / this.steps; + // avoid OutOfMemoryError when allocating an array + Preconditions.checkState(numIntervals <= MAX_INTERVALS, + "Too many distribution intervals (maxSize/step): " + numIntervals + + ", should be less than " + (MAX_INTERVALS+1) + "."); this.distribution = new int[1 + (int) (numIntervals)]; - Preconditions.checkState(numIntervals < Integer.MAX_VALUE, - "Too many distribution intervals"); } void visit(RandomAccessFile file) throws IOException { Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java?rev=1573694&r1=1573693&r2=1573694&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java Mon Mar 3 19:40:45 2014 @@ -101,9 +101,8 @@ public class OfflineImageViewerPB { options.addOption("p", "processor", true, ""); options.addOption("h", "help", false, ""); - options.addOption("skipBlocks", false, ""); - options.addOption("printToScreen", false, ""); - options.addOption("delimiter", true, ""); + options.addOption("maxSize", true, ""); + options.addOption("step", true, ""); return options; } @@ -118,10 +117,15 @@ public class OfflineImageViewerPB { * @throws IOException */ public static void main(String[] args) throws IOException { + int status = run(args); + System.exit(status); + } + + public static int run(String[] args) throws IOException { Options options = buildOptions(); if (args.length == 0) { printUsage(); - return; + return 0; } CommandLineParser parser = new PosixParser(); @@ -132,12 +136,12 @@ public class OfflineImageViewerPB { } catch (ParseException e) { System.out.println("Error parsing command-line options: "); printUsage(); - return; + return -1; } if (cmd.hasOption("h")) { // print help and exit printUsage(); - return; + return 0; } String inputFile = cmd.getOptionValue("i"); @@ -160,6 +164,7 @@ public class OfflineImageViewerPB { } else { new LsrPBImage(conf, out).visit(new RandomAccessFile(inputFile, "r")); } + return 0; } catch (EOFException e) { System.err.println("Input file ended unexpectedly. Exiting"); } catch (IOException e) { @@ -167,7 +172,7 @@ public class OfflineImageViewerPB { } finally { IOUtils.cleanup(null, out); } - + return -1; } /** Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1573694&r1=1573693&r2=1573694&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Mon Mar 3 19:40:45 2014 @@ -278,6 +278,14 @@ public class TestOfflineImageViewer { } @Test + public void testFileDistributionCalculatorWithOptions() throws IOException { + int status = OfflineImageViewerPB.run(new String[] {"-i", + originalFsimage.getAbsolutePath(), "-o", "-", "-p", "FileDistribution", + "-maxSize", "512", "-step", "8"}); + assertEquals(0, status); + } + + @Test public void testPBImageXmlWriter() throws IOException, SAXException, ParserConfigurationException { StringWriter output = new StringWriter();