svn commit: r1528502 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Author: jlowe
Date: Wed Oct 2 14:48:12 2013
New Revision: 1528502
URL: http://svn.apache.org/r1528502
Log:
HADOOP-9063. enhance unit-test coverage of class org.apache.hadoop.fs.FileUtil.
Contributed by Ivan A. Veselovsky
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1528502r1=1528501r2=1528502view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct
2 14:48:12 2013
@@ -339,6 +339,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-10006. Compilation failure in trunk for
o.a.h.fs.swift.util.JSONUtil (Junping Du via stevel)
+HADOOP-9063. enhance unit-test coverage of class
+org.apache.hadoop.fs.FileUtil (Ivan A. Veselovsky via jlowe)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1528502r1=1528501r2=1528502view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Wed Oct 2 14:48:12 2013
@@ -24,6 +24,8 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,15 +34,20 @@ import java.util.List;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarOutputStream;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.*;
public class TestFileUtil {
private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
@@ -48,14 +55,14 @@ public class TestFileUtil {
private static final String TEST_ROOT_DIR = System.getProperty(
test.build.data, /tmp) + /fu;
private static final File TEST_DIR = new File(TEST_ROOT_DIR);
- private static String FILE = x;
- private static String LINK = y;
- private static String DIR = dir;
- private File del = new File(TEST_DIR, del);
- private File tmp = new File(TEST_DIR, tmp);
- private File dir1 = new File(del, DIR + 1);
- private File dir2 = new File(del, DIR + 2);
- private File partitioned = new File(TEST_DIR, partitioned);
+ private static final String FILE = x;
+ private static final String LINK = y;
+ private static final String DIR = dir;
+ private final File del = new File(TEST_DIR, del);
+ private final File tmp = new File(TEST_DIR, tmp);
+ private final File dir1 = new File(del, DIR + 1);
+ private final File dir2 = new File(del, DIR + 2);
+ private final File partitioned = new File(TEST_DIR, partitioned);
/**
* Creates multiple directories for testing.
@@ -116,17 +123,17 @@ public class TestFileUtil {
* @param contents String non-null file contents.
* @throws IOException if an I/O error occurs.
*/
- private void createFile(File directory, String name, String contents)
+ private File createFile(File directory, String name, String contents)
throws IOException {
File newFile = new File(directory, name);
PrintWriter pw = new PrintWriter(newFile);
-
try {
pw.println(contents);
}
finally {
pw.close();
}
+return newFile;
}
@Test (timeout = 3)
@@ -553,14 +560,283 @@ public class TestFileUtil {
* @throws IOException
*/
@Test (timeout = 3)
- public void testGetDU() throws IOException {
+ public void testGetDU() throws Exception {
setupDirs();
long du = FileUtil.getDU(TEST_DIR);
// Only two files (in partitioned). Each has 3 characters +
system-specific
// line separator.
-long expected = 2 * (3 + System.getProperty(line.separator).length());
+final
svn commit: r1528506 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Author: jlowe
Date: Wed Oct 2 14:53:28 2013
New Revision: 1528506
URL: http://svn.apache.org/r1528506
Log:
svn merge -c 1528502 FIXES: HADOOP-9063. enhance unit-test coverage of class
org.apache.hadoop.fs.FileUtil. Contributed by Ivan A. Veselovsky
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1528506r1=1528505r2=1528506view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Oct 2 14:53:28 2013
@@ -52,6 +52,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9998. Provide methods to clear only part of the DNSToSwitchMapping.
(Junping Du via Colin Patrick McCabe)
+HADOOP-9063. enhance unit-test coverage of class
+org.apache.hadoop.fs.FileUtil (Ivan A. Veselovsky via jlowe)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1528506r1=1528505r2=1528506view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Wed Oct 2 14:53:28 2013
@@ -24,6 +24,8 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -32,15 +34,20 @@ import java.util.List;
import java.util.jar.Attributes;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
+import org.apache.tools.tar.TarEntry;
+import org.apache.tools.tar.TarOutputStream;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import static org.junit.Assert.*;
public class TestFileUtil {
private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
@@ -48,14 +55,14 @@ public class TestFileUtil {
private static final String TEST_ROOT_DIR = System.getProperty(
test.build.data, /tmp) + /fu;
private static final File TEST_DIR = new File(TEST_ROOT_DIR);
- private static String FILE = x;
- private static String LINK = y;
- private static String DIR = dir;
- private File del = new File(TEST_DIR, del);
- private File tmp = new File(TEST_DIR, tmp);
- private File dir1 = new File(del, DIR + 1);
- private File dir2 = new File(del, DIR + 2);
- private File partitioned = new File(TEST_DIR, partitioned);
+ private static final String FILE = x;
+ private static final String LINK = y;
+ private static final String DIR = dir;
+ private final File del = new File(TEST_DIR, del);
+ private final File tmp = new File(TEST_DIR, tmp);
+ private final File dir1 = new File(del, DIR + 1);
+ private final File dir2 = new File(del, DIR + 2);
+ private final File partitioned = new File(TEST_DIR, partitioned);
/**
* Creates multiple directories for testing.
@@ -116,17 +123,17 @@ public class TestFileUtil {
* @param contents String non-null file contents.
* @throws IOException if an I/O error occurs.
*/
- private void createFile(File directory, String name, String contents)
+ private File createFile(File directory, String name, String contents)
throws IOException {
File newFile = new File(directory, name);
PrintWriter pw = new PrintWriter(newFile);
-
try {
pw.println(contents);
}
finally {
pw.close();
}
+return newFile;
}
@Test (timeout = 3)
@@ -553,14 +560,283 @@ public class TestFileUtil {
* @throws IOException
*/
@Test (timeout = 3)
- public void testGetDU() throws IOException {
+ public void testGetDU() throws Exception {
setupDirs();
long du = FileUtil.getDU(TEST_DIR);
// Only two files (in
svn commit: r1528621 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/test/java/org/apache/hadoop/util/bloom/ src/test/java/org/apache/hadoop/util/hash/
Author: jlowe Date: Wed Oct 2 20:55:26 2013 New Revision: 1528621 URL: http://svn.apache.org/r1528621 Log: svn merge -c 1528620 FIXES: HADOOP-9254. Cover packages org.apache.hadoop.util.bloom, org.apache.hadoop.util.hash. Contributed by Vadim Bondarev Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/ - copied from r1528620, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java - copied unchanged from r1528620, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java - copied unchanged from r1528620, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/ - copied from r1528620, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/TestHash.java - copied unchanged from r1528620, hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/TestHash.java Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1528621r1=1528620r2=1528621view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct 2 20:55:26 2013 @@ -55,6 +55,9 @@ Release 2.3.0 - UNRELEASED HADOOP-9063. enhance unit-test coverage of class org.apache.hadoop.fs.FileUtil (Ivan A. Veselovsky via jlowe) +HADOOP-9254. Cover packages org.apache.hadoop.util.bloom, +org.apache.hadoop.util.hash (Vadim Bondarev via jlowe) + OPTIMIZATIONS HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
svn commit: r1528620 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/test/java/org/apache/hadoop/util/bloom/ src/test/java/org/apache/hadoop/util/hash/
Author: jlowe
Date: Wed Oct 2 20:50:42 2013
New Revision: 1528620
URL: http://svn.apache.org/r1528620
Log:
HADOOP-9254. Cover packages org.apache.hadoop.util.bloom,
org.apache.hadoop.util.hash. Contributed by Vadim Bondarev
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java
(with props)
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/TestBloomFilters.java
(with props)
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/hash/TestHash.java
(with props)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1528620r1=1528619r2=1528620view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Oct
2 20:50:42 2013
@@ -342,6 +342,9 @@ Release 2.3.0 - UNRELEASED
HADOOP-9063. enhance unit-test coverage of class
org.apache.hadoop.fs.FileUtil (Ivan A. Veselovsky via jlowe)
+HADOOP-9254. Cover packages org.apache.hadoop.util.bloom,
+org.apache.hadoop.util.hash (Vadim Bondarev via jlowe)
+
OPTIMIZATIONS
HADOOP-9748. Reduce blocking on UGI.ensureInitialized (daryn)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java?rev=1528620view=auto
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java
(added)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/bloom/BloomFilterCommonTester.java
Wed Oct 2 20:50:42 2013
@@ -0,0 +1,533 @@
+package org.apache.hadoop.util.bloom;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.AbstractCollection;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.util.hash.Hash;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+public class BloomFilterCommonTesterT extends Filter {
+
+ private static final double LN2 = Math.log(2);
+ private static final double LN2_SQUARED = LN2 * LN2;
+
+ private final int hashType;
+ private final int numInsertions;
+
+ private final ImmutableList.BuilderT builder = ImmutableList.builder();
+
+ private ImmutableSetBloomFilterTestStrategy filterTestStrateges;
+
+ private final PreAssertionHelper preAssertionHelper;
+
+ static int optimalNumOfBits(int n, double p) {
+return (int) (-n * Math.log(p) / LN2_SQUARED);
+ }
+
+ public static T extends Filter BloomFilterCommonTesterT of(int hashId,
+ int numInsertions) {
+return new BloomFilterCommonTesterT(hashId, numInsertions);
+ }
+
+ public BloomFilterCommonTesterT withFilterInstance(T filter) {
+builder.add(filter);
+return this;
+ }
+
+ private BloomFilterCommonTester(int hashId, int numInsertions) {
+this.hashType = hashId;
+this.numInsertions = numInsertions;
+
+this.preAssertionHelper = new PreAssertionHelper() {
+
+ @Override
+
