HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the correct namenode (Contributed by Wei-Chiu Chuang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6702e7d6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6702e7d6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6702e7d6 Branch: refs/heads/HDFS-1312 Commit: 6702e7d66a4949a40bcdbd9f51ca1e746c8991e1 Parents: 34cd7cd Author: Vinayakumar B <[email protected]> Authored: Thu Jan 7 14:20:18 2016 +0530 Committer: Vinayakumar B <[email protected]> Committed: Thu Jan 7 14:20:18 2016 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/datanode/SimulatedFSDataset.java | 3 +- .../server/datanode/TestSimulatedFSDataset.java | 47 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dfa9701..c451ea6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2567,6 +2567,9 @@ Release 2.8.0 - UNRELEASED HDFS-9605. Add links to failed volumes to explorer.html in HDFS Web UI. (Archana T via wheat9) + HDFS-9619. SimulatedFSDataset sometimes can not find blockpool for the + correct namenode (Wei-Chiu Chuang via vinayakumarb) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 4316854..7a10379 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -29,6 +29,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import javax.management.NotCompliantMBeanException; import javax.management.ObjectName; @@ -521,7 +522,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> { } private final Map<String, Map<Block, BInfo>> blockMap - = new HashMap<String, Map<Block,BInfo>>(); + = new ConcurrentHashMap<String, Map<Block,BInfo>>(); private final SimulatedStorage storage; private final SimulatedVolume volume; private final String datanodeUuid; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6702e7d6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index 8dc80d5..dd7d239 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -38,6 +39,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetFactory; import org.apache.hadoop.util.DataChecksum; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -338,4 +340,49 @@ public class TestSimulatedFSDataset { fsdataset.addBlockPool(bpid, conf); return fsdataset; } + + @Test + public void testConcurrentAddBlockPool() throws InterruptedException, + IOException { + final String[] bpids = {"BP-TEST1-", "BP-TEST2-"}; + final SimulatedFSDataset fsdataset = new SimulatedFSDataset(null, conf); + class AddBlockPoolThread extends Thread { + private int id; + private IOException ioe; + public AddBlockPoolThread(int id) { + super(); + this.id = id; + } + public void test() throws InterruptedException, IOException { + this.join(); + if (ioe != null) { + throw ioe; + } + } + public void run() { + for (int i=0; i < 10000; i++) { + // add different block pools concurrently + String newbpid = bpids[id] + i; + fsdataset.addBlockPool(newbpid, conf); + // and then add a block into the pool + ExtendedBlock block = new ExtendedBlock(newbpid,1); + try { + // it will throw an exception if the block pool is not found + fsdataset.createTemporary(StorageType.DEFAULT, block); + } catch (IOException ioe) { + // JUnit does not capture exception in non-main thread, + // so cache it and then let main thread throw later. + this.ioe = ioe; + } + assert(fsdataset.getReplicaString(newbpid,1) != "null"); + } + } + }; + AddBlockPoolThread t1 = new AddBlockPoolThread(0); + AddBlockPoolThread t2 = new AddBlockPoolThread(1); + t1.start(); + t2.start(); + t1.test(); + t2.test(); + } }
