Fetch no more than 128 splits in parallel patch by Patricio Echague; reviewed by jbellis for CASSANDRA-6169
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/eee485eb Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/eee485eb Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/eee485eb Branch: refs/heads/trunk Commit: eee485eb63233de3bc2718d67b0284c1e94793b4 Parents: 5d5925a Author: Jonathan Ellis <jbel...@apache.org> Authored: Wed Oct 9 13:35:48 2013 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Wed Oct 9 13:35:48 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/hadoop/AbstractColumnFamilyInputFormat.java | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/eee485eb/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index f0ee993..7d63a4b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -20,6 +20,7 @@ * Log UnknownColumnfamilyException when closing socket (CASSANDRA-5725) * Properly error out on CREATE INDEX for counters table (CASSANDRA-6160) * Handle JMX notification failure for repair (CASSANDRA-6097) + * (Hadoop) Fetch no more than 128 splits in parallel (CASSANDRA-6169) 1.2.10 http://git-wip-us.apache.org/repos/asf/cassandra/blob/eee485eb/src/java/org/apache/cassandra/hadoop/AbstractColumnFamilyInputFormat.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hadoop/AbstractColumnFamilyInputFormat.java b/src/java/org/apache/cassandra/hadoop/AbstractColumnFamilyInputFormat.java index 1c8fd0b..16d03f9 100644 --- a/src/java/org/apache/cassandra/hadoop/AbstractColumnFamilyInputFormat.java +++ b/src/java/org/apache/cassandra/hadoop/AbstractColumnFamilyInputFormat.java @@ -27,8 +27,10 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import org.apache.cassandra.auth.IAuthenticator; import org.apache.cassandra.dht.IPartitioner; @@ -123,7 +125,7 @@ public abstract class AbstractColumnFamilyInputFormat<K, Y> extends InputFormat< logger.debug("partitioner is " + partitioner); // cannonical ranges, split into pieces, fetching the splits in parallel - ExecutorService executor = Executors.newCachedThreadPool(); + ExecutorService executor = new ThreadPoolExecutor(0, 128, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); List<InputSplit> splits = new ArrayList<InputSplit>(); try
