seojangho closed pull request #8: [NEMO-36] Make Executor Do Not Care About
Capacity
URL: https://github.com/apache/incubator-nemo/pull/8
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
b/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
index c539d7ab..255e8bbe 100644
--- a/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
+++ b/conf/src/main/java/edu/snu/nemo/conf/JobConf.java
@@ -234,14 +234,12 @@
}
public static final RequiredParameter<String> EXECUTOR_ID = new
RequiredParameter<>();
- public static final OptionalParameter<Integer> EXECUTOR_CAPACITY = new
OptionalParameter<>();
public static final RequiredParameter<String> JOB_ID = new
RequiredParameter<>();
public static final OptionalParameter<String> LOCAL_DISK_DIRECTORY = new
OptionalParameter<>();
public static final OptionalParameter<String> GLUSTER_DISK_DIRECTORY = new
OptionalParameter<>();
public static final ConfigurationModule EXECUTOR_CONF = new JobConf()
.bindNamedParameter(ExecutorId.class, EXECUTOR_ID)
- .bindNamedParameter(ExecutorCapacity.class, EXECUTOR_CAPACITY)
.bindNamedParameter(JobId.class, JOB_ID)
.bindNamedParameter(FileDirectory.class, LOCAL_DISK_DIRECTORY)
.bindNamedParameter(GlusterVolumeDirectory.class, GLUSTER_DISK_DIRECTORY)
diff --git a/runtime/driver/src/main/java/edu/snu/nemo/driver/NemoDriver.java
b/runtime/driver/src/main/java/edu/snu/nemo/driver/NemoDriver.java
index 33b2984a..60e455ef 100644
--- a/runtime/driver/src/main/java/edu/snu/nemo/driver/NemoDriver.java
+++ b/runtime/driver/src/main/java/edu/snu/nemo/driver/NemoDriver.java
@@ -121,9 +121,8 @@ public void onNext(final StartTime startTime) {
@Override
public void onNext(final AllocatedEvaluator allocatedEvaluator) {
final String executorId = RuntimeIdGenerator.generateExecutorId();
- final int numOfCores =
allocatedEvaluator.getEvaluatorDescriptor().getNumberOfCores();
runtimeMaster.onContainerAllocated(executorId, allocatedEvaluator,
- getExecutorConfiguration(executorId, numOfCores));
+ getExecutorConfiguration(executorId));
}
}
@@ -185,10 +184,9 @@ public void onNext(final StopTime stopTime) {
}
}
- private Configuration getExecutorConfiguration(final String executorId,
final int executorCapacity) {
+ private Configuration getExecutorConfiguration(final String executorId) {
final Configuration executorConfiguration = JobConf.EXECUTOR_CONF
.set(JobConf.EXECUTOR_ID, executorId)
- .set(JobConf.EXECUTOR_CAPACITY, executorCapacity)
.set(JobConf.GLUSTER_DISK_DIRECTORY, glusterDirectory)
.set(JobConf.LOCAL_DISK_DIRECTORY, localDirectory)
.set(JobConf.JOB_ID, jobId)
diff --git
a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/Executor.java
b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/Executor.java
index bb31edf2..06a07d20 100644
--- a/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/Executor.java
+++ b/runtime/executor/src/main/java/edu/snu/nemo/runtime/executor/Executor.java
@@ -69,14 +69,13 @@
@Inject
public Executor(@Parameter(JobConf.ExecutorId.class) final String executorId,
- @Parameter(JobConf.ExecutorCapacity.class) final int
executorCapacity,
final PersistentConnectionToMasterMap
persistentConnectionToMasterMap,
final MessageEnvironment messageEnvironment,
final SerializerManager serializerManager,
final DataTransferFactory dataTransferFactory,
final MetricManagerWorker metricMessageSender) {
this.executorId = executorId;
- this.executorService = Executors.newFixedThreadPool(executorCapacity);
+ this.executorService = Executors.newCachedThreadPool();
this.persistentConnectionToMasterMap = persistentConnectionToMasterMap;
this.serializerManager = serializerManager;
this.dataTransferFactory = dataTransferFactory;
diff --git
a/tests/src/test/java/edu/snu/nemo/tests/runtime/executor/datatransfer/DataTransferTest.java
b/tests/src/test/java/edu/snu/nemo/tests/runtime/executor/datatransfer/DataTransferTest.java
index 70379eef..73a1b5d2 100644
---
a/tests/src/test/java/edu/snu/nemo/tests/runtime/executor/datatransfer/DataTransferTest.java
+++
b/tests/src/test/java/edu/snu/nemo/tests/runtime/executor/datatransfer/DataTransferTest.java
@@ -98,8 +98,6 @@
SourceVertex.class})
public final class DataTransferTest {
private static final String EXECUTOR_ID_PREFIX = "Executor";
- private static final int EXECUTOR_CAPACITY = 1;
- private static final int MAX_SCHEDULE_ATTEMPT = 2;
private static final int SCHEDULE_TIMEOUT = 1000;
private static final DataStoreProperty.Value MEMORY_STORE =
DataStoreProperty.Value.MemoryStore;
private static final DataStoreProperty.Value SER_MEMORY_STORE =
DataStoreProperty.Value.SerializedMemoryStore;
@@ -198,7 +196,6 @@ private BlockManagerWorker createWorker(final String
executorId, final LocalMess
// Unused, but necessary for wiring up the message environments
final Executor executor = new Executor(
executorId,
- EXECUTOR_CAPACITY,
conToMaster,
messageEnvironment,
serializerManager,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
