This is an automated email from the ASF dual-hosted git repository. zhouky pushed a commit to branch testmain in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 94e3337f5d34041604c236924f2f029187e26d44 Author: zky.zhoukeyong <[email protected]> AuthorDate: Fri Jun 16 18:40:39 2023 +0800 batch revive --- .../flink/readclient/FlinkShuffleClientImpl.java | 8 +- .../org/apache/celeborn/client/ReviveManager.java | 125 ++++++++ .../apache/celeborn/client/ShuffleClientImpl.java | 352 ++++++++++++++------- .../celeborn/client/ChangePartitionManager.scala | 13 +- .../apache/celeborn/client/LifecycleManager.scala | 108 ++++--- .../client/RequestLocationCallContext.scala | 44 ++- .../celeborn/client/ShuffleClientHelper.scala | 7 +- .../celeborn/common/protocol/ReviveRequest.java | 46 +++ .../common/protocol/message/StatusCode.java | 3 +- common/src/main/proto/TransportMessages.proto | 25 +- .../common/protocol/message/ControlMessages.scala | 56 +++- .../celeborn/tests/spark/RetryReviveTest.scala | 3 +- 12 files changed, 592 insertions(+), 198 deletions(-) diff --git a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java index 5d2d1a081..7d883b40f 100644 --- a/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java +++ b/client-flink/common/src/main/java/org/apache/celeborn/plugin/flink/readclient/FlinkShuffleClientImpl.java @@ -49,6 +49,7 @@ import org.apache.celeborn.common.network.protocol.RegionFinish; import org.apache.celeborn.common.network.protocol.RegionStart; import org.apache.celeborn.common.network.util.TransportConf; import org.apache.celeborn.common.protocol.PartitionLocation; +import org.apache.celeborn.common.protocol.PbChangeLocationPartitionInfo; import org.apache.celeborn.common.protocol.PbChangeLocationResponse; import org.apache.celeborn.common.protocol.TransportModuleConstants; import org.apache.celeborn.common.protocol.message.ControlMessages; @@ -388,7 +389,6 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl { applicationId, shuffleId, mapId, - attemptId, location.getId(), location.getEpoch(), location, @@ -396,9 +396,11 @@ public class FlinkShuffleClientImpl extends ShuffleClientImpl { conf.clientRpcRequestPartitionLocationRpcAskTimeout(), ClassTag$.MODULE$.apply(PbChangeLocationResponse.class)); // per partitionKey only serve single PartitionLocation in Client Cache. - StatusCode respStatus = Utils.toStatusCode(response.getStatus()); + PbChangeLocationPartitionInfo partitionInfo = response.getPartitionInfo(0); + StatusCode respStatus = Utils.toStatusCode(partitionInfo.getStatus()); if (StatusCode.SUCCESS.equals(respStatus)) { - return Optional.of(PbSerDeUtils.fromPbPartitionLocation(response.getLocation())); + return Optional.of( + PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition())); } else { // throw exception logger.error( diff --git a/client/src/main/java/org/apache/celeborn/client/ReviveManager.java b/client/src/main/java/org/apache/celeborn/client/ReviveManager.java new file mode 100644 index 000000000..8e889639d --- /dev/null +++ b/client/src/main/java/org/apache/celeborn/client/ReviveManager.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.client; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.celeborn.common.protocol.PartitionLocation; +import org.apache.celeborn.common.protocol.ReviveRequest; +import org.apache.celeborn.common.protocol.message.StatusCode; +import org.apache.celeborn.common.util.ThreadUtils; + +class ReviveManager { + // appid -> shuffleId -> requests + HashMap<String, HashMap<Integer, Set<ReviveRequest>>> pendingRequests = new HashMap<>(); + HashMap<String, HashMap<Integer, Set<ReviveRequest>>> inProcessRequests = null; + ShuffleClientImpl shuffleClient; + private ScheduledExecutorService batchReviveRequestScheduler = + ThreadUtils.newDaemonSingleThreadScheduledExecutor("worker-forward-message-scheduler"); + + public ReviveManager(ShuffleClientImpl shuffleClient) { + this.shuffleClient = shuffleClient; + + batchReviveRequestScheduler.scheduleAtFixedRate( + new Runnable() { + @Override + public void run() { + // We must ensure the swap is atomic + synchronized (this) { + inProcessRequests = pendingRequests; + pendingRequests = new HashMap<>(); + } + if (!inProcessRequests.isEmpty()) { + for (Map.Entry<String, HashMap<Integer, Set<ReviveRequest>>> entry : + inProcessRequests.entrySet()) { + String appId = entry.getKey(); + HashMap<Integer, Set<ReviveRequest>> shuffleMap = entry.getValue(); + for (Map.Entry<Integer, Set<ReviveRequest>> shuffleEntry : shuffleMap.entrySet()) { + // Call reviveBatch for requests in the same (appId, shuffleId) + int shuffleId = shuffleEntry.getKey(); + Set<ReviveRequest> requests = shuffleEntry.getValue(); + Set<Integer> mapIds = new HashSet<>(); + ArrayList<ReviveRequest> filteredRequests = new ArrayList<>(); + Map<Integer, ReviveRequest> requestsToSend = new HashMap<>(); + + // Insert request that is not MapperEnded and with the max epoch + // into requestsToSend + Iterator<ReviveRequest> iter = requests.iterator(); + while (iter.hasNext()) { + ReviveRequest req = iter.next(); + if (shuffleClient.checkRevivedLocation( + shuffleId, req.mapId, req.partitionId, req.loc.getEpoch(), false)) { + req.reviveStatus = StatusCode.SUCCESS; + } else { + filteredRequests.add(req); + mapIds.add(req.mapId); + PartitionLocation loc = req.loc; + if (!requestsToSend.containsKey(loc.getId()) + || requestsToSend.get(loc.getId()).epoch < loc.getEpoch()) { + requestsToSend.put(loc.getId(), req); + } + } + } + + if (!requestsToSend.isEmpty()) { + // Call reviveBatch. Return null means Exception caught or + // SHUFFLE_NOT_REGISTERED + Map<Integer, StatusCode> results = + shuffleClient.reviveBatch( + appId, shuffleId, mapIds, requestsToSend.values()); + if (results == null) { + for (ReviveRequest req : filteredRequests) { + req.reviveStatus = StatusCode.REVIVE_FAILED; + } + } else { + for (ReviveRequest req : filteredRequests) { + if (shuffleClient.mapperEnded(shuffleId, req.mapId)) { + req.reviveStatus = StatusCode.MAP_ENDED; + } else { + req.reviveStatus = results.get(req.partitionId); + } + } + } + } + } // End foreach shuffleMap.entrySet + } // End foreach inProcessRequests.entrySet + } + + inProcessRequests.clear(); + } + }, + 100, + 100, + TimeUnit.MILLISECONDS); + } + + public void addRequest(String appId, int shuffleId, ReviveRequest request) { + shuffleClient.excludeWorkerByCause(request.cause, request.loc); + // This sync is necessary to ensure the add action is atomic + synchronized (this) { + HashMap<Integer, Set<ReviveRequest>> shuffleMap = + pendingRequests.computeIfAbsent(appId, (id) -> new HashMap()); + Set<ReviveRequest> requests = + shuffleMap.computeIfAbsent(shuffleId, (id) -> ConcurrentHashMap.newKeySet()); + requests.add(request); + } + } +} diff --git a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java index 799ba6a75..160fd807d 100644 --- a/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java +++ b/client/src/main/java/org/apache/celeborn/client/ShuffleClientImpl.java @@ -20,10 +20,7 @@ package org.apache.celeborn.client; import java.io.IOException; import java.nio.ByteBuffer; import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import scala.reflect.ClassTag$; @@ -109,7 +106,7 @@ public class ShuffleClientImpl extends ShuffleClient { private final ExecutorService partitionSplitPool; private final Map<Integer, Set<Integer>> splitting = JavaUtils.newConcurrentHashMap(); - ThreadLocal<Compressor> compressorThreadLocal = + private ThreadLocal<Compressor> compressorThreadLocal = new ThreadLocal<Compressor>() { @Override protected Compressor initialValue() { @@ -117,6 +114,8 @@ public class ShuffleClientImpl extends ShuffleClient { } }; + private ReviveManager reviveManager = new ReviveManager(this); + protected static class ReduceFileGroups { public Map<Integer, Set<PartitionLocation>> partitionGroups; public int[] mapAttempts; @@ -202,21 +201,34 @@ public class ShuffleClientImpl extends ShuffleClient { private void submitRetryPushData( String applicationId, int shuffleId, - int mapId, - int attemptId, byte[] body, int batchId, - PartitionLocation loc, RpcResponseCallback wrappedCallback, PushState pushState, - StatusCode cause, + ReviveRequest request, int remainReviveTimes) { + int mapId = request.mapId; + int attemptId = request.attemptId; + PartitionLocation loc = request.loc; + StatusCode cause = request.cause; int partitionId = loc.getId(); - if (!revive( - applicationId, shuffleId, mapId, attemptId, partitionId, loc.getEpoch(), loc, cause)) { - wrappedCallback.onFailure( - new CelebornIOException(cause + " then revive but " + StatusCode.REVIVE_FAILED)); - } else if (mapperEnded(shuffleId, mapId)) { + long reviveWaitTime = + conf.clientRpcRequestPartitionLocationRpcAskTimeout().duration().toMillis() + 1000; + final long delta = 50; + long accumulatedTime = 0; + while (request.reviveStatus == StatusCode.REVIVE_INITIALIZED + && accumulatedTime <= reviveWaitTime) { + try { + Thread.sleep(50); + accumulatedTime += delta; + } catch (InterruptedException e) { + logger.error("Interrupted while waiting for Revive result!"); + wrappedCallback.onFailure( + new CelebornIOException(cause + " then revive but " + StatusCode.REVIVE_FAILED)); + return; + } + } + if (mapperEnded(shuffleId, mapId)) { logger.debug( "Revive for push data success, but the mapper already ended for shuffle {} map {} attempt {} partition {} batch {} location {}.", shuffleId, @@ -226,6 +238,14 @@ public class ShuffleClientImpl extends ShuffleClient { batchId, loc); pushState.removeBatch(batchId, loc.hostAndPushPort()); + } else if (request.reviveStatus != StatusCode.SUCCESS) { + wrappedCallback.onFailure( + new CelebornIOException( + cause + + " then revive but " + + StatusCode.REVIVE_FAILED + + ", revive status " + + request.reviveStatus)); } else { PartitionLocation newLoc = reducePartitionMap.get(shuffleId).get(partitionId); logger.info( @@ -270,6 +290,25 @@ public class ShuffleClientImpl extends ShuffleClient { } } + public ReviveRequest[] addAndGetReviveRequests( + String appId, + int shuffleId, + int mapId, + int attemptId, + ArrayList<DataBatches.DataBatch> batches, + StatusCode cause) { + ReviveRequest[] reviveRequests = new ReviveRequest[batches.size()]; + for (int i = 0; i < batches.size(); i++) { + DataBatches.DataBatch batch = batches.get(i); + PartitionLocation loc = batch.loc; + ReviveRequest reviveRequest = + new ReviveRequest(mapId, attemptId, loc.getId(), loc.getEpoch(), loc, cause); + reviveManager.addRequest(appId, shuffleId, reviveRequest); + reviveRequests[i] = reviveRequest; + } + return reviveRequests; + } + private void submitRetryPushMergedData( PushState pushState, String applicationId, @@ -279,56 +318,77 @@ public class ShuffleClientImpl extends ShuffleClient { ArrayList<DataBatches.DataBatch> batches, StatusCode cause, Integer oldGroupedBatchId, + ReviveRequest[] reviveRequests, int remainReviveTimes) { HashMap<String, DataBatches> newDataBatchesMap = new HashMap<>(); ArrayList<DataBatches.DataBatch> reviveFailedBatchesMap = new ArrayList<>(); - for (DataBatches.DataBatch batch : batches) { - int partitionId = batch.loc.getId(); - if (!revive( - applicationId, - shuffleId, - mapId, - attemptId, - partitionId, - batch.loc.getEpoch(), - batch.loc, - cause)) { - if (remainReviveTimes > 0) { - reviveFailedBatchesMap.add(batch); + long reviveWaitTime = + conf.clientRpcRequestPartitionLocationRpcAskTimeout().duration().toMillis() + 1000; + final long delta = 50; + long accumulatedTime = 0; + int index = 0; + while (index < reviveRequests.length && accumulatedTime <= reviveWaitTime) { + ReviveRequest request = reviveRequests[index]; + DataBatches.DataBatch batch = batches.get(index); + if (request.reviveStatus != StatusCode.REVIVE_INITIALIZED) { + if (request.reviveStatus == StatusCode.SUCCESS) { + PartitionLocation newLoc = reducePartitionMap.get(shuffleId).get(request.partitionId); + DataBatches newDataBatches = + newDataBatchesMap.computeIfAbsent(genAddressPair(newLoc), (s) -> new DataBatches()); + newDataBatches.addDataBatch(newLoc, batch.batchId, batch.body); + } else if (request.reviveStatus == StatusCode.MAP_ENDED) { + logger.debug( + "Revive for push merged data success, but the mapper already ended for shuffle {} map {} attempt {} partition {} batch {}.", + shuffleId, + mapId, + attemptId, + request.partitionId, + oldGroupedBatchId); } else { - String errorMsg = - String.format( - "Revive failed while pushing merged for shuffle %d map %d attempt %d partition %d batch %d location %s.", - shuffleId, mapId, attemptId, partitionId, oldGroupedBatchId, batch.loc); - pushState.exception.compareAndSet( - null, - new CelebornIOException( - errorMsg, - new CelebornIOException(cause + " then revive but " + StatusCode.REVIVE_FAILED))); - return; + if (remainReviveTimes > 0) { + reviveFailedBatchesMap.add(batch); + } else { + String errorMsg = + String.format( + "Revive failed while pushing merged for shuffle %d map %d attempt %d partition %d batch %d location %s.", + shuffleId, mapId, attemptId, request.partitionId, oldGroupedBatchId, batch.loc); + pushState.exception.compareAndSet( + null, + new CelebornIOException( + errorMsg, + new CelebornIOException( + cause + " then revive but " + StatusCode.REVIVE_FAILED))); + return; + } } - } else if (mapperEnded(shuffleId, mapId)) { - logger.debug( - "Revive for push merged data success, but the mapper already ended for shuffle {} map {} attempt {} partition {} batch {}.", - shuffleId, - mapId, - attemptId, - partitionId, - oldGroupedBatchId); + index++; } else { - PartitionLocation newLoc = reducePartitionMap.get(shuffleId).get(partitionId); - logger.info( - "Revive for push merged data success, new location for shuffle {} map {} attempt {} partition {} batch {} is location {}.", - shuffleId, - mapId, - attemptId, - partitionId, - oldGroupedBatchId, - newLoc); - DataBatches newDataBatches = - newDataBatchesMap.computeIfAbsent(genAddressPair(newLoc), (s) -> new DataBatches()); - newDataBatches.addDataBatch(newLoc, batch.batchId, batch.body); + try { + Thread.sleep(delta); + } catch (InterruptedException e) { + logger.warn("Interrupted while waiting for Revive result!"); + } + accumulatedTime += delta; + } + } + + for (int i = index; i < reviveRequests.length; i++) { + ReviveRequest request = reviveRequests[index]; + DataBatches.DataBatch batch = batches.get(i); + if (remainReviveTimes > 0) { + reviveFailedBatchesMap.add(batch); + } else { + String errorMsg = + String.format( + "Revive failed while pushing merged for shuffle %d map %d attempt %d partition %d batch %d location %s.", + shuffleId, mapId, attemptId, request.partitionId, oldGroupedBatchId, batch.loc); + pushState.exception.compareAndSet( + null, + new CelebornIOException( + errorMsg, + new CelebornIOException(cause + " then revive but " + StatusCode.REVIVE_FAILED))); + return; } } @@ -348,6 +408,9 @@ public class ShuffleClientImpl extends ShuffleClient { if (reviveFailedBatchesMap.isEmpty()) { pushState.removeBatch(oldGroupedBatchId, batches.get(0).loc.hostAndPushPort()); } else { + ReviveRequest[] requests = + addAndGetReviveRequests( + applicationId, shuffleId, mapId, attemptId, reviveFailedBatchesMap, cause); pushDataRetryPool.submit( () -> submitRetryPushMergedData( @@ -359,6 +422,7 @@ public class ShuffleClientImpl extends ShuffleClient { reviveFailedBatchesMap, cause, oldGroupedBatchId, + requests, remainReviveTimes - 1)); } } @@ -503,36 +567,31 @@ public class ShuffleClientImpl extends ShuffleClient { } } - private boolean waitRevivedLocation( - ConcurrentHashMap<Integer, PartitionLocation> map, int partitionId, int epoch) { + boolean checkRevivedLocation(int shuffleId, int mapId, int partitionId, int epoch, boolean wait) { + ConcurrentHashMap<Integer, PartitionLocation> map = reducePartitionMap.get(shuffleId); PartitionLocation currentLocation = map.get(partitionId); - if (currentLocation != null && currentLocation.getEpoch() > epoch) { + if (mapperEnded(shuffleId, mapId) + || (currentLocation != null && currentLocation.getEpoch() > epoch)) { return true; - } - - long sleepTimeMs = RND.nextInt(50); - if (sleepTimeMs > 30) { - try { - TimeUnit.MILLISECONDS.sleep(sleepTimeMs); - } catch (InterruptedException e) { - logger.error("Waiting revived location was interrupted.", e); - Thread.currentThread().interrupt(); + } else if (wait) { + long sleepTimeMs = RND.nextInt(50); + if (sleepTimeMs > 30) { + try { + TimeUnit.MILLISECONDS.sleep(sleepTimeMs); + } catch (InterruptedException e) { + logger.error("Waiting revived location was interrupted.", e); + Thread.currentThread().interrupt(); + } } - } - currentLocation = map.get(partitionId); - return currentLocation != null && currentLocation.getEpoch() > epoch; + currentLocation = map.get(partitionId); + return currentLocation != null && currentLocation.getEpoch() > epoch; + } else { + return false; + } } - private boolean revive( - String applicationId, - int shuffleId, - int mapId, - int attemptId, - int partitionId, - int epoch, - PartitionLocation oldLocation, - StatusCode cause) { + void excludeWorkerByCause(StatusCode cause, PartitionLocation oldLocation) { // Add ShuffleClient side blacklist if (shuffleClientPushBlacklistEnabled && oldLocation != null) { if (cause == StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_MASTER) { @@ -549,9 +608,22 @@ public class ShuffleClientImpl extends ShuffleClient { blacklist.add(oldLocation.getPeer().hostAndPushPort()); } } + } + + private boolean revive( + String applicationId, + int shuffleId, + int mapId, + int attemptId, + int partitionId, + int epoch, + PartitionLocation oldLocation, + StatusCode cause) { + excludeWorkerByCause(cause, oldLocation); ConcurrentHashMap<Integer, PartitionLocation> map = reducePartitionMap.get(shuffleId); - if (waitRevivedLocation(map, partitionId, epoch)) { + + if (checkRevivedLocation(shuffleId, mapId, partitionId, epoch, true)) { logger.debug( "Revive already success for shuffle {} map {} attempt {} partition {} epoch {}, just return true(Assume revive successfully).", shuffleId, @@ -561,34 +633,19 @@ public class ShuffleClientImpl extends ShuffleClient { epoch); return true; } - if (mapperEnded(shuffleId, mapId)) { - logger.debug( - "Revive success, but the mapper ended for shuffle {} map {} attempt {} partition {}, just return true(Assume revive successfully).", - shuffleId, - mapId, - attemptId, - partitionId); - return true; - } try { PbChangeLocationResponse response = driverRssMetaService.askSync( Revive$.MODULE$.apply( - applicationId, - shuffleId, - mapId, - attemptId, - partitionId, - epoch, - oldLocation, - cause), + applicationId, shuffleId, mapId, partitionId, epoch, oldLocation, cause), conf.clientRpcRequestPartitionLocationRpcAskTimeout(), ClassTag$.MODULE$.apply(PbChangeLocationResponse.class)); // per partitionKey only serve single PartitionLocation in Client Cache. - StatusCode respStatus = Utils.toStatusCode(response.getStatus()); + PbChangeLocationPartitionInfo partitionInfo = response.getPartitionInfo(0); + StatusCode respStatus = Utils.toStatusCode(partitionInfo.getStatus()); if (StatusCode.SUCCESS.equals(respStatus)) { - map.put(partitionId, PbSerDeUtils.fromPbPartitionLocation(response.getLocation())); + map.put(partitionId, PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition())); return true; } else if (StatusCode.MAP_ENDED.equals(respStatus)) { logger.debug( @@ -615,6 +672,65 @@ public class ShuffleClientImpl extends ShuffleClient { } } + /** @return partitionId -> StatusCode */ + Map<Integer, StatusCode> reviveBatch( + String applicationId, + int shuffleId, + Set<Integer> mapIds, + Collection<ReviveRequest> requests) { + Map<Integer, StatusCode> results = new HashMap<>(); + + ConcurrentHashMap<Integer, PartitionLocation> map = reducePartitionMap.get(shuffleId); + + try { + PbChangeLocationResponse response = + driverRssMetaService.askSync( + Revive$.MODULE$.apply(applicationId, shuffleId, mapIds, requests), + conf.clientRpcRequestPartitionLocationRpcAskTimeout(), + ClassTag$.MODULE$.apply(PbChangeLocationResponse.class)); + + for (int i = 0; i < response.getEndedMapIdCount(); i++) { + int mapId = response.getEndedMapId(i); + mapperEndMap.computeIfAbsent(shuffleId, (id) -> ConcurrentHashMap.newKeySet()).add(mapId); + } + + for (int i = 0; i < response.getPartitionInfoCount(); i++) { + PbChangeLocationPartitionInfo partitionInfo = response.getPartitionInfo(i); + int partitionId = partitionInfo.getPartitionId(); + StatusCode statusCode = Utils.toStatusCode(partitionInfo.getStatus()); + + if (StatusCode.SUCCESS.equals(statusCode)) { + PartitionLocation loc = + PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition()); + map.put(partitionId, loc); + } else if (StatusCode.STAGE_ENDED.equals(statusCode)) { + stageEnded(shuffleId); + return results; + } else if (StatusCode.SHUFFLE_NOT_REGISTERED.equals(statusCode)) { + return null; + } + results.put(partitionId, statusCode); + } + + return results; + } catch (Exception e) { + StringBuilder partitionIds = new StringBuilder(); + StringBuilder epochs = new StringBuilder(); + requests.forEach( + (req) -> { + partitionIds.append(req.partitionId).append(","); + epochs.append(req.epoch).append(","); + }); + logger.error( + "Exception raised while reviving for shuffle {} partitionIds {} epochs {}.", + shuffleId, + partitionIds, + epochs, + e); + return null; + } + } + public int pushOrMergeData( String applicationId, int shuffleId, @@ -782,19 +898,25 @@ public class ShuffleClientImpl extends ShuffleClient { attemptId, partitionId, nextBatchId); + ReviveRequest reviveRequest = + new ReviveRequest( + mapId, + attemptId, + partitionId, + loc.getEpoch(), + loc, + StatusCode.HARD_SPLIT); + reviveManager.addRequest(applicationId, shuffleId, reviveRequest); pushDataRetryPool.submit( () -> submitRetryPushData( applicationId, shuffleId, - mapId, - attemptId, body, nextBatchId, - loc, this, pushState, - StatusCode.HARD_SPLIT, + reviveRequest, remainReviveTimes)); } else if (reason == StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue()) { logger.debug( @@ -863,19 +985,19 @@ public class ShuffleClientImpl extends ShuffleClient { if (!pushStatusIsBlacklisted(cause)) { remainReviveTimes = remainReviveTimes - 1; } + ReviveRequest reviveRequest = + new ReviveRequest(mapId, attemptId, partitionId, loc.getEpoch(), loc, cause); + reviveManager.addRequest(applicationId, shuffleId, reviveRequest); pushDataRetryPool.submit( () -> submitRetryPushData( applicationId, shuffleId, - mapId, - attemptId, body, nextBatchId, - loc, this, pushState, - cause, + reviveRequest, remainReviveTimes)); } else { pushState.removeBatch(nextBatchId, loc.hostAndPushPort()); @@ -1167,6 +1289,10 @@ public class ShuffleClientImpl extends ShuffleClient { Arrays.toString(partitionIds), groupedBatchId, Arrays.toString(batchIds)); + + ReviveRequest[] requests = + addAndGetReviveRequests( + applicationId, shuffleId, mapId, attemptId, batches, StatusCode.HARD_SPLIT); pushDataRetryPool.submit( () -> submitRetryPushMergedData( @@ -1178,6 +1304,7 @@ public class ShuffleClientImpl extends ShuffleClient { batches, StatusCode.HARD_SPLIT, groupedBatchId, + requests, remainReviveTimes)); } else if (reason == StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue()) { logger.debug( @@ -1248,6 +1375,10 @@ public class ShuffleClientImpl extends ShuffleClient { tmpRemainReviveTimes = tmpRemainReviveTimes - 1; } int finalRemainReviveTimes = tmpRemainReviveTimes; + + ReviveRequest[] requests = + addAndGetReviveRequests( + applicationId, shuffleId, mapId, attemptId, batches, cause); pushDataRetryPool.submit( () -> submitRetryPushMergedData( @@ -1259,6 +1390,7 @@ public class ShuffleClientImpl extends ShuffleClient { batches, cause, groupedBatchId, + requests, finalRemainReviveTimes)); } else { pushState.removeBatch(groupedBatchId, hostPort); @@ -1525,7 +1657,7 @@ public class ShuffleClientImpl extends ShuffleClient { driverRssMetaService = endpointRef; } - protected boolean mapperEnded(int shuffleId, int mapId) { + boolean mapperEnded(int shuffleId, int mapId) { return (mapperEndMap.containsKey(shuffleId) && mapperEndMap.get(shuffleId).contains(mapId)) || stageEnded(shuffleId); } diff --git a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala index fd42f51d6..0a1838711 100644 --- a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala +++ b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala @@ -164,7 +164,7 @@ class ChangePartitionManager( // If new slot for the partition has been allocated, reply and return. // Else register and allocate for it. getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { latestLoc => - context.reply(StatusCode.SUCCESS, Some(latestLoc)) + context.reply(partitionId, StatusCode.SUCCESS, Some(latestLoc)) logDebug(s"New partition found, old partition $partitionId-$oldEpoch return it." + s" shuffleId: $shuffleId $latestLoc") return @@ -226,9 +226,11 @@ class ChangePartitionManager( location -> Option(requestsMap.remove(location.getId)) } }.foreach { case (newLocation, requests) => - requests.map(_.asScala.toList.foreach(_.context.reply( - StatusCode.SUCCESS, - Option(newLocation)))) + requests.map(_.asScala.toList.foreach(req => + req.context.reply( + newLocation.getId, + StatusCode.SUCCESS, + Option(newLocation)))) } } @@ -242,7 +244,8 @@ class ChangePartitionManager( Option(requestsMap.remove(changePartition.partitionId)) } }.foreach { requests => - requests.map(_.asScala.toList.foreach(_.context.reply(status, None))) + requests.map(_.asScala.toList.foreach(req => + req.context.reply(req.partitionId, status, None))) } } diff --git a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala index 77dcfc559..9cf4119bf 100644 --- a/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala +++ b/client/src/main/scala/org/apache/celeborn/client/LifecycleManager.scala @@ -240,30 +240,31 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin case pb: PbRevive => val applicationId = pb.getApplicationId val shuffleId = pb.getShuffleId - val mapId = pb.getMapId - val attemptId = pb.getAttemptId - val partitionId = pb.getPartitionId - val epoch = pb.getEpoch - val oldPartition = - if (pb.hasOldPartition) { - PbSerDeUtils.fromPbPartitionLocation(pb.getOldPartition) - } else { - null - } - val cause = Utils.toStatusCode(pb.getStatus) - logTrace(s"Received Revive request, " + - s"$applicationId, $shuffleId, $mapId, $attemptId ,$partitionId," + - s" $epoch, $oldPartition, $cause.") + val mapIds = pb.getMapIdList + val partitionInfos = pb.getPartitionInfoList + + val partitionIds = new util.ArrayList[Integer](partitionInfos.size) + val epochs = new util.ArrayList[Integer](partitionInfos.size()) + val oldPartitions = new util.ArrayList[PartitionLocation](partitionInfos.size()); + val causes = new util.ArrayList[StatusCode](partitionIds.size()); + (0 until partitionInfos.size()).foreach { idx => + val info = partitionInfos.get(idx) + partitionIds.add(info.getPartitionId) + epochs.add(info.getEpoch) + oldPartitions.add(PbSerDeUtils.fromPbPartitionLocation(info.getPartition)) + causes.add(Utils.toStatusCode(info.getStatus)) + } + logInfo( + s"Received Revive request, shuffleId $shuffleId, partitions ${partitionIds.size()}, partitionIds ${partitionIds.asScala.mkString(",")}") handleRevive( context, applicationId, shuffleId, - mapId, - attemptId, - partitionId, - epoch, - oldPartition, - cause) + mapIds, + partitionIds, + epochs, + oldPartitions, + causes) case pb: PbPartitionSplit => val applicationId = pb.getApplicationId @@ -274,7 +275,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin logTrace(s"Received split request, " + s"$applicationId, $shuffleId, $partitionId, $epoch, $oldPartition") changePartitionManager.handleRequestPartitionLocation( - ChangeLocationCallContext(context), + ChangeLocationsCallContext(context, 1), applicationId, shuffleId, partitionId, @@ -497,43 +498,50 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin context: RpcCallContext, applicationId: String, shuffleId: Int, - mapId: Int, - attemptId: Int, - partitionId: Int, - oldEpoch: Int, - oldPartition: PartitionLocation, - cause: StatusCode): Unit = { + mapIds: util.List[Integer], + partitionIds: util.List[Integer], + oldEpochs: util.List[Integer], + oldPartitions: util.List[PartitionLocation], + causes: util.List[StatusCode]): Unit = { + val contextWrapper = + ChangeLocationsCallContext(context, partitionIds.size()) // If shuffle not registered, reply ShuffleNotRegistered and return if (!registeredShuffle.contains(shuffleId)) { logError(s"[handleRevive] shuffle $shuffleId not registered!") - context.reply(ChangeLocationResponse(StatusCode.SHUFFLE_NOT_REGISTERED, None)) - return - } - - if (getPartitionType(shuffleId) == PartitionType.MAP) { - logError(s"[handleRevive] shuffle $shuffleId revived filed, because map partition don't support revive!") - context.reply(ChangeLocationResponse(StatusCode.REVIVE_FAILED, None)) + contextWrapper.reply( + -1, + StatusCode.SHUFFLE_NOT_REGISTERED, + None) return } - if (commitManager.isMapperEnded(shuffleId, mapId)) { - logWarning(s"[handleRevive] Mapper ended, mapId $mapId, current attemptId $attemptId, " + - s"ended attemptId ${commitManager.getMapperAttempts(shuffleId)(mapId)}, shuffleId $shuffleId.") - context.reply(ChangeLocationResponse(StatusCode.MAP_ENDED, None)) + if (commitManager.isStageEnd(shuffleId)) { + logError(s"[handleRevive] shuffle $shuffleId stage ended!") + contextWrapper.reply( + -1, + StatusCode.STAGE_ENDED, + None) return } - logWarning(s"Do Revive for shuffle ${Utils.makeShuffleKey(applicationId, shuffleId)}, " + - s"oldPartition: $oldPartition, cause: $cause") + mapIds.asScala foreach (mapId => { + if (commitManager.isMapperEnded(shuffleId, mapId)) { + logWarning(s"[handleRevive] Mapper ended, mapId $mapId, ended attemptId ${commitManager.getMapperAttempts( + shuffleId)(mapId)}, shuffleId $shuffleId") + contextWrapper.markMapperEnd(mapId) + } + }) - changePartitionManager.handleRequestPartitionLocation( - ChangeLocationCallContext(context), - applicationId, - shuffleId, - partitionId, - oldEpoch, - oldPartition, - Some(cause)) + 0 until partitionIds.size() foreach (idx => { + changePartitionManager.handleRequestPartitionLocation( + contextWrapper, + applicationId, + shuffleId, + partitionIds.get(idx), + oldEpochs.get(idx), + oldPartitions.get(idx), + Some(causes.get(idx))) + }) } private def handleMapperEnd( @@ -764,7 +772,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin } if (!destroyResource.isEmpty) { destroySlotsWithRetry(applicationId, shuffleId, destroyResource) - logInfo(s"Destroyed peer partitions for reserve buffer failed workers " + + logDebug(s"Destroyed peer partitions for reserve buffer failed workers " + s"${Utils.makeShuffleKey(applicationId, shuffleId)}, $destroyResource") val workerIds = new util.ArrayList[String]() @@ -776,7 +784,7 @@ class LifecycleManager(appId: String, val conf: CelebornConf) extends RpcEndpoin } val msg = ReleaseSlots(applicationId, shuffleId, workerIds, workerSlotsPerDisk) requestMasterReleaseSlots(msg) - logInfo(s"Released slots for reserve buffer failed workers " + + logDebug(s"Released slots for reserve buffer failed workers " + s"${workerIds.asScala.mkString(",")}" + s"${slots.asScala.mkString(",")}" + s"${Utils.makeShuffleKey(applicationId, shuffleId)}, ") } diff --git a/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala b/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala index 91cbf6df6..ebbee6cfb 100644 --- a/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala +++ b/client/src/main/scala/org/apache/celeborn/client/RequestLocationCallContext.scala @@ -17,23 +17,57 @@ package org.apache.celeborn.client +import java.util +import java.util.concurrent.ConcurrentHashMap + +import org.apache.celeborn.common.internal.Logging import org.apache.celeborn.common.protocol.PartitionLocation import org.apache.celeborn.common.protocol.message.ControlMessages.{ChangeLocationResponse, RegisterShuffleResponse} import org.apache.celeborn.common.protocol.message.StatusCode import org.apache.celeborn.common.rpc.RpcCallContext trait RequestLocationCallContext { - def reply(status: StatusCode, partitionLocationOpt: Option[PartitionLocation]): Unit + def reply( + partitionId: Int, + status: StatusCode, + partitionLocationOpt: Option[PartitionLocation]): Unit } -case class ChangeLocationCallContext(context: RpcCallContext) extends RequestLocationCallContext { - override def reply(status: StatusCode, partitionLocationOpt: Option[PartitionLocation]): Unit = { - context.reply(ChangeLocationResponse(status, partitionLocationOpt)) +case class ChangeLocationsCallContext( + context: RpcCallContext, + partitionCount: Int) + extends RequestLocationCallContext with Logging { + val endedMapIds = new util.HashSet[Integer]() + val newLocs = new ConcurrentHashMap[Integer, (StatusCode, PartitionLocation)](partitionCount) + + def markMapperEnd(mapId: Int): Unit = this.synchronized { + endedMapIds.add(mapId) + } + + override def reply( + partitionId: Int, + status: StatusCode, + partitionLocationOpt: Option[PartitionLocation]): Unit = { + newLocs.putIfAbsent( + partitionId, + (status, partitionLocationOpt.getOrElse(null))) + + if (newLocs.size() == partitionCount || StatusCode.SHUFFLE_NOT_REGISTERED == status + || StatusCode.STAGE_ENDED == status) { + doReply() + } + } + + private def doReply(): Unit = this.synchronized { + context.reply(ChangeLocationResponse(endedMapIds, newLocs)) } } case class ApplyNewLocationCallContext(context: RpcCallContext) extends RequestLocationCallContext { - override def reply(status: StatusCode, partitionLocationOpt: Option[PartitionLocation]): Unit = { + override def reply( + partitionId: Int, + status: StatusCode, + partitionLocationOpt: Option[PartitionLocation]): Unit = { partitionLocationOpt match { case Some(partitionLocation) => context.reply(RegisterShuffleResponse(status, Array(partitionLocation))) diff --git a/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala b/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala index d6b9be376..16c5c5198 100644 --- a/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala +++ b/client/src/main/scala/org/apache/celeborn/client/ShuffleClientHelper.scala @@ -42,9 +42,12 @@ object ShuffleClientHelper extends Logging { req, conf.clientRpcRequestPartitionLocationRpcAskTimeout).onComplete { case Success(resp) => - val respStatus = Utils.toStatusCode(resp.getStatus) + val partitionInfo = resp.getPartitionInfo(0) + val respStatus = Utils.toStatusCode(partitionInfo.getStatus) if (respStatus == StatusCode.SUCCESS) { - shuffleLocs.put(partitionId, PbSerDeUtils.fromPbPartitionLocation(resp.getLocation)) + shuffleLocs.put( + partitionId, + PbSerDeUtils.fromPbPartitionLocation(partitionInfo.getPartition)) } else { logInfo(s"split failed for $respStatus, " + s"shuffle file can be larger than expected, try split again"); diff --git a/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java b/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java new file mode 100644 index 000000000..77eb9a520 --- /dev/null +++ b/common/src/main/java/org/apache/celeborn/common/protocol/ReviveRequest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.common.protocol; + +import org.apache.celeborn.common.protocol.message.StatusCode; + +public class ReviveRequest { + public int mapId; + public int attemptId; + public int partitionId; + public int epoch; + public PartitionLocation loc; + public StatusCode cause; + public volatile StatusCode reviveStatus; + + public ReviveRequest( + int mapId, + int attemptId, + int partitionId, + int epoch, + PartitionLocation loc, + StatusCode cause) { + this.mapId = mapId; + this.attemptId = attemptId; + this.partitionId = partitionId; + this.epoch = epoch; + this.loc = loc; + this.cause = cause; + reviveStatus = StatusCode.REVIVE_INITIALIZED; + } +} diff --git a/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java b/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java index 688f0def8..5ad47bed8 100644 --- a/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java +++ b/common/src/main/java/org/apache/celeborn/common/protocol/message/StatusCode.java @@ -77,7 +77,8 @@ public enum StatusCode { PUSH_DATA_MASTER_BLACKLISTED(44), PUSH_DATA_SLAVE_BLACKLISTED(45), - FETCH_DATA_TIMEOUT(46); + FETCH_DATA_TIMEOUT(46), + REVIVE_INITIALIZED(47); private final byte value; diff --git a/common/src/main/proto/TransportMessages.proto b/common/src/main/proto/TransportMessages.proto index d3f26e4e9..c326d156b 100644 --- a/common/src/main/proto/TransportMessages.proto +++ b/common/src/main/proto/TransportMessages.proto @@ -206,20 +206,29 @@ message PbRequestSlotsResponse { map<string, PbWorkerResource> workerResource = 2; } +message PbRevivePartitionInfo { + int32 partitionId = 1; + int32 epoch = 2; + PbPartitionLocation partition = 3; + int32 status = 4; +} + message PbRevive { string applicationId = 1; int32 shuffleId = 2; - int32 mapId = 3; - int32 attemptId = 4; - int32 partitionId = 5; - int32 epoch = 6; - PbPartitionLocation oldPartition = 7; - int32 status = 8; + repeated int32 mapId = 3; + repeated PbRevivePartitionInfo partitionInfo = 4; +} + +message PbChangeLocationPartitionInfo { + int32 partitionId = 1; + int32 status = 2; + PbPartitionLocation partition = 3; } message PbChangeLocationResponse { - int32 status = 1; - PbPartitionLocation location = 2; + repeated int32 endedMapId = 1; + repeated PbChangeLocationPartitionInfo partitionInfo = 2; } message PbPartitionSplit { diff --git a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala index 438369a42..61c4de4a7 100644 --- a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala +++ b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala @@ -188,27 +188,51 @@ object ControlMessages extends Logging { extends MasterMessage object Revive { + def apply( + appId: String, + shuffleId: Int, + mapIds: util.Set[Integer], + reviveRequests: util.Collection[ReviveRequest]): PbRevive = { + val builder = PbRevive.newBuilder() + .setApplicationId(appId) + .setShuffleId(shuffleId) + .addAllMapId(mapIds) + + reviveRequests.asScala.foreach(req => { + val partitionInfoBuilder = PbRevivePartitionInfo.newBuilder() + .setPartitionId(req.partitionId) + .setEpoch(req.epoch) + .setStatus(req.cause.getValue) + if (req.loc != null) { + partitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(req.loc)) + } + builder.addPartitionInfo(partitionInfoBuilder.build()) + }) + + builder.build() + } + def apply( appId: String, shuffleId: Int, mapId: Int, - attemptId: Int, partitionId: Int, epoch: Int, oldPartition: PartitionLocation, cause: StatusCode): PbRevive = { - val builder = PbRevive.newBuilder() - builder.setApplicationId(appId) - .setShuffleId(shuffleId) - .setMapId(mapId) - .setAttemptId(attemptId) + val partitionInfoBuilder = PbRevivePartitionInfo.newBuilder() .setPartitionId(partitionId) .setEpoch(epoch) .setStatus(cause.getValue) if (oldPartition != null) { - builder.setOldPartition(PbSerDeUtils.toPbPartitionLocation(oldPartition)) + partitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(oldPartition)) } - builder.build() + val builder = PbRevive.newBuilder() + builder.setApplicationId(appId) + .setShuffleId(shuffleId) + .addMapId(mapId) + .addPartitionInfo(partitionInfoBuilder.build()) + .build() } } @@ -230,12 +254,18 @@ object ControlMessages extends Logging { object ChangeLocationResponse { def apply( - status: StatusCode, - partitionLocationOpt: Option[PartitionLocation]): PbChangeLocationResponse = { + mapIds: util.Set[Integer], + newLocs: util.Map[Integer, (StatusCode, PartitionLocation)]): PbChangeLocationResponse = { val builder = PbChangeLocationResponse.newBuilder() - builder.setStatus(status.getValue) - partitionLocationOpt.foreach { partitionLocation => - builder.setLocation(PbSerDeUtils.toPbPartitionLocation(partitionLocation)) + builder.addAllEndedMapId(mapIds) + newLocs.asScala.foreach { case (partitionId, (status, loc)) => + val pbChangeLocationPartitionInfoBuilder = PbChangeLocationPartitionInfo.newBuilder() + .setPartitionId(partitionId) + .setStatus(status.getValue) + if (loc != null) { + pbChangeLocationPartitionInfoBuilder.setPartition(PbSerDeUtils.toPbPartitionLocation(loc)) + } + builder.addPartitionInfo(pbChangeLocationPartitionInfoBuilder.build()) } builder.build() } diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala index 791ac05df..b5c2c3599 100644 --- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala +++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryReviveTest.scala @@ -50,8 +50,9 @@ class RetryReviveTest extends AnyFunSuite val ss = SparkSession.builder() .config(updateSparkConf(sparkConf, ShuffleMode.HASH)) .getOrCreate() - ss.sparkContext.parallelize(1 to 1000, 2) + val result = ss.sparkContext.parallelize(1 to 1000, 2) .map { i => (i, Range(1, 1000).mkString(",")) }.groupByKey(16).collect() + assert(result.size == 1000) ss.stop() } }
