Repository: giraph Updated Branches: refs/heads/trunk 0dab32741 -> b9f414296
GIRAPH-996: Large requests degrade performance. Print out warnings. (dlogothetis via majakabiljo) Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b9f41429 Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b9f41429 Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b9f41429 Branch: refs/heads/trunk Commit: b9f41429619590e8e1c06d47c2a8268fa3bc07db Parents: 0dab327 Author: Maja Kabiljo <[email protected]> Authored: Wed Mar 4 10:39:41 2015 -0800 Committer: Maja Kabiljo <[email protected]> Committed: Wed Mar 4 10:39:41 2015 -0800 ---------------------------------------------------------------------- CHANGELOG | 2 ++ .../java/org/apache/giraph/comm/netty/NettyClient.java | 10 ++++++++++ .../main/java/org/apache/giraph/conf/GiraphConstants.java | 7 +++++++ 3 files changed, 19 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/CHANGELOG ---------------------------------------------------------------------- diff --git a/CHANGELOG b/CHANGELOG index 331bed6..3c31eb0 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ Giraph Change Log Release 1.2.0 - unreleased + GIRAPH-996: Large requests degrade performance. Print out warnings. (dlogothetis via majakabiljo) + GIRAPH-990. Current trunk will build for hadoop 1.2.0 not 0.20.203 as stated by documentation GIRAPH-992: Zookeeper logs have too many NodeExists (edunov) http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java index 78e318e..40c6ab9 100644 --- a/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java +++ b/giraph-core/src/main/java/org/apache/giraph/comm/netty/NettyClient.java @@ -159,6 +159,8 @@ public class NettyClient implements ResetSuperstepMetricsObserver { private final int receiveBufferSize; /** Do we have a limit on number of open requests */ private final boolean limitNumberOfOpenRequests; + /** Warn if request size is bigger than the buffer size by this factor */ + private final float requestSizeWarningThreshold; /** Maximum number of requests without confirmation we can have */ private final int maxNumberOfOpenRequests; /** @@ -221,6 +223,8 @@ public class NettyClient implements ResetSuperstepMetricsObserver { this.channelsPerServer = GiraphConstants.CHANNELS_PER_SERVER.get(conf); sendBufferSize = CLIENT_SEND_BUFFER_SIZE.get(conf); receiveBufferSize = CLIENT_RECEIVE_BUFFER_SIZE.get(conf); + this.requestSizeWarningThreshold = + GiraphConstants.REQUEST_SIZE_WARNING_THRESHOLD.get(conf); limitNumberOfOpenRequests = LIMIT_NUMBER_OF_OPEN_REQUESTS.get(conf); if (limitNumberOfOpenRequests) { @@ -728,6 +732,12 @@ public class NettyClient implements ResetSuperstepMetricsObserver { "request info of " + oldRequestInfo); } } + if (request.getSerializedSize() > + requestSizeWarningThreshold * sendBufferSize) { + LOG.warn("Creating large request of type " + request.getClass() + + ", size " + request.getSerializedSize() + + " bytes. Check netty buffer size."); + } ChannelFuture writeFuture = channel.write(request); newRequestInfo.setWriteFuture(writeFuture); writeFuture.addListener(logErrorListener); http://git-wip-us.apache.org/repos/asf/giraph/blob/b9f41429/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java index f1cf520..65b7892 100644 --- a/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java +++ b/giraph-core/src/main/java/org/apache/giraph/conf/GiraphConstants.java @@ -732,6 +732,13 @@ public interface GiraphConstants { "request size is M, and a worker has P partitions, than its " + "initial partition buffer size will be (M / P) * (1 + A)."); + + /** Warn if msg request size exceeds default size by this factor */ + FloatConfOption REQUEST_SIZE_WARNING_THRESHOLD = new FloatConfOption( + "giraph.msgRequestWarningThreshold", 2.0f, + "If request sizes are bigger than the buffer size by this factor " + + "warnings are printed to the log and to the command line"); + /** Maximum size of vertices (in bytes) per peer before flush */ IntConfOption MAX_VERTEX_REQUEST_SIZE = new IntConfOption("giraph.vertexRequestSize", 512 * ONE_KB,
