randomkang commented on code in PR #3144:
URL: https://github.com/apache/brpc/pull/3144#discussion_r4247787345
##########
src/brpc/rdma_transport.cpp:
##########
@@ -30,10 +30,25 @@ DECLARE_bool(usercode_in_pthread);
extern SocketVarsCollector *g_vars;
-void RdmaTransport::Init(Socket *socket, const SocketOptions &options) {
+void RdmaTransport::DoInit(Socket *socket, const SocketOptions &options, bool
use_gdr) {
CHECK(_rdma_ep == nullptr);
- if (options.socket_mode == SOCKET_MODE_RDMA) {
- _rdma_ep = new rdma::RdmaEndpoint(socket);
+ // gdr mode is a special mode of rdma mode.
+ // both rdma mode and gdr mode need init rdma::RdmaEndpoint.
+ if (options.socket_mode == SOCKET_MODE_RDMA ||
+ options.socket_mode == SOCKET_MODE_GDR) {
+ _rdma_ep = new(std::nothrow)rdma::RdmaEndpoint(socket, use_gdr);
+ if (!_rdma_ep) {
+ const int saved_errno = errno;
+ PLOG(ERROR) << "Fail to create RdmaEndpoint";
+ socket->SetFailed(
+ saved_errno, "Fail to create RdmaEndpoint: %s",
berror(saved_errno));
+ // No endpoint: later code paths must not assume RDMA is active
+ // and dereference _rdma_ep. Mark the transport as not using RDMA.
+ _rdma_state = RDMA_OFF;
+ _socket = socket;
+ _default_connect = options.app_connect;
+ return;
Review Comment:
ok
##########
src/brpc/rdma/rdma_helper.cpp:
##########
@@ -588,7 +593,41 @@ static void GlobalRdmaInitializeOrDieImpl() {
g_rdma_available.store(true, butil::memory_order_relaxed);
}
+static void GlobalGdrInitializeOrDieImpl() {
+#if BRPC_WITH_GDR
+ // GPUDirect RDMA requires all received data to be written into the GPU
+ // memory buffer specified at post-receive time. When MLX5_SCATTER_TO_CQE
+ // is enabled, small messages are inlined directly into host memory, which
+ // breaks this requirement and causes operations to fail. Force it to 0
+ // regardless of the user-provided value.
+ if (setenv("MLX5_SCATTER_TO_CQE", "0", 1) != 0) {
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
