osaf/libs/core/mds/include/mds_dt_tcp_disc.h | 1 -
osaf/libs/core/mds/mds_dt_tcp.c | 20 +++++-
osaf/services/infrastructure/dtms/config/dtmd.conf | 23 +++++++
osaf/services/infrastructure/dtms/dtm/dtm_intra.c | 13 +--
osaf/services/infrastructure/dtms/dtm/dtm_node_sockets.c | 44 ++++++-------
osaf/services/infrastructure/dtms/dtm/dtm_read_config.c | 30 +++++++++
osaf/services/infrastructure/dtms/include/dtm.h | 2 +
osaf/services/infrastructure/dtms/include/dtm_cb.h | 3 +
osaf/services/infrastructure/dtms/include/dtm_intra_disc.h | 2 -
osaf/services/infrastructure/nid/scripts/opensafd.in | 1 +
10 files changed, 104 insertions(+), 35 deletions(-)
1) Their is NO issue with MDS TCP in-service Upgrade ,The value of
SO_RCVBUF & SO_SNDBUF of TCP sockets was set to 64000 (MDS_SND_RCV_SIZE)
using setsockopt() ,the kernel doubles this value ,and this doubled value is
returned by
getsockopt(). So MDTM_NORMAL_MSG_FRAG_SIZE_MAX size is used to send for
for both previous & current versions of Opensaf nodes which are using MDS TCP
as transport
2) Made DTM-TCP inter-node & intra-node socket SO_SNDBUF and SO_RCVBUF buffer
in bytes configurable .
The kernel doubles this value (to allow space for bookkeeping over-head)
when it is set using setsockopt(),and this doubled value is returned by
getsockopt(). The default value is set to DTM_SOCK_SND_RCV_BUF_SIZE=126976
3) Made TCP_NODELAY configurable to specifi whether MDS TCP transport should
follow the Nagle algorithm for
deciding when to send data.By default, TCP will follow the Nagle algorithm is
disable this behavior ,
can be configured to TRUE(1) or FALSE(0)
diff --git a/osaf/libs/core/mds/include/mds_dt_tcp_disc.h
b/osaf/libs/core/mds/include/mds_dt_tcp_disc.h
--- a/osaf/libs/core/mds/include/mds_dt_tcp_disc.h
+++ b/osaf/libs/core/mds/include/mds_dt_tcp_disc.h
@@ -70,7 +70,6 @@ server_instance_upper + sub_ref_val + su
#define MDS_SND_VERSION 1
#define MDS_RCV_IDENTIFIRE 0x56123456
#define MDS_RCV_VERSION 1
-#define DTM_INTRANODE_MSG_SIZE 1500
#define DTM_INTRANODE_UNSENT_MSG 200
#define MDS_MDTM_LOWER_INSTANCE 0x00000000
diff --git a/osaf/libs/core/mds/mds_dt_tcp.c b/osaf/libs/core/mds/mds_dt_tcp.c
--- a/osaf/libs/core/mds/mds_dt_tcp.c
+++ b/osaf/libs/core/mds/mds_dt_tcp.c
@@ -38,7 +38,6 @@
#include <configmake.h>
#define MDS_MDTM_SUN_PATH 255
-#define MDS_SND_RCV_SIZE 64000
#define MDS_MDTM_CONNECT_PATH PKGLOCALSTATEDIR "/osaf_dtm_intra_server"
#ifndef MDS_PORT_NUMBER
@@ -53,6 +52,9 @@
/* Send_buffer_size + MDS_MDTM_DTM_PID_BUFFER_SIZE */
#define MDS_MDTM_DTM_PID_BUFFER_SIZE (2 + MDS_MDTM_DTM_PID_SIZE)
+/* The default value is set by the rmem_default/wmem_default */
+#define MDS_SOCK_SND_RCV_BUF_SIZE 126976
+
extern uint32_t mdtm_num_subscriptions;
extern MDS_SUBTN_REF_VAL mdtm_handle;
extern uint32_t mdtm_global_frag_num_tcp;
@@ -79,11 +81,12 @@ uint32_t mdtm_process_recv_events_tcp(vo
uint32_t mds_mdtm_init_tcp(NODE_ID nodeid, uint32_t *mds_tcp_ref)
{
uint32_t flags;
- uint32_t size = MDS_SND_RCV_SIZE;
+ uint32_t size = MDS_SOCK_SND_RCV_BUF_SIZE;
struct sockaddr_un server_addr_un, dhserver_addr_un;
struct sockaddr_in server_addr_in;
struct sockaddr_in6 server_addr_in6;
uint8_t buffer[MDS_MDTM_DTM_PID_BUFFER_SIZE];
+ char *ptr;
mdtm_pid = getpid();
@@ -116,6 +119,19 @@ uint32_t mds_mdtm_init_tcp(NODE_ID nodei
syslog(LOG_ERR, "MDS:MDTM:TCP ncs_patricia_tree_init failed
MDTM_INIT\n");
return NCSCC_RC_FAILURE;
}
+
+ /* setting MDS_SOCK_SND_RCV_BUF_SIZE from environment variable if
given.
+ The default value is set to MDS_SOCK_SND_RCV_BUF_SIZE (126976).
+ based on application requirements user need to export
MDS_SOCK_SND_RCV_BUF_SIZE
+ varible.
+
+ If MDS_SOCK_SND_RCV_BUF_SIZE exported to new value
+ it is also mandatory to change `DTM_SOCK_SND_RCV_BUF_SIZE=` with
the same value of
+ for example if we export MDS_SOCK_SND_RCV_BUF_SIZE=16777216
+ DTM_SOCK_SND_RCV_BUF_SIZE=16777216 also need to be changed in
/etc/opensaf/dtm.conf */
+ if ((ptr = getenv("MDS_SOCK_SND_RCV_BUF_SIZE")) != NULL) {
+ size = atoi(ptr);
+ }
/* Create the sockets required for Binding, Send, receive and Discovery
*/
diff --git a/osaf/services/infrastructure/dtms/config/dtmd.conf
b/osaf/services/infrastructure/dtms/config/dtmd.conf
--- a/osaf/services/infrastructure/dtms/config/dtmd.conf
+++ b/osaf/services/infrastructure/dtms/config/dtmd.conf
@@ -72,4 +72,27 @@ DTM_TCP_KEEPALIVE_INTVL=1
# Optional
DTM_TCP_KEEPALIVE_PROBES=2
+#
+#Used to Set the dtm internode & intranode socket SO_SNDBUF/SO_RCVBUF buffer
in bytes.
+#The kernel doubles this value (to allow space for bookkeeping over-head)
+#when it is set using setsockopt(),and this doubled value is returned by
+#getsockopt(). The default value is set by the rmem_default/wmem_default
+#sysctl and the maximum allowed value is set by the rmem_max/wmem_max sysctl.
+#
+# IMP NOTE : If DTM_SOCK_SND_RCV_BUF_SIZE is configured according to
application requirements ,
+#it is also mandatory to export `MDS_SOCK_SND_RCV_BUF_SIZE=` with the same
value of
+#for example if set as DTM_SOCK_SND_RCV_BUF_SIZE=16777216 need to do :
+#export MDS_SOCK_SND_RCV_BUF_SIZE=16777216
+DTM_SOCK_SND_RCV_BUF_SIZE=126976
+#
+#Specifies whether MDS TCP transport should follow the Nagle algorithm for
+#deciding when to send data.By default, TCP will follow the Nagle algorithm is
disable
+#this behavior,MDS TCP transport can enable TCP_NODELAY to force
+#TCP to always send data immediately.For example,TCP_NODELAY should be used
+#when there is an application using TCP for a request/response.
+#This option is only supported for sockets with an address family
+#of AF_INET or AF_INET6 (internode )and type of SOCK_STREAM
+# TRUE(1) or FALSE(0)
+DTM_TCP_NODELAY_FLAG=1
+
diff --git a/osaf/services/infrastructure/dtms/dtm/dtm_intra.c
b/osaf/services/infrastructure/dtms/dtm/dtm_intra.c
--- a/osaf/services/infrastructure/dtms/dtm/dtm_intra.c
+++ b/osaf/services/infrastructure/dtms/dtm/dtm_intra.c
@@ -42,7 +42,6 @@ DTM_INTRANODE_CB *dtm_intranode_cb = NUL
#define DTM_INTRANODE_MAX_PROCESSES 100
#define DTM_INTRANODE_POLL_TIMEOUT 20000
-#define DTM_RECV_BUFFER_SIZE 1500
#define DTM_INTRANODE_TASKNAME "DTM_INTRANODE"
#define DTM_INTRANODE_STACKSIZE NCS_STACKSIZE_HUGE
@@ -58,7 +57,7 @@ static struct pollfd pfd_list[DTM_INTRAN
static int dtm_intranode_max_fd;
-static uint32_t dtm_intra_processing_init(char *node_ip, DTM_IP_ADDR_TYPE
i_addr_family);
+static uint32_t dtm_intra_processing_init(char *node_ip, DTM_IP_ADDR_TYPE
i_addr_family, int32_t sock_snd_rcv_buf_size);
static void dtm_intranode_processing(void);
static uint32_t dtm_intranode_add_poll_fdlist(int fd, uint16_t event);
static uint32_t dtm_intranode_create_rcv_task(int task_hdl);
@@ -79,10 +78,9 @@ uint32_t dtm_socket_domain = AF_UNIX;
*/
uint32_t dtm_service_discovery_init(DTM_INTERNODE_CB *dtms_cb)
{
- return dtm_intra_processing_init(dtms_cb->ip_addr,
dtms_cb->i_addr_family);
+ return dtm_intra_processing_init(dtms_cb->ip_addr,
dtms_cb->i_addr_family, dtms_cb->sock_snd_rcv_buf_size);
}
-#define DTM_INTRANODE_SOCK_SIZE 64000
/**
* Function to init the intranode processing
@@ -92,10 +90,10 @@ uint32_t dtm_service_discovery_init(DTM_
* @return NCSCC_RC_FAILURE
*
*/
-uint32_t dtm_intra_processing_init(char *node_ip, DTM_IP_ADDR_TYPE
i_addr_family)
+uint32_t dtm_intra_processing_init(char *node_ip, DTM_IP_ADDR_TYPE
i_addr_family, int32_t sock_snd_rcv_buf_size)
{
- int servlen, size = DTM_INTRANODE_SOCK_SIZE; /* For socket fd and
server len */
+ int servlen, size = sock_snd_rcv_buf_size; /* For socket fd and
server len */
struct sockaddr_un serv_addr; /* For Unix Sock address */
char server_ux_name[255];
NCS_PATRICIA_PARAMS pat_tree_params;
@@ -112,6 +110,7 @@ uint32_t dtm_intra_processing_init(char
return NCSCC_RC_FAILURE;
}
dtm_intranode_cb->sock_domain = dtm_socket_domain;
+ dtm_intranode_cb->sock_snd_rcv_buf_size = sock_snd_rcv_buf_size;
/* Open a socket, If socket opens to fail return Error */
dtm_intranode_cb->server_sockfd = socket(dtm_socket_domain,
SOCK_STREAM, 0);
@@ -894,7 +893,7 @@ static uint32_t dtm_intranode_process_in
{
int flags;
/* Accept processing */
- int accept_fd = 0, retry_count = 0, size = DTM_INTRANODE_SOCK_SIZE;
+ int accept_fd = 0, retry_count = 0, size =
dtm_intranode_cb->sock_snd_rcv_buf_size;
socklen_t len = sizeof(struct sockaddr_un);
struct sockaddr_un cli_addr;
/* Accept should be non_block */
diff --git a/osaf/services/infrastructure/dtms/dtm/dtm_node_sockets.c
b/osaf/services/infrastructure/dtms/dtm/dtm_node_sockets.c
--- a/osaf/services/infrastructure/dtms/dtm/dtm_node_sockets.c
+++ b/osaf/services/infrastructure/dtms/dtm/dtm_node_sockets.c
@@ -26,7 +26,6 @@
#define MYPORT "6900"
#define MAXBUFLEN 100
-#define DTM_INTERNODE_SOCK_SIZE 64000
struct addrinfo *mcast_sender_addr; /* Holder for mcast_sender_addr address
*/
@@ -431,7 +430,7 @@ uint32_t dtm_comm_socket_send(int sock_d
int rtn = 0;
int err = 0;
int rc = NCSCC_RC_SUCCESS;
- rtn = send(sock_desc, (raw_type *) buffer, buffer_len, 0);
+ rtn = send(sock_desc, (raw_type *) buffer, buffer_len, MSG_NOSIGNAL);
err = errno;
if (rtn < 0) {
if (!IS_BLOCKIN_ERROR(err)) {
@@ -508,7 +507,7 @@ static char *comm_get_foreign_address(in
int comm_socket_setup_new(DTM_INTERNODE_CB * dtms_cb, const char
*foreign_address, in_port_t foreign_port,
DTM_IP_ADDR_TYPE ip_addr_type)
{
- int sock_desc = -1, size = DTM_INTERNODE_SOCK_SIZE;
+ int sock_desc = -1, size = dtms_cb->sock_snd_rcv_buf_size;
int err = 0, rv;
char local_port_str[INET6_ADDRSTRLEN];
struct addrinfo *addr_list;
@@ -578,12 +577,12 @@ int comm_socket_setup_new(DTM_INTERNODE_
goto done;
}
- int flag = 1;
- if (setsockopt(sock_desc, IPPROTO_TCP, TCP_NODELAY, (void *)&flag,
sizeof(flag)) != 0) {
- LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
- dtm_comm_socket_close(&sock_desc);
- goto done;
- }
+ int flag = dtms_cb->tcp_nodelay_flag;
+ if (setsockopt(sock_desc, IPPROTO_TCP, TCP_NODELAY, (void *)&flag,
sizeof(flag)) != 0) {
+ LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
+ dtm_comm_socket_close(&sock_desc);
+ goto done;
+ }
if (NCSCC_RC_SUCCESS != set_keepalive(dtms_cb, sock_desc)) {
LOG_ER("DTM :set_keepalive failed ");
@@ -660,7 +659,7 @@ uint32_t dtm_stream_nonblocking_listener
struct addrinfo addr_criteria; /* Criteria for address match */
char local_port_str[6];
struct addrinfo *addr_list = NULL, *p;; /* List of serv addresses */
- int size = DTM_INTERNODE_SOCK_SIZE;
+ int size = dtms_cb->sock_snd_rcv_buf_size;
int rv;
char ip_addr_eth[INET6_ADDRSTRLEN + IFNAMSIZ];
dtms_cb->stream_sock = -1;
@@ -728,11 +727,11 @@ uint32_t dtm_stream_nonblocking_listener
LOG_ER("DTM:Socket snd buf size set failed err :%s",
strerror(errno));
}
- int flag = 1;
- if (setsockopt(dtms_cb->stream_sock, IPPROTO_TCP, TCP_NODELAY, (void
*)&flag, sizeof(flag)) != 0) {
- LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
- dtm_sockdesc_close(dtms_cb->stream_sock);
- return NCSCC_RC_FAILURE;
+ int flag = dtms_cb->tcp_nodelay_flag;
+ if (setsockopt(dtms_cb->stream_sock, IPPROTO_TCP, TCP_NODELAY, (void
*)&flag, sizeof(flag)) != 0) {
+ LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
+ dtm_sockdesc_close(dtms_cb->stream_sock);
+ return NCSCC_RC_FAILURE;
}
if (set_keepalive(dtms_cb, dtms_cb->stream_sock) != NCSCC_RC_SUCCESS) {
@@ -1249,7 +1248,6 @@ int dtm_process_connect(DTM_INTERNODE_CB
}
-#define DTM_INTERNODE_SOCK_SIZE 64000
/**
* Function for dtm accept the connection
@@ -1270,7 +1268,7 @@ int dtm_process_accept(DTM_INTERNODE_CB
int err = 0;
DTM_NODE_DB node;
DTM_NODE_DB *new_node;
- int new_conn_sd, size = DTM_INTERNODE_SOCK_SIZE;
+ int new_conn_sd, size = dtms_cb->sock_snd_rcv_buf_size;
const struct sockaddr *clnt_addr1 = (struct sockaddr *)&clnt_addr;
TRACE_ENTER();
@@ -1298,12 +1296,12 @@ int dtm_process_accept(DTM_INTERNODE_CB
goto done;
}
- int flag = 1;
- if (setsockopt(new_conn_sd, IPPROTO_TCP, TCP_NODELAY, (void *)&flag,
sizeof(flag)) != 0) {
- LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
- dtm_comm_socket_close(&new_conn_sd);
- goto done;
- }
+ int flag = dtms_cb->tcp_nodelay_flag;
+ if (setsockopt(new_conn_sd, IPPROTO_TCP, TCP_NODELAY, (void *)&flag,
sizeof(flag)) != 0) {
+ LOG_ER("DTM:Socket TCP_NODELAY set failed err :%s",
strerror(errno));
+ dtm_comm_socket_close(&new_conn_sd);
+ goto done;
+ }
if (NCSCC_RC_SUCCESS != set_keepalive(dtms_cb, new_conn_sd)) {
LOG_ER("DTM:set_keepalive failed ");
diff --git a/osaf/services/infrastructure/dtms/dtm/dtm_read_config.c
b/osaf/services/infrastructure/dtms/dtm/dtm_read_config.c
--- a/osaf/services/infrastructure/dtms/dtm/dtm_read_config.c
+++ b/osaf/services/infrastructure/dtms/dtm/dtm_read_config.c
@@ -94,6 +94,10 @@ void dtm_print_config(DTM_INTERNODE_CB *
TRACE(" %d", config->initial_dis_timeout);
TRACE(" DTM_BCAST_FRE_MSECS: ");
TRACE(" %d", config->bcast_msg_freq);
+ TRACE(" DTM_SOCK_SND_RCV_BUF_SIZE: ");
+ TRACE(" %d", config->sock_snd_rcv_buf_size);
+ TRACE(" DTM_TCP_NODELAY_FLAG: ");
+ TRACE(" %d", config->tcp_nodelay_flag);
TRACE("DTM : ");
}
@@ -220,6 +224,8 @@ int dtm_read_config(DTM_INTERNODE_CB * c
config->i_addr_family = DTM_IP_ADDR_TYPE_IPV4;
config->bcast_msg_freq = BCAST_FRE;
config->initial_dis_timeout = DIS_TIME_OUT;
+ config->sock_snd_rcv_buf_size = DTM_SOCK_SND_RCV_BUF_SIZE;
+ config->tcp_nodelay_flag = true;
config->mcast_flag = false;
config->scope_link = false;
config->node_id = m_NCS_GET_NODE_ID;
@@ -424,6 +430,30 @@ int dtm_read_config(DTM_INTERNODE_CB * c
tag_len = 0;
}
+ if (strncmp(line, "DTM_SOCK_SND_RCV_BUF_SIZE=",
strlen("DTM_SOCK_SND_RCV_BUF_SIZE=")) == 0) {
+ tag_len = strlen("DTM_SOCK_SND_RCV_BUF_SIZE=");
+ config->sock_snd_rcv_buf_size =
atoi(&line[tag_len]);
+ if (config->sock_snd_rcv_buf_size < 126976) {
+ LOG_ER("DTM: DTM_SOCK_SND_RCV_BUF_SIZE
must be greater than rmem_default/wmem_default sysctl of dtmd.conf");
+ return -1;
+ }
+
+ tag = 0;
+ tag_len = 0;
+
+ }
+
+ if (strncmp(line, "DTM_TCP_NODELAY_FLAG=",
strlen("DTM_TCP_NODELAY_FLAG=")) == 0) {
+ tag_len = strlen("DTM_TCP_NODELAY_FLAG=");
+ config->tcp_nodelay_flag = atoi(&line[tag_len]);
+ if (config->tcp_nodelay_flag < 0 ||
config->tcp_nodelay_flag > 1) {
+ LOG_ER("DTM: DTM_TCP_NODELAY_FLAG
needs to be 0 or 1 of dtmd.conf ");
+ return -1;
+ }
+ tag = 0;
+ tag_len = 0;
+
+ }
}
memset(line, 0, DTM_MAX_TAG_LEN);
diff --git a/osaf/services/infrastructure/dtms/include/dtm.h
b/osaf/services/infrastructure/dtms/include/dtm.h
--- a/osaf/services/infrastructure/dtms/include/dtm.h
+++ b/osaf/services/infrastructure/dtms/include/dtm.h
@@ -37,6 +37,8 @@ extern uint8_t initial_discovery_phase;
#define m_NODE_DISCOVERY_TASKNAME "NODE_DISCOVERY"
#define m_NODE_DISCOVERY_STACKSIZE NCS_STACKSIZE_HUGE
+/* The default value of SO_RCVBUF & SO_SNDBUF , it is set by the
rmem_default/wmem_default */
+#define DTM_SOCK_SND_RCV_BUF_SIZE 126976
#define DTM_MAX_TAG_LEN 256
typedef enum {
diff --git a/osaf/services/infrastructure/dtms/include/dtm_cb.h
b/osaf/services/infrastructure/dtms/include/dtm_cb.h
--- a/osaf/services/infrastructure/dtms/include/dtm_cb.h
+++ b/osaf/services/infrastructure/dtms/include/dtm_cb.h
@@ -90,6 +90,8 @@ typedef struct dtm_internode_cb {
int comm_keepidle_time;
int comm_keepalive_intvl;
int comm_keepalive_probes;
+ int32_t sock_snd_rcv_buf_size; /* The value of SO_RCVBUF & SO_SNDBUF */
+ uint8_t tcp_nodelay_flag; /* The value of TCP_NODELAY Flag */
SYSF_MBX mbx;
int mbx_fd;
} DTM_INTERNODE_CB;
@@ -116,6 +118,7 @@ typedef struct dtm_intranode_cb {
SYSF_MBX mbx;
int mbx_fd;
int sock_domain;
+ int32_t sock_snd_rcv_buf_size; /* The value of SO_RCVBUF & SO_SNDBUF */
} DTM_INTRANODE_CB;
extern DTM_INTRANODE_CB *dtm_intranode_cb;
diff --git a/osaf/services/infrastructure/dtms/include/dtm_intra_disc.h
b/osaf/services/infrastructure/dtms/include/dtm_intra_disc.h
--- a/osaf/services/infrastructure/dtms/include/dtm_intra_disc.h
+++ b/osaf/services/infrastructure/dtms/include/dtm_intra_disc.h
@@ -17,8 +17,6 @@
#ifndef DTM_INTRA_DISC_H
#define DTM_INTRA_DISC_H
-#define DTM_INTRANODE_MSG_SIZE 1400
-#define DTM_INTERNODE_MSG_SIZE DTM_INTRANODE_MSG_SIZE
/* 2 -len(0), 4 - iden(2), 1- ver(6), 1-msg type(7), 4- server_type (8),
4 -server lower(12), 4 -server lower(16), 8 -ref(20), 4
- nodeid(28),
diff --git a/osaf/services/infrastructure/nid/scripts/opensafd.in
b/osaf/services/infrastructure/nid/scripts/opensafd.in
--- a/osaf/services/infrastructure/nid/scripts/opensafd.in
+++ b/osaf/services/infrastructure/nid/scripts/opensafd.in
@@ -184,6 +184,7 @@ generate_nodeid() {
}
check_transport() {
+ rm -f $pkglocalstatedir/osaf_dtm_intra_server
if [ "$MDS_TRANSPORT" = "TIPC" ]; then
# Transport TIPC
check_tipc
------------------------------------------------------------------------------
WatchGuard Dimension instantly turns raw network data into actionable
security intelligence. It gives you real-time visual feedback on key
security issues and trends. Skip the complicated setup - simply import
a virtual appliance and go from zero to informed in seconds.
http://pubads.g.doubleclick.net/gampad/clk?id=123612991&iu=/4140/ostg.clktrk
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
