Hi David,
On Wed, Jan 08, 2014 at 04:59:21PM +0000, David Harrold wrote:
> Hi,
>
> I notice that recent HAProxy has support for being a TCP Fast Open Server,
> and I have a requirement to also add support for HAProxy to be a TCP Fast
> Open Client for those upstream servers that support it.
>
> This is for a scenario where there is a satellite link between HAProxy and
> the servers, so every Round Trip we can remove helps a lot.
>
>
> Before I dive in, is anybody already working on this, or has anybody already
> investigated whether this is feasible?
Yes, I experimented with it in 1.5-dev17 exactly one year ago. I'm appending
the patch I used. I got mixed results (my kernel included a backport of the
TFO code which didn't seem very reliable).
The code is clearly not in a mergeable state, though it can help you figure
out what to touch. There were a few layering violations at this time, but I
think that the current architecture will make this easier to implement.
I'm appending the patch in case you manage to update it to latest version.
Please keep us informed!
Regards,
Willy
>From eee84abe901a9f605f9973016969f89546f7aa29 Mon Sep 17 00:00:00 2001
From: Willy Tarreau <w...@1wt.eu>
Date: Sun, 6 Jan 2013 21:25:20 +0100
Subject: EXP: connection: add support for server-side TFO (TCP FastOpen)
This adds the "tfo" server keyword which passes the FASTOPEN flag
all along the connection chain. A call to si_chksnd() is performed
after tcp_connect_server() if the connect() was skipped due to the
flag.
This is still quite ugly overall, and as of linux 3.7.1, TCP fastopen
doesn't work well enough : it's quickly disabled for an unknown reason,
and there is no way to have the server respond with the data in the
SYN/ACK, nor to close with an RST without sending a FIN/ACK.
---
include/common/compat.h | 5 +++++
include/proto/stream_interface.h | 6 ++++++
include/types/connection.h | 2 ++
include/types/server.h | 2 +-
src/proto_tcp.c | 7 ++++++-
src/raw_sock.c | 9 +++++++--
src/server.c | 8 ++++++++
7 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/include/common/compat.h b/include/common/compat.h
index c4ac08c..824f3b3 100644
--- a/include/common/compat.h
+++ b/include/common/compat.h
@@ -69,6 +69,11 @@
#define MAXPATHLEN 128
#endif
+/* TCP Fast Open on Linux */
+#ifndef MSG_FASTOPEN
+#define MSG_FASTOPEN 0x20000000
+#endif
+
/* On Linux, allows pipes to be resized */
#ifndef F_SETPIPE_SZ
#define F_SETPIPE_SZ (1024 + 7)
diff --git a/include/proto/stream_interface.h b/include/proto/stream_interface.h
index c43ef57..96f705b 100644
--- a/include/proto/stream_interface.h
+++ b/include/proto/stream_interface.h
@@ -115,6 +115,12 @@ static inline int si_connect(struct stream_interface *si)
if (ret != SN_ERR_NONE)
return ret;
+ if (si->conn->flags & CO_FL_FAST_OPEN) {
+ si_chk_snd(si);
+ if (si->flags & SI_FL_ERR)
+ return SN_ERR_SRVCL; /* assume it's a connect() error */
+ }
+
/* needs src ip/port for logging */
if (si->flags & SI_FL_SRC_ADDR)
conn_get_from_addr(si->conn);
diff --git a/include/types/connection.h b/include/types/connection.h
index 255811c..2a1c1e8 100644
--- a/include/types/connection.h
+++ b/include/types/connection.h
@@ -82,6 +82,8 @@ enum {
CO_FL_CURR_WR_ENA = 0x00000040, /* sending is currently desired */
CO_FL_WAIT_WR = 0x00000080, /* sending needs to poll first */
+ CO_FL_FAST_OPEN = 0x00000100, /* connect using TCP Fast Open */
+
/* These flags are used by data layers to indicate they had to stop
* sending data because a buffer was empty (WAIT_DATA) or stop receiving
* data because a buffer was full (WAIT_ROOM). The connection handler
diff --git a/include/types/server.h b/include/types/server.h
index b58a062..ba02efd 100644
--- a/include/types/server.h
+++ b/include/types/server.h
@@ -47,7 +47,7 @@
#define SRV_RUNNING 0x0001 /* the server is UP */
#define SRV_BACKUP 0x0002 /* this server is a backup server */
#define SRV_MAPPORTS 0x0004 /* this server uses mapped ports */
-/* unused: 0x0008 */
+#define SRV_FASTOPEN 0x0008 /* Use TCP Fast Open to connect to server */
#define SRV_CHECKED 0x0010 /* this server needs to be checked */
#define SRV_GOINGDOWN 0x0020 /* this server says that it's going down (404)
*/
#define SRV_WARMINGUP 0x0040 /* this server is warming up after a failure */
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 7285457..d51705a 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -248,6 +248,7 @@ int tcp_connect_server(struct connection *conn, int data,
int delack)
struct server *srv;
struct proxy *be;
struct conn_src *src;
+ int use_fastopen = 0;
switch (obj_type(conn->target)) {
case OBJ_TYPE_PROXY:
@@ -257,6 +258,7 @@ int tcp_connect_server(struct connection *conn, int data,
int delack)
case OBJ_TYPE_SERVER:
srv = objt_server(conn->target);
be = srv->proxy;
+ use_fastopen = (srv->state & SRV_FASTOPEN);
break;
default:
return SN_ERR_INTERNAL;
@@ -402,7 +404,8 @@ int tcp_connect_server(struct connection *conn, int data,
int delack)
if (global.tune.server_rcvbuf)
setsockopt(fd, SOL_SOCKET, SO_RCVBUF,
&global.tune.server_rcvbuf, sizeof(global.tune.server_rcvbuf));
- if ((connect(fd, (struct sockaddr *)&conn->addr.to,
get_addr_len(&conn->addr.to)) == -1) &&
+ if (!use_fastopen &&
+ (connect(fd, (struct sockaddr *)&conn->addr.to,
get_addr_len(&conn->addr.to)) == -1) &&
(errno != EINPROGRESS) && (errno != EALREADY) && (errno !=
EISCONN)) {
if (errno == EAGAIN || errno == EADDRINUSE || errno ==
EADDRNOTAVAIL) {
@@ -437,6 +440,8 @@ int tcp_connect_server(struct connection *conn, int data,
int delack)
fdtab[fd].owner = conn;
conn->flags = CO_FL_WAIT_L4_CONN; /* connection in progress */
conn->flags |= CO_FL_ADDR_TO_SET;
+ if (use_fastopen)
+ conn->flags |= CO_FL_FAST_OPEN;
fdtab[fd].iocb = conn_fd_handler;
fd_insert(fd);
diff --git a/src/raw_sock.c b/src/raw_sock.c
index 20a867a..5368463 100644
--- a/src/raw_sock.c
+++ b/src/raw_sock.c
@@ -325,7 +325,12 @@ static int raw_sock_from_buf(struct connection *conn,
struct buffer *buf, int fl
if (try < buf->o)
send_flag = MSG_MORE;
- ret = send(conn->t.sock.fd, bo_ptr(buf), try, send_flag |
flags);
+ if ((conn->flags & (CO_FL_FAST_OPEN|CO_FL_WAIT_L4_CONN)) ==
+ (CO_FL_FAST_OPEN|CO_FL_WAIT_L4_CONN))
+ ret = sendto(conn->t.sock.fd, bo_ptr(buf), try,
send_flag | flags | MSG_FASTOPEN,
+ (struct sockaddr *)&conn->addr.to,
get_addr_len(&conn->addr.to));
+ else
+ ret = send(conn->t.sock.fd, bo_ptr(buf), try, send_flag
| flags);
if (ret > 0) {
buf->o -= ret;
@@ -339,7 +344,7 @@ static int raw_sock_from_buf(struct connection *conn,
struct buffer *buf, int fl
if (ret < try)
break;
}
- else if (ret == 0 || errno == EAGAIN || errno == ENOTCONN) {
+ else if (ret == 0 || errno == EAGAIN || errno == ENOTCONN ||
errno == EINPROGRESS) {
/* nothing written, we need to poll for write first */
__conn_data_poll_send(conn);
break;
diff --git a/src/server.c b/src/server.c
index 568d042..9ab4c1b 100644
--- a/src/server.c
+++ b/src/server.c
@@ -138,6 +138,13 @@ static int srv_parse_id(char **args, int *cur_arg, struct
proxy *curproxy, struc
return 0;
}
+/* parse the "tfo" server keyword */
+static int srv_parse_tfo(char **args, int *cur_arg, struct proxy *px, struct
server *newsrv, char **err)
+{
+ newsrv->state |= SRV_FASTOPEN;
+ return 0;
+}
+
/* Note: must not be declared <const> as its list will be overwritten.
* Please take care of keeping this list alphabetically sorted, doing so helps
* all code contributors.
@@ -147,6 +154,7 @@ static int srv_parse_id(char **args, int *cur_arg, struct
proxy *curproxy, struc
*/
static struct srv_kw_list srv_kws = { "ALL", { }, {
{ "id", srv_parse_id, 1, 0 }, /* set id# of server
*/
+ { "tfo", srv_parse_tfo, 0, 0 }, /* enable TCP Fast
Open of server */
{ NULL, NULL, 0 },
}};
--
1.7.12.2.21.g234cd45.dirty
