diff --git a/src/server.c b/src/server.c
index 8e49f79..0d1ac16 100644
--- a/src/server.c
+++ b/src/server.c
@@ -7469,6 +7469,8 @@ int srv_add_to_idle_list(struct server *srv, struct connection *conn, int is_saf
 	       (conn->flags & CO_FL_REVERSED)))) &&
 	    !conn->mux->used_streams(conn) && conn->mux->avail_streams(conn)) {
 		int retadd;
+		int sched = 0;
+		int when;
 
 		retadd = _HA_ATOMIC_ADD_FETCH(&srv->curr_idle_conns, 1);
 		if (retadd > srv->max_idle_conns) {
@@ -7502,12 +7504,15 @@ int srv_add_to_idle_list(struct server *srv, struct connection *conn, int is_saf
 				eb32_insert(&idle_conn_srv, &srv->idle_node);
 				if (!task_in_wq(idle_conn_task) && !
 				    task_in_rq(idle_conn_task)) {
-					task_schedule(idle_conn_task,
-					              srv->idle_node.key);
+					sched = 1;
+					when = srv->idle_node.key;
 				}
 				BUG_ON_STRESS(!mt_list_isempty(&conn->toremove_list));
 			}
 			HA_SPIN_UNLOCK(OTHER_LOCK, &idle_conn_srv_lock);
+
+			if (sched)
+				task_schedule(idle_conn_task, when);
 		}
 		return 1;
 	}