This pair of function is implemented on top of spin_lock_bh() that
is going to handle a softirq mask in order to apply finegrained vector
disablement. The lock function is going to return the previous vectors
enabled mask prior to the last call to local_bh_disable(), following a
similar model to that of local_irq_save/restore. Subsequent calls to
local_bh_disable() and friends can then stack up:
bh = local_bh_disable(vec_mask);
lock_sock_fast(&bh2) {
*bh2 = spin_lock_bh(...)
}
...
unlock_sock_fast(bh2) {
spin_unlock_bh(bh2, ...);
}
local_bh_enable(bh);
To prepare for that, make lock_sock_fast() able to return a saved vector
enabled mask and pass it back to unlock_sock_fast(). We'll plug it to
spin_lock_bh() in a subsequent patch.
Signed-off-by: Frederic Weisbecker <frede...@kernel.org>
Cc: Ingo Molnar <mi...@kernel.org>
Cc: Sebastian Andrzej Siewior <bige...@linutronix.de>
Cc: Thomas Gleixner <t...@linutronix.de>
Cc: Peter Zijlstra <pet...@infradead.org>
Cc: Linus Torvalds <torva...@linux-foundation.org>
Cc: David S. Miller <da...@davemloft.net>
Cc: Mauro Carvalho Chehab <mche...@s-opensource.com>
Cc: Paul E. McKenney <paul...@linux.vnet.ibm.com>
---
include/net/sock.h | 5 +++--
net/core/datagram.c | 5 +++--
net/core/sock.c | 2 +-
net/ipv4/tcp.c | 10 ++++++----
net/ipv4/udp.c | 11 +++++++----
5 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/include/net/sock.h b/include/net/sock.h
index 433f45f..7bba619 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1500,7 +1500,7 @@ void release_sock(struct sock *sk);
SINGLE_DEPTH_NESTING)
#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
-bool lock_sock_fast(struct sock *sk);
+bool lock_sock_fast(struct sock *sk, unsigned int *bh);
/**
* unlock_sock_fast - complement of lock_sock_fast
* @sk: socket
@@ -1509,7 +1509,8 @@ bool lock_sock_fast(struct sock *sk);
* fast unlock socket for user context.
* If slow mode is on, we call regular release_sock()
*/
-static inline void unlock_sock_fast(struct sock *sk, bool slow)
+static inline void unlock_sock_fast(struct sock *sk, bool slow,
+ unsigned int bh)
{
if (slow)
release_sock(sk);
diff --git a/net/core/datagram.c b/net/core/datagram.c
index 9aac0d6..0cdee87 100644
--- a/net/core/datagram.c
+++ b/net/core/datagram.c
@@ -334,17 +334,18 @@ EXPORT_SYMBOL(skb_free_datagram);
void __skb_free_datagram_locked(struct sock *sk, struct sk_buff *skb, int len)
{
bool slow;
+ unsigned int bh;
if (!skb_unref(skb)) {
sk_peek_offset_bwd(sk, len);
return;
}
- slow = lock_sock_fast(sk);
+ slow = lock_sock_fast(sk, &bh);
sk_peek_offset_bwd(sk, len);
skb_orphan(skb);
sk_mem_reclaim_partial(sk);
- unlock_sock_fast(sk, slow);
+ unlock_sock_fast(sk, slow, bh);
/* skb is now orphaned, can be freed outside of locked section */
__kfree_skb(skb);
diff --git a/net/core/sock.c b/net/core/sock.c
index 3730eb8..b886b86 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2874,7 +2874,7 @@ EXPORT_SYMBOL(release_sock);
*
* sk_lock.slock unlocked, owned = 1, BH enabled
*/
-bool lock_sock_fast(struct sock *sk)
+bool lock_sock_fast(struct sock *sk, unsigned int *bh)
{
might_sleep();
spin_lock_bh(&sk->sk_lock.slock);
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index dfd9bae..31b391a 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -605,6 +605,7 @@ EXPORT_SYMBOL(tcp_poll);
int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
{
struct tcp_sock *tp = tcp_sk(sk);
+ unsigned int bh;
int answ;
bool slow;
@@ -613,9 +614,9 @@ int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
if (sk->sk_state == TCP_LISTEN)
return -EINVAL;
- slow = lock_sock_fast(sk);
+ slow = lock_sock_fast(sk, &bh);
answ = tcp_inq(sk);
- unlock_sock_fast(sk, slow);
+ unlock_sock_fast(sk, slow, bh);
break;
case SIOCATMARK:
answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
@@ -3101,6 +3102,7 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info)
{
const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
const struct inet_connection_sock *icsk = inet_csk(sk);
+ unsigned int bh;
u32 now;
u64 rate64;
bool slow;
@@ -3134,7 +3136,7 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info)
return;
}
- slow = lock_sock_fast(sk);
+ slow = lock_sock_fast(sk, &bh);
info->tcpi_ca_state = icsk->icsk_ca_state;
info->tcpi_retransmits = icsk->icsk_retransmits;
@@ -3208,7 +3210,7 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info)
info->tcpi_bytes_retrans = tp->bytes_retrans;
info->tcpi_dsack_dups = tp->dsack_dups;
info->tcpi_reord_seen = tp->reord_seen;
- unlock_sock_fast(sk, slow);
+ unlock_sock_fast(sk, slow, bh);
}
EXPORT_SYMBOL_GPL(tcp_get_info);
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 7d69dd6..8148896 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1452,11 +1452,13 @@ EXPORT_SYMBOL_GPL(udp_init_sock);
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
{
+ unsigned int bh;
+
if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) {
- bool slow = lock_sock_fast(sk);
+ bool slow = lock_sock_fast(sk, &bh);
sk_peek_offset_bwd(sk, len);
- unlock_sock_fast(sk, slow);
+ unlock_sock_fast(sk, slow, bh);
}
if (!skb_unref(skb))
@@ -2378,10 +2380,11 @@ int udp_rcv(struct sk_buff *skb)
void udp_destroy_sock(struct sock *sk)
{
+ unsigned int bh;
struct udp_sock *up = udp_sk(sk);
- bool slow = lock_sock_fast(sk);
+ bool slow = lock_sock_fast(sk, &bh);
udp_flush_pending_frames(sk);
- unlock_sock_fast(sk, slow);
+ unlock_sock_fast(sk, slow, bh);
if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) {
void (*encap_destroy)(struct sock *sk);
encap_destroy = READ_ONCE(up->encap_destroy);
--
2.7.4
