averello/linux-stable-mirror
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2026-08-09 06:14:34 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
185a4caeecabc150106deda1da170b09f2ad803f
linux-stable-mirror/net/ipv4/udp_tunnel_core.c
T
Kuniyuki IwashimaandJakub Kicinski 72d3b9a4c2 udp_tunnel: Remove synchronize_rcu() in udp_tunnel_sock_release(). 
Commit 3cf7203ca6 ("net/tunnel: wait until all sk_user_data
reader finish before releasing the sock") added synchronize_rcu()
in udp_tunnel_sock_release().

This was intended to protect the fast path of a dying vxlan device
from dereferencing vxlan_sock->sock->sk after sock_orphan() has set
sock->sk to NULL.

However, vxlan does not need to access struct socket itself
in the fast path; it only reads struct sock, and struct socket
is only used for tunnel setup and teardown.

This applies to all other UDP tunnel users, and they have been
converted to access struct sock directly.

In addition, each device-specific struct used in their fast paths
is freed after one RCU grace period.  Since this occurs after
udp_tunnel_sock_release(), the struct is guaranteed to be freed
after struct udp_sock.

Therefore, synchronize_rcu() in udp_tunnel_sock_release() is
now redundant.

Let's remove it.

Tested:

A script creating/upping vxlan devices in 4000 netns runs 10x
faster with this change.  We can see the same improvement with
other UDP tunnel devices as well.

  $ cat vxlan.sh
  for i in `seq 1 40`
  do
      (for j in `seq 1 100` ; do
            unshare -n bash -c "ip link add vxlan0 type vxlan id 100 local 127.0.0.1 dstport 4789 && ip link set vxlan0 up";
       done) &
  done
  wait

With bpftrace, we can see vxlan_stop() is significantly faster.

  bpftrace -e '
  kprobe:vxlan_stop {
          @start[tid] = nsecs;
  }

  kretprobe:vxlan_stop /@start[tid]/ {
          @duration_us = hist((nsecs - @start[tid]) / 1000);
          delete(@start[tid]);
  }

  END {
          printf("\nExecution time of vxlan_stop (us):\n");
  }'

Before:

  # time ./vxlan.sh // without bpftrace
  real	0m50.615s
  user	0m8.171s
  sys	1m45.101s

  @duration_us:
  [4K, 8K)            1266 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@                   |
  [8K, 16K)           1957 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [16K, 32K)           764 |@@@@@@@@@@@@@@@@@@@@                                |
  [32K, 64K)             6 |                                                    |
  [64K, 128K)            4 |                                                    |
  [128K, 256K)           3 |                                                    |

After:

  # time ./vxlan.sh // without bpftrace
  real	0m5.247s
  user	0m7.956s
  sys	1m47.404s

  @duration_us:
  [16, 32)            3411 |@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@|
  [32, 64)             383 |@@@@@                                               |
  [64, 128)            107 |@                                                   |
  [128, 256)            79 |@                                                   |
  [256, 512)            16 |                                                    |
  [512, 1K)              2 |                                                    |
  [1K, 2K)               2 |                                                    |

Next step is to remove another synchronize_net() in vxlan_stop()
and variants in other devices.

Signed-off-by: Kuniyuki Iwashima <kuniyu@google.com>
Link: https://patch.msgid.link/20260502031401.3557229-16-kuniyu@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2026-05-05 17:47:06 -07:00

276 lines
6.7 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0-only
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/socket.h>
#include <linux/kernel.h>
#include <net/dst_metadata.h>
#include <net/flow.h>
#include <net/udp.h>
#include <net/udp_tunnel.h>
#include <net/inet_dscp.h>
int udp_sock_create4(struct net *net, struct udp_port_cfg *cfg,
struct socket **sockp)
{
int err;
struct socket *sock = NULL;
struct sockaddr_in udp_addr;
err = sock_create_kern(net, AF_INET, SOCK_DGRAM, 0, &sock);
if (err < 0)
goto error;
if (cfg->bind_ifindex) {
err = sock_bindtoindex(sock->sk, cfg->bind_ifindex, true);
if (err < 0)
goto error;
}
udp_addr.sin_family = AF_INET;
udp_addr.sin_addr = cfg->local_ip;
udp_addr.sin_port = cfg->local_udp_port;
err = kernel_bind(sock, (struct sockaddr_unsized *)&udp_addr,
sizeof(udp_addr));
if (err < 0)
goto error;
if (cfg->peer_udp_port) {
udp_addr.sin_family = AF_INET;
udp_addr.sin_addr = cfg->peer_ip;
udp_addr.sin_port = cfg->peer_udp_port;
err = kernel_connect(sock, (struct sockaddr_unsized *)&udp_addr,
sizeof(udp_addr), 0);
if (err < 0)
goto error;
}
sock->sk->sk_no_check_tx = !cfg->use_udp_checksums;
*sockp = sock;
return 0;
error:
if (sock) {
kernel_sock_shutdown(sock, SHUT_RDWR);
sock_release(sock);
}
*sockp = NULL;
return err;
}
EXPORT_SYMBOL(udp_sock_create4);
static bool sk_saddr_any(struct sock *sk)
{
#if IS_ENABLED(CONFIG_IPV6)
return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
#else
return !sk->sk_rcv_saddr;
#endif
}
void setup_udp_tunnel_sock(struct net *net, struct sock *sk,
struct udp_tunnel_sock_cfg *cfg)
{
/* Disable multicast loopback */
inet_clear_bit(MC_LOOP, sk);
/* Enable CHECKSUM_UNNECESSARY to CHECKSUM_COMPLETE conversion */
inet_inc_convert_csum(sk);
rcu_assign_sk_user_data(sk, cfg->sk_user_data);
udp_sk(sk)->encap_type = cfg->encap_type;
udp_sk(sk)->encap_rcv = cfg->encap_rcv;
udp_sk(sk)->encap_err_rcv = cfg->encap_err_rcv;
udp_sk(sk)->encap_err_lookup = cfg->encap_err_lookup;
udp_sk(sk)->encap_destroy = cfg->encap_destroy;
udp_sk(sk)->gro_receive = cfg->gro_receive;
udp_sk(sk)->gro_complete = cfg->gro_complete;
udp_tunnel_encap_enable(sk);
udp_tunnel_update_gro_rcv(sk, true);
if (!sk->sk_dport && !sk->sk_bound_dev_if && sk_saddr_any(sk) &&
sk->sk_kern_sock)
udp_tunnel_update_gro_lookup(net, sk, true);
}
EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
void udp_tunnel_push_rx_port(struct net_device *dev, struct sock *sk,
unsigned short type)
{
struct udp_tunnel_info ti;
ti.type = type;
ti.sa_family = sk->sk_family;
ti.port = inet_sk(sk)->inet_sport;
udp_tunnel_nic_add_port(dev, &ti);
}
EXPORT_SYMBOL_GPL(udp_tunnel_push_rx_port);
void udp_tunnel_drop_rx_port(struct net_device *dev, struct sock *sk,
unsigned short type)
{
struct udp_tunnel_info ti;
ti.type = type;
ti.sa_family = sk->sk_family;
ti.port = inet_sk(sk)->inet_sport;
udp_tunnel_nic_del_port(dev, &ti);
}
EXPORT_SYMBOL_GPL(udp_tunnel_drop_rx_port);
/* Notify netdevs that UDP port started listening */
void udp_tunnel_notify_add_rx_port(struct sock *sk, unsigned short type)
{
struct net *net = sock_net(sk);
struct udp_tunnel_info ti;
struct net_device *dev;
ASSERT_RTNL();
ti.type = type;
ti.sa_family = sk->sk_family;
ti.port = inet_sk(sk)->inet_sport;
for_each_netdev(net, dev) {
udp_tunnel_nic_lock(dev);
udp_tunnel_nic_add_port(dev, &ti);
udp_tunnel_nic_unlock(dev);
}
}
EXPORT_SYMBOL_GPL(udp_tunnel_notify_add_rx_port);
/* Notify netdevs that UDP port is no more listening */
void udp_tunnel_notify_del_rx_port(struct sock *sk, unsigned short type)
{
struct net *net = sock_net(sk);
struct udp_tunnel_info ti;
struct net_device *dev;
ASSERT_RTNL();
ti.type = type;
ti.sa_family = sk->sk_family;
ti.port = inet_sk(sk)->inet_sport;
for_each_netdev(net, dev) {
udp_tunnel_nic_lock(dev);
udp_tunnel_nic_del_port(dev, &ti);
udp_tunnel_nic_unlock(dev);
}
}
EXPORT_SYMBOL_GPL(udp_tunnel_notify_del_rx_port);
void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
__be32 src, __be32 dst, __u8 tos, __u8 ttl,
__be16 df, __be16 src_port, __be16 dst_port,
bool xnet, bool nocheck, u16 ipcb_flags)
{
struct udphdr *uh;
__skb_push(skb, sizeof(*uh));
skb_reset_transport_header(skb);
uh = udp_hdr(skb);
uh->dest = dst_port;
uh->source = src_port;
uh->len = htons(skb->len);
memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
udp_set_csum(nocheck, skb, src, dst, skb->len);
iptunnel_xmit(sk, rt, skb, src, dst, IPPROTO_UDP, tos, ttl, df, xnet,
ipcb_flags);
}
EXPORT_SYMBOL_GPL(udp_tunnel_xmit_skb);
void udp_tunnel_sock_release(struct sock *sk)
{
struct socket *sock = sk->sk_socket;
rcu_assign_sk_user_data(sk, NULL);
kernel_sock_shutdown(sock, SHUT_RDWR);
sock_release(sock);
}
EXPORT_SYMBOL_GPL(udp_tunnel_sock_release);
struct metadata_dst *udp_tun_rx_dst(struct sk_buff *skb, unsigned short family,
const unsigned long *flags,
__be64 tunnel_id, int md_size)
{
struct metadata_dst *tun_dst;
struct ip_tunnel_info *info;
if (family == AF_INET)
tun_dst = ip_tun_rx_dst(skb, flags, tunnel_id, md_size);
else
tun_dst = ipv6_tun_rx_dst(skb, flags, tunnel_id, md_size);
if (!tun_dst)
return NULL;
info = &tun_dst->u.tun_info;
info->key.tp_src = udp_hdr(skb)->source;
info->key.tp_dst = udp_hdr(skb)->dest;
if (udp_hdr(skb)->check)
__set_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
return tun_dst;
}
EXPORT_SYMBOL_GPL(udp_tun_rx_dst);
struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb,
struct net_device *dev,
struct net *net, int oif,
__be32 *saddr,
const struct ip_tunnel_key *key,
__be16 sport, __be16 dport, u8 tos,
struct dst_cache *dst_cache)
{
struct rtable *rt = NULL;
struct flowi4 fl4;
#ifdef CONFIG_DST_CACHE
if (dst_cache) {
rt = dst_cache_get_ip4(dst_cache, saddr);
if (rt)
return rt;
}
#endif
memset(&fl4, 0, sizeof(fl4));
fl4.flowi4_mark = skb->mark;
fl4.flowi4_proto = IPPROTO_UDP;
fl4.flowi4_oif = oif;
fl4.daddr = key->u.ipv4.dst;
fl4.saddr = key->u.ipv4.src;
fl4.fl4_dport = dport;
fl4.fl4_sport = sport;
fl4.flowi4_dscp = inet_dsfield_to_dscp(tos);
fl4.flowi4_flags = key->flow_flags;
rt = ip_route_output_key(net, &fl4);
if (IS_ERR(rt)) {
netdev_dbg(dev, "no route to %pI4\n", &fl4.daddr);
return ERR_PTR(-ENETUNREACH);
}
if (rt->dst.dev == dev) { /* is this necessary? */
netdev_dbg(dev, "circular route to %pI4\n", &fl4.daddr);
ip_rt_put(rt);
return ERR_PTR(-ELOOP);
}
#ifdef CONFIG_DST_CACHE
if (dst_cache)
dst_cache_set_ip4(dst_cache, &rt->dst, fl4.saddr);
#endif
*saddr = fl4.saddr;
return rt;
}
EXPORT_SYMBOL_GPL(udp_tunnel_dst_lookup);
MODULE_DESCRIPTION("IPv4 Foo over UDP tunnel driver");
MODULE_LICENSE("GPL");
Reference in New Issue View Git Blame Copy Permalink