net: Use a percpu_counter for sockets_allocated

Instead of using one atomic_t per protocol, use a percpu_counter
for "sockets_allocated", to reduce cache line contention on
heavy duty network servers.

Note : We revert commit (248969ae31e1b3276fc4399d67ce29a5d81e6fd9
net: af_unix can make unix_nr_socks visbile in /proc),
since it is not anymore used after sock_prot_inuse_add() addition

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/sctp/sctp.h b/include/net/sctp/sctp.h
index 23797506f593e87af3df9171ab75929513085278..bbb7742195b0df31e4498b19e909e7a97a7b47e7 100644 (file)
--- a/include/net/sctp/sctp.h
+++ b/include/net/sctp/sctp.h
@@ -138,6 +138,7 @@ void sctp_write_space(struct sock *sk);
 unsigned int sctp_poll(struct file *file, struct socket *sock,
                poll_table *wait);
 void sctp_sock_rfree(struct sk_buff *skb);
+extern struct percpu_counter sctp_sockets_allocated;
 
 /*
  * sctp/primitive.c

diff --git a/include/net/sock.h b/include/net/sock.h
index 00cd486d362fc973b94a0e47de65d08187da3eca..a2a3890959c4e2a525967de900d81a8be5540d21 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -649,7 +649,7 @@ struct proto {
        /* Memory pressure */
        void                    (*enter_memory_pressure)(struct sock *sk);
        atomic_t                *memory_allocated;      /* Current allocated memory. */
-       atomic_t                *sockets_allocated;     /* Current number of sockets. */
+       struct percpu_counter   *sockets_allocated;     /* Current number of sockets. */
        /*
         * Pressure flag: try to collapse.
         * Technical note: it is used by multiple contexts non atomically.

diff --git a/include/net/tcp.h b/include/net/tcp.h
index e8ae90a8c35e1c5727cc595a59674a488c4d09ee..cbca3b8a133d58d26ca49a414c5a1ee348f7d5ce 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -238,7 +238,7 @@ extern int sysctl_tcp_slow_start_after_idle;
 extern int sysctl_tcp_max_ssthresh;
 
 extern atomic_t tcp_memory_allocated;
-extern atomic_t tcp_sockets_allocated;
+extern struct percpu_counter tcp_sockets_allocated;
 extern int tcp_memory_pressure;
 
 /*

diff --git a/net/core/sock.c b/net/core/sock.c
index a4e840e5a05335aeb0517f5e741f5149f6f919cf..7a081b647bf9d07370f69ba97e951fdc7599f65f 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1071,7 +1071,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
                newsk->sk_sleep  = NULL;
 
                if (newsk->sk_prot->sockets_allocated)
-                       atomic_inc(newsk->sk_prot->sockets_allocated);
+                       percpu_counter_inc(newsk->sk_prot->sockets_allocated);
        }
 out:
        return newsk;
@@ -1463,8 +1463,12 @@ int __sk_mem_schedule(struct sock *sk, int size, int kind)
        }
 
        if (prot->memory_pressure) {
-               if (!*prot->memory_pressure ||
-                   prot->sysctl_mem[2] > atomic_read(prot->sockets_allocated) *
+               int alloc;
+
+               if (!*prot->memory_pressure)
+                       return 1;
+               alloc = percpu_counter_read_positive(prot->sockets_allocated);
+               if (prot->sysctl_mem[2] > alloc *
                    sk_mem_pages(sk->sk_wmem_queued +
                                 atomic_read(&sk->sk_rmem_alloc) +
                                 sk->sk_forward_alloc))

diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
index 731789bb499feee9af50211c16cd35a89308215c..4944b47ad6287deeb277692798ff15471593d069 100644 (file)
--- a/net/ipv4/proc.c
+++ b/net/ipv4/proc.c
@@ -55,7 +55,8 @@ static int sockstat_seq_show(struct seq_file *seq, void *v)
        seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %d\n",
                   sock_prot_inuse_get(net, &tcp_prot),
                   atomic_read(&tcp_orphan_count),
-                  tcp_death_row.tw_count, atomic_read(&tcp_sockets_allocated),
+                  tcp_death_row.tw_count,
+                  (int)percpu_counter_sum_positive(&tcp_sockets_allocated),
                   atomic_read(&tcp_memory_allocated));
        seq_printf(seq, "UDP: inuse %d mem %d\n",
                   sock_prot_inuse_get(net, &udp_prot),

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 044224a341eb10b6cb476c2b0b43d130a4af5916..e6fade9ebf621c38c056a7d90c900c66bbf0bc33 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -290,9 +290,12 @@ EXPORT_SYMBOL(sysctl_tcp_rmem);
 EXPORT_SYMBOL(sysctl_tcp_wmem);
 
 atomic_t tcp_memory_allocated; /* Current allocated memory. */
-atomic_t tcp_sockets_allocated;        /* Current number of TCP sockets. */
-
 EXPORT_SYMBOL(tcp_memory_allocated);
+
+/*
+ * Current number of TCP sockets.
+ */
+struct percpu_counter tcp_sockets_allocated;
 EXPORT_SYMBOL(tcp_sockets_allocated);
 
 /*
@@ -2685,6 +2688,7 @@ void __init tcp_init(void)
 
        BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));
 
+       percpu_counter_init(&tcp_sockets_allocated, 0);
        tcp_hashinfo.bind_bucket_cachep =
                kmem_cache_create("tcp_bind_bucket",
                                  sizeof(struct inet_bind_bucket), 0,

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index cab2458f86fdd7f5c74c67abfa4b700c9e5629ac..26b9030747cc97b61eeca8554ca6792863a9d6fe 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -1797,7 +1797,7 @@ static int tcp_v4_init_sock(struct sock *sk)
        sk->sk_sndbuf = sysctl_tcp_wmem[1];
        sk->sk_rcvbuf = sysctl_tcp_rmem[1];
 
-       atomic_inc(&tcp_sockets_allocated);
+       percpu_counter_inc(&tcp_sockets_allocated);
 
        return 0;
 }
@@ -1845,7 +1845,7 @@ void tcp_v4_destroy_sock(struct sock *sk)
                sk->sk_sndmsg_page = NULL;
        }
 
-       atomic_dec(&tcp_sockets_allocated);
+       percpu_counter_dec(&tcp_sockets_allocated);
 }
 
 EXPORT_SYMBOL(tcp_v4_destroy_sock);

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index f259c9671f3ec0f714cd9ae72c56a8f94558870a..8702b06cb60a9dbe6b1d7490bd30656ed79e141c 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -1830,7 +1830,7 @@ static int tcp_v6_init_sock(struct sock *sk)
        sk->sk_sndbuf = sysctl_tcp_wmem[1];
        sk->sk_rcvbuf = sysctl_tcp_rmem[1];
 
-       atomic_inc(&tcp_sockets_allocated);
+       percpu_counter_inc(&tcp_sockets_allocated);
 
        return 0;
 }

diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c
index a8ca743241ee19379c54f3d96635e2133e4d0183..d5ea232c9126799654d23b49aefc77a09763adc4 100644 (file)
--- a/net/sctp/protocol.c
+++ b/net/sctp/protocol.c
@@ -102,6 +102,8 @@ struct sock *sctp_get_ctl_sock(void)
 /* Set up the proc fs entry for the SCTP protocol. */
 static __init int sctp_proc_init(void)
 {
+       if (percpu_counter_init(&sctp_sockets_allocated, 0))
+               goto out_nomem;
 #ifdef CONFIG_PROC_FS
        if (!proc_net_sctp) {
                struct proc_dir_entry *ent;
@@ -110,7 +112,7 @@ static __init int sctp_proc_init(void)
                        ent->owner = THIS_MODULE;
                        proc_net_sctp = ent;
                } else
-                       goto out_nomem;
+                       goto out_free_percpu;
        }
 
        if (sctp_snmp_proc_init())
@@ -135,6 +137,8 @@ out_snmp_proc_init:
                proc_net_sctp = NULL;
                remove_proc_entry("sctp", init_net.proc_net);
        }
+out_free_percpu:
+       percpu_counter_destroy(&sctp_sockets_allocated);
 out_nomem:
        return -ENOMEM;
 #else

diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index ba81fe3ccab83c7268ee685f375c01b75632d862..a2de585888d0a53cef7e53745ccb4f033abd61a1 100644 (file)
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -114,7 +114,7 @@ extern int sysctl_sctp_wmem[3];
 
 static int sctp_memory_pressure;
 static atomic_t sctp_memory_allocated;
-static atomic_t sctp_sockets_allocated;
+struct percpu_counter sctp_sockets_allocated;
 
 static void sctp_enter_memory_pressure(struct sock *sk)
 {
@@ -3613,7 +3613,7 @@ SCTP_STATIC int sctp_init_sock(struct sock *sk)
        sp->hmac = NULL;
 
        SCTP_DBG_OBJCNT_INC(sock);
-       atomic_inc(&sctp_sockets_allocated);
+       percpu_counter_inc(&sctp_sockets_allocated);
 
        local_bh_disable();
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
@@ -3632,7 +3632,7 @@ SCTP_STATIC void sctp_destroy_sock(struct sock *sk)
        /* Release our hold on the endpoint. */
        ep = sctp_sk(sk)->ep;
        sctp_endpoint_free(ep);
-       atomic_dec(&sctp_sockets_allocated);
+       percpu_counter_dec(&sctp_sockets_allocated);
        local_bh_disable();
        sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
        local_bh_enable();

diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 3a35a6e8bf918aca54689fe945bd7e667fecd973..5aaf23e43f1de87740641ef22fa169d9045f7ed3 100644 (file)
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -571,7 +571,6 @@ static const struct proto_ops unix_seqpacket_ops = {
 static struct proto unix_proto = {
        .name                   = "UNIX",
        .owner                  = THIS_MODULE,
-       .sockets_allocated      = &unix_nr_socks,
        .obj_size               = sizeof(struct unix_sock),
 };