From 66018506e15bea62de4eefc3298f170b4bfcf5ef Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 7 Jun 2010 03:12:08 +0000 Subject: ip: Router Alert RCU conversion Straightforward conversion to RCU. One rwlock becomes a spinlock, and is static. Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/ip.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/net/ip.h') diff --git a/include/net/ip.h b/include/net/ip.h index 452f229c380..9982c97f0bd 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -62,10 +62,10 @@ struct ip_ra_chain { struct ip_ra_chain *next; struct sock *sk; void (*destructor)(struct sock *); + struct rcu_head rcu; }; extern struct ip_ra_chain *ip_ra_chain; -extern rwlock_t ip_ra_lock; /* IP flags. */ #define IP_CE 0x8000 /* Flag: "Congestion" */ -- cgit v1.2.3-70-g09d2 From 592fcb9dfafaa02dd0edc207bf5d3a0ee7a1f8df Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 9 Jun 2010 16:21:07 +0000 Subject: ip: ip_ra_control() rcu fix commit 66018506e15b (ip: Router Alert RCU conversion) introduced RCU lookups to ip_call_ra_chain(). It missed proper deinit phase : When ip_ra_control() deletes an ip_ra_chain, it should make sure ip_call_ra_chain() users can not start to use socket during the rcu grace period. It should also delay the sock_put() after the grace period, or we risk a premature socket freeing and corruptions, as raw sockets are not rcu protected yet. This delay avoids using expensive atomic_inc_not_zero() in ip_call_ra_chain(). Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/ip.h | 5 ++++- net/ipv4/ip_sockglue.c | 19 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) (limited to 'include/net/ip.h') diff --git a/include/net/ip.h b/include/net/ip.h index 9982c97f0bd..d52f0118036 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -61,7 +61,10 @@ struct ipcm_cookie { struct ip_ra_chain { struct ip_ra_chain *next; struct sock *sk; - void (*destructor)(struct sock *); + union { + void (*destructor)(struct sock *); + struct sock *saved_sk; + }; struct rcu_head rcu; }; diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 08b9519a24f..47fff528ff3 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -241,9 +241,13 @@ int ip_cmsg_send(struct net *net, struct msghdr *msg, struct ipcm_cookie *ipc) struct ip_ra_chain *ip_ra_chain; static DEFINE_SPINLOCK(ip_ra_lock); -static void ip_ra_free_rcu(struct rcu_head *head) + +static void ip_ra_destroy_rcu(struct rcu_head *head) { - kfree(container_of(head, struct ip_ra_chain, rcu)); + struct ip_ra_chain *ra = container_of(head, struct ip_ra_chain, rcu); + + sock_put(ra->saved_sk); + kfree(ra); } int ip_ra_control(struct sock *sk, unsigned char on, @@ -264,13 +268,20 @@ int ip_ra_control(struct sock *sk, unsigned char on, kfree(new_ra); return -EADDRINUSE; } + /* dont let ip_call_ra_chain() use sk again */ + ra->sk = NULL; rcu_assign_pointer(*rap, ra->next); spin_unlock_bh(&ip_ra_lock); if (ra->destructor) ra->destructor(sk); - sock_put(sk); - call_rcu(&ra->rcu, ip_ra_free_rcu); + /* + * Delay sock_put(sk) and kfree(ra) after one rcu grace + * period. This guarantee ip_call_ra_chain() dont need + * to mess with socket refcounts. + */ + ra->saved_sk = sk; + call_rcu(&ra->rcu, ip_ra_destroy_rcu); return 0; } } -- cgit v1.2.3-70-g09d2 From 1823e4c80eeae2a774c75569ce3035070e5ee009 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Tue, 22 Jun 2010 20:58:41 +0000 Subject: snmp: add align parameter to snmp_mib_init() In preparation for 64bit snmp counters for some mibs, add an 'align' parameter to snmp_mib_init(), instead of assuming mibs only contain 'unsigned long' fields. Callers can use __alignof__(type) to provide correct alignment. Signed-off-by: Eric Dumazet CC: Herbert Xu CC: Arnaldo Carvalho de Melo CC: Hideaki YOSHIFUJI CC: Vlad Yasevich Signed-off-by: David S. Miller --- include/net/ip.h | 2 +- net/dccp/proto.c | 3 ++- net/ipv4/af_inet.c | 27 +++++++++++++++++---------- net/ipv6/addrconf.c | 9 ++++++--- net/ipv6/af_inet6.c | 15 ++++++++++----- net/sctp/protocol.c | 3 ++- net/xfrm/xfrm_policy.c | 3 ++- 7 files changed, 40 insertions(+), 22 deletions(-) (limited to 'include/net/ip.h') diff --git a/include/net/ip.h b/include/net/ip.h index d52f0118036..3b524df7ddd 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -178,7 +178,7 @@ extern struct ipv4_config ipv4_config; #define NET_ADD_STATS_USER(net, field, adnd) SNMP_ADD_STATS_USER((net)->mib.net_statistics, field, adnd) extern unsigned long snmp_fold_field(void __percpu *mib[], int offt); -extern int snmp_mib_init(void __percpu *ptr[2], size_t mibsize); +extern int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align); extern void snmp_mib_free(void __percpu *ptr[2]); extern struct local_ports { diff --git a/net/dccp/proto.c b/net/dccp/proto.c index f79bcef5088..096250d1323 100644 --- a/net/dccp/proto.c +++ b/net/dccp/proto.c @@ -1002,7 +1002,8 @@ EXPORT_SYMBOL_GPL(dccp_shutdown); static inline int dccp_mib_init(void) { return snmp_mib_init((void __percpu **)dccp_statistics, - sizeof(struct dccp_mib)); + sizeof(struct dccp_mib), + __alignof__(struct dccp_mib)); } static inline void dccp_mib_exit(void) diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index b4c0969137c..640db9b9033 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1427,13 +1427,13 @@ unsigned long snmp_fold_field(void __percpu *mib[], int offt) } EXPORT_SYMBOL_GPL(snmp_fold_field); -int snmp_mib_init(void __percpu *ptr[2], size_t mibsize) +int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align) { BUG_ON(ptr == NULL); - ptr[0] = __alloc_percpu(mibsize, __alignof__(unsigned long)); + ptr[0] = __alloc_percpu(mibsize, align); if (!ptr[0]) goto err0; - ptr[1] = __alloc_percpu(mibsize, __alignof__(unsigned long)); + ptr[1] = __alloc_percpu(mibsize, align); if (!ptr[1]) goto err1; return 0; @@ -1490,25 +1490,32 @@ static const struct net_protocol icmp_protocol = { static __net_init int ipv4_mib_init_net(struct net *net) { if (snmp_mib_init((void __percpu **)net->mib.tcp_statistics, - sizeof(struct tcp_mib)) < 0) + sizeof(struct tcp_mib), + __alignof__(struct tcp_mib)) < 0) goto err_tcp_mib; if (snmp_mib_init((void __percpu **)net->mib.ip_statistics, - sizeof(struct ipstats_mib)) < 0) + sizeof(struct ipstats_mib), + __alignof__(struct ipstats_mib)) < 0) goto err_ip_mib; if (snmp_mib_init((void __percpu **)net->mib.net_statistics, - sizeof(struct linux_mib)) < 0) + sizeof(struct linux_mib), + __alignof__(struct linux_mib)) < 0) goto err_net_mib; if (snmp_mib_init((void __percpu **)net->mib.udp_statistics, - sizeof(struct udp_mib)) < 0) + sizeof(struct udp_mib), + __alignof__(struct udp_mib)) < 0) goto err_udp_mib; if (snmp_mib_init((void __percpu **)net->mib.udplite_statistics, - sizeof(struct udp_mib)) < 0) + sizeof(struct udp_mib), + __alignof__(struct udp_mib)) < 0) goto err_udplite_mib; if (snmp_mib_init((void __percpu **)net->mib.icmp_statistics, - sizeof(struct icmp_mib)) < 0) + sizeof(struct icmp_mib), + __alignof__(struct icmp_mib)) < 0) goto err_icmp_mib; if (snmp_mib_init((void __percpu **)net->mib.icmpmsg_statistics, - sizeof(struct icmpmsg_mib)) < 0) + sizeof(struct icmpmsg_mib), + __alignof__(struct icmpmsg_mib)) < 0) goto err_icmpmsg_mib; tcp_mib_init(net); diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index b97bb1f3080..c20a7c260a8 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -284,13 +284,16 @@ static void addrconf_mod_timer(struct inet6_ifaddr *ifp, static int snmp6_alloc_dev(struct inet6_dev *idev) { if (snmp_mib_init((void __percpu **)idev->stats.ipv6, - sizeof(struct ipstats_mib)) < 0) + sizeof(struct ipstats_mib), + __alignof__(struct ipstats_mib)) < 0) goto err_ip; if (snmp_mib_init((void __percpu **)idev->stats.icmpv6, - sizeof(struct icmpv6_mib)) < 0) + sizeof(struct icmpv6_mib), + __alignof__(struct icmpv6_mib)) < 0) goto err_icmp; if (snmp_mib_init((void __percpu **)idev->stats.icmpv6msg, - sizeof(struct icmpv6msg_mib)) < 0) + sizeof(struct icmpv6msg_mib), + __alignof__(struct icmpv6msg_mib)) < 0) goto err_icmpmsg; return 0; diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 94b1b9c954b..e830cd4f9d0 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -971,19 +971,24 @@ static void ipv6_packet_cleanup(void) static int __net_init ipv6_init_mibs(struct net *net) { if (snmp_mib_init((void __percpu **)net->mib.udp_stats_in6, - sizeof (struct udp_mib)) < 0) + sizeof(struct udp_mib), + __alignof__(struct udp_mib)) < 0) return -ENOMEM; if (snmp_mib_init((void __percpu **)net->mib.udplite_stats_in6, - sizeof (struct udp_mib)) < 0) + sizeof(struct udp_mib), + __alignof__(struct udp_mib)) < 0) goto err_udplite_mib; if (snmp_mib_init((void __percpu **)net->mib.ipv6_statistics, - sizeof(struct ipstats_mib)) < 0) + sizeof(struct ipstats_mib), + __alignof__(struct ipstats_mib)) < 0) goto err_ip_mib; if (snmp_mib_init((void __percpu **)net->mib.icmpv6_statistics, - sizeof(struct icmpv6_mib)) < 0) + sizeof(struct icmpv6_mib), + __alignof__(struct icmpv6_mib)) < 0) goto err_icmp_mib; if (snmp_mib_init((void __percpu **)net->mib.icmpv6msg_statistics, - sizeof(struct icmpv6msg_mib)) < 0) + sizeof(struct icmpv6msg_mib), + __alignof__(struct icmpv6msg_mib)) < 0) goto err_icmpmsg_mib; return 0; diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index a0e1a7fdebb..c0e162aeb0b 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -1002,7 +1002,8 @@ int sctp_register_pf(struct sctp_pf *pf, sa_family_t family) static inline int init_sctp_mibs(void) { return snmp_mib_init((void __percpu **)sctp_statistics, - sizeof(struct sctp_mib)); + sizeof(struct sctp_mib), + __alignof__(struct sctp_mib)); } static inline void cleanup_sctp_mibs(void) diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 4bf27d90133..593c06be6b6 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -2480,7 +2480,8 @@ static int __net_init xfrm_statistics_init(struct net *net) int rv; if (snmp_mib_init((void __percpu **)net->mib.xfrm_statistics, - sizeof(struct linux_xfrm_mib)) < 0) + sizeof(struct linux_xfrm_mib), + __alignof__(struct linux_xfrm_mib)) < 0) return -ENOMEM; rv = xfrm_proc_init(net); if (rv < 0) -- cgit v1.2.3-70-g09d2 From 4ce3c183fcade7f4b30a33dae90cd774c3d9e094 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Wed, 30 Jun 2010 13:31:19 -0700 Subject: snmp: 64bit ipstats_mib for all arches /proc/net/snmp and /proc/net/netstat expose SNMP counters. Width of these counters is either 32 or 64 bits, depending on the size of "unsigned long" in kernel. This means user program parsing these files must already be prepared to deal with 64bit values, regardless of user program being 32 or 64 bit. This patch introduces 64bit snmp values for IPSTAT mib, where some counters can wrap pretty fast if they are 32bit wide. # netstat -s|egrep "InOctets|OutOctets" InOctets: 244068329096 OutOctets: 244069348848 Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- include/net/ip.h | 20 +++++++++----- include/net/ipv6.h | 12 ++++----- include/net/snmp.h | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++--- net/ipv4/af_inet.c | 36 +++++++++++++++++++++++++ net/ipv4/proc.c | 15 ++++++----- net/ipv6/addrconf.c | 18 ++++++++++++- net/ipv6/proc.c | 17 +++++++++--- 7 files changed, 167 insertions(+), 26 deletions(-) (limited to 'include/net/ip.h') diff --git a/include/net/ip.h b/include/net/ip.h index 3b524df7ddd..890f9725d68 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -165,12 +165,12 @@ struct ipv4_config { }; extern struct ipv4_config ipv4_config; -#define IP_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.ip_statistics, field) -#define IP_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.ip_statistics, field) -#define IP_ADD_STATS(net, field, val) SNMP_ADD_STATS((net)->mib.ip_statistics, field, val) -#define IP_ADD_STATS_BH(net, field, val) SNMP_ADD_STATS_BH((net)->mib.ip_statistics, field, val) -#define IP_UPD_PO_STATS(net, field, val) SNMP_UPD_PO_STATS((net)->mib.ip_statistics, field, val) -#define IP_UPD_PO_STATS_BH(net, field, val) SNMP_UPD_PO_STATS_BH((net)->mib.ip_statistics, field, val) +#define IP_INC_STATS(net, field) SNMP_INC_STATS64((net)->mib.ip_statistics, field) +#define IP_INC_STATS_BH(net, field) SNMP_INC_STATS64_BH((net)->mib.ip_statistics, field) +#define IP_ADD_STATS(net, field, val) SNMP_ADD_STATS64((net)->mib.ip_statistics, field, val) +#define IP_ADD_STATS_BH(net, field, val) SNMP_ADD_STATS64_BH((net)->mib.ip_statistics, field, val) +#define IP_UPD_PO_STATS(net, field, val) SNMP_UPD_PO_STATS64((net)->mib.ip_statistics, field, val) +#define IP_UPD_PO_STATS_BH(net, field, val) SNMP_UPD_PO_STATS64_BH((net)->mib.ip_statistics, field, val) #define NET_INC_STATS(net, field) SNMP_INC_STATS((net)->mib.net_statistics, field) #define NET_INC_STATS_BH(net, field) SNMP_INC_STATS_BH((net)->mib.net_statistics, field) #define NET_INC_STATS_USER(net, field) SNMP_INC_STATS_USER((net)->mib.net_statistics, field) @@ -178,6 +178,14 @@ extern struct ipv4_config ipv4_config; #define NET_ADD_STATS_USER(net, field, adnd) SNMP_ADD_STATS_USER((net)->mib.net_statistics, field, adnd) extern unsigned long snmp_fold_field(void __percpu *mib[], int offt); +#if BITS_PER_LONG==32 +extern u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t sync_off); +#else +static inline u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t syncp_off) +{ + return snmp_fold_field(mib, offt); +} +#endif extern int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align); extern void snmp_mib_free(void __percpu *ptr[2]); diff --git a/include/net/ipv6.h b/include/net/ipv6.h index f5808d596aa..1f841241099 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -136,17 +136,17 @@ extern struct ctl_path net_ipv6_ctl_path[]; /* MIBs */ #define IP6_INC_STATS(net, idev,field) \ - _DEVINC(net, ipv6, , idev, field) + _DEVINC(net, ipv6, 64, idev, field) #define IP6_INC_STATS_BH(net, idev,field) \ - _DEVINC(net, ipv6, _BH, idev, field) + _DEVINC(net, ipv6, 64_BH, idev, field) #define IP6_ADD_STATS(net, idev,field,val) \ - _DEVADD(net, ipv6, , idev, field, val) + _DEVADD(net, ipv6, 64, idev, field, val) #define IP6_ADD_STATS_BH(net, idev,field,val) \ - _DEVADD(net, ipv6, _BH, idev, field, val) + _DEVADD(net, ipv6, 64_BH, idev, field, val) #define IP6_UPD_PO_STATS(net, idev,field,val) \ - _DEVUPD(net, ipv6, , idev, field, val) + _DEVUPD(net, ipv6, 64, idev, field, val) #define IP6_UPD_PO_STATS_BH(net, idev,field,val) \ - _DEVUPD(net, ipv6, _BH, idev, field, val) + _DEVUPD(net, ipv6, 64_BH, idev, field, val) #define ICMP6_INC_STATS(net, idev, field) \ _DEVINC(net, icmpv6, , idev, field) #define ICMP6_INC_STATS_BH(net, idev, field) \ diff --git a/include/net/snmp.h b/include/net/snmp.h index 899003d18db..a0e61806d48 100644 --- a/include/net/snmp.h +++ b/include/net/snmp.h @@ -47,15 +47,16 @@ struct snmp_mib { } /* - * We use all unsigned longs. Linux will soon be so reliable that even - * these will rapidly get too small 8-). Seriously consider the IpInReceives - * count on the 20Gb/s + networks people expect in a few years time! + * We use unsigned longs for most mibs but u64 for ipstats. */ +#include /* IPstats */ #define IPSTATS_MIB_MAX __IPSTATS_MIB_MAX struct ipstats_mib { - unsigned long mibs[IPSTATS_MIB_MAX]; + /* mibs[] must be first field of struct ipstats_mib */ + u64 mibs[IPSTATS_MIB_MAX]; + struct u64_stats_sync syncp; }; /* ICMP */ @@ -155,4 +156,70 @@ struct linux_xfrm_mib { ptr->mibs[basefield##PKTS]++; \ ptr->mibs[basefield##OCTETS] += addend;\ } while (0) + + +#if BITS_PER_LONG==32 + +#define SNMP_ADD_STATS64_BH(mib, field, addend) \ + do { \ + __typeof__(*mib[0]) *ptr = __this_cpu_ptr((mib)[0]); \ + u64_stats_update_begin(&ptr->syncp); \ + ptr->mibs[field] += addend; \ + u64_stats_update_end(&ptr->syncp); \ + } while (0) +#define SNMP_ADD_STATS64_USER(mib, field, addend) \ + do { \ + __typeof__(*mib[0]) *ptr; \ + preempt_disable(); \ + ptr = __this_cpu_ptr((mib)[1]); \ + u64_stats_update_begin(&ptr->syncp); \ + ptr->mibs[field] += addend; \ + u64_stats_update_end(&ptr->syncp); \ + preempt_enable(); \ + } while (0) +#define SNMP_ADD_STATS64(mib, field, addend) \ + do { \ + __typeof__(*mib[0]) *ptr; \ + preempt_disable(); \ + ptr = __this_cpu_ptr((mib)[!in_softirq()]); \ + u64_stats_update_begin(&ptr->syncp); \ + ptr->mibs[field] += addend; \ + u64_stats_update_end(&ptr->syncp); \ + preempt_enable(); \ + } while (0) +#define SNMP_INC_STATS64_BH(mib, field) SNMP_ADD_STATS64_BH(mib, field, 1) +#define SNMP_INC_STATS64_USER(mib, field) SNMP_ADD_STATS64_USER(mib, field, 1) +#define SNMP_INC_STATS64(mib, field) SNMP_ADD_STATS64(mib, field, 1) +#define SNMP_UPD_PO_STATS64(mib, basefield, addend) \ + do { \ + __typeof__(*mib[0]) *ptr; \ + preempt_disable(); \ + ptr = __this_cpu_ptr((mib)[!in_softirq()]); \ + u64_stats_update_begin(&ptr->syncp); \ + ptr->mibs[basefield##PKTS]++; \ + ptr->mibs[basefield##OCTETS] += addend; \ + u64_stats_update_end(&ptr->syncp); \ + preempt_enable(); \ + } while (0) +#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) \ + do { \ + __typeof__(*mib[0]) *ptr; \ + ptr = __this_cpu_ptr((mib)[!in_softirq()]); \ + u64_stats_update_begin(&ptr->syncp); \ + ptr->mibs[basefield##PKTS]++; \ + ptr->mibs[basefield##OCTETS] += addend; \ + u64_stats_update_end(&ptr->syncp); \ + } while (0) +#else +#define SNMP_INC_STATS64_BH(mib, field) SNMP_INC_STATS_BH(mib, field) +#define SNMP_INC_STATS64_USER(mib, field) SNMP_INC_STATS_USER(mib, field) +#define SNMP_INC_STATS64(mib, field) SNMP_INC_STATS(mib, field) +#define SNMP_DEC_STATS64(mib, field) SNMP_DEC_STATS(mib, field) +#define SNMP_ADD_STATS64_BH(mib, field, addend) SNMP_ADD_STATS_BH(mib, field, addend) +#define SNMP_ADD_STATS64_USER(mib, field, addend) SNMP_ADD_STATS_USER(mib, field, addend) +#define SNMP_ADD_STATS64(mib, field, addend) SNMP_ADD_STATS(mib, field, addend) +#define SNMP_UPD_PO_STATS64(mib, basefield, addend) SNMP_UPD_PO_STATS(mib, basefield, addend) +#define SNMP_UPD_PO_STATS64_BH(mib, basefield, addend) SNMP_UPD_PO_STATS_BH(mib, basefield, addend) +#endif + #endif diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 640db9b9033..3ceb025b16f 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -1427,6 +1427,42 @@ unsigned long snmp_fold_field(void __percpu *mib[], int offt) } EXPORT_SYMBOL_GPL(snmp_fold_field); +#if BITS_PER_LONG==32 + +u64 snmp_fold_field64(void __percpu *mib[], int offt, size_t syncp_offset) +{ + u64 res = 0; + int cpu; + + for_each_possible_cpu(cpu) { + void *bhptr, *userptr; + struct u64_stats_sync *syncp; + u64 v_bh, v_user; + unsigned int start; + + /* first mib used by softirq context, we must use _bh() accessors */ + bhptr = per_cpu_ptr(SNMP_STAT_BHPTR(mib), cpu); + syncp = (struct u64_stats_sync *)(bhptr + syncp_offset); + do { + start = u64_stats_fetch_begin_bh(syncp); + v_bh = *(((u64 *) bhptr) + offt); + } while (u64_stats_fetch_retry_bh(syncp, start)); + + /* second mib used in USER context */ + userptr = per_cpu_ptr(SNMP_STAT_USRPTR(mib), cpu); + syncp = (struct u64_stats_sync *)(userptr + syncp_offset); + do { + start = u64_stats_fetch_begin(syncp); + v_user = *(((u64 *) userptr) + offt); + } while (u64_stats_fetch_retry(syncp, start)); + + res += v_bh + v_user; + } + return res; +} +EXPORT_SYMBOL_GPL(snmp_fold_field64); +#endif + int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align) { BUG_ON(ptr == NULL); diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c index e320ca6b3ef..4ae1f203f7c 100644 --- a/net/ipv4/proc.c +++ b/net/ipv4/proc.c @@ -343,10 +343,12 @@ static int snmp_seq_show(struct seq_file *seq, void *v) IPV4_DEVCONF_ALL(net, FORWARDING) ? 1 : 2, sysctl_ip_default_ttl); + BUILD_BUG_ON(offsetof(struct ipstats_mib, mibs) != 0); for (i = 0; snmp4_ipstats_list[i].name != NULL; i++) - seq_printf(seq, " %lu", - snmp_fold_field((void __percpu **)net->mib.ip_statistics, - snmp4_ipstats_list[i].entry)); + seq_printf(seq, " %llu", + snmp_fold_field64((void __percpu **)net->mib.ip_statistics, + snmp4_ipstats_list[i].entry, + offsetof(struct ipstats_mib, syncp))); icmp_put(seq); /* RFC 2011 compatibility */ icmpmsg_put(seq); @@ -432,9 +434,10 @@ static int netstat_seq_show(struct seq_file *seq, void *v) seq_puts(seq, "\nIpExt:"); for (i = 0; snmp4_ipextstats_list[i].name != NULL; i++) - seq_printf(seq, " %lu", - snmp_fold_field((void __percpu **)net->mib.ip_statistics, - snmp4_ipextstats_list[i].entry)); + seq_printf(seq, " %llu", + snmp_fold_field64((void __percpu **)net->mib.ip_statistics, + snmp4_ipextstats_list[i].entry, + offsetof(struct ipstats_mib, syncp))); seq_putc(seq, '\n'); return 0; diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 2514adf5251..e81155d2f25 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -3862,12 +3862,28 @@ static inline void __snmp6_fill_stats(u64 *stats, void __percpu **mib, memset(&stats[items], 0, pad); } +static inline void __snmp6_fill_stats64(u64 *stats, void __percpu **mib, + int items, int bytes, size_t syncpoff) +{ + int i; + int pad = bytes - sizeof(u64) * items; + BUG_ON(pad < 0); + + /* Use put_unaligned() because stats may not be aligned for u64. */ + put_unaligned(items, &stats[0]); + for (i = 1; i < items; i++) + put_unaligned(snmp_fold_field64(mib, i, syncpoff), &stats[i]); + + memset(&stats[items], 0, pad); +} + static void snmp6_fill_stats(u64 *stats, struct inet6_dev *idev, int attrtype, int bytes) { switch (attrtype) { case IFLA_INET6_STATS: - __snmp6_fill_stats(stats, (void __percpu **)idev->stats.ipv6, IPSTATS_MIB_MAX, bytes); + __snmp6_fill_stats64(stats, (void __percpu **)idev->stats.ipv6, + IPSTATS_MIB_MAX, bytes, offsetof(struct ipstats_mib, syncp)); break; case IFLA_INET6_ICMP6STATS: __snmp6_fill_stats(stats, (void __percpu **)idev->stats.icmpv6, ICMP6_MIB_MAX, bytes); diff --git a/net/ipv6/proc.c b/net/ipv6/proc.c index 566798d69f3..d082eaeefa2 100644 --- a/net/ipv6/proc.c +++ b/net/ipv6/proc.c @@ -174,17 +174,28 @@ static void snmp6_seq_show_item(struct seq_file *seq, void __percpu **mib, const struct snmp_mib *itemlist) { int i; - for (i=0; itemlist[i].name; i++) + + for (i = 0; itemlist[i].name; i++) seq_printf(seq, "%-32s\t%lu\n", itemlist[i].name, snmp_fold_field(mib, itemlist[i].entry)); } +static void snmp6_seq_show_item64(struct seq_file *seq, void __percpu **mib, + const struct snmp_mib *itemlist, size_t syncpoff) +{ + int i; + + for (i = 0; itemlist[i].name; i++) + seq_printf(seq, "%-32s\t%llu\n", itemlist[i].name, + snmp_fold_field64(mib, itemlist[i].entry, syncpoff)); +} + static int snmp6_seq_show(struct seq_file *seq, void *v) { struct net *net = (struct net *)seq->private; - snmp6_seq_show_item(seq, (void __percpu **)net->mib.ipv6_statistics, - snmp6_ipstats_list); + snmp6_seq_show_item64(seq, (void __percpu **)net->mib.ipv6_statistics, + snmp6_ipstats_list, offsetof(struct ipstats_mib, syncp)); snmp6_seq_show_item(seq, (void __percpu **)net->mib.icmpv6_statistics, snmp6_icmp6_list); snmp6_seq_show_icmpv6msg(seq, -- cgit v1.2.3-70-g09d2