diff options
Diffstat (limited to 'net/core/dst.c')
| -rw-r--r-- | net/core/dst.c | 224 |
1 files changed, 148 insertions, 76 deletions
diff --git a/net/core/dst.c b/net/core/dst.c index 57bc4d5b8d0..a028409ee43 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -12,11 +12,14 @@ #include <linux/workqueue.h> #include <linux/mm.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/netdevice.h> #include <linux/skbuff.h> #include <linux/string.h> #include <linux/types.h> #include <net/net_namespace.h> +#include <linux/sched.h> +#include <linux/prefetch.h> #include <net/dst.h> @@ -31,9 +34,6 @@ * 3) This list is guarded by a mutex, * so that the gc_task and dst_dev_event() can be synchronized. */ -#if RT_CACHE_DEBUG >= 2 -static atomic_t dst_total = ATOMIC_INIT(0); -#endif /* * We want to keep lock & list close together @@ -42,7 +42,7 @@ static atomic_t dst_total = ATOMIC_INIT(0); */ static struct { spinlock_t lock; - struct dst_entry *list; + struct dst_entry *list; unsigned long timer_inc; unsigned long timer_expires; } dst_garbage = { @@ -50,7 +50,7 @@ static struct { .timer_inc = DST_GC_MAX, }; static void dst_gc_task(struct work_struct *work); -static void ___dst_free(struct dst_entry * dst); +static void ___dst_free(struct dst_entry *dst); static DECLARE_DELAYED_WORK(dst_gc_work, dst_gc_task); @@ -67,10 +67,6 @@ static void dst_gc_task(struct work_struct *work) unsigned long expires = ~0L; struct dst_entry *dst, *next, head; struct dst_entry *last = &head; -#if RT_CACHE_DEBUG >= 2 - ktime_t time_start = ktime_get(); - struct timespec elapsed; -#endif mutex_lock(&dst_gc_mutex); next = dst_busy_list; @@ -79,6 +75,7 @@ loop: while ((dst = next) != NULL) { next = dst->next; prefetch(&next->next); + cond_resched(); if (likely(atomic_read(&dst->__refcnt))) { last->next = dst; last = dst; @@ -97,7 +94,7 @@ loop: * But we do not have state "obsoleted, but * referenced by parent", so it is right. */ - if (dst->obsolete > 1) + if (dst->obsolete > 0) continue; ___dst_free(dst); @@ -133,8 +130,8 @@ loop: } expires = dst_garbage.timer_expires; /* - * if the next desired timer is more than 4 seconds in the future - * then round the timer to whole seconds + * if the next desired timer is more than 4 seconds in the + * future then round the timer to whole seconds */ if (expires > 4*HZ) expires = round_jiffies_relative(expires); @@ -143,58 +140,83 @@ loop: spin_unlock_bh(&dst_garbage.lock); mutex_unlock(&dst_gc_mutex); -#if RT_CACHE_DEBUG >= 2 - elapsed = ktime_to_timespec(ktime_sub(ktime_get(), time_start)); - printk(KERN_DEBUG "dst_total: %d delayed: %d work_perf: %d" - " expires: %lu elapsed: %lu us\n", - atomic_read(&dst_total), delayed, work_performed, - expires, - elapsed.tv_sec * USEC_PER_SEC + elapsed.tv_nsec / NSEC_PER_USEC); -#endif } -int dst_discard(struct sk_buff *skb) +int dst_discard_sk(struct sock *sk, struct sk_buff *skb) { kfree_skb(skb); return 0; } -EXPORT_SYMBOL(dst_discard); +EXPORT_SYMBOL(dst_discard_sk); + +const u32 dst_default_metrics[RTAX_MAX + 1] = { + /* This initializer is needed to force linker to place this variable + * into const section. Otherwise it might end into bss section. + * We really want to avoid false sharing on this variable, and catch + * any writes on it. + */ + [RTAX_MAX] = 0xdeadbeef, +}; -void * dst_alloc(struct dst_ops * ops) + +void *dst_alloc(struct dst_ops *ops, struct net_device *dev, + int initial_ref, int initial_obsolete, unsigned short flags) { - struct dst_entry * dst; + struct dst_entry *dst; - if (ops->gc && atomic_read(&ops->entries) > ops->gc_thresh) { + if (ops->gc && dst_entries_get_fast(ops) > ops->gc_thresh) { if (ops->gc(ops)) return NULL; } - dst = kmem_cache_zalloc(ops->kmem_cachep, GFP_ATOMIC); + dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC); if (!dst) return NULL; - atomic_set(&dst->__refcnt, 0); + dst->child = NULL; + dst->dev = dev; + if (dev) + dev_hold(dev); dst->ops = ops; - dst->lastuse = jiffies; + dst_init_metrics(dst, dst_default_metrics, true); + dst->expires = 0UL; dst->path = dst; - dst->input = dst->output = dst_discard; -#if RT_CACHE_DEBUG >= 2 - atomic_inc(&dst_total); + dst->from = NULL; +#ifdef CONFIG_XFRM + dst->xfrm = NULL; +#endif + dst->input = dst_discard; + dst->output = dst_discard_sk; + dst->error = 0; + dst->obsolete = initial_obsolete; + dst->header_len = 0; + dst->trailer_len = 0; +#ifdef CONFIG_IP_ROUTE_CLASSID + dst->tclassid = 0; #endif - atomic_inc(&ops->entries); + atomic_set(&dst->__refcnt, initial_ref); + dst->__use = 0; + dst->lastuse = jiffies; + dst->flags = flags; + dst->pending_confirm = 0; + dst->next = NULL; + if (!(flags & DST_NOCOUNT)) + dst_entries_add(ops, 1); return dst; } +EXPORT_SYMBOL(dst_alloc); -static void ___dst_free(struct dst_entry * dst) +static void ___dst_free(struct dst_entry *dst) { /* The first case (dev==NULL) is required, when protocol module is unloaded. */ if (dst->dev == NULL || !(dst->dev->flags&IFF_UP)) { - dst->input = dst->output = dst_discard; + dst->input = dst_discard; + dst->output = dst_discard_sk; } - dst->obsolete = 2; + dst->obsolete = DST_OBSOLETE_DEAD; } -void __dst_free(struct dst_entry * dst) +void __dst_free(struct dst_entry *dst) { spin_lock_bh(&dst_garbage.lock); ___dst_free(dst); @@ -203,43 +225,29 @@ void __dst_free(struct dst_entry * dst) if (dst_garbage.timer_inc > DST_GC_INC) { dst_garbage.timer_inc = DST_GC_INC; dst_garbage.timer_expires = DST_GC_MIN; - cancel_delayed_work(&dst_gc_work); - schedule_delayed_work(&dst_gc_work, dst_garbage.timer_expires); + mod_delayed_work(system_wq, &dst_gc_work, + dst_garbage.timer_expires); } spin_unlock_bh(&dst_garbage.lock); } +EXPORT_SYMBOL(__dst_free); struct dst_entry *dst_destroy(struct dst_entry * dst) { struct dst_entry *child; - struct neighbour *neigh; - struct hh_cache *hh; smp_rmb(); again: - neigh = dst->neighbour; - hh = dst->hh; child = dst->child; - dst->hh = NULL; - if (hh && atomic_dec_and_test(&hh->hh_refcnt)) - kfree(hh); - - if (neigh) { - dst->neighbour = NULL; - neigh_release(neigh); - } - - atomic_dec(&dst->ops->entries); + if (!(dst->flags & DST_NOCOUNT)) + dst_entries_add(dst->ops, -1); if (dst->ops->destroy) dst->ops->destroy(dst); if (dst->dev) dev_put(dst->dev); -#if RT_CACHE_DEBUG >= 2 - atomic_dec(&dst_total); -#endif kmem_cache_free(dst->ops->kmem_cachep, dst); dst = child; @@ -259,19 +267,90 @@ again: } return NULL; } +EXPORT_SYMBOL(dst_destroy); + +static void dst_destroy_rcu(struct rcu_head *head) +{ + struct dst_entry *dst = container_of(head, struct dst_entry, rcu_head); + + dst = dst_destroy(dst); + if (dst) + __dst_free(dst); +} void dst_release(struct dst_entry *dst) { if (dst) { - int newrefcnt; + int newrefcnt; - smp_mb__before_atomic_dec(); - newrefcnt = atomic_dec_return(&dst->__refcnt); - WARN_ON(newrefcnt < 0); + newrefcnt = atomic_dec_return(&dst->__refcnt); + WARN_ON(newrefcnt < 0); + if (unlikely(dst->flags & DST_NOCACHE) && !newrefcnt) + call_rcu(&dst->rcu_head, dst_destroy_rcu); } } EXPORT_SYMBOL(dst_release); +u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old) +{ + u32 *p = kmalloc(sizeof(u32) * RTAX_MAX, GFP_ATOMIC); + + if (p) { + u32 *old_p = __DST_METRICS_PTR(old); + unsigned long prev, new; + + memcpy(p, old_p, sizeof(u32) * RTAX_MAX); + + new = (unsigned long) p; + prev = cmpxchg(&dst->_metrics, old, new); + + if (prev != old) { + kfree(p); + p = __DST_METRICS_PTR(prev); + if (prev & DST_METRICS_READ_ONLY) + p = NULL; + } + } + return p; +} +EXPORT_SYMBOL(dst_cow_metrics_generic); + +/* Caller asserts that dst_metrics_read_only(dst) is false. */ +void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old) +{ + unsigned long prev, new; + + new = ((unsigned long) dst_default_metrics) | DST_METRICS_READ_ONLY; + prev = cmpxchg(&dst->_metrics, old, new); + if (prev == old) + kfree(__DST_METRICS_PTR(old)); +} +EXPORT_SYMBOL(__dst_destroy_metrics_generic); + +/** + * __skb_dst_set_noref - sets skb dst, without a reference + * @skb: buffer + * @dst: dst entry + * @force: if force is set, use noref version even for DST_NOCACHE entries + * + * Sets skb dst, assuming a reference was not taken on dst + * skb_dst_drop() should not dst_release() this dst + */ +void __skb_dst_set_noref(struct sk_buff *skb, struct dst_entry *dst, bool force) +{ + WARN_ON(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); + /* If dst not in cache, we must take a reference, because + * dst_release() will destroy dst as soon as its refcount becomes zero + */ + if (unlikely((dst->flags & DST_NOCACHE) && !force)) { + dst_hold(dst); + skb_dst_set(skb, dst); + } else { + skb->_skb_refdst = (unsigned long)dst | SKB_DST_NOREF; + } +} +EXPORT_SYMBOL(__skb_dst_set_noref); + /* Dirty hack. We did it in 2.2 (in __dst_free), * we have _very_ good reasons not to repeat * this mistake in 2.3, but we have no choice @@ -280,8 +359,8 @@ EXPORT_SYMBOL(dst_release); * * Commented and originally written by Alexey. */ -static inline void dst_ifdown(struct dst_entry *dst, struct net_device *dev, - int unregister) +static void dst_ifdown(struct dst_entry *dst, struct net_device *dev, + int unregister) { if (dst->ops->ifdown) dst->ops->ifdown(dst, dev, unregister); @@ -290,26 +369,23 @@ static inline void dst_ifdown(struct dst_entry *dst, struct net_device *dev, return; if (!unregister) { - dst->input = dst->output = dst_discard; + dst->input = dst_discard; + dst->output = dst_discard_sk; } else { dst->dev = dev_net(dst->dev)->loopback_dev; dev_hold(dst->dev); dev_put(dev); - if (dst->neighbour && dst->neighbour->dev == dev) { - dst->neighbour->dev = dst->dev; - dev_hold(dst->dev); - dev_put(dev); - } } } -static int dst_dev_event(struct notifier_block *this, unsigned long event, void *ptr) +static int dst_dev_event(struct notifier_block *this, unsigned long event, + void *ptr) { - struct net_device *dev = ptr; + struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct dst_entry *dst, *last = NULL; switch (event) { - case NETDEV_UNREGISTER: + case NETDEV_UNREGISTER_FINAL: case NETDEV_DOWN: mutex_lock(&dst_gc_mutex); for (dst = dst_busy_list; dst; dst = dst->next) { @@ -326,9 +402,8 @@ static int dst_dev_event(struct notifier_block *this, unsigned long event, void last->next = dst; else dst_busy_list = dst; - for (; dst; dst = dst->next) { + for (; dst; dst = dst->next) dst_ifdown(dst, dev, event != NETDEV_DOWN); - } mutex_unlock(&dst_gc_mutex); break; } @@ -337,13 +412,10 @@ static int dst_dev_event(struct notifier_block *this, unsigned long event, void static struct notifier_block dst_dev_notifier = { .notifier_call = dst_dev_event, + .priority = -10, /* must be called after other network notifiers */ }; void __init dst_init(void) { register_netdevice_notifier(&dst_dev_notifier); } - -EXPORT_SYMBOL(__dst_free); -EXPORT_SYMBOL(dst_alloc); -EXPORT_SYMBOL(dst_destroy); |