/spare/repo/netdev-2.6 branch 'master'

1 files changed, 772 insertions, 856 deletions

diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
index a701405fab0..b2dea4e5da7 100644
--- a/net/ipv4/fib_trie.c
+++ b/net/ipv4/fib_trie.c
@@ -43,7 +43,7 @@
  *		2 of the License, or (at your option) any later version.
  */
 
-#define VERSION "0.325"
+#define VERSION "0.402"
 
 #include <linux/config.h>
 #include <asm/uaccess.h>
@@ -62,6 +62,7 @@
 #include <linux/netdevice.h>
 #include <linux/if_arp.h>
 #include <linux/proc_fs.h>
+#include <linux/rcupdate.h>
 #include <linux/skbuff.h>
 #include <linux/netlink.h>
 #include <linux/init.h>
@@ -77,56 +78,55 @@
 #undef CONFIG_IP_FIB_TRIE_STATS
 #define MAX_CHILDS 16384
 
-#define EXTRACT(p, n, str) ((str)<<(p)>>(32-(n)))
 #define KEYLENGTH (8*sizeof(t_key))
 #define MASK_PFX(k, l) (((l)==0)?0:(k >> (KEYLENGTH-l)) << (KEYLENGTH-l))
 #define TKEY_GET_MASK(offset, bits) (((bits)==0)?0:((t_key)(-1) << (KEYLENGTH - bits) >> offset))
 
-static DEFINE_RWLOCK(fib_lock);
-
 typedef unsigned int t_key;
 
 #define T_TNODE 0
 #define T_LEAF  1
 #define NODE_TYPE_MASK	0x1UL
-#define NODE_PARENT(_node) \
-	((struct tnode *)((_node)->_parent & ~NODE_TYPE_MASK))
-#define NODE_SET_PARENT(_node, _ptr) \
-	((_node)->_parent = (((unsigned long)(_ptr)) | \
-                     ((_node)->_parent & NODE_TYPE_MASK)))
-#define NODE_INIT_PARENT(_node, _type) \
-	((_node)->_parent = (_type))
-#define NODE_TYPE(_node) \
-	((_node)->_parent & NODE_TYPE_MASK)
-
-#define IS_TNODE(n) (!(n->_parent & T_LEAF))
-#define IS_LEAF(n) (n->_parent & T_LEAF)
+#define NODE_PARENT(node) \
+	((struct tnode *)rcu_dereference(((node)->parent & ~NODE_TYPE_MASK)))
+
+#define NODE_TYPE(node) ((node)->parent & NODE_TYPE_MASK)
+
+#define NODE_SET_PARENT(node, ptr)		\
+	rcu_assign_pointer((node)->parent,	\
+			   ((unsigned long)(ptr)) | NODE_TYPE(node))
+
+#define IS_TNODE(n) (!(n->parent & T_LEAF))
+#define IS_LEAF(n) (n->parent & T_LEAF)
 
 struct node {
-        t_key key;
-	unsigned long _parent;
+	t_key key;
+	unsigned long parent;
 };
 
 struct leaf {
-        t_key key;
-	unsigned long _parent;
+	t_key key;
+	unsigned long parent;
 	struct hlist_head list;
+	struct rcu_head rcu;
 };
 
 struct leaf_info {
 	struct hlist_node hlist;
+	struct rcu_head rcu;
 	int plen;
 	struct list_head falh;
 };
 
 struct tnode {
-        t_key key;
-	unsigned long _parent;
-        unsigned short pos:5;        /* 2log(KEYLENGTH) bits needed */
-        unsigned short bits:5;       /* 2log(KEYLENGTH) bits needed */
-        unsigned short full_children;  /* KEYLENGTH bits needed */
-        unsigned short empty_children; /* KEYLENGTH bits needed */
-        struct node *child[0];
+	t_key key;
+	unsigned long parent;
+	unsigned short pos:5;		/* 2log(KEYLENGTH) bits needed */
+	unsigned short bits:5;		/* 2log(KEYLENGTH) bits needed */
+	unsigned short full_children;	/* KEYLENGTH bits needed */
+	unsigned short empty_children;	/* KEYLENGTH bits needed */
+	struct rcu_head rcu;
+	struct node *child[0];
 };
 
 #ifdef CONFIG_IP_FIB_TRIE_STATS
@@ -150,77 +150,45 @@ struct trie_stat {
 };
 
 struct trie {
-        struct node *trie;
+	struct node *trie;
 #ifdef CONFIG_IP_FIB_TRIE_STATS
 	struct trie_use_stats stats;
 #endif
-        int size;
+	int size;
 	unsigned int revision;
 };
 
-static int trie_debug = 0;
-
-static int tnode_full(struct tnode *tn, struct node *n);
 static void put_child(struct trie *t, struct tnode *tn, int i, struct node *n);
 static void tnode_put_child_reorg(struct tnode *tn, int i, struct node *n, int wasfull);
-static int tnode_child_length(struct tnode *tn);
 static struct node *resize(struct trie *t, struct tnode *tn);
-static struct tnode *inflate(struct trie *t, struct tnode *tn, int *err);
-static struct tnode *halve(struct trie *t, struct tnode *tn, int *err);
+static struct tnode *inflate(struct trie *t, struct tnode *tn);
+static struct tnode *halve(struct trie *t, struct tnode *tn);
 static void tnode_free(struct tnode *tn);
 static void trie_dump_seq(struct seq_file *seq, struct trie *t);
-extern struct fib_alias *fib_find_alias(struct list_head *fah, u8 tos, u32 prio);
-extern int fib_detect_death(struct fib_info *fi, int order,
-                            struct fib_info **last_resort, int *last_idx, int *dflt);
-
-extern void rtmsg_fib(int event, u32 key, struct fib_alias *fa, int z, int tb_id,
-               struct nlmsghdr *n, struct netlink_skb_parms *req);
 
-static kmem_cache_t *fn_alias_kmem;
+static kmem_cache_t *fn_alias_kmem __read_mostly;
 static struct trie *trie_local = NULL, *trie_main = NULL;
 
-static void trie_bug(char *err)
-{
-	printk("Trie Bug: %s\n", err);
-	BUG();
-}
+
+/* rcu_read_lock needs to be hold by caller from readside */
 
 static inline struct node *tnode_get_child(struct tnode *tn, int i)
 {
-        if (i >= 1<<tn->bits)
-                trie_bug("tnode_get_child");
+	BUG_ON(i >= 1 << tn->bits);
 
-        return tn->child[i];
+	return rcu_dereference(tn->child[i]);
 }
 
-static inline int tnode_child_length(struct tnode *tn)
+static inline int tnode_child_length(const struct tnode *tn)
 {
-        return 1<<tn->bits;
+	return 1 << tn->bits;
 }
 
-/*
-  _________________________________________________________________
-  | i | i | i | i | i | i | i | N | N | N | S | S | S | S | S | C |
-  ----------------------------------------------------------------
-    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15
-
-  _________________________________________________________________
-  | C | C | C | u | u | u | u | u | u | u | u | u | u | u | u | u |
-  -----------------------------------------------------------------
-   16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31
-
-  tp->pos = 7
-  tp->bits = 3
-  n->pos = 15
-  n->bits=4
-  KEYLENGTH=32
-*/
-
 static inline t_key tkey_extract_bits(t_key a, int offset, int bits)
 {
-        if (offset < KEYLENGTH)
+	if (offset < KEYLENGTH)
 		return ((t_key)(a << offset)) >> (KEYLENGTH - bits);
-        else
+	else
 		return 0;
 }
 
@@ -233,8 +201,8 @@ static inline int tkey_sub_equals(t_key a, int offset, int bits, t_key b)
 {
 	if (bits == 0 || offset >= KEYLENGTH)
 		return 1;
-        bits = bits > KEYLENGTH ? KEYLENGTH : bits;
-        return ((a ^ b) << offset) >> (KEYLENGTH - bits) == 0;
+	bits = bits > KEYLENGTH ? KEYLENGTH : bits;
+	return ((a ^ b) << offset) >> (KEYLENGTH - bits) == 0;
 }
 
 static inline int tkey_mismatch(t_key a, int offset, t_key b)
@@ -249,14 +217,6 @@ static inline int tkey_mismatch(t_key a, int offset, t_key b)
 	return i;
 }
 
-/* Candiate for fib_semantics */
-
-static void fn_free_alias(struct fib_alias *fa)
-{
-	fib_release_info(fa->fa_info);
-	kmem_cache_free(fn_alias_kmem, fa);
-}
-
 /*
   To understand this stuff, an understanding of keys and all their bits is 
   necessary. Every node in the trie has a key associated with it, but not 
@@ -295,7 +255,7 @@ static void fn_free_alias(struct fib_alias *fa)
   tp->pos = 7
   tp->bits = 3
   n->pos = 15
-  n->bits=4
+  n->bits = 4
 
   First, let's just ignore the bits that come before the parent tp, that is 
   the bits from 0 to (tp->pos-1). They are *known* but at this point we do 
@@ -320,60 +280,65 @@ static void fn_free_alias(struct fib_alias *fa)
 
 */
 
-static void check_tnode(struct tnode *tn)
+static inline void check_tnode(const struct tnode *tn)
 {
-	if (tn && tn->pos+tn->bits > 32) {
-		printk("TNODE ERROR tn=%p, pos=%d, bits=%d\n", tn, tn->pos, tn->bits);
-	}
+	WARN_ON(tn && tn->pos+tn->bits > 32);
 }
 
 static int halve_threshold = 25;
 static int inflate_threshold = 50;
 
-static struct leaf *leaf_new(void)
+
+static void __alias_free_mem(struct rcu_head *head)
 {
-	struct leaf *l = kmalloc(sizeof(struct leaf),  GFP_KERNEL);
-	if (l) {
-		NODE_INIT_PARENT(l, T_LEAF);
-		INIT_HLIST_HEAD(&l->list);
-	}
-	return l;
+	struct fib_alias *fa = container_of(head, struct fib_alias, rcu);
+	kmem_cache_free(fn_alias_kmem, fa);
 }
 
-static struct leaf_info *leaf_info_new(int plen)
+static inline void alias_free_mem_rcu(struct fib_alias *fa)
 {
-	struct leaf_info *li = kmalloc(sizeof(struct leaf_info),  GFP_KERNEL);
-	if (li) {
-		li->plen = plen;
-		INIT_LIST_HEAD(&li->falh);
-	}
-	return li;
+	call_rcu(&fa->rcu, __alias_free_mem);
+}
+
+static void __leaf_free_rcu(struct rcu_head *head)
+{
+	kfree(container_of(head, struct leaf, rcu));
+}
+
+static inline void free_leaf(struct leaf *leaf)
+{
+	call_rcu(&leaf->rcu, __leaf_free_rcu);
 }
 
-static inline void free_leaf(struct leaf *l)
+static void __leaf_info_free_rcu(struct rcu_head *head)
 {
-	kfree(l);
+	kfree(container_of(head, struct leaf_info, rcu));
 }
 
-static inline void free_leaf_info(struct leaf_info *li)
+static inline void free_leaf_info(struct leaf_info *leaf)
 {
-	kfree(li);
+	call_rcu(&leaf->rcu, __leaf_info_free_rcu);
 }
 
 static struct tnode *tnode_alloc(unsigned int size)
 {
-	if (size <= PAGE_SIZE) {
-		return kmalloc(size, GFP_KERNEL);
-	} else {
-		return (struct tnode *)
-			__get_free_pages(GFP_KERNEL, get_order(size));
-	}
+	struct page *pages;
+
+	if (size <= PAGE_SIZE)
+		return kcalloc(size, 1, GFP_KERNEL);
+
+	pages = alloc_pages(GFP_KERNEL|__GFP_ZERO, get_order(size));
+	if (!pages)
+		return NULL;
+
+	return page_address(pages);
 }
 
-static void __tnode_free(struct tnode *tn)
+static void __tnode_free_rcu(struct rcu_head *head)
 {
+	struct tnode *tn = container_of(head, struct tnode, rcu);
 	unsigned int size = sizeof(struct tnode) +
-	                    (1<<tn->bits) * sizeof(struct node *);
+		(1 << tn->bits) * sizeof(struct node *);
 
 	if (size <= PAGE_SIZE)
 		kfree(tn);
@@ -381,15 +346,40 @@ static void __tnode_free(struct tnode *tn)
 		free_pages((unsigned long)tn, get_order(size));
 }
 
+static inline void tnode_free(struct tnode *tn)
+{
+	call_rcu(&tn->rcu, __tnode_free_rcu);
+}
+
+static struct leaf *leaf_new(void)
+{
+	struct leaf *l = kmalloc(sizeof(struct leaf),  GFP_KERNEL);
+	if (l) {
+		l->parent = T_LEAF;
+		INIT_HLIST_HEAD(&l->list);
+	}
+	return l;
+}
+
+static struct leaf_info *leaf_info_new(int plen)
+{
+	struct leaf_info *li = kmalloc(sizeof(struct leaf_info),  GFP_KERNEL);
+	if (li) {
+		li->plen = plen;
+		INIT_LIST_HEAD(&li->falh);
+	}
+	return li;
+}
+
 static struct tnode* tnode_new(t_key key, int pos, int bits)
 {
 	int nchildren = 1<<bits;
 	int sz = sizeof(struct tnode) + nchildren * sizeof(struct node *);
 	struct tnode *tn = tnode_alloc(sz);
 
-	if (tn)  {
+	if (tn) {
 		memset(tn, 0, sz);
-		NODE_INIT_PARENT(tn, T_TNODE);
+		tn->parent = T_TNODE;
 		tn->pos = pos;
 		tn->bits = bits;
 		tn->key = key;
@@ -397,38 +387,17 @@ static struct tnode* tnode_new(t_key key, int pos, int bits)
 		tn->empty_children = 1<<bits;
 	}
 
-	if (trie_debug > 0)
-		printk("AT %p s=%u %u\n", tn, (unsigned int) sizeof(struct tnode),
-		       (unsigned int) (sizeof(struct node) * 1<<bits));
+	pr_debug("AT %p s=%u %u\n", tn, (unsigned int) sizeof(struct tnode),
+		 (unsigned int) (sizeof(struct node) * 1<<bits));
 	return tn;
 }
 
-static void tnode_free(struct tnode *tn)
-{
-	if (!tn) {
-		trie_bug("tnode_free\n");
-	}
-	if (IS_LEAF(tn)) {
-		free_leaf((struct leaf *)tn);
-		if (trie_debug > 0 )
-			printk("FL %p \n", tn);
-	}
-	else if (IS_TNODE(tn)) {
-		__tnode_free(tn);
-		if (trie_debug > 0 )
-			printk("FT %p \n", tn);
-	}
-	else {
-		trie_bug("tnode_free\n");
-	}
-}
-
 /*
  * Check whether a tnode 'n' is "full", i.e. it is an internal node
  * and no bits are skipped. See discussion in dyntree paper p. 6
  */
 
-static inline int tnode_full(struct tnode *tn, struct node *n)
+static inline int tnode_full(const struct tnode *tn, const struct node *n)
 {
 	if (n == NULL || IS_LEAF(n))
 		return 0;
@@ -448,15 +417,11 @@ static inline void put_child(struct trie *t, struct tnode *tn, int i, struct nod
 
 static void tnode_put_child_reorg(struct tnode *tn, int i, struct node *n, int wasfull)
 {
-	struct node *chi;
+	struct node *chi = tn->child[i];
 	int isfull;
 
-	if (i >= 1<<tn->bits) {
-		printk("bits=%d, i=%d\n", tn->bits, i);
-		trie_bug("tnode_put_child_reorg bits");
-	}
-	write_lock_bh(&fib_lock);
-	chi = tn->child[i];
+	BUG_ON(i >= 1<<tn->bits);
+
 
 	/* update emptyChildren */
 	if (n == NULL && chi != NULL)
@@ -465,33 +430,32 @@ static void tnode_put_child_reorg(struct tnode *tn, int i, struct node *n, int w
 		tn->empty_children--;
 
 	/* update fullChildren */
-        if (wasfull == -1)
+	if (wasfull == -1)
 		wasfull = tnode_full(tn, chi);
 
 	isfull = tnode_full(tn, n);
 	if (wasfull && !isfull)
 		tn->full_children--;
-
 	else if (!wasfull && isfull)
 		tn->full_children++;
+
 	if (n)
 		NODE_SET_PARENT(n, tn);
 
-	tn->child[i] = n;
-	write_unlock_bh(&fib_lock);
+	rcu_assign_pointer(tn->child[i], n);
 }
 
 static struct node *resize(struct trie *t, struct tnode *tn)
 {
 	int i;
 	int err = 0;
+	struct tnode *old_tn;
 
  	if (!tn)
 		return NULL;
 
-	if (trie_debug)
-		printk("In tnode_resize %p inflate_threshold=%d threshold=%d\n",
-		      tn, inflate_threshold, halve_threshold);
+	pr_debug("In tnode_resize %p inflate_threshold=%d threshold=%d\n",
+		 tn, inflate_threshold, halve_threshold);
 
 	/* No children */
 	if (tn->empty_children == tnode_child_length(tn)) {
@@ -501,20 +465,16 @@ static struct node *resize(struct trie *t, struct tnode *tn)
 	/* One child */
 	if (tn->empty_children == tnode_child_length(tn) - 1)
 		for (i = 0; i < tnode_child_length(tn); i++) {
+			struct node *n;
 
-			write_lock_bh(&fib_lock);
-			if (tn->child[i] != NULL) {
-
-				/* compress one level */
-				struct node *n = tn->child[i];
-				if (n)
-					NODE_INIT_PARENT(n, NODE_TYPE(n));
+			n = tn->child[i];
+			if (!n)
+				continue;
 
-				write_unlock_bh(&fib_lock);
-				tnode_free(tn);
-				return n;
-			}
-			write_unlock_bh(&fib_lock);
+			/* compress one level */
+			NODE_SET_PARENT(n, NULL);
+			tnode_free(tn);
+			return n;
 		}
 	/*
 	 * Double as long as the resulting node has a number of
@@ -566,16 +526,16 @@ static struct node *resize(struct trie *t, struct tnode *tn)
 	 *
 	 * expand not_to_be_doubled and to_be_doubled, and shorten:
 	 * 100 * (tnode_child_length(tn) - tn->empty_children +
-	 *    tn->full_children ) >= inflate_threshold * new_child_length
+	 *    tn->full_children) >= inflate_threshold * new_child_length
 	 *
 	 * expand new_child_length:
 	 * 100 * (tnode_child_length(tn) - tn->empty_children +
-	 *    tn->full_children ) >=
+	 *    tn->full_children) >=
 	 *      inflate_threshold * tnode_child_length(tn) * 2
 	 *
 	 * shorten again:
 	 * 50 * (tn->full_children + tnode_child_length(tn) -
-	 *    tn->empty_children ) >= inflate_threshold *
+	 *    tn->empty_children) >= inflate_threshold *
 	 *    tnode_child_length(tn)
 	 *
 	 */
@@ -587,9 +547,10 @@ static struct node *resize(struct trie *t, struct tnode *tn)
 	       50 * (tn->full_children + tnode_child_length(tn) - tn->empty_children) >=
 				inflate_threshold * tnode_child_length(tn))) {
 
-		tn = inflate(t, tn, &err);
-
-		if (err) {
+		old_tn = tn;
+		tn = inflate(t, tn);
+		if (IS_ERR(tn)) {
+			tn = old_tn;
 #ifdef CONFIG_IP_FIB_TRIE_STATS
 			t->stats.resize_node_skipped++;
 #endif
@@ -609,9 +570,10 @@ static struct node *resize(struct trie *t, struct tnode *tn)
 	       100 * (tnode_child_length(tn) - tn->empty_children) <
 	       halve_threshold * tnode_child_length(tn)) {
 
-		tn = halve(t, tn, &err);
-
-		if (err) {
+		old_tn = tn;
+		tn = halve(t, tn);
+		if (IS_ERR(tn)) {
+			tn = old_tn;
 #ifdef CONFIG_IP_FIB_TRIE_STATS
 			t->stats.resize_node_skipped++;
 #endif
@@ -621,44 +583,37 @@ static struct node *resize(struct trie *t, struct tnode *tn)
 
 
 	/* Only one child remains */
-
 	if (tn->empty_children == tnode_child_length(tn) - 1)
 		for (i = 0; i < tnode_child_length(tn); i++) {
-		
-			write_lock_bh(&fib_lock);
-			if (tn->child[i] != NULL) {
-				/* compress one level */
-				struct node *n = tn->child[i];
-
-				if (n)
-					NODE_INIT_PARENT(n, NODE_TYPE(n));
-
-				write_unlock_bh(&fib_lock);
-				tnode_free(tn);
-				return n;
-			}
-			write_unlock_bh(&fib_lock);
+			struct node *n;
+
+			n = tn->child[i];
+			if (!n)
+				continue;
+
+			/* compress one level */
+
+			NODE_SET_PARENT(n, NULL);
+			tnode_free(tn);
+			return n;
 		}
 
 	return (struct node *) tn;
 }
 
-static struct tnode *inflate(struct trie *t, struct tnode *tn, int *err)
+static struct tnode *inflate(struct trie *t, struct tnode *tn)
 {
 	struct tnode *inode;
 	struct tnode *oldtnode = tn;
 	int olen = tnode_child_length(tn);
 	int i;
 
-  	if (trie_debug)
-		printk("In inflate\n");
+	pr_debug("In inflate\n");
 
 	tn = tnode_new(oldtnode->key, oldtnode->pos, oldtnode->bits + 1);
 
-	if (!tn) {
-		*err = -ENOMEM;
-		return oldtnode;
-	}
+	if (!tn)
+		return ERR_PTR(-ENOMEM);
 
 	/*
 	 * Preallocate and store tnodes before the actual work so we
@@ -666,8 +621,8 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn, int *err)
 	 * fails. In case of failure we return the oldnode and  inflate
 	 * of tnode is ignored.
 	 */
-		
-	for(i = 0; i < olen; i++) {
+
+	for (i = 0; i < olen; i++) {
 		struct tnode *inode = (struct tnode *) tnode_get_child(oldtnode, i);
 
 		if (inode &&
@@ -675,46 +630,30 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn, int *err)
 		    inode->pos == oldtnode->pos + oldtnode->bits &&
 		    inode->bits > 1) {
 			struct tnode *left, *right;
-
 			t_key m = TKEY_GET_MASK(inode->pos, 1);
 
 			left = tnode_new(inode->key&(~m), inode->pos + 1,
 					 inode->bits - 1);
+			if (!left)
+				goto nomem;
 
-			if (!left) {
-				*err = -ENOMEM;
-				break;
-			}
-		
 			right = tnode_new(inode->key|m, inode->pos + 1,
 					  inode->bits - 1);
 
-			if (!right) {
-				*err = -ENOMEM;
-				break;
-			}
+                        if (!right) {
+				tnode_free(left);
+				goto nomem;
+                        }
 
 			put_child(t, tn, 2*i, (struct node *) left);
 			put_child(t, tn, 2*i+1, (struct node *) right);
 		}
 	}
 
-	if (*err) {
-		int size = tnode_child_length(tn);
-		int j;
-
-		for(j = 0; j < size; j++)
-			if (tn->child[j])
-				tnode_free((struct tnode *)tn->child[j]);
-
-		tnode_free(tn);
-	
-		*err = -ENOMEM;
-		return oldtnode;
-	}
-
-	for(i = 0; i < olen; i++) {
+	for (i = 0; i < olen; i++) {
 		struct node *node = tnode_get_child(oldtnode, i);
+		struct tnode *left, *right;
+		int size, j;
 
 		/* An empty child */
 		if (node == NULL)
@@ -740,76 +679,82 @@ static struct tnode *inflate(struct trie *t, struct tnode *tn, int *err)
 			put_child(t, tn, 2*i+1, inode->child[1]);
 
 			tnode_free(inode);
+			continue;
 		}
 
-			/* An internal node with more than two children */
-		else {
-			struct tnode *left, *right;
-			int size, j;
-
-			/* We will replace this node 'inode' with two new
-			 * ones, 'left' and 'right', each with half of the
-			 * original children. The two new nodes will have
-			 * a position one bit further down the key and this
-			 * means that the "significant" part of their keys
-			 * (see the discussion near the top of this file)
-			 * will differ by one bit, which will be "0" in
-			 * left's key and "1" in right's key. Since we are
-			 * moving the key position by one step, the bit that
-			 * we are moving away from - the bit at position
-			 * (inode->pos) - is the one that will differ between
-			 * left and right. So... we synthesize that bit in the
-			 * two  new keys.
-			 * The mask 'm' below will be a single "one" bit at
-			 * the position (inode->pos)
-			 */
-
-			/* Use the old key, but set the new significant
-			 *   bit to zero.
-			 */
+		/* An internal node with more than two children */
+
+		/* We will replace this node 'inode' with two new
+		 * ones, 'left' and 'right', each with half of the
+		 * original children. The two new nodes will have
+		 * a position one bit further down the key and this
+		 * means that the "significant" part of their keys
+		 * (see the discussion near the top of this file)
+		 * will differ by one bit, which will be "0" in
+		 * left's key and "1" in right's key. Since we are
+		 * moving the key position by one step, the bit that
+		 * we are moving away from - the bit at position
+		 * (inode->pos) - is the one that will differ between
+		 * left and right. So... we synthesize that bit in the
+		 * two  new keys.
+		 * The mask 'm' below will be a single "one" bit at
+		 * the position (inode->pos)
+		 */
 
-			left = (struct tnode *) tnode_get_child(tn, 2*i);
-			put_child(t, tn, 2*i, NULL);
+		/* Use the old key, but set the new significant
+		 *   bit to zero.
+		 */
 
-			if (!left)
-				BUG();
+		left = (struct tnode *) tnode_get_child(tn, 2*i);
+		put_child(t, tn, 2*i, NULL);
 
-			right = (struct tnode *) tnode_get_child(tn, 2*i+1);
-			put_child(t, tn, 2*i+1, NULL);
+		BUG_ON(!left);
 
-			if (!right)
-				BUG();
+		right = (struct tnode *) tnode_get_child(tn, 2*i+1);
+		put_child(t, tn, 2*i+1, NULL);
 
-			size = tnode_child_length(left);
-			for(j = 0; j < size; j++) {
-				put_child(t, left, j, inode->child[j]);
-				put_child(t, right, j, inode->child[j + size]);
-			}
-			put_child(t, tn, 2*i, resize(t, left));
-			put_child(t, tn, 2*i+1, resize(t, right));
+		BUG_ON(!right);
 
-			tnode_free(inode);
+		size = tnode_child_length(left);
+		for (j = 0; j < size; j++) {
+			put_child(t, left, j, inode->child[j]);
+			put_child(t, right, j, inode->child[j + size]);
 		}
+		put_child(t, tn, 2*i, resize(t, left));
+		put_child(t, tn, 2*i+1, resize(t, right));
+
+		tnode_free(inode);
 	}
 	tnode_free(oldtnode);
 	return tn;
+nomem:
+	{
+		int size = tnode_child_length(tn);
+		int j;
+
+		for (j = 0; j < size; j++)
+			if (tn->child[j])
+				tnode_free((struct tnode *)tn->child[j]);
+
+		tnode_free(tn);
+
+		return ERR_PTR(-ENOMEM);
+	}
 }
 
-static struct tnode *halve(struct trie *t, struct tnode *tn, int *err)
+static struct tnode *halve(struct trie *t, struct tnode *tn)
 {
 	struct tnode *oldtnode = tn;
 	struct node *left, *right;
 	int i;
 	int olen = tnode_child_length(tn);
 
-	if (trie_debug) printk("In halve\n");
+	pr_debug("In halve\n");
 
 	tn = tnode_new(oldtnode->key, oldtnode->pos, oldtnode->bits - 1);
 
-	if (!tn) {
-		*err = -ENOMEM;
-		return oldtnode;
-	}
+	if (!tn)
+		return ERR_PTR(-ENOMEM);
 
 	/*
 	 * Preallocate and store tnodes before the actual work so we
@@ -818,38 +763,27 @@ static struct tnode *halve(struct trie *t, struct tnode *tn, int *err)
 	 * of tnode is ignored.
 	 */
 
-	for(i = 0; i < olen; i += 2) {
+	for (i = 0; i < olen; i += 2) {
 		left = tnode_get_child(oldtnode, i);
 		right = tnode_get_child(oldtnode, i+1);
 
 		/* Two nonempty children */
-		if (left && right)  {
-			struct tnode *newBinNode =
-				tnode_new(left->key, tn->pos + tn->bits, 1);
+		if (left && right) {
+			struct tnode *newn;
 
-			if (!newBinNode) {
-				*err = -ENOMEM;
-				break;
-			}
-			put_child(t, tn, i/2, (struct node *)newBinNode);
-		}
-	}
+			newn = tnode_new(left->key, tn->pos + tn->bits, 1);
 
-	if (*err) {
-		int size = tnode_child_length(tn);
-		int j;
+			if (!newn)
+				goto nomem;
 
-		for(j = 0; j < size; j++)
-			if (tn->child[j])
-				tnode_free((struct tnode *)tn->child[j]);
+			put_child(t, tn, i/2, (struct node *)newn);
+		}
 
-		tnode_free(tn);
-	
-		*err = -ENOMEM;
-		return oldtnode;
 	}
 
-	for(i = 0; i < olen; i += 2) {
+	for (i = 0; i < olen; i += 2) {
+		struct tnode *newBinNode;
+
 		left = tnode_get_child(oldtnode, i);
 		right = tnode_get_child(oldtnode, i+1);
 
@@ -858,88 +792,99 @@ static struct tnode *halve(struct trie *t, struct tnode *tn, int *err)
 			if (right == NULL)    /* Both are empty */
 				continue;
 			put_child(t, tn, i/2, right);
-		} else if (right == NULL)
+			continue;
+		}
+
+		if (right == NULL) {
 			put_child(t, tn, i/2, left);
+			continue;
+		}
 
 		/* Two nonempty children */
-		else {
-			struct tnode *newBinNode =
-				(struct tnode *) tnode_get_child(tn, i/2);
-			put_child(t, tn, i/2, NULL);
-
-			if (!newBinNode)
-				BUG();
-
-			put_child(t, newBinNode, 0, left);
-			put_child(t, newBinNode, 1, right);
-			put_child(t, tn, i/2, resize(t, newBinNode));
-		}
+		newBinNode = (struct tnode *) tnode_get_child(tn, i/2);
+		put_child(t, tn, i/2, NULL);
+		put_child(t, newBinNode, 0, left);
+		put_child(t, newBinNode, 1, right);
+		put_child(t, tn, i/2, resize(t, newBinNode));
 	}
 	tnode_free(oldtnode);
 	return tn;
+nomem:
+	{
+		int size = tnode_child_length(tn);
+		int j;
+
+		for (j = 0; j < size; j++)
+			if (tn->child[j])
+				tnode_free((struct tnode *)tn->child[j]);
+
+		tnode_free(tn);
+
+		return ERR_PTR(-ENOMEM);
+	}
 }
 
-static void *trie_init(struct trie *t)
+static void trie_init(struct trie *t)
 {
-	if (t) {
-		t->size = 0;
-		t->trie = NULL;
-		t->revision = 0;
+	if (!t)
+		return;
+
+	t->size = 0;
+	rcu_assign_pointer(t->trie, NULL);
+	t->revision = 0;
 #ifdef CONFIG_IP_FIB_TRIE_STATS
-       		memset(&t->stats, 0, sizeof(struct trie_use_stats));
+	memset(&t->stats, 0, sizeof(struct trie_use_stats));
 #endif
-	}
-	return t;
 }
 
+/* readside most use rcu_read_lock currently dump routines
+ via get_fa_head and dump */
+
 static struct leaf_info *find_leaf_info(struct hlist_head *head, int plen)
 {
 	struct hlist_node *node;
 	struct leaf_info *li;
 
-	hlist_for_each_entry(li, node, head, hlist) {
+	hlist_for_each_entry_rcu(li, node, head, hlist)
 		if (li->plen == plen)
 			return li;
-	}
+
 	return NULL;
 }
 
 static inline struct list_head * get_fa_head(struct leaf *l, int plen)
 {
-	struct list_head *fa_head = NULL;
 	struct leaf_info *li = find_leaf_info(&l->list, plen);
 
-	if (li)
-		fa_head = &li->falh;
+	if (!li)
+		return NULL;
 
-	return fa_head;
+	return &li->falh;
 }
 
 static void insert_leaf_info(struct hlist_head *head, struct leaf_info *new)
 {
-	struct leaf_info *li = NULL, *last = NULL;
-	struct hlist_node *node, *tmp;
-
-	write_lock_bh(&fib_lock);
-
-	if (hlist_empty(head))
-		hlist_add_head(&new->hlist, head);
-	else {
-		hlist_for_each_entry_safe(li, node, tmp, head, hlist) {
-		
-			if (new->plen > li->plen)
-				break;
-		
-			last = li;
-		}
-		if (last)
-			hlist_add_after(&last->hlist, &new->hlist);
-		else
-			hlist_add_before(&new->hlist, &li->hlist);
-	}
-	write_unlock_bh(&fib_lock);
+        struct leaf_info *li = NULL, *last = NULL;
+        struct hlist_node *node;
+
+        if (hlist_empty(head)) {
+                hlist_add_head_rcu(&new->hlist, head);
+        } else {
+                hlist_for_each_entry(li, node, head, hlist) {
+                        if (new->plen > li->plen)
+                                break;
+
+                        last = li;
+                }
+                if (last)
+                        hlist_add_after_rcu(&last->hlist, &new->hlist);
+                else
+                        hlist_add_before_rcu(&new->hlist, &li->hlist);
+        }
 }
 
+/* rcu_read_lock needs to be hold by caller from readside */
+
 static struct leaf *
 fib_find_node(struct trie *t, u32 key)
 {
@@ -948,61 +893,43 @@ fib_find_node(struct trie *t, u32 key)
 	struct node *n;
 
 	pos = 0;
-	n = t->trie;
+	n = rcu_dereference(t->trie);
 
 	while (n != NULL &&  NODE_TYPE(n) == T_TNODE) {
 		tn = (struct tnode *) n;
-		
+
 		check_tnode(tn);
-		
+
 		if (tkey_sub_equals(tn->key, pos, tn->pos-pos, key)) {
-			pos=tn->pos + tn->bits;
+			pos = tn->pos + tn->bits;
 			n = tnode_get_child(tn, tkey_extract_bits(key, tn->pos, tn->bits));
-		}
-		else
+		} else
 			break;
 	}
 	/* Case we have found a leaf. Compare prefixes */
 
-	if (n != NULL && IS_LEAF(n) && tkey_equals(key, n->key)) {
-		struct leaf *l = (struct leaf *) n;
-		return l;
-	}
+	if (n != NULL && IS_LEAF(n) && tkey_equals(key, n->key))
+		return (struct leaf *)n;
+
 	return NULL;
 }
 
 static struct node *trie_rebalance(struct trie *t, struct tnode *tn)
 {
-	int i = 0;
 	int wasfull;
 	t_key cindex, key;
 	struct tnode *tp = NULL;
 
-	if (!tn)
-		BUG();
-
 	key = tn->key;
-	i = 0;
 
 	while (tn != NULL && NODE_PARENT(tn) != NULL) {
 
-		if (i > 10) {
-			printk("Rebalance tn=%p \n", tn);
-			if (tn) 		printk("tn->parent=%p \n", NODE_PARENT(tn));
-		
-			printk("Rebalance tp=%p \n", tp);
-			if (tp) 		printk("tp->parent=%p \n", NODE_PARENT(tp));
-		}
-
-		if (i > 12) BUG();
-		i++;
-
 		tp = NODE_PARENT(tn);
 		cindex = tkey_extract_bits(key, tp->pos, tp->bits);
 		wasfull = tnode_full(tp, tnode_get_child(tp, cindex));
 		tn = (struct tnode *) resize (t, (struct tnode *)tn);
 		tnode_put_child_reorg((struct tnode *)tp, cindex,(struct node*)tn, wasfull);
-	
+
 		if (!NODE_PARENT(tn))
 			break;
 
@@ -1015,6 +942,8 @@ static struct node *trie_rebalance(struct trie *t, struct tnode *tn)
 	return (struct node*) tn;
 }
 
+/* only used from updater-side */
+
 static  struct list_head *
 fib_insert_node(struct trie *t, int *err, u32 key, int plen)
 {
@@ -1050,20 +979,16 @@ fib_insert_node(struct trie *t, int *err, u32 key, int plen)
 
 	while (n != NULL &&  NODE_TYPE(n) == T_TNODE) {
 		tn = (struct tnode *) n;
-		
+
 		check_tnode(tn);
-	
+
 		if (tkey_sub_equals(tn->key, pos, tn->pos-pos, key)) {
 			tp = tn;
-			pos=tn->pos + tn->bits;
+			pos = tn->pos + tn->bits;
 			n = tnode_get_child(tn, tkey_extract_bits(key, tn->pos, tn->bits));
 
-			if (n && NODE_PARENT(n) != tn) {
-				printk("BUG tn=%p, n->parent=%p\n", tn, NODE_PARENT(n));
-				BUG();
-			}
-		}
-		else
+			BUG_ON(n && NODE_PARENT(n) != tn);
+		} else
 			break;
 	}
 
@@ -1073,17 +998,15 @@ fib_insert_node(struct trie *t, int *err, u32 key, int plen)
 	 * tp is n's (parent) ----> NULL or TNODE
 	 */
 
-	if (tp && IS_LEAF(tp))
-		BUG();
-
+	BUG_ON(tp && IS_LEAF(tp));
 
 	/* Case 1: n is a leaf. Compare prefixes */
 
 	if (n != NULL && IS_LEAF(n) && tkey_equals(key, n->key)) {
-		struct leaf *l = ( struct leaf *)  n;
-	
+		struct leaf *l = (struct leaf *) n;
+
 		li = leaf_info_new(plen);
-	
+
 		if (!li) {
 			*err = -ENOMEM;
 			goto err;
@@ -1113,35 +1036,29 @@ fib_insert_node(struct trie *t, int *err, u32 key, int plen)
 	fa_head = &li->falh;
 	insert_leaf_info(&l->list, li);
 
-	/* Case 2: n is NULL, and will just insert a new leaf */
 	if (t->trie && n == NULL) {
+		/* Case 2: n is NULL, and will just insert a new leaf */
 
 		NODE_SET_PARENT(l, tp);
-	
-		if (!tp)
-			BUG();
 
-		else {
-			cindex = tkey_extract_bits(key, tp->pos, tp->bits);
-			put_child(t, (struct tnode *)tp, cindex, (struct node *)l);
-		}
-	}
-	/* Case 3: n is a LEAF or a TNODE and the key doesn't match. */
-	else {
+		cindex = tkey_extract_bits(key, tp->pos, tp->bits);
+		put_child(t, (struct tnode *)tp, cindex, (struct node *)l);
+	} else {
+		/* Case 3: n is a LEAF or a TNODE and the key doesn't match. */
 		/*
 		 *  Add a new tnode here
 		 *  first tnode need some special handling
 		 */
 
 		if (tp)
-			pos=tp->pos+tp->bits;
+			pos = tp->pos+tp->bits;
 		else
-			pos=0;
+			pos = 0;
+
 		if (n) {
 			newpos = tkey_mismatch(key, pos, n->key);
 			tn = tnode_new(n->key, newpos, 1);
-		}
-		else {
+		} else {
 			newpos = 0;
 			tn = tnode_new(key, newpos, 1); /* First tnode */
 		}
@@ -1151,32 +1068,33 @@ fib_insert_node(struct trie *t, int *err, u32 key, int plen)
 			tnode_free((struct tnode *) l);
 			*err = -ENOMEM;
 			goto err;
-		}		
-		
+		}
+
 		NODE_SET_PARENT(tn, tp);