nilfs2: B-tree based block mapping

This adds declarations and functions of NILFS2 B-tree.

Two variants are integrated in the NILFS2 B-tree.  The B-tree for the most
files points to the child nodes or data blocks with virtual block
addresses, whereas the B-tree of the DAT uses actual block addresses.

Signed-off-by: Koji Sato <sato.koji@lab.ntt.co.jp>
Signed-off-by: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 2276 insertions, 0 deletions

diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c
new file mode 100644
index 00000000000..893f0190a61
--- /dev/null
+++ b/fs/nilfs2/btree.c
@@ -0,0 +1,2276 @@
+/*
+ * btree.c - NILFS B-tree.
+ *
+ * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * Written by Koji Sato <koji@osrg.net>.
+ */
+
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/pagevec.h>
+#include "nilfs.h"
+#include "page.h"
+#include "btnode.h"
+#include "btree.h"
+#include "alloc.h"
+
+/**
+ * struct nilfs_btree_path - A path on which B-tree operations are executed
+ * @bp_bh: buffer head of node block
+ * @bp_sib_bh: buffer head of sibling node block
+ * @bp_index: index of child node
+ * @bp_oldreq: ptr end request for old ptr
+ * @bp_newreq: ptr alloc request for new ptr
+ * @bp_op: rebalance operation
+ */
+struct nilfs_btree_path {
+	struct buffer_head *bp_bh;
+	struct buffer_head *bp_sib_bh;
+	int bp_index;
+	union nilfs_bmap_ptr_req bp_oldreq;
+	union nilfs_bmap_ptr_req bp_newreq;
+	struct nilfs_btnode_chkey_ctxt bp_ctxt;
+	void (*bp_op)(struct nilfs_btree *, struct nilfs_btree_path *,
+		      int, __u64 *, __u64 *);
+};
+
+/*
+ * B-tree path operations
+ */
+
+static struct kmem_cache *nilfs_btree_path_cache;
+
+int __init nilfs_btree_path_cache_init(void)
+{
+	nilfs_btree_path_cache =
+		kmem_cache_create("nilfs2_btree_path_cache",
+				  sizeof(struct nilfs_btree_path) *
+				  NILFS_BTREE_LEVEL_MAX, 0, 0, NULL);
+	return (nilfs_btree_path_cache != NULL) ? 0 : -ENOMEM;
+}
+
+void nilfs_btree_path_cache_destroy(void)
+{
+	kmem_cache_destroy(nilfs_btree_path_cache);
+}
+
+static inline struct nilfs_btree_path *
+nilfs_btree_alloc_path(const struct nilfs_btree *btree)
+{
+	return (struct nilfs_btree_path *)
+		kmem_cache_alloc(nilfs_btree_path_cache, GFP_NOFS);
+}
+
+static inline void nilfs_btree_free_path(const struct nilfs_btree *btree,
+					 struct nilfs_btree_path *path)
+{
+	kmem_cache_free(nilfs_btree_path_cache, path);
+}
+
+static void nilfs_btree_init_path(const struct nilfs_btree *btree,
+				  struct nilfs_btree_path *path)
+{
+	int level;
+
+	for (level = NILFS_BTREE_LEVEL_DATA;
+	     level < NILFS_BTREE_LEVEL_MAX;
+	     level++) {
+		path[level].bp_bh = NULL;
+		path[level].bp_sib_bh = NULL;
+		path[level].bp_index = 0;
+		path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
+		path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
+		path[level].bp_op = NULL;
+	}
+}
+
+static void nilfs_btree_clear_path(const struct nilfs_btree *btree,
+				   struct nilfs_btree_path *path)
+{
+	int level;
+
+	for (level = NILFS_BTREE_LEVEL_DATA;
+	     level < NILFS_BTREE_LEVEL_MAX;
+	     level++) {
+		if (path[level].bp_bh != NULL) {
+			nilfs_bmap_put_block(&btree->bt_bmap,
+					     path[level].bp_bh);
+			path[level].bp_bh = NULL;
+		}
+		/* sib_bh is released or deleted by prepare or commit
+		 * operations. */
+		path[level].bp_sib_bh = NULL;
+		path[level].bp_index = 0;
+		path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
+		path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
+		path[level].bp_op = NULL;
+	}
+}
+
+
+/*
+ * B-tree node operations
+ */
+
+static inline int
+nilfs_btree_node_get_flags(const struct nilfs_btree *btree,
+			   const struct nilfs_btree_node *node)
+{
+	return node->bn_flags;
+}
+
+static inline void
+nilfs_btree_node_set_flags(struct nilfs_btree *btree,
+			   struct nilfs_btree_node *node,
+			   int flags)
+{
+	node->bn_flags = flags;
+}
+
+static inline int nilfs_btree_node_root(const struct nilfs_btree *btree,
+					const struct nilfs_btree_node *node)
+{
+	return nilfs_btree_node_get_flags(btree, node) & NILFS_BTREE_NODE_ROOT;
+}
+
+static inline int
+nilfs_btree_node_get_level(const struct nilfs_btree *btree,
+			   const struct nilfs_btree_node *node)
+{
+	return node->bn_level;
+}
+
+static inline void
+nilfs_btree_node_set_level(struct nilfs_btree *btree,
+			   struct nilfs_btree_node *node,
+			   int level)
+{
+	node->bn_level = level;
+}
+
+static inline int
+nilfs_btree_node_get_nchildren(const struct nilfs_btree *btree,
+			       const struct nilfs_btree_node *node)
+{
+	return le16_to_cpu(node->bn_nchildren);
+}
+
+static inline void
+nilfs_btree_node_set_nchildren(struct nilfs_btree *btree,
+			       struct nilfs_btree_node *node,
+			       int nchildren)
+{
+	node->bn_nchildren = cpu_to_le16(nchildren);
+}
+
+static inline int
+nilfs_btree_node_size(const struct nilfs_btree *btree)
+{
+	return 1 << btree->bt_bmap.b_inode->i_blkbits;
+}
+
+static inline int
+nilfs_btree_node_nchildren_min(const struct nilfs_btree *btree,
+			       const struct nilfs_btree_node *node)
+{
+	return nilfs_btree_node_root(btree, node) ?
+		NILFS_BTREE_ROOT_NCHILDREN_MIN :
+		NILFS_BTREE_NODE_NCHILDREN_MIN(nilfs_btree_node_size(btree));
+}
+
+static inline int
+nilfs_btree_node_nchildren_max(const struct nilfs_btree *btree,
+			       const struct nilfs_btree_node *node)
+{
+	return nilfs_btree_node_root(btree, node) ?
+		NILFS_BTREE_ROOT_NCHILDREN_MAX :
+		NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(btree));
+}
+
+static inline __le64 *
+nilfs_btree_node_dkeys(const struct nilfs_btree *btree,
+		       const struct nilfs_btree_node *node)
+{
+	return (__le64 *)((char *)(node + 1) +
+			  (nilfs_btree_node_root(btree, node) ?
+			   0 : NILFS_BTREE_NODE_EXTRA_PAD_SIZE));
+}
+
+static inline __le64 *
+nilfs_btree_node_dptrs(const struct nilfs_btree *btree,
+		       const struct nilfs_btree_node *node)
+{
+	return (__le64 *)(nilfs_btree_node_dkeys(btree, node) +
+			  nilfs_btree_node_nchildren_max(btree, node));
+}
+
+static inline __u64
+nilfs_btree_node_get_key(const struct nilfs_btree *btree,
+			 const struct nilfs_btree_node *node, int index)
+{
+	return nilfs_bmap_dkey_to_key(*(nilfs_btree_node_dkeys(btree, node) +
+					index));
+}
+
+static inline void
+nilfs_btree_node_set_key(struct nilfs_btree *btree,
+			 struct nilfs_btree_node *node, int index, __u64 key)
+{
+	*(nilfs_btree_node_dkeys(btree, node) + index) =
+		nilfs_bmap_key_to_dkey(key);
+}
+
+static inline __u64
+nilfs_btree_node_get_ptr(const struct nilfs_btree *btree,
+			 const struct nilfs_btree_node *node,
+			 int index)
+{
+	return nilfs_bmap_dptr_to_ptr(*(nilfs_btree_node_dptrs(btree, node) +
+					index));
+}
+
+static inline void
+nilfs_btree_node_set_ptr(struct nilfs_btree *btree,
+			 struct nilfs_btree_node *node,
+			 int index,
+			 __u64 ptr)
+{
+	*(nilfs_btree_node_dptrs(btree, node) + index) =
+		nilfs_bmap_ptr_to_dptr(ptr);
+}
+
+static void nilfs_btree_node_init(struct nilfs_btree *btree,
+				  struct nilfs_btree_node *node,
+				  int flags, int level, int nchildren,
+				  const __u64 *keys, const __u64 *ptrs)
+{
+	__le64 *dkeys;
+	__le64 *dptrs;
+	int i;
+
+	nilfs_btree_node_set_flags(btree, node, flags);
+	nilfs_btree_node_set_level(btree, node, level);
+	nilfs_btree_node_set_nchildren(btree, node, nchildren);
+
+	dkeys = nilfs_btree_node_dkeys(btree, node);
+	dptrs = nilfs_btree_node_dptrs(btree, node);
+	for (i = 0; i < nchildren; i++) {
+		dkeys[i] = nilfs_bmap_key_to_dkey(keys[i]);
+		dptrs[i] = nilfs_bmap_ptr_to_dptr(ptrs[i]);
+	}
+}
+
+/* Assume the buffer heads corresponding to left and right are locked. */
+static void nilfs_btree_node_move_left(struct nilfs_btree *btree,
+				       struct nilfs_btree_node *left,
+				       struct nilfs_btree_node *right,
+				       int n)
+{
+	__le64 *ldkeys, *rdkeys;
+	__le64 *ldptrs, *rdptrs;
+	int lnchildren, rnchildren;
+
+	ldkeys = nilfs_btree_node_dkeys(btree, left);
+	ldptrs = nilfs_btree_node_dptrs(btree, left);
+	lnchildren = nilfs_btree_node_get_nchildren(btree, left);
+
+	rdkeys = nilfs_btree_node_dkeys(btree, right);
+	rdptrs = nilfs_btree_node_dptrs(btree, right);
+	rnchildren = nilfs_btree_node_get_nchildren(btree, right);
+
+	memcpy(ldkeys + lnchildren, rdkeys, n * sizeof(*rdkeys));
+	memcpy(ldptrs + lnchildren, rdptrs, n * sizeof(*rdptrs));
+	memmove(rdkeys, rdkeys + n, (rnchildren - n) * sizeof(*rdkeys));
+	memmove(rdptrs, rdptrs + n, (rnchildren - n) * sizeof(*rdptrs));
+
+	lnchildren += n;
+	rnchildren -= n;
+	nilfs_btree_node_set_nchildren(btree, left, lnchildren);
+	nilfs_btree_node_set_nchildren(btree, right, rnchildren);
+}
+
+/* Assume that the buffer heads corresponding to left and right are locked. */
+static void nilfs_btree_node_move_right(struct nilfs_btree *btree,
+					struct nilfs_btree_node *left,
+					struct nilfs_btree_node *right,
+					int n)
+{
+	__le64 *ldkeys, *rdkeys;
+	__le64 *ldptrs, *rdptrs;
+	int lnchildren, rnchildren;
+
+	ldkeys = nilfs_btree_node_dkeys(btree, left);
+	ldptrs = nilfs_btree_node_dptrs(btree, left);
+	lnchildren = nilfs_btree_node_get_nchildren(btree, left);
+
+	rdkeys = nilfs_btree_node_dkeys(btree, right);
+	rdptrs = nilfs_btree_node_dptrs(btree, right);
+	rnchildren = nilfs_btree_node_get_nchildren(btree, right);
+
+	memmove(rdkeys + n, rdkeys, rnchildren * sizeof(*rdkeys));
+	memmove(rdptrs + n, rdptrs, rnchildren * sizeof(*rdptrs));
+	memcpy(rdkeys, ldkeys + lnchildren - n, n * sizeof(*rdkeys));
+	memcpy(rdptrs, ldptrs + lnchildren - n, n * sizeof(*rdptrs));
+
+	lnchildren -= n;
+	rnchildren += n;
+	nilfs_btree_node_set_nchildren(btree, left, lnchildren);
+	nilfs_btree_node_set_nchildren(btree, right, rnchildren);
+}
+
+/* Assume that the buffer head corresponding to node is locked. */
+static void nilfs_btree_node_insert(struct nilfs_btree *btree,
+				    struct nilfs_btree_node *node,
+				    __u64 key, __u64 ptr, int index)
+{
+	__le64 *dkeys;
+	__le64 *dptrs;
+	int nchildren;
+
+	dkeys = nilfs_btree_node_dkeys(btree, node);
+	dptrs = nilfs_btree_node_dptrs(btree, node);
+	nchildren = nilfs_btree_node_get_nchildren(btree, node);
+	if (index < nchildren) {
+		memmove(dkeys + index + 1, dkeys + index,
+			(nchildren - index) * sizeof(*dkeys));
+		memmove(dptrs + index + 1, dptrs + index,
+			(nchildren - index) * sizeof(*dptrs));
+	}
+	dkeys[index] = nilfs_bmap_key_to_dkey(key);
+	dptrs[index] = nilfs_bmap_ptr_to_dptr(ptr);
+	nchildren++;
+	nilfs_btree_node_set_nchildren(btree, node, nchildren);
+}
+
+/* Assume that the buffer head corresponding to node is locked. */
+static void nilfs_btree_node_delete(struct nilfs_btree *btree,
+				    struct nilfs_btree_node *node,
+				    __u64 *keyp, __u64 *ptrp, int index)
+{
+	__u64 key;
+	__u64 ptr;
+	__le64 *dkeys;
+	__le64 *dptrs;
+	int nchildren;
+
+	dkeys = nilfs_btree_node_dkeys(btree, node);
+	dptrs = nilfs_btree_node_dptrs(btree, node);
+	key = nilfs_bmap_dkey_to_key(dkeys[index]);
+	ptr = nilfs_bmap_dptr_to_ptr(dptrs[index]);
+	nchildren = nilfs_btree_node_get_nchildren(btree, node);
+	if (keyp != NULL)
+		*keyp = key;
+	if (ptrp != NULL)
+		*ptrp = ptr;
+
+	if (index < nchildren - 1) {
+		memmove(dkeys + index, dkeys + index + 1,
+			(nchildren - index - 1) * sizeof(*dkeys));
+		memmove(dptrs + index, dptrs + index + 1,
+			(nchildren - index - 1) * sizeof(*dptrs));
+	}
+	nchildren--;
+	nilfs_btree_node_set_nchildren(btree, node, nchildren);
+}
+
+static int nilfs_btree_node_lookup(const struct nilfs_btree *btree,
+				   const struct nilfs_btree_node *node,
+				   __u64 key, int *indexp)
+{
+	__u64 nkey;
+	int index, low, high, s;
+
+	/* binary search */
+	low = 0;
+	high = nilfs_btree_node_get_nchildren(btree, node) - 1;
+	index = 0;
+	s = 0;
+	while (low <= high) {
+		index = (low + high) / 2;
+		nkey = nilfs_btree_node_get_key(btree, node, index);
+		if (nkey == key) {
+			s = 0;
+			goto out;
+		} else if (nkey < key) {
+			low = index + 1;
+			s = -1;
+		} else {
+			high = index - 1;
+			s = 1;
+		}
+	}
+
+	/* adjust index */
+	if (nilfs_btree_node_get_level(btree, node) >
+	    NILFS_BTREE_LEVEL_NODE_MIN) {
+		if ((s > 0) && (index > 0))
+			index--;
+	} else if (s < 0)
+		index++;
+
+ out:
+	BUG_ON(indexp == NULL);
+	*indexp = index;
+
+	return s == 0;
+}
+
+static inline struct nilfs_btree_node *
+nilfs_btree_get_root(const struct nilfs_btree *btree)
+{
+	return (struct nilfs_btree_node *)btree->bt_bmap.b_u.u_data;
+}
+
+static inline struct nilfs_btree_node *
+nilfs_btree_get_nonroot_node(const struct nilfs_btree *btree,
+			     const struct nilfs_btree_path *path,
+			     int level)
+{
+	return (struct nilfs_btree_node *)path[level].bp_bh->b_data;
+}
+
+static inline struct nilfs_btree_node *
+nilfs_btree_get_sib_node(const struct nilfs_btree *btree,
+			 const struct nilfs_btree_path *path,
+			 int level)
+{
+	return (struct nilfs_btree_node *)path[level].bp_sib_bh->b_data;
+}
+
+static inline int nilfs_btree_height(const struct nilfs_btree *btree)
+{
+	return nilfs_btree_node_get_level(btree, nilfs_btree_get_root(btree))
+		+ 1;
+}
+
+static inline struct nilfs_btree_node *
+nilfs_btree_get_node(const struct nilfs_btree *btree,
+		     const struct nilfs_btree_path *path,
+		     int level)
+{
+	return (level == nilfs_btree_height(btree) - 1) ?
+		nilfs_btree_get_root(btree) :
+		nilfs_btree_get_nonroot_node(btree, path, level);
+}
+
+static int nilfs_btree_do_lookup(const struct nilfs_btree *btree,
+				 struct nilfs_btree_path *path,
+				 __u64 key, __u64 *ptrp, int minlevel)
+{
+	struct nilfs_btree_node *node;
+	__u64 ptr;
+	int level, index, found, ret;
+
+	BUG_ON(minlevel <= NILFS_BTREE_LEVEL_DATA);
+
+	node = nilfs_btree_get_root(btree);
+	level = nilfs_btree_node_get_level(btree, node);
+	if ((level < minlevel) ||
+	    (nilfs_btree_node_get_nchildren(btree, node) <= 0))
+		return -ENOENT;
+
+	found = nilfs_btree_node_lookup(btree, node, key, &index);
+	ptr = nilfs_btree_node_get_ptr(btree, node, index);
+	path[level].bp_bh = NULL;
+	path[level].bp_index = index;
+
+	for (level--; level >= minlevel; level--) {
+		ret = nilfs_bmap_get_block(&btree->bt_bmap, ptr,
+					   &path[level].bp_bh);
+		if (ret < 0)
+			return ret;
+		node = nilfs_btree_get_nonroot_node(btree, path, level);
+		BUG_ON(level != nilfs_btree_node_get_level(btree, node));
+		if (!found)
+			found = nilfs_btree_node_lookup(btree, node, key,
+							&index);
+		else
+			index = 0;
+		if (index < nilfs_btree_node_nchildren_max(btree, node))
+			ptr = nilfs_btree_node_get_ptr(btree, node, index);
+		else {
+			BUG_ON(found || level != NILFS_BTREE_LEVEL_NODE_MIN);
+			/* insert */
+			ptr = NILFS_BMAP_INVALID_PTR;
+		}
+		path[level].bp_index = index;
+	}
+	if (!found)
+		return -ENOENT;
+
+	if (ptrp != NULL)
+		*ptrp = ptr;
+
+	return 0;
+}
+
+static int nilfs_btree_do_lookup_last(const struct nilfs_btree *btree,
+				      struct nilfs_btree_path *path,
+				      __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node;
+	__u64 ptr;
+	int index, level, ret;
+
+	node = nilfs_btree_get_root(btree);
+	index = nilfs_btree_node_get_nchildren(btree, node) - 1;
+	if (index < 0)
+		return -ENOENT;
+	level = nilfs_btree_node_get_level(btree, node);
+	ptr = nilfs_btree_node_get_ptr(btree, node, index);
+	path[level].bp_bh = NULL;
+	path[level].bp_index = index;
+
+	for (level--; level > 0; level--) {
+		ret = nilfs_bmap_get_block(&btree->bt_bmap, ptr,
+					   &path[level].bp_bh);
+		if (ret < 0)
+			return ret;
+		node = nilfs_btree_get_nonroot_node(btree, path, level);
+		BUG_ON(level != nilfs_btree_node_get_level(btree, node));
+		index = nilfs_btree_node_get_nchildren(btree, node) - 1;
+		ptr = nilfs_btree_node_get_ptr(btree, node, index);
+		path[level].bp_index = index;
+	}
+
+	if (keyp != NULL)
+		*keyp = nilfs_btree_node_get_key(btree, node, index);
+	if (ptrp != NULL)
+		*ptrp = ptr;
+
+	return 0;
+}
+
+static int nilfs_btree_lookup(const struct nilfs_bmap *bmap,
+			      __u64 key, int level, __u64 *ptrp)
+{
+	struct nilfs_btree *btree;
+	struct nilfs_btree_path *path;
+	__u64 ptr;
+	int ret;
+
+	btree = (struct nilfs_btree *)bmap;
+	path = nilfs_btree_alloc_path(btree);
+	if (path == NULL)
+		return -ENOMEM;
+	nilfs_btree_init_path(btree, path);
+
+	ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level);
+
+	if (ptrp != NULL)
+		*ptrp = ptr;
+
+	nilfs_btree_clear_path(btree, path);
+	nilfs_btree_free_path(btree, path);
+
+	return ret;
+}
+
+static void nilfs_btree_promote_key(struct nilfs_btree *btree,
+				    struct nilfs_btree_path *path,
+				    int level, __u64 key)
+{
+	if (level < nilfs_btree_height(btree) - 1) {
+		do {
+			lock_buffer(path[level].bp_bh);
+			nilfs_btree_node_set_key(
+				btree,
+				nilfs_btree_get_nonroot_node(
+					btree, path, level),
+				path[level].bp_index, key);
+			if (!buffer_dirty(path[level].bp_bh))
+				nilfs_btnode_mark_dirty(path[level].bp_bh);
+			unlock_buffer(path[level].bp_bh);
+		} while ((path[level].bp_index == 0) &&
+			 (++level < nilfs_btree_height(btree) - 1));
+	}
+
+	/* root */
+	if (level == nilfs_btree_height(btree) - 1) {
+		nilfs_btree_node_set_key(btree,
+					 nilfs_btree_get_root(btree),
+					 path[level].bp_index, key);
+	}
+}
+
+static void nilfs_btree_do_insert(struct nilfs_btree *btree,
+				  struct nilfs_btree_path *path,
+				  int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node;
+
+	if (level < nilfs_btree_height(btree) - 1) {
+		lock_buffer(path[level].bp_bh);
+		node = nilfs_btree_get_nonroot_node(btree, path, level);
+		nilfs_btree_node_insert(btree, node, *keyp, *ptrp,
+					path[level].bp_index);
+		if (!buffer_dirty(path[level].bp_bh))
+			nilfs_btnode_mark_dirty(path[level].bp_bh);
+		unlock_buffer(path[level].bp_bh);
+
+		if (path[level].bp_index == 0)
+			nilfs_btree_promote_key(btree, path, level + 1,
+						nilfs_btree_node_get_key(
+							btree, node, 0));
+	} else {
+		node = nilfs_btree_get_root(btree);
+		nilfs_btree_node_insert(btree, node, *keyp, *ptrp,
+					path[level].bp_index);
+	}
+}
+
+static void nilfs_btree_carry_left(struct nilfs_btree *btree,
+				   struct nilfs_btree_path *path,
+				   int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node, *left;
+	int nchildren, lnchildren, n, move;
+
+	lock_buffer(path[level].bp_bh);
+	lock_buffer(path[level].bp_sib_bh);
+
+	node = nilfs_btree_get_nonroot_node(btree, path, level);
+	left = nilfs_btree_get_sib_node(btree, path, level);
+	nchildren = nilfs_btree_node_get_nchildren(btree, node);
+	lnchildren = nilfs_btree_node_get_nchildren(btree, left);
+	move = 0;
+
+	n = (nchildren + lnchildren + 1) / 2 - lnchildren;
+	if (n > path[level].bp_index) {
+		/* move insert point */
+		n--;
+		move = 1;
+	}
+
+	nilfs_btree_node_move_left(btree, left, node, n);
+
+	if (!buffer_dirty(path[level].bp_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_bh);
+	if (!buffer_dirty(path[level].bp_sib_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
+
+	unlock_buffer(path[level].bp_bh);
+	unlock_buffer(path[level].bp_sib_bh);
+
+	nilfs_btree_promote_key(btree, path, level + 1,
+				nilfs_btree_node_get_key(btree, node, 0));
+
+	if (move) {
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_bh);
+		path[level].bp_bh = path[level].bp_sib_bh;
+		path[level].bp_sib_bh = NULL;
+		path[level].bp_index += lnchildren;
+		path[level + 1].bp_index--;
+	} else {
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_sib_bh);
+		path[level].bp_sib_bh = NULL;
+		path[level].bp_index -= n;
+	}
+
+	nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
+}
+
+static void nilfs_btree_carry_right(struct nilfs_btree *btree,
+				    struct nilfs_btree_path *path,
+				    int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node, *right;
+	int nchildren, rnchildren, n, move;
+
+	lock_buffer(path[level].bp_bh);
+	lock_buffer(path[level].bp_sib_bh);
+
+	node = nilfs_btree_get_nonroot_node(btree, path, level);
+	right = nilfs_btree_get_sib_node(btree, path, level);
+	nchildren = nilfs_btree_node_get_nchildren(btree, node);
+	rnchildren = nilfs_btree_node_get_nchildren(btree, right);
+	move = 0;
+
+	n = (nchildren + rnchildren + 1) / 2 - rnchildren;
+	if (n > nchildren - path[level].bp_index) {
+		/* move insert point */
+		n--;
+		move = 1;
+	}
+
+	nilfs_btree_node_move_right(btree, node, right, n);
+
+	if (!buffer_dirty(path[level].bp_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_bh);
+	if (!buffer_dirty(path[level].bp_sib_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
+
+	unlock_buffer(path[level].bp_bh);
+	unlock_buffer(path[level].bp_sib_bh);
+
+	path[level + 1].bp_index++;
+	nilfs_btree_promote_key(btree, path, level + 1,
+				nilfs_btree_node_get_key(btree, right, 0));
+	path[level + 1].bp_index--;
+
+	if (move) {
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_bh);
+		path[level].bp_bh = path[level].bp_sib_bh;
+		path[level].bp_sib_bh = NULL;
+		path[level].bp_index -=
+			nilfs_btree_node_get_nchildren(btree, node);
+		path[level + 1].bp_index++;
+	} else {
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_sib_bh);
+		path[level].bp_sib_bh = NULL;
+	}
+
+	nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
+}
+
+static void nilfs_btree_split(struct nilfs_btree *btree,
+			      struct nilfs_btree_path *path,
+			      int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node, *right;
+	__u64 newkey;
+	__u64 newptr;
+	int nchildren, n, move;
+
+	lock_buffer(path[level].bp_bh);
+	lock_buffer(path[level].bp_sib_bh);
+
+	node = nilfs_btree_get_nonroot_node(btree, path, level);
+	right = nilfs_btree_get_sib_node(btree, path, level);
+	nchildren = nilfs_btree_node_get_nchildren(btree, node);
+	move = 0;
+
+	n = (nchildren + 1) / 2;
+	if (n > nchildren - path[level].bp_index) {
+		n--;
+		move = 1;
+	}
+
+	nilfs_btree_node_move_right(btree, node, right, n);
+
+	if (!buffer_dirty(path[level].bp_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_bh);
+	if (!buffer_dirty(path[level].bp_sib_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
+
+	unlock_buffer(path[level].bp_bh);
+	unlock_buffer(path[level].bp_sib_bh);
+
+	newkey = nilfs_btree_node_get_key(btree, right, 0);
+	newptr = path[level].bp_newreq.bpr_ptr;
+
+	if (move) {
+		path[level].bp_index -=
+			nilfs_btree_node_get_nchildren(btree, node);
+		nilfs_btree_node_insert(btree, right, *keyp, *ptrp,
+					path[level].bp_index);
+
+		*keyp = nilfs_btree_node_get_key(btree, right, 0);
+		*ptrp = path[level].bp_newreq.bpr_ptr;
+
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_bh);
+		path[level].bp_bh = path[level].bp_sib_bh;
+		path[level].bp_sib_bh = NULL;
+	} else {
+		nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
+
+		*keyp = nilfs_btree_node_get_key(btree, right, 0);
+		*ptrp = path[level].bp_newreq.bpr_ptr;
+
+		nilfs_bmap_put_block(&btree->bt_bmap, path[level].bp_sib_bh);
+		path[level].bp_sib_bh = NULL;
+	}
+
+	path[level + 1].bp_index++;
+}
+
+static void nilfs_btree_grow(struct nilfs_btree *btree,
+			     struct nilfs_btree_path *path,
+			     int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *root, *child;
+	int n;
+
+	lock_buffer(path[level].bp_sib_bh);
+
+	root = nilfs_btree_get_root(btree);
+	child = nilfs_btree_get_sib_node(btree, path, level);
+
+	n = nilfs_btree_node_get_nchildren(btree, root);
+
+	nilfs_btree_node_move_right(btree, root, child, n);
+	nilfs_btree_node_set_level(btree, root, level + 1);
+
+	if (!buffer_dirty(path[level].bp_sib_bh))
+		nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
+
+	unlock_buffer(path[level].bp_sib_bh);
+
+	path[level].bp_bh = path[level].bp_sib_bh;
+	path[level].bp_sib_bh = NULL;
+
+	nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
+
+	*keyp = nilfs_btree_node_get_key(btree, child, 0);
+	*ptrp = path[level].bp_newreq.bpr_ptr;
+}
+
+static __u64 nilfs_btree_find_near(const struct nilfs_btree *btree,
+				   const struct nilfs_btree_path *path)
+{
+	struct nilfs_btree_node *node;
+	int level;
+
+	if (path == NULL)
+		return NILFS_BMAP_INVALID_PTR;
+
+	/* left sibling */
+	level = NILFS_BTREE_LEVEL_NODE_MIN;
+	if (path[level].bp_index > 0) {
+		node = nilfs_btree_get_node(btree, path, level);
+		return nilfs_btree_node_get_ptr(btree, node,
+						path[level].bp_index - 1);
+	}
+
+	/* parent */
+	level = NILFS_BTREE_LEVEL_NODE_MIN + 1;
+	if (level <= nilfs_btree_height(btree) - 1) {
+		node = nilfs_btree_get_node(btree, path, level);
+		return nilfs_btree_node_get_ptr(btree, node,
+						path[level].bp_index);
+	}
+
+	return NILFS_BMAP_INVALID_PTR;
+}
+
+static __u64 nilfs_btree_find_target_v(const struct nilfs_btree *btree,
+				       const struct nilfs_btree_path *path,
+				       __u64 key)
+{
+	__u64 ptr;
+
+	ptr = nilfs_bmap_find_target_seq(&btree->bt_bmap, key);
+	if (ptr != NILFS_BMAP_INVALID_PTR)
+		/* sequential access */
+		return ptr;
+	else {
+		ptr = nilfs_btree_find_near(btree, path);
+		if (ptr != NILFS_BMAP_INVALID_PTR)
+			/* near */
+			return ptr;
+	}
+	/* block group */
+	return nilfs_bmap_find_target_in_group(&btree->bt_bmap);
+}
+
+static void nilfs_btree_set_target_v(struct nilfs_btree *btree, __u64 key,
+				     __u64 ptr)
+{
+	btree->bt_bmap.b_last_allocated_key = key;
+	btree->bt_bmap.b_last_allocated_ptr = ptr;
+}
+
+static int nilfs_btree_prepare_insert(struct nilfs_btree *btree,
+				      struct nilfs_btree_path *path,
+				      int *levelp, __u64 key, __u64 ptr,
+				      struct nilfs_bmap_stats *stats)
+{
+	struct buffer_head *bh;
+	struct nilfs_btree_node *node, *parent, *sib;
+	__u64 sibptr;
+	int pindex, level, ret;
+
+	stats->bs_nblocks = 0;
+	level = NILFS_BTREE_LEVEL_DATA;
+
+	/* allocate a new ptr for data block */
+	if (btree->bt_ops->btop_find_target != NULL)
+		path[level].bp_newreq.bpr_ptr =
+			(*btree->bt_ops->btop_find_target)(btree, path, key);
+
+	ret = (*btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr)(
+		&btree->bt_bmap, &path[level].bp_newreq);
+	if (ret < 0)
+		goto err_out_data;
+
+	for (level = NILFS_BTREE_LEVEL_NODE_MIN;
+	     level < nilfs_btree_height(btree) - 1;
+	     level++) {
+		node = nilfs_btree_get_nonroot_node(btree, path, level);
+		if (nilfs_btree_node_get_nchildren(btree, node) <
+		    nilfs_btree_node_nchildren_max(btree, node)) {
+			path[level].bp_op = nilfs_btree_do_insert;
+			stats->bs_nblocks++;
+			goto out;
+		}
+
+		parent = nilfs_btree_get_node(btree, path, level + 1);
+		pindex = path[level + 1].bp_index;
+
+		/* left sibling */
+		if (pindex > 0) {
+			sibptr = nilfs_btree_node_get_ptr(btree, parent,
+							  pindex - 1);
+			ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
+						   &bh);
+			if (ret < 0)
+				goto err_out_child_node;
+			sib = (struct nilfs_btree_node *)bh->b_data;
+			if (nilfs_btree_node_get_nchildren(btree, sib) <
+			    nilfs_btree_node_nchildren_max(btree, sib)) {
+				path[level].bp_sib_bh = bh;
+				path[level].bp_op = nilfs_btree_carry_left;
+				stats->bs_nblocks++;
+				goto out;
+			} else
+				nilfs_bmap_put_block(&btree->bt_bmap, bh);
+		}
+
+		/* right sibling */
+		if (pindex <
+		    nilfs_btree_node_get_nchildren(btree, parent) - 1) {
+			sibptr = nilfs_btree_node_get_ptr(btree, parent,
+							  pindex + 1);
+			ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
+						   &bh);
+			if (ret < 0)
+				goto err_out_child_node;
+			sib = (struct nilfs_btree_node *)bh->b_data;
+			if (nilfs_btree_node_get_nchildren(btree, sib) <
+			    nilfs_btree_node_nchildren_max(btree, sib)) {
+				path[level].bp_sib_bh = bh;
+				path[level].bp_op = nilfs_btree_carry_right;
+				stats->bs_nblocks++;
+				goto out;
+			} else
+				nilfs_bmap_put_block(&btree->bt_bmap, bh);
+		}
+
+		/* split */
+		path[level].bp_newreq.bpr_ptr =
+			path[level - 1].bp_newreq.bpr_ptr + 1;
+		ret = (*btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr)(
+			&btree->bt_bmap, &path[level].bp_newreq);
+		if (ret < 0)
+			goto err_out_child_node;
+		ret = nilfs_bmap_get_new_block(&btree->bt_bmap,
+					       path[level].bp_newreq.bpr_ptr,
+					       &bh);
+		if (ret < 0)
+			goto err_out_curr_node;
+
+		stats->bs_nblocks++;
+
+		lock_buffer(bh);
+		nilfs_btree_node_init(btree,
+				      (struct nilfs_btree_node *)bh->b_data,
+				      0, level, 0, NULL, NULL);
+		unlock_buffer(bh);
+		path[level].bp_sib_bh = bh;
+		path[level].bp_op = nilfs_btree_split;
+	}
+
+	/* root */
+	node = nilfs_btree_get_root(btree);
+	if (nilfs_btree_node_get_nchildren(btree, node) <
+	    nilfs_btree_node_nchildren_max(btree, node)) {
+		path[level].bp_op = nilfs_btree_do_insert;
+		stats->bs_nblocks++;
+		goto out;
+	}
+
+	/* grow */
+	path[level].bp_newreq.bpr_ptr = path[level - 1].bp_newreq.bpr_ptr + 1;
+	ret = (*btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr)(
+		&btree->bt_bmap, &path[level].bp_newreq);
+	if (ret < 0)
+		goto err_out_child_node;
+	ret = nilfs_bmap_get_new_block(&btree->bt_bmap,
+				       path[level].bp_newreq.bpr_ptr, &bh);
+	if (ret < 0)
+		goto err_out_curr_node;
+
+	lock_buffer(bh);
+	nilfs_btree_node_init(btree, (struct nilfs_btree_node *)bh->b_data,
+			      0, level, 0, NULL, NULL);
+	unlock_buffer(bh);
+	path[level].bp_sib_bh = bh;
+	path[level].bp_op = nilfs_btree_grow;
+
+	level++;
+	path[level].bp_op = nilfs_btree_do_insert;
+
+	/* a newly-created node block and a data block are added */
+	stats->bs_nblocks += 2;
+
+	/* success */
+ out:
+	*levelp = level;
+	return ret;
+
+	/* error */
+ err_out_curr_node:
+	(*btree->bt_bmap.b_pops->bpop_abort_alloc_ptr)(&btree->bt_bmap,
+						       &path[level].bp_newreq);
+ err_out_child_node:
+	for (level--; level > NILFS_BTREE_LEVEL_DATA; level--) {
+		nilfs_bmap_delete_block(&btree->bt_bmap, path[level].bp_sib_bh);
+		(*btree->bt_bmap.b_pops->bpop_abort_alloc_ptr)(
+			&btree->bt_bmap, &path[level].bp_newreq);
+
+	}
+
+	(*btree->bt_bmap.b_pops->bpop_abort_alloc_ptr)(&btree->bt_bmap,
+						       &path[level].bp_newreq);
+ err_out_data:
+	*levelp = level;
+	stats->bs_nblocks = 0;
+	return ret;
+}
+
+static void nilfs_btree_commit_insert(struct nilfs_btree *btree,
+				      struct nilfs_btree_path *path,
+				      int maxlevel, __u64 key, __u64 ptr)
+{
+	int level;
+
+	set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
+	ptr = path[NILFS_BTREE_LEVEL_DATA].bp_newreq.bpr_ptr;
+	if (btree->bt_ops->btop_set_target != NULL)
+		(*btree->bt_ops->btop_set_target)(btree, key, ptr);
+
+	for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
+		if (btree->bt_bmap.b_pops->bpop_commit_alloc_ptr != NULL) {
+			(*btree->bt_bmap.b_pops->bpop_commit_alloc_ptr)(
+				&btree->bt_bmap, &path[level - 1].bp_newreq);
+		}
+		(*path[level].bp_op)(btree, path, level, &key, &ptr);
+	}
+
+	if (!nilfs_bmap_dirty(&btree->bt_bmap))
+		nilfs_bmap_set_dirty(&btree->bt_bmap);
+}
+
+static int nilfs_btree_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr)
+{
+	struct nilfs_btree *btree;
+	struct nilfs_btree_path *path;
+	struct nilfs_bmap_stats stats;
+	int level, ret;
+
+	btree = (struct nilfs_btree *)bmap;
+	path = nilfs_btree_alloc_path(btree);
+	if (path == NULL)
+		return -ENOMEM;
+	nilfs_btree_init_path(btree, path);
+
+	ret = nilfs_btree_do_lookup(btree, path, key, NULL,
+				    NILFS_BTREE_LEVEL_NODE_MIN);
+	if (ret != -ENOENT) {
+		if (ret == 0)
+			ret = -EEXIST;
+		goto out;
+	}
+
+	ret = nilfs_btree_prepare_insert(btree, path, &level, key, ptr, &stats);
+	if (ret < 0)
+		goto out;
+	nilfs_btree_commit_insert(btree, path, level, key, ptr);
+	nilfs_bmap_add_blocks(bmap, stats.bs_nblocks);
+
+ out:
+	nilfs_btree_clear_path(btree, path);
+	nilfs_btree_free_path(btree, path);
+	return ret;
+}
+
+static void nilfs_btree_do_delete(struct nilfs_btree *btree,
+				  struct nilfs_btree_path *path,
+				  int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_node *node;
+
+	if (level < nilfs_btree_height(btree) - 1) {
+		lock_buffer(path[level].bp_bh);
+		node = nilfs_btree_get_nonroot_node(btree, path, level);
+		nilfs_btree_node_delete(btree, node, keyp, ptrp,
+					path[level].bp_index);
+		if (!buffer_dirty(path[level].bp_bh))
+			nilfs_btnode_mark_dirty(path[level].bp_bh);
+		unlock_buffer(path[level].bp_bh);
+		if (path[level].bp_index == 0)
+			nilfs_btree_promote_key(btree, path, level + 1,
+				nilfs_btree_node_get_key(btree, node, 0));
+	} else {
+		node = nilfs_btree_get_root(btree);
+		nilfs_btree_node_delete(btree, node, keyp, ptrp,
+					path[level].bp_index);
+	}
+}
+
+static void nilfs_btree_borrow_left(struct nilfs_btree *btree,
+				    struct nilfs_btree_path *path,
+				    int level, __u64 *keyp, __u64 *ptrp)
+{
+	struct nilfs_btree_no