libceph: keep source rather than message osd op array

An osd request keeps a pointer to the osd operations (ops) array
that it builds in its request message.

In order to allow each op in the array to have its own distinct
data, we will need to keep track of each op's data, and that
information does not go over the wire.

As long as we're tracking the data we might as well just track the
entire (source) op definition for each of the ops.  And if we're
doing that, we'll have no more need to keep a pointer to the
wire-encoded version.

This patch makes the array of source ops be kept with the osd
request structure, and uses that instead of the version encoded in
the message in places where that was previously used.  The array
will be embedded in the request structure, and the maximum number of
ops we ever actually use is currently 2.  So reduce CEPH_OSD_MAX_OP
to 2 to reduce the size of the structure.

The result of doing this sort of ripples back up, and as a result
various function parameters and local variables become unnecessary.

Make r_num_ops be unsigned, and move the definition of struct
ceph_osd_req_op earlier to ensure it's defined where needed.

It does not yet add per-op data, that's coming soon.

This resolves:
    http://tracker.ceph.com/issues/4656

Signed-off-by: Alex Elder <elder@inktank.com>
Reviewed-by: Josh Durgin <josh.durgin@inktank.com>

2 files changed, 11 insertions, 16 deletions

diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 127be29a6c2..c9da074f0fe 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -288,7 +288,6 @@ static int start_read(struct inode *inode, struct list_head *page_list, int max)
 	struct page *page = list_entry(page_list->prev, struct page, lru);
 	struct ceph_vino vino;
 	struct ceph_osd_request *req;
-	struct ceph_osd_req_op op;
 	u64 off;
 	u64 len;
 	int i;
@@ -314,7 +313,7 @@ static int start_read(struct inode *inode, struct list_head *page_list, int max)
 	     off, len);
 	vino = ceph_vino(inode);
 	req = ceph_osdc_new_request(osdc, &ci->i_layout, vino, off, &len,
-				    1, &op, CEPH_OSD_OP_READ,
+				    1, CEPH_OSD_OP_READ,
 				    CEPH_OSD_FLAG_READ, NULL,
 				    ci->i_truncate_seq, ci->i_truncate_size,
 				    false);
@@ -349,7 +348,7 @@ static int start_read(struct inode *inode, struct list_head *page_list, int max)
 	req->r_callback = finish_read;
 	req->r_inode = inode;
 
-	ceph_osdc_build_request(req, off, 1, &op, NULL, vino.snap, NULL);
+	ceph_osdc_build_request(req, off, NULL, vino.snap, NULL);
 
 	dout("start_read %p starting %p %lld~%lld\n", inode, req, off, len);
 	ret = ceph_osdc_start_request(osdc, req, false);
@@ -567,7 +566,7 @@ static void writepages_finish(struct ceph_osd_request *req,
 	struct ceph_snap_context *snapc = req->r_snapc;
 	struct address_space *mapping = inode->i_mapping;
 	int rc = req->r_result;
-	u64 bytes = le64_to_cpu(req->r_request_ops[0].extent.length);
+	u64 bytes = req->r_ops[0].extent.length;
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	long writeback_stat;
 	unsigned issued = ceph_caps_issued(ci);
@@ -635,8 +634,7 @@ static void writepages_finish(struct ceph_osd_request *req,
 
 static struct ceph_osd_request *
 ceph_writepages_osd_request(struct inode *inode, u64 offset, u64 *len,
-				struct ceph_snap_context *snapc,
-				int num_ops, struct ceph_osd_req_op *ops)
+				struct ceph_snap_context *snapc, int num_ops)
 {
 	struct ceph_fs_client *fsc;
 	struct ceph_inode_info *ci;
@@ -648,7 +646,7 @@ ceph_writepages_osd_request(struct inode *inode, u64 offset, u64 *len,
 	/* BUG_ON(vino.snap != CEPH_NOSNAP); */
 
 	return ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
-			vino, offset, len, num_ops, ops, CEPH_OSD_OP_WRITE,
+			vino, offset, len, num_ops, CEPH_OSD_OP_WRITE,
 			CEPH_OSD_FLAG_WRITE|CEPH_OSD_FLAG_ONDISK,
 			snapc, ci->i_truncate_seq, ci->i_truncate_size, true);
 }
@@ -738,7 +736,6 @@ retry:
 	last_snapc = snapc;
 
 	while (!done && index <= end) {
-		struct ceph_osd_req_op ops[2];
 		int num_ops = do_sync ? 2 : 1;
 		struct ceph_vino vino;
 		unsigned i;
@@ -846,7 +843,7 @@ get_more_pages:
 				len = wsize;
 				req = ceph_writepages_osd_request(inode,
 							offset, &len, snapc,
-							num_ops, ops);
+							num_ops);
 
 				if (IS_ERR(req)) {
 					rc = PTR_ERR(req);
@@ -927,11 +924,11 @@ get_more_pages:
 
 		/* Update the write op length in case we changed it */
 
-		osd_req_op_extent_update(&ops[0], len);
+		osd_req_op_extent_update(&req->r_ops[0], len);
 
 		vino = ceph_vino(inode);
-		ceph_osdc_build_request(req, offset, num_ops, ops,
-					snapc, vino.snap, &inode->i_mtime);
+		ceph_osdc_build_request(req, offset, snapc, vino.snap,
+					&inode->i_mtime);
 
 		rc = ceph_osdc_start_request(&fsc->client->osdc, req, true);
 		BUG_ON(rc);
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index da642af14a2..a12f47642c4 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -478,7 +478,6 @@ static ssize_t ceph_sync_write(struct file *file, const char __user *data,
 	struct ceph_snap_context *snapc;
 	struct ceph_vino vino;
 	struct ceph_osd_request *req;
-	struct ceph_osd_req_op ops[2];
 	int num_ops = 1;
 	struct page **pages;
 	int num_pages;
@@ -534,7 +533,7 @@ more:
 	snapc = ci->i_snap_realm->cached_context;
 	vino = ceph_vino(inode);
 	req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
-				    vino, pos, &len, num_ops, ops,
+				    vino, pos, &len, num_ops,
 				    CEPH_OSD_OP_WRITE, flags, snapc,
 				    ci->i_truncate_seq, ci->i_truncate_size,
 				    false);
@@ -579,8 +578,7 @@ more:
 					false, own_pages);
 
 	/* BUG_ON(vino.snap != CEPH_NOSNAP); */
-	ceph_osdc_build_request(req, pos, num_ops, ops,
-				snapc, vino.snap, &mtime);
+	ceph_osdc_build_request(req, pos, snapc, vino.snap, &mtime);
 
 	ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!ret) {