Commit 3499e8a5d4dbb083324efd942e2c4fb7eb65f27c

Authored by Yehuda Sadeh
Committed by Sage Weil
1 parent 7669a2c95e

ceph: refactor osdc requests creation functions

The osd requests creation are being decoupled from the
vino parameter, allowing clients using the osd to use
other arbitrary object names that are not necessarily
vino based. Also, calc_raw_layout now takes a snap id.

Signed-off-by: Yehuda Sadeh <yehuda@hq.newdream.net>
Signed-off-by: Sage Weil <sage@newdream.net>

Showing 2 changed files with 155 additions and 57 deletions Side-by-side Diff

fs/ceph/osd_client.c
... ... @@ -22,6 +22,35 @@
22 22  
23 23 static void kick_requests(struct ceph_osd_client *osdc, struct ceph_osd *osd);
24 24  
  25 +void ceph_calc_raw_layout(struct ceph_osd_client *osdc,
  26 + struct ceph_file_layout *layout,
  27 + u64 snapid,
  28 + u64 off, u64 len, u64 *bno,
  29 + struct ceph_osd_request *req)
  30 +{
  31 + struct ceph_osd_request_head *reqhead = req->r_request->front.iov_base;
  32 + struct ceph_osd_op *op = (void *)(reqhead + 1);
  33 + u64 orig_len = len;
  34 + u64 objoff, objlen; /* extent in object */
  35 +
  36 + reqhead->snapid = cpu_to_le64(snapid);
  37 +
  38 + /* object extent? */
  39 + ceph_calc_file_object_mapping(layout, off, &len, bno,
  40 + &objoff, &objlen);
  41 + if (len < orig_len)
  42 + dout(" skipping last %llu, final file extent %llu~%llu\n",
  43 + orig_len - len, off, len);
  44 +
  45 + op->extent.offset = cpu_to_le64(objoff);
  46 + op->extent.length = cpu_to_le64(objlen);
  47 + req->r_num_pages = calc_pages_for(off, len);
  48 +
  49 + dout("calc_layout bno=%llx %llu~%llu (%d pages)\n",
  50 + *bno, objoff, objlen, req->r_num_pages);
  51 +
  52 +}
  53 +
25 54 /*
26 55 * Implement client access to distributed object storage cluster.
27 56 *
28 57  
29 58  
30 59  
31 60  
... ... @@ -48,34 +77,17 @@
48 77 * fill osd op in request message.
49 78 */
50 79 static void calc_layout(struct ceph_osd_client *osdc,
51   - struct ceph_vino vino, struct ceph_file_layout *layout,
  80 + struct ceph_vino vino,
  81 + struct ceph_file_layout *layout,
52 82 u64 off, u64 *plen,
53 83 struct ceph_osd_request *req)
54 84 {
55   - struct ceph_osd_request_head *reqhead = req->r_request->front.iov_base;
56   - struct ceph_osd_op *op = (void *)(reqhead + 1);
57   - u64 orig_len = *plen;
58   - u64 objoff, objlen; /* extent in object */
59 85 u64 bno;
60 86  
61   - reqhead->snapid = cpu_to_le64(vino.snap);
  87 + ceph_calc_raw_layout(osdc, layout, vino.snap, off, *plen, &bno, req);
62 88  
63   - /* object extent? */
64   - ceph_calc_file_object_mapping(layout, off, plen, &bno,
65   - &objoff, &objlen);
66   - if (*plen < orig_len)
67   - dout(" skipping last %llu, final file extent %llu~%llu\n",
68   - orig_len - *plen, off, *plen);
69   -
70 89 sprintf(req->r_oid, "%llx.%08llx", vino.ino, bno);
71 90 req->r_oid_len = strlen(req->r_oid);
72   -
73   - op->extent.offset = cpu_to_le64(objoff);
74   - op->extent.length = cpu_to_le64(objlen);
75   - req->r_num_pages = calc_pages_for(off, *plen);
76   -
77   - dout("calc_layout %s (%d) %llu~%llu (%d pages)\n",
78   - req->r_oid, req->r_oid_len, objoff, objlen, req->r_num_pages);
79 91 }
80 92  
81 93 /*
82 94  
83 95  
84 96  
85 97  
86 98  
87 99  
... ... @@ -108,44 +120,35 @@
108 120 kfree(req);
109 121 }
110 122  
111   -/*
112   - * build new request AND message, calculate layout, and adjust file
113   - * extent as needed.
114   - *
115   - * if the file was recently truncated, we include information about its
116   - * old and new size so that the object can be updated appropriately. (we
117   - * avoid synchronously deleting truncated objects because it's slow.)
118   - *
119   - * if @do_sync, include a 'startsync' command so that the osd will flush
120   - * data quickly.
121   - */
122   -struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
123   - struct ceph_file_layout *layout,
124   - struct ceph_vino vino,
125   - u64 off, u64 *plen,
126   - int opcode, int flags,
  123 +struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc,
  124 + int flags,
127 125 struct ceph_snap_context *snapc,
128 126 int do_sync,
129   - u32 truncate_seq,
130   - u64 truncate_size,
131   - struct timespec *mtime,
132   - bool use_mempool, int num_reply)
  127 + bool use_mempool,
  128 + gfp_t gfp_flags,
  129 + struct page **pages)
133 130 {
134 131 struct ceph_osd_request *req;
135 132 struct ceph_msg *msg;
136   - struct ceph_osd_request_head *head;
137   - struct ceph_osd_op *op;
138   - void *p;
139 133 int num_op = 1 + do_sync;
140   - size_t msg_size = sizeof(*head) + num_op*sizeof(*op);
141   - int i;
  134 + size_t msg_size = sizeof(struct ceph_osd_request_head) +
  135 + num_op*sizeof(struct ceph_osd_op);
142 136  
143 137 if (use_mempool) {
144   - req = mempool_alloc(osdc->req_mempool, GFP_NOFS);
  138 + req = mempool_alloc(osdc->req_mempool, gfp_flags);
145 139 memset(req, 0, sizeof(*req));
146 140 } else {
147   - req = kzalloc(sizeof(*req), GFP_NOFS);
  141 + req = kzalloc(sizeof(*req), gfp_flags);
148 142 }
  143 + if (!req)
  144 + return NULL;
  145 +
  146 + if (use_mempool) {
  147 + req = mempool_alloc(osdc->req_mempool, gfp_flags);
  148 + memset(req, 0, sizeof(*req));
  149 + } else {
  150 + req = kzalloc(sizeof(*req), gfp_flags);
  151 + }
149 152 if (req == NULL)
150 153 return NULL;
151 154  
... ... @@ -164,7 +167,7 @@
164 167 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, 0);
165 168 else
166 169 msg = ceph_msg_new(CEPH_MSG_OSD_OPREPLY,
167   - OSD_OPREPLY_FRONT_LEN, GFP_NOFS);
  170 + OSD_OPREPLY_FRONT_LEN, gfp_flags);
168 171 if (!msg) {
169 172 ceph_osdc_put_request(req);
170 173 return NULL;
171 174  
172 175  
... ... @@ -178,18 +181,48 @@
178 181 if (use_mempool)
179 182 msg = ceph_msgpool_get(&osdc->msgpool_op, 0);
180 183 else
181   - msg = ceph_msg_new(CEPH_MSG_OSD_OP, msg_size, GFP_NOFS);
  184 + msg = ceph_msg_new(CEPH_MSG_OSD_OP, msg_size, gfp_flags);
182 185 if (!msg) {
183 186 ceph_osdc_put_request(req);
184 187 return NULL;
185 188 }
186 189 msg->hdr.type = cpu_to_le16(CEPH_MSG_OSD_OP);
187 190 memset(msg->front.iov_base, 0, msg->front.iov_len);
  191 +
  192 + req->r_request = msg;
  193 + req->r_pages = pages;
  194 +
  195 + return req;
  196 +}
  197 +
  198 +/*
  199 + * build new request AND message
  200 + *
  201 + */
  202 +void ceph_osdc_build_request(struct ceph_osd_request *req,
  203 + u64 off, u64 *plen,
  204 + int opcode,
  205 + struct ceph_snap_context *snapc,
  206 + int do_sync,
  207 + u32 truncate_seq,
  208 + u64 truncate_size,
  209 + struct timespec *mtime,
  210 + const char *oid,
  211 + int oid_len)
  212 +{
  213 + struct ceph_msg *msg = req->r_request;
  214 + struct ceph_osd_request_head *head;
  215 + struct ceph_osd_op *op;
  216 + void *p;
  217 + int num_op = 1 + do_sync;
  218 + size_t msg_size = sizeof(*head) + num_op*sizeof(*op);
  219 + int i;
  220 + int flags = req->r_flags;
  221 +
188 222 head = msg->front.iov_base;
189 223 op = (void *)(head + 1);
190 224 p = (void *)(op + num_op);
191 225  
192   - req->r_request = msg;
193 226 req->r_snapc = ceph_get_snap_context(snapc);
194 227  
195 228 head->client_inc = cpu_to_le32(1); /* always, for now. */
... ... @@ -199,10 +232,6 @@
199 232 head->num_ops = cpu_to_le16(num_op);
200 233 op->op = cpu_to_le16(opcode);
201 234  
202   - /* calculate max write size */
203   - calc_layout(osdc, vino, layout, off, plen, req);
204   - req->r_file_layout = *layout; /* keep a copy */
205   -
206 235 if (flags & CEPH_OSD_FLAG_WRITE) {
207 236 req->r_request->hdr.data_off = cpu_to_le16(off);
208 237 req->r_request->hdr.data_len = cpu_to_le32(*plen);
... ... @@ -212,9 +241,9 @@
212 241 op->extent.truncate_seq = cpu_to_le32(truncate_seq);
213 242  
214 243 /* fill in oid */
215   - head->object_len = cpu_to_le32(req->r_oid_len);
216   - memcpy(p, req->r_oid, req->r_oid_len);
217   - p += req->r_oid_len;
  244 + head->object_len = cpu_to_le32(oid_len);
  245 + memcpy(p, oid, oid_len);
  246 + p += oid_len;
218 247  
219 248 if (do_sync) {
220 249 op++;
... ... @@ -233,6 +262,50 @@
233 262 msg_size = p - msg->front.iov_base;
234 263 msg->front.iov_len = msg_size;
235 264 msg->hdr.front_len = cpu_to_le32(msg_size);
  265 + return;
  266 +}
  267 +
  268 +/*
  269 + * build new request AND message, calculate layout, and adjust file
  270 + * extent as needed.
  271 + *
  272 + * if the file was recently truncated, we include information about its
  273 + * old and new size so that the object can be updated appropriately. (we
  274 + * avoid synchronously deleting truncated objects because it's slow.)
  275 + *
  276 + * if @do_sync, include a 'startsync' command so that the osd will flush
  277 + * data quickly.
  278 + */
  279 +struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
  280 + struct ceph_file_layout *layout,
  281 + struct ceph_vino vino,
  282 + u64 off, u64 *plen,
  283 + int opcode, int flags,
  284 + struct ceph_snap_context *snapc,
  285 + int do_sync,
  286 + u32 truncate_seq,
  287 + u64 truncate_size,
  288 + struct timespec *mtime,
  289 + bool use_mempool, int num_reply)
  290 +{
  291 + struct ceph_osd_request *req =
  292 + ceph_osdc_alloc_request(osdc, flags,
  293 + snapc, do_sync,
  294 + use_mempool,
  295 + GFP_NOFS, NULL);
  296 + if (IS_ERR(req))
  297 + return req;
  298 +
  299 + /* calculate max write size */
  300 + calc_layout(osdc, vino, layout, off, plen, req);
  301 + req->r_file_layout = *layout; /* keep a copy */
  302 +
  303 + ceph_osdc_build_request(req, off, plen, opcode,
  304 + snapc, do_sync,
  305 + truncate_seq, truncate_size,
  306 + mtime,
  307 + req->r_oid, req->r_oid_len);
  308 +
236 309 return req;
237 310 }
238 311  
fs/ceph/osd_client.h
... ... @@ -119,6 +119,31 @@
119 119 extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
120 120 struct ceph_msg *msg);
121 121  
  122 +extern void ceph_calc_raw_layout(struct ceph_osd_client *osdc,
  123 + struct ceph_file_layout *layout,
  124 + u64 snapid,
  125 + u64 off, u64 len, u64 *bno,
  126 + struct ceph_osd_request *req);
  127 +
  128 +extern struct ceph_osd_request *ceph_osdc_alloc_request(struct ceph_osd_client *osdc,
  129 + int flags,
  130 + struct ceph_snap_context *snapc,
  131 + int do_sync,
  132 + bool use_mempool,
  133 + gfp_t gfp_flags,
  134 + struct page **pages);
  135 +
  136 +extern void ceph_osdc_build_request(struct ceph_osd_request *req,
  137 + u64 off, u64 *plen,
  138 + int opcode,
  139 + struct ceph_snap_context *snapc,
  140 + int do_sync,
  141 + u32 truncate_seq,
  142 + u64 truncate_size,
  143 + struct timespec *mtime,
  144 + const char *oid,
  145 + int oid_len);
  146 +
122 147 extern struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *,
123 148 struct ceph_file_layout *layout,
124 149 struct ceph_vino vino,