ceph: fix file lock interruption

When a lock operation is interrupted, current code sends a unlock request to MDS to undo the lock operation. This method does not work as expected because the unlock request can drop locks that have already been acquired. The fix is use the newly introduced CEPH_LOCK_FCNTL_INTR/CEPH_LOCK_FLOCK_INTR requests to interrupt blocked file lock request. These requests do not drop locks that have alread been acquired, they only interrupt blocked file lock request. Signed-off-by: Yan, Zheng <zyan@redhat.com>

ceph: fix file lock interruption
When a lock operation is interrupted, current code sends a unlock request to MDS to undo the lock operation. This method does not work as expected because the unlock request can drop locks that have already been acquired. The fix is use the newly introduced CEPH_LOCK_FCNTL_INTR/CEPH_LOCK_FLOCK_INTR requests to interrupt blocked file lock request. These requests do not drop locks that have alread been acquired, they only interrupt blocked file lock request. Signed-off-by: Yan, Zheng <zyan@redhat.com>
Yan, Zheng · Ilya Dryomov
1 parent b2776bf714
Showing 4 changed files with 67 additions and 12 deletions Side-by-side Diff
fs/ceph/locks.c
fs/ceph/mds_client.c
fs/ceph/mds_client.h
include/linux/ceph/ceph_fs.h
@@ -9,6 +9,8 @@
 #include <linux/ceph/pagelist.h>
  
 static u64 lock_secret;
+static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc,
+                                         struct ceph_mds_request *req);
  
 static inline u64 secure_addr(void *addr)
 {
@@ -40,6 +42,9 @@
 	u64 length = 0;
 	u64 owner;
  
+	if (operation != CEPH_MDS_OP_SETFILELOCK || cmd == CEPH_LOCK_UNLOCK)
+		wait = 0;
+
 	req = ceph_mdsc_create_request(mdsc, operation, USE_AUTH_MDS);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
@@ -68,6 +73,9 @@
 	req->r_args.filelock_change.length = cpu_to_le64(length);
 	req->r_args.filelock_change.wait = wait;
  
+	if (wait)
+		req->r_wait_for_completion = ceph_lock_wait_for_completion;
+
 	err = ceph_mdsc_do_request(mdsc, inode, req);
  
 	if (operation == CEPH_MDS_OP_GETFILELOCK) {
@@ -96,6 +104,52 @@
 	return err;
 }
  
+static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc,
+                                         struct ceph_mds_request *req)
+{
+	struct ceph_mds_request *intr_req;
+	struct inode *inode = req->r_inode;
+	int err, lock_type;
+
+	BUG_ON(req->r_op != CEPH_MDS_OP_SETFILELOCK);
+	if (req->r_args.filelock_change.rule == CEPH_LOCK_FCNTL)
+		lock_type = CEPH_LOCK_FCNTL_INTR;
+	else if (req->r_args.filelock_change.rule == CEPH_LOCK_FLOCK)
+		lock_type = CEPH_LOCK_FLOCK_INTR;
+	else
+		BUG_ON(1);
+	BUG_ON(req->r_args.filelock_change.type == CEPH_LOCK_UNLOCK);
+
+	err = wait_for_completion_interruptible(&req->r_completion);
+	if (!err)
+		return 0;
+
+	dout("ceph_lock_wait_for_completion: request %llu was interrupted\n",
+	     req->r_tid);
+
+	intr_req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETFILELOCK,
+					    USE_AUTH_MDS);
+	if (IS_ERR(intr_req))
+		return PTR_ERR(intr_req);
+
+	intr_req->r_inode = inode;
+	ihold(inode);
+	intr_req->r_num_caps = 1;
+
+	intr_req->r_args.filelock_change = req->r_args.filelock_change;
+	intr_req->r_args.filelock_change.rule = lock_type;
+	intr_req->r_args.filelock_change.type = CEPH_LOCK_UNLOCK;
+
+	err = ceph_mdsc_do_request(mdsc, inode, intr_req);
+	ceph_mdsc_put_request(intr_req);
+
+	if (err && err != -ERESTARTSYS)
+		return err;
+
+	wait_for_completion(&req->r_completion);
+	return 0;
+}
+
 /**
  * Attempt to set an fcntl lock.
  * For now, this just goes away to the server. Later it may be more awesome.
@@ -143,11 +197,6 @@
 				     err);
 			}
 		}
-
-	} else if (err == -ERESTARTSYS) {
-		dout("undoing lock\n");
-		ceph_lock_message(CEPH_LOCK_FCNTL, op, file,
-				  CEPH_LOCK_UNLOCK, 0, fl);
 	}
 	return err;
 }
@@ -186,11 +235,6 @@
 					  file, CEPH_LOCK_UNLOCK, 0, fl);
 			dout("got %d on flock_lock_file_wait, undid lock", err);
 		}
-	} else if (err == -ERESTARTSYS) {
-		dout("undoing lock\n");
-		ceph_lock_message(CEPH_LOCK_FLOCK,
-				  CEPH_MDS_OP_SETFILELOCK,
-				  file, CEPH_LOCK_UNLOCK, 0, fl);
 	}
 	return err;
 }
@@ -2208,6 +2208,8 @@
 			&req->r_completion, req->r_timeout);
 		if (err == 0)
 			err = -EIO;
+	} else if (req->r_wait_for_completion) {
+		err = req->r_wait_for_completion(mdsc, req);
 	} else {
 		err = wait_for_completion_killable(&req->r_completion);
 	}
@@ -166,6 +166,11 @@
  */
 typedef void (*ceph_mds_request_callback_t) (struct ceph_mds_client *mdsc,
 					     struct ceph_mds_request *req);
+/*
+ * wait for request completion callback
+ */
+typedef int (*ceph_mds_request_wait_callback_t) (struct ceph_mds_client *mdsc,
+						 struct ceph_mds_request *req);
  
 /*
  * an in-flight mds request
@@ -239,6 +244,7 @@
 	struct completion r_completion;
 	struct completion r_safe_completion;
 	ceph_mds_request_callback_t r_callback;
+	ceph_mds_request_wait_callback_t r_wait_for_completion;
 	struct list_head  r_unsafe_item;  /* per-session unsafe list item */
 	bool		  r_got_unsafe, r_got_safe, r_got_result;
  
@@ -522,8 +522,11 @@
 	__le32 dist[];
 } __attribute__ ((packed));
  
-#define CEPH_LOCK_FCNTL    1
-#define CEPH_LOCK_FLOCK    2
+#define CEPH_LOCK_FCNTL		1
+#define CEPH_LOCK_FLOCK		2
+#define CEPH_LOCK_FCNTL_INTR    3
+#define CEPH_LOCK_FLOCK_INTR    4
+
  
 #define CEPH_LOCK_SHARED   1
 #define CEPH_LOCK_EXCL     2
...	...	@@ -9,6 +9,8 @@
9	9	#include <linux/ceph/pagelist.h>
10	10
11	11	static u64 lock_secret;
	12	+static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc,
	13	+ struct ceph_mds_request *req);
12	14
13	15	static inline u64 secure_addr(void *addr)
14	16	{
...	...	@@ -40,6 +42,9 @@
40	42	u64 length = 0;
41	43	u64 owner;
42	44
	45	+ if (operation != CEPH_MDS_OP_SETFILELOCK \|\| cmd == CEPH_LOCK_UNLOCK)
	46	+ wait = 0;
	47	+
43	48	req = ceph_mdsc_create_request(mdsc, operation, USE_AUTH_MDS);
44	49	if (IS_ERR(req))
45	50	return PTR_ERR(req);
...	...	@@ -68,6 +73,9 @@
68	73	req->r_args.filelock_change.length = cpu_to_le64(length);
69	74	req->r_args.filelock_change.wait = wait;
70	75
	76	+ if (wait)
	77	+ req->r_wait_for_completion = ceph_lock_wait_for_completion;
	78	+
71	79	err = ceph_mdsc_do_request(mdsc, inode, req);
72	80
73	81	if (operation == CEPH_MDS_OP_GETFILELOCK) {
...	...	@@ -96,6 +104,52 @@
96	104	return err;
97	105	}
98	106
	107	+static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc,
	108	+ struct ceph_mds_request *req)
	109	+{
	110	+ struct ceph_mds_request *intr_req;
	111	+ struct inode *inode = req->r_inode;
	112	+ int err, lock_type;
	113	+
	114	+ BUG_ON(req->r_op != CEPH_MDS_OP_SETFILELOCK);
	115	+ if (req->r_args.filelock_change.rule == CEPH_LOCK_FCNTL)
	116	+ lock_type = CEPH_LOCK_FCNTL_INTR;
	117	+ else if (req->r_args.filelock_change.rule == CEPH_LOCK_FLOCK)
	118	+ lock_type = CEPH_LOCK_FLOCK_INTR;
	119	+ else
	120	+ BUG_ON(1);
	121	+ BUG_ON(req->r_args.filelock_change.type == CEPH_LOCK_UNLOCK);
	122	+
	123	+ err = wait_for_completion_interruptible(&req->r_completion);
	124	+ if (!err)
	125	+ return 0;
	126	+
	127	+ dout("ceph_lock_wait_for_completion: request %llu was interrupted\n",
	128	+ req->r_tid);
	129	+
	130	+ intr_req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETFILELOCK,
	131	+ USE_AUTH_MDS);
	132	+ if (IS_ERR(intr_req))
	133	+ return PTR_ERR(intr_req);
	134	+
	135	+ intr_req->r_inode = inode;
	136	+ ihold(inode);
	137	+ intr_req->r_num_caps = 1;
	138	+
	139	+ intr_req->r_args.filelock_change = req->r_args.filelock_change;
	140	+ intr_req->r_args.filelock_change.rule = lock_type;
	141	+ intr_req->r_args.filelock_change.type = CEPH_LOCK_UNLOCK;
	142	+
	143	+ err = ceph_mdsc_do_request(mdsc, inode, intr_req);
	144	+ ceph_mdsc_put_request(intr_req);
	145	+
	146	+ if (err && err != -ERESTARTSYS)
	147	+ return err;
	148	+
	149	+ wait_for_completion(&req->r_completion);
	150	+ return 0;
	151	+}
	152	+
99	153	/**
100	154	* Attempt to set an fcntl lock.
101	155	* For now, this just goes away to the server. Later it may be more awesome.
...	...	@@ -143,11 +197,6 @@
143	197	err);
144	198	}
145	199	}
146		-
147		- } else if (err == -ERESTARTSYS) {
148		- dout("undoing lock\n");
149		- ceph_lock_message(CEPH_LOCK_FCNTL, op, file,
150		- CEPH_LOCK_UNLOCK, 0, fl);
151	200	}
152	201	return err;
153	202	}
...	...	@@ -186,11 +235,6 @@
186	235	file, CEPH_LOCK_UNLOCK, 0, fl);
187	236	dout("got %d on flock_lock_file_wait, undid lock", err);
188	237	}
189		- } else if (err == -ERESTARTSYS) {
190		- dout("undoing lock\n");
191		- ceph_lock_message(CEPH_LOCK_FLOCK,
192		- CEPH_MDS_OP_SETFILELOCK,
193		- file, CEPH_LOCK_UNLOCK, 0, fl);
194	238	}
195	239	return err;
196	240	}
...	...	@@ -2208,6 +2208,8 @@
2208	2208	&req->r_completion, req->r_timeout);
2209	2209	if (err == 0)
2210	2210	err = -EIO;
	2211	+ } else if (req->r_wait_for_completion) {
	2212	+ err = req->r_wait_for_completion(mdsc, req);
2211	2213	} else {
2212	2214	err = wait_for_completion_killable(&req->r_completion);
2213	2215	}
...	...	@@ -166,6 +166,11 @@
166	166	*/
167	167	typedef void (ceph_mds_request_callback_t) (struct ceph_mds_client mdsc,
168	168	struct ceph_mds_request *req);
	169	+/*
	170	+ * wait for request completion callback
	171	+ */
	172	+typedef int (ceph_mds_request_wait_callback_t) (struct ceph_mds_client mdsc,
	173	+ struct ceph_mds_request *req);
169	174
170	175	/*
171	176	* an in-flight mds request
...	...	@@ -239,6 +244,7 @@
239	244	struct completion r_completion;
240	245	struct completion r_safe_completion;
241	246	ceph_mds_request_callback_t r_callback;
	247	+ ceph_mds_request_wait_callback_t r_wait_for_completion;
242	248	struct list_head r_unsafe_item; /* per-session unsafe list item */
243	249	bool r_got_unsafe, r_got_safe, r_got_result;
244	250
...	...	@@ -522,8 +522,11 @@
522	522	__le32 dist[];
523	523	} __attribute__ ((packed));
524	524
525		-#define CEPH_LOCK_FCNTL 1
526		-#define CEPH_LOCK_FLOCK 2
	525	+#define CEPH_LOCK_FCNTL 1
	526	+#define CEPH_LOCK_FLOCK 2
	527	+#define CEPH_LOCK_FCNTL_INTR 3
	528	+#define CEPH_LOCK_FLOCK_INTR 4
	529	+
527	530
528	531	#define CEPH_LOCK_SHARED 1
529	532	#define CEPH_LOCK_EXCL 2