mm: extend prefault helpers to fault in more than PAGE_SIZE

drm/i915 wants to read/write more than one page in its fastpath and hence needs to prefault more than PAGE_SIZE bytes. Add new functions in filemap.h to make that possible. Also kill a copy&pasted spurious space in both functions while at it. v2: As suggested by Andrew Morton, add a multipage parameter to both functions to avoid the additional branch for the pagemap.c hotpath. My gcc 4.6 here seems to dtrt and indeed reap these branches where not needed. v3: Becaus I couldn't find a way around adding a uaddr += PAGE_SIZE to the filemap.c hotpaths (that the compiler couldn't remove again), let's go with separate new functions for the multipage use-case. v4: Adjust comment to CodingStlye and fix spelling. Acked-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>

mm: extend prefault helpers to fault in more than PAGE_SIZE
drm/i915 wants to read/write more than one page in its fastpath and hence needs to prefault more than PAGE_SIZE bytes. Add new functions in filemap.h to make that possible. Also kill a copy&pasted spurious space in both functions while at it. v2: As suggested by Andrew Morton, add a multipage parameter to both functions to avoid the additional branch for the pagemap.c hotpath. My gcc 4.6 here seems to dtrt and indeed reap these branches where not needed. v3: Becaus I couldn't find a way around adding a uaddr += PAGE_SIZE to the filemap.c hotpaths (that the compiler couldn't remove again), let's go with separate new functions for the multipage use-case. v4: Adjust comment to CodingStlye and fix spelling. Acked-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Daniel Vetter
1 parent d174bd6472
Showing 3 changed files with 66 additions and 6 deletions Side-by-side Diff
drivers/gpu/drm/i915/i915_gem.c
drivers/gpu/drm/i915/i915_gem_execbuffer.c
include/linux/pagemap.h
@@ -416,7 +416,7 @@
 		mutex_unlock(&dev->struct_mutex);
  
 		if (!prefaulted) {
-			ret = fault_in_pages_writeable(user_data, remain);
+			ret = fault_in_multipages_writeable(user_data, remain);
 			/* Userspace is tricking us, but we've already clobbered
 			 * its pages with the prefault and promised to write the
 			 * data up to the first fault. Hence ignore any errors
@@ -809,8 +809,8 @@
 		       args->size))
 		return -EFAULT;
  
-	ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
-				      args->size);
+	ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
+					   args->size);
 	if (ret)
 		return -EFAULT;
  
@@ -997,7 +997,7 @@
 		if (!access_ok(VERIFY_WRITE, ptr, length))
 			return -EFAULT;
  
-		if (fault_in_pages_readable(ptr, length))
+		if (fault_in_multipages_readable(ptr, length))
 			return -EFAULT;
 	}
  
@@ -426,7 +426,7 @@
 		 */
 		if (((unsigned long)uaddr & PAGE_MASK) !=
 				((unsigned long)end & PAGE_MASK))
-		 	ret = __put_user(0, end);
+			ret = __put_user(0, end);
 	}
 	return ret;
 }
  
@@ -445,10 +445,70 @@
  
 		if (((unsigned long)uaddr & PAGE_MASK) !=
 				((unsigned long)end & PAGE_MASK)) {
-		 	ret = __get_user(c, end);
+			ret = __get_user(c, end);
 			(void)c;
 		}
 	}
+	return ret;
+}
+
+/*
+ * Multipage variants of the above prefault helpers, useful if more than
+ * PAGE_SIZE of data needs to be prefaulted. These are separate from the above
+ * functions (which only handle up to PAGE_SIZE) to avoid clobbering the
+ * filemap.c hotpaths.
+ */
+static inline int fault_in_multipages_writeable(char __user *uaddr, int size)
+{
+	int ret;
+	const char __user *end = uaddr + size - 1;
+
+	if (unlikely(size == 0))
+		return 0;
+
+	/*
+	 * Writing zeroes into userspace here is OK, because we know that if
+	 * the zero gets there, we'll be overwriting it.
+	 */
+	while (uaddr <= end) {
+		ret = __put_user(0, uaddr);
+		if (ret != 0)
+			return ret;
+		uaddr += PAGE_SIZE;
+	}
+
+	/* Check whether the range spilled into the next page. */
+	if (((unsigned long)uaddr & PAGE_MASK) ==
+			((unsigned long)end & PAGE_MASK))
+		ret = __put_user(0, end);
+
+	return ret;
+}
+
+static inline int fault_in_multipages_readable(const char __user *uaddr,
+					       int size)
+{
+	volatile char c;
+	int ret;
+	const char __user *end = uaddr + size - 1;
+
+	if (unlikely(size == 0))
+		return 0;
+
+	while (uaddr <= end) {
+		ret = __get_user(c, uaddr);
+		if (ret != 0)
+			return ret;
+		uaddr += PAGE_SIZE;
+	}
+
+	/* Check whether the range spilled into the next page. */
+	if (((unsigned long)uaddr & PAGE_MASK) ==
+			((unsigned long)end & PAGE_MASK)) {
+		ret = __get_user(c, end);
+		(void)c;
+	}
+
 	return ret;
 }
...	...	@@ -416,7 +416,7 @@
416	416	mutex_unlock(&dev->struct_mutex);
417	417
418	418	if (!prefaulted) {
419		- ret = fault_in_pages_writeable(user_data, remain);
	419	+ ret = fault_in_multipages_writeable(user_data, remain);
420	420	/* Userspace is tricking us, but we've already clobbered
421	421	* its pages with the prefault and promised to write the
422	422	* data up to the first fault. Hence ignore any errors
...	...	@@ -809,8 +809,8 @@
809	809	args->size))
810	810	return -EFAULT;
811	811
812		- ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
813		- args->size);
	812	+ ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
	813	+ args->size);
814	814	if (ret)
815	815	return -EFAULT;
816	816
...	...	@@ -997,7 +997,7 @@
997	997	if (!access_ok(VERIFY_WRITE, ptr, length))
998	998	return -EFAULT;
999	999
1000		- if (fault_in_pages_readable(ptr, length))
	1000	+ if (fault_in_multipages_readable(ptr, length))
1001	1001	return -EFAULT;
1002	1002	}
1003	1003
...	...	@@ -426,7 +426,7 @@
426	426	*/
427	427	if (((unsigned long)uaddr & PAGE_MASK) !=
428	428	((unsigned long)end & PAGE_MASK))
429		- ret = __put_user(0, end);
	429	+ ret = __put_user(0, end);
430	430	}
431	431	return ret;
432	432	}
433	433
...	...	@@ -445,10 +445,70 @@
445	445
446	446	if (((unsigned long)uaddr & PAGE_MASK) !=
447	447	((unsigned long)end & PAGE_MASK)) {
448		- ret = __get_user(c, end);
	448	+ ret = __get_user(c, end);
449	449	(void)c;
450	450	}
451	451	}
	452	+ return ret;
	453	+}
	454	+
	455	+/*
	456	+ * Multipage variants of the above prefault helpers, useful if more than
	457	+ * PAGE_SIZE of data needs to be prefaulted. These are separate from the above
	458	+ * functions (which only handle up to PAGE_SIZE) to avoid clobbering the
	459	+ * filemap.c hotpaths.
	460	+ */
	461	+static inline int fault_in_multipages_writeable(char __user *uaddr, int size)
	462	+{
	463	+ int ret;
	464	+ const char __user *end = uaddr + size - 1;
	465	+
	466	+ if (unlikely(size == 0))
	467	+ return 0;
	468	+
	469	+ /*
	470	+ * Writing zeroes into userspace here is OK, because we know that if
	471	+ * the zero gets there, we'll be overwriting it.
	472	+ */
	473	+ while (uaddr <= end) {
	474	+ ret = __put_user(0, uaddr);
	475	+ if (ret != 0)
	476	+ return ret;
	477	+ uaddr += PAGE_SIZE;
	478	+ }
	479	+
	480	+ /* Check whether the range spilled into the next page. */
	481	+ if (((unsigned long)uaddr & PAGE_MASK) ==
	482	+ ((unsigned long)end & PAGE_MASK))
	483	+ ret = __put_user(0, end);
	484	+
	485	+ return ret;
	486	+}
	487	+
	488	+static inline int fault_in_multipages_readable(const char __user *uaddr,
	489	+ int size)
	490	+{
	491	+ volatile char c;
	492	+ int ret;
	493	+ const char __user *end = uaddr + size - 1;
	494	+
	495	+ if (unlikely(size == 0))
	496	+ return 0;
	497	+
	498	+ while (uaddr <= end) {
	499	+ ret = __get_user(c, uaddr);
	500	+ if (ret != 0)
	501	+ return ret;
	502	+ uaddr += PAGE_SIZE;
	503	+ }
	504	+
	505	+ /* Check whether the range spilled into the next page. */
	506	+ if (((unsigned long)uaddr & PAGE_MASK) ==
	507	+ ((unsigned long)end & PAGE_MASK)) {
	508	+ ret = __get_user(c, end);
	509	+ (void)c;
	510	+ }
	511	+
452	512	return ret;
453	513	}
454	514