hugetlbfs: fix mmap failure in unaligned size request

The current kernel returns -EINVAL unless a given mmap length is "almost" hugepage aligned. This is because in sys_mmap_pgoff() the given length is passed to vm_mmap_pgoff() as it is without being aligned with hugepage boundary. This is a regression introduced in commit 40716e29243d ("hugetlbfs: fix alignment of huge page requests"), where alignment code is pushed into hugetlb_file_setup() and the variable len in caller side is not changed. To fix this, this patch partially reverts that commit, and adds alignment code in caller side. And it also introduces hstate_sizelog() in order to get proper hstate to specified hugepage size. Addresses https://bugzilla.kernel.org/show_bug.cgi?id=56881 [akpm@linux-foundation.org: fix warning when CONFIG_HUGETLB_PAGE=n] Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reported-by: <iceman_dvd@yahoo.com> Cc: Steven Truelove <steven.truelove@utoronto.ca> Cc: Jianguo Wu <wujianguo@huawei.com> Cc: Hugh Dickins <hughd@google.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

hugetlbfs: fix mmap failure in unaligned size request
The current kernel returns -EINVAL unless a given mmap length is "almost" hugepage aligned. This is because in sys_mmap_pgoff() the given length is passed to vm_mmap_pgoff() as it is without being aligned with hugepage boundary. This is a regression introduced in commit 40716e29243d ("hugetlbfs: fix alignment of huge page requests"), where alignment code is pushed into hugetlb_file_setup() and the variable len in caller side is not changed. To fix this, this patch partially reverts that commit, and adds alignment code in caller side. And it also introduces hstate_sizelog() in order to get proper hstate to specified hugepage size. Addresses https://bugzilla.kernel.org/show_bug.cgi?id=56881 [akpm@linux-foundation.org: fix warning when CONFIG_HUGETLB_PAGE=n] Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Reported-by: <iceman_dvd@yahoo.com> Cc: Steven Truelove <steven.truelove@utoronto.ca> Cc: Jianguo Wu <wujianguo@huawei.com> Cc: Hugh Dickins <hughd@google.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Naoya Horiguchi · Linus Torvalds
1 parent 1ab4ce7623
Showing 4 changed files with 34 additions and 22 deletions Side-by-side Diff
fs/hugetlbfs/inode.c
include/linux/hugetlb.h
ipc/shm.c
mm/mmap.c
@@ -909,11 +909,8 @@
  
 static int get_hstate_idx(int page_size_log)
 {
-	struct hstate *h;
+	struct hstate *h = hstate_sizelog(page_size_log);
  
-	if (!page_size_log)
-		return default_hstate_idx;
-	h = size_to_hstate(1 << page_size_log);
 	if (!h)
 		return -1;
 	return h - hstates;
@@ -929,9 +926,12 @@
 	.d_dname = hugetlb_dname
 };
  
-struct file *hugetlb_file_setup(const char *name, unsigned long addr,
-				size_t size, vm_flags_t acctflag,
-				struct user_struct **user,
+/*
+ * Note that size should be aligned to proper hugepage size in caller side,
+ * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
+ */
+struct file *hugetlb_file_setup(const char *name, size_t size,
+				vm_flags_t acctflag, struct user_struct **user,
 				int creat_flags, int page_size_log)
 {
 	struct file *file = ERR_PTR(-ENOMEM);
@@ -939,8 +939,6 @@
 	struct path path;
 	struct super_block *sb;
 	struct qstr quick_string;
-	struct hstate *hstate;
-	unsigned long num_pages;
 	int hstate_idx;
  
 	hstate_idx = get_hstate_idx(page_size_log);
  
@@ -980,12 +978,10 @@
 	if (!inode)
 		goto out_dentry;
  
-	hstate = hstate_inode(inode);
-	size += addr & ~huge_page_mask(hstate);
-	num_pages = ALIGN(size, huge_page_size(hstate)) >>
-			huge_page_shift(hstate);
 	file = ERR_PTR(-ENOMEM);
-	if (hugetlb_reserve_pages(inode, 0, num_pages, NULL, acctflag))
+	if (hugetlb_reserve_pages(inode, 0,
+			size >> huge_page_shift(hstate_inode(inode)), NULL,
+			acctflag))
 		goto out_inode;
  
 	d_instantiate(path.dentry, inode);
@@ -189,8 +189,7 @@
  
 extern const struct file_operations hugetlbfs_file_operations;
 extern const struct vm_operations_struct hugetlb_vm_ops;
-struct file *hugetlb_file_setup(const char *name, unsigned long addr,
-				size_t size, vm_flags_t acct,
+struct file *hugetlb_file_setup(const char *name, size_t size, vm_flags_t acct,
 				struct user_struct **user, int creat_flags,
 				int page_size_log);
  
@@ -209,8 +208,8 @@
  
 #define is_file_hugepages(file)			0
 static inline struct file *
-hugetlb_file_setup(const char *name, unsigned long addr, size_t size,
-		vm_flags_t acctflag, struct user_struct **user, int creat_flags,
+hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
+		struct user_struct **user, int creat_flags,
 		int page_size_log)
 {
 	return ERR_PTR(-ENOSYS);
@@ -288,6 +287,13 @@
 	return hstate_inode(file_inode(f));
 }
  
+static inline struct hstate *hstate_sizelog(int page_size_log)
+{
+	if (!page_size_log)
+		return &default_hstate;
+	return size_to_hstate(1 << page_size_log);
+}
+
 static inline struct hstate *hstate_vma(struct vm_area_struct *vma)
 {
 	return hstate_file(vma->vm_file);
  
@@ -352,11 +358,12 @@
 	return h - hstates;
 }
  
-#else
+#else	/* CONFIG_HUGETLB_PAGE */
 struct hstate {};
 #define alloc_huge_page_node(h, nid) NULL
 #define alloc_bootmem_huge_page(h) NULL
 #define hstate_file(f) NULL
+#define hstate_sizelog(s) NULL
 #define hstate_vma(v) NULL
 #define hstate_inode(i) NULL
 #define huge_page_size(h) PAGE_SIZE
@@ -371,7 +378,7 @@
 }
 #define hstate_index_to_shift(index) 0
 #define hstate_index(h) 0
-#endif
+#endif	/* CONFIG_HUGETLB_PAGE */
  
 #endif /* _LINUX_HUGETLB_H */
@@ -491,10 +491,14 @@
  
 	sprintf (name, "SYSV%08x", key);
 	if (shmflg & SHM_HUGETLB) {
+		struct hstate *hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT)
+						& SHM_HUGE_MASK);
+		size_t hugesize = ALIGN(size, huge_page_size(hs));
+
 		/* hugetlb_file_setup applies strict accounting */
 		if (shmflg & SHM_NORESERVE)
 			acctflag = VM_NORESERVE;
-		file = hugetlb_file_setup(name, 0, size, acctflag,
+		file = hugetlb_file_setup(name, hugesize, acctflag,
 				  &shp->mlock_user, HUGETLB_SHMFS_INODE,
 				(shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
 	} else {
@@ -1363,15 +1363,20 @@
 		file = fget(fd);
 		if (!file)
 			goto out;
+		if (is_file_hugepages(file))
+			len = ALIGN(len, huge_page_size(hstate_file(file)));
 	} else if (flags & MAP_HUGETLB) {
 		struct user_struct *user = NULL;
+
+		len = ALIGN(len, huge_page_size(hstate_sizelog(
+			(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK)));
 		/*
 		 * VM_NORESERVE is used because the reservations will be
 		 * taken when vm_ops->mmap() is called
 		 * A dummy user value is used because we are not locking
 		 * memory so no accounting is necessary
 		 */
-		file = hugetlb_file_setup(HUGETLB_ANON_FILE, addr, len,
+		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
 				VM_NORESERVE,
 				&user, HUGETLB_ANONHUGE_INODE,
 				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
...	...	@@ -909,11 +909,8 @@
909	909
910	910	static int get_hstate_idx(int page_size_log)
911	911	{
912		- struct hstate *h;
	912	+ struct hstate *h = hstate_sizelog(page_size_log);
913	913
914		- if (!page_size_log)
915		- return default_hstate_idx;
916		- h = size_to_hstate(1 << page_size_log);
917	914	if (!h)
918	915	return -1;
919	916	return h - hstates;
...	...	@@ -929,9 +926,12 @@
929	926	.d_dname = hugetlb_dname
930	927	};
931	928
932		-struct file hugetlb_file_setup(const char name, unsigned long addr,
933		- size_t size, vm_flags_t acctflag,
934		- struct user_struct **user,
	929	+/*
	930	+ * Note that size should be aligned to proper hugepage size in caller side,
	931	+ * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
	932	+ */
	933	+struct file hugetlb_file_setup(const char name, size_t size,
	934	+ vm_flags_t acctflag, struct user_struct **user,
935	935	int creat_flags, int page_size_log)
936	936	{
937	937	struct file *file = ERR_PTR(-ENOMEM);
...	...	@@ -939,8 +939,6 @@
939	939	struct path path;
940	940	struct super_block *sb;
941	941	struct qstr quick_string;
942		- struct hstate *hstate;
943		- unsigned long num_pages;
944	942	int hstate_idx;
945	943
946	944	hstate_idx = get_hstate_idx(page_size_log);
947	945
...	...	@@ -980,12 +978,10 @@
980	978	if (!inode)
981	979	goto out_dentry;
982	980
983		- hstate = hstate_inode(inode);
984		- size += addr & ~huge_page_mask(hstate);
985		- num_pages = ALIGN(size, huge_page_size(hstate)) >>
986		- huge_page_shift(hstate);
987	981	file = ERR_PTR(-ENOMEM);
988		- if (hugetlb_reserve_pages(inode, 0, num_pages, NULL, acctflag))
	982	+ if (hugetlb_reserve_pages(inode, 0,
	983	+ size >> huge_page_shift(hstate_inode(inode)), NULL,
	984	+ acctflag))
989	985	goto out_inode;
990	986
991	987	d_instantiate(path.dentry, inode);
...	...	@@ -189,8 +189,7 @@
189	189
190	190	extern const struct file_operations hugetlbfs_file_operations;
191	191	extern const struct vm_operations_struct hugetlb_vm_ops;
192		-struct file hugetlb_file_setup(const char name, unsigned long addr,
193		- size_t size, vm_flags_t acct,
	192	+struct file hugetlb_file_setup(const char name, size_t size, vm_flags_t acct,
194	193	struct user_struct **user, int creat_flags,
195	194	int page_size_log);
196	195
...	...	@@ -209,8 +208,8 @@
209	208
210	209	#define is_file_hugepages(file) 0
211	210	static inline struct file *
212		-hugetlb_file_setup(const char *name, unsigned long addr, size_t size,
213		- vm_flags_t acctflag, struct user_struct **user, int creat_flags,
	211	+hugetlb_file_setup(const char *name, size_t size, vm_flags_t acctflag,
	212	+ struct user_struct **user, int creat_flags,
214	213	int page_size_log)
215	214	{
216	215	return ERR_PTR(-ENOSYS);
...	...	@@ -288,6 +287,13 @@
288	287	return hstate_inode(file_inode(f));
289	288	}
290	289
	290	+static inline struct hstate *hstate_sizelog(int page_size_log)
	291	+{
	292	+ if (!page_size_log)
	293	+ return &default_hstate;
	294	+ return size_to_hstate(1 << page_size_log);
	295	+}
	296	+
291	297	static inline struct hstate hstate_vma(struct vm_area_struct vma)
292	298	{
293	299	return hstate_file(vma->vm_file);
294	300
...	...	@@ -352,11 +358,12 @@
352	358	return h - hstates;
353	359	}
354	360
355		-#else
	361	+#else /* CONFIG_HUGETLB_PAGE */
356	362	struct hstate {};
357	363	#define alloc_huge_page_node(h, nid) NULL
358	364	#define alloc_bootmem_huge_page(h) NULL
359	365	#define hstate_file(f) NULL
	366	+#define hstate_sizelog(s) NULL
360	367	#define hstate_vma(v) NULL
361	368	#define hstate_inode(i) NULL
362	369	#define huge_page_size(h) PAGE_SIZE
...	...	@@ -371,7 +378,7 @@
371	378	}
372	379	#define hstate_index_to_shift(index) 0
373	380	#define hstate_index(h) 0
374		-#endif
	381	+#endif /* CONFIG_HUGETLB_PAGE */
375	382
376	383	#endif /* _LINUX_HUGETLB_H */
...	...	@@ -491,10 +491,14 @@
491	491
492	492	sprintf (name, "SYSV%08x", key);
493	493	if (shmflg & SHM_HUGETLB) {
	494	+ struct hstate *hs = hstate_sizelog((shmflg >> SHM_HUGE_SHIFT)
	495	+ & SHM_HUGE_MASK);
	496	+ size_t hugesize = ALIGN(size, huge_page_size(hs));
	497	+
494	498	/* hugetlb_file_setup applies strict accounting */
495	499	if (shmflg & SHM_NORESERVE)
496	500	acctflag = VM_NORESERVE;
497		- file = hugetlb_file_setup(name, 0, size, acctflag,
	501	+ file = hugetlb_file_setup(name, hugesize, acctflag,
498	502	&shp->mlock_user, HUGETLB_SHMFS_INODE,
499	503	(shmflg >> SHM_HUGE_SHIFT) & SHM_HUGE_MASK);
500	504	} else {
...	...	@@ -1363,15 +1363,20 @@
1363	1363	file = fget(fd);
1364	1364	if (!file)
1365	1365	goto out;
	1366	+ if (is_file_hugepages(file))
	1367	+ len = ALIGN(len, huge_page_size(hstate_file(file)));
1366	1368	} else if (flags & MAP_HUGETLB) {
1367	1369	struct user_struct *user = NULL;
	1370	+
	1371	+ len = ALIGN(len, huge_page_size(hstate_sizelog(
	1372	+ (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK)));
1368	1373	/*
1369	1374	* VM_NORESERVE is used because the reservations will be
1370	1375	* taken when vm_ops->mmap() is called
1371	1376	* A dummy user value is used because we are not locking
1372	1377	* memory so no accounting is necessary
1373	1378	*/
1374		- file = hugetlb_file_setup(HUGETLB_ANON_FILE, addr, len,
	1379	+ file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1375	1380	VM_NORESERVE,
1376	1381	&user, HUGETLB_ANONHUGE_INODE,
1377	1382	(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);