Eric Lee / smarc-fsl-linux-kernel

Blame view

mm/gup_benchmark.c 4.52 KB

64c349f4a Kirill A. Shutemov mm: add infrastru...	1 2 3 4 5 6 7 8	#include <linux/kernel.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/uaccess.h> #include <linux/ktime.h> #include <linux/debugfs.h> #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark)
657d4f799 Barry Song mm/gup_benchmark:...	9 10 11 12	#define GUP_BENCHMARK _IOWR('g', 2, struct gup_benchmark) #define PIN_FAST_BENCHMARK _IOWR('g', 3, struct gup_benchmark) #define PIN_BENCHMARK _IOWR('g', 4, struct gup_benchmark) #define PIN_LONGTERM_BENCHMARK _IOWR('g', 5, struct gup_benchmark)
64c349f4a Kirill A. Shutemov mm: add infrastru...	13 14	struct gup_benchmark {
26db3d09d Keith Busch mm/gup_benchmark....	15 16	__u64 get_delta_usec; __u64 put_delta_usec;
64c349f4a Kirill A. Shutemov mm: add infrastru...	17 18 19 20	__u64 addr; __u64 size; __u32 nr_pages_per_call; __u32 flags;
26db3d09d Keith Busch mm/gup_benchmark....	21	__u64 expansion[10]; /* For future use */
64c349f4a Kirill A. Shutemov mm: add infrastru...	22	};
41c45d37b John Hubbard mm/gup_benchmark:...	23 24 25 26 27 28 29	static void put_back_pages(unsigned int cmd, struct page **pages, unsigned long nr_pages) { unsigned long i; switch (cmd) { case GUP_FAST_BENCHMARK:
41c45d37b John Hubbard mm/gup_benchmark:...	30 31 32 33 34 35 36	case GUP_BENCHMARK: for (i = 0; i < nr_pages; i++) put_page(pages[i]); break; case PIN_FAST_BENCHMARK: case PIN_BENCHMARK:
657d4f799 Barry Song mm/gup_benchmark:...	37	case PIN_LONGTERM_BENCHMARK:
41c45d37b John Hubbard mm/gup_benchmark:...	38 39 40 41 42 43 44 45 46 47 48 49 50 51	unpin_user_pages(pages, nr_pages); break; } } static void verify_dma_pinned(unsigned int cmd, struct page *pages, unsigned long nr_pages) { unsigned long i; struct page page; switch (cmd) { case PIN_FAST_BENCHMARK: case PIN_BENCHMARK:
657d4f799 Barry Song mm/gup_benchmark:...	52	case PIN_LONGTERM_BENCHMARK:
41c45d37b John Hubbard mm/gup_benchmark:...	53 54 55 56 57 58 59 60 61 62 63 64 65	for (i = 0; i < nr_pages; i++) { page = pages[i]; if (WARN(!page_maybe_dma_pinned(page), "pages[%lu] is NOT dma-pinned ", i)) { dump_page(page, "gup_benchmark failure"); break; } } break; } }
64c349f4a Kirill A. Shutemov mm: add infrastru...	66 67 68 69	static int __gup_benchmark_ioctl(unsigned int cmd, struct gup_benchmark *gup) { ktime_t start_time, end_time;
518968645 YueHaibing mm/gup_benchmark:...	70 71	unsigned long i, nr_pages, addr, next; int nr;
64c349f4a Kirill A. Shutemov mm: add infrastru...	72	struct page **pages;
a7c46c0c0 Navid Emamdoost mm/gup: fix memor...	73	int ret = 0;
f3964599c Jann Horn mm/gup_benchmark:...	74 75	bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
64c349f4a Kirill A. Shutemov mm: add infrastru...	76
4b408c74e Dan Carpenter mm/gup_benchmark....	77 78	if (gup->size > ULONG_MAX) return -EINVAL;
64c349f4a Kirill A. Shutemov mm: add infrastru...	79	nr_pages = gup->size / PAGE_SIZE;
778e1cdd8 Kees Cook treewide: kvzallo...	80	pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
64c349f4a Kirill A. Shutemov mm: add infrastru...	81 82	if (!pages) return -ENOMEM;
f3964599c Jann Horn mm/gup_benchmark:...	83 84 85 86	if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) { ret = -EINTR; goto free_pages; }
64c349f4a Kirill A. Shutemov mm: add infrastru...	87 88 89 90 91 92 93 94 95 96 97 98	i = 0; nr = gup->nr_pages_per_call; start_time = ktime_get(); for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { if (nr != gup->nr_pages_per_call) break; next = addr + nr * PAGE_SIZE; if (next > gup->addr + gup->size) { next = gup->addr + gup->size; nr = (next - addr) / PAGE_SIZE; }
bdffe23ee John Hubbard mm/gup_benchmark:...	99 100	/* Filter out most gup flags: only allow a tiny subset here: */ gup->flags &= FOLL_WRITE;
714a3a1eb Keith Busch mm/gup_benchmark....	101 102	switch (cmd) { case GUP_FAST_BENCHMARK:
bdffe23ee John Hubbard mm/gup_benchmark:...	103	nr = get_user_pages_fast(addr, nr, gup->flags,
714a3a1eb Keith Busch mm/gup_benchmark....	104 105	pages + i); break;
714a3a1eb Keith Busch mm/gup_benchmark....	106	case GUP_BENCHMARK:
bdffe23ee John Hubbard mm/gup_benchmark:...	107	nr = get_user_pages(addr, nr, gup->flags, pages + i,
714a3a1eb Keith Busch mm/gup_benchmark....	108 109	NULL); break;
41c45d37b John Hubbard mm/gup_benchmark:...	110 111 112 113 114 115 116 117	case PIN_FAST_BENCHMARK: nr = pin_user_pages_fast(addr, nr, gup->flags, pages + i); break; case PIN_BENCHMARK: nr = pin_user_pages(addr, nr, gup->flags, pages + i, NULL); break;
657d4f799 Barry Song mm/gup_benchmark:...	118 119 120 121 122	case PIN_LONGTERM_BENCHMARK: nr = pin_user_pages(addr, nr, gup->flags \| FOLL_LONGTERM, pages + i, NULL); break;
714a3a1eb Keith Busch mm/gup_benchmark....	123	default:
a7c46c0c0 Navid Emamdoost mm/gup: fix memor...	124	ret = -EINVAL;
f3964599c Jann Horn mm/gup_benchmark:...	125	goto unlock;
714a3a1eb Keith Busch mm/gup_benchmark....	126	}
09e35a4a1 Michael S. Tsirkin mm/gup_benchmark:...	127 128	if (nr <= 0) break;
64c349f4a Kirill A. Shutemov mm: add infrastru...	129 130 131	i += nr; } end_time = ktime_get();
41c45d37b John Hubbard mm/gup_benchmark:...	132 133	/* Shifting the meaning of nr_pages: now it is actual number pinned: */ nr_pages = i;
26db3d09d Keith Busch mm/gup_benchmark....	134	gup->get_delta_usec = ktime_us_delta(end_time, start_time);
64c349f4a Kirill A. Shutemov mm: add infrastru...	135	gup->size = addr - gup->addr;
41c45d37b John Hubbard mm/gup_benchmark:...	136 137 138 139 140	/* * Take an un-benchmark-timed moment to verify DMA pinned * state: print a warning if any non-dma-pinned pages are found: */ verify_dma_pinned(cmd, pages, nr_pages);
26db3d09d Keith Busch mm/gup_benchmark....	141	start_time = ktime_get();
41c45d37b John Hubbard mm/gup_benchmark:...	142 143	put_back_pages(cmd, pages, nr_pages);
26db3d09d Keith Busch mm/gup_benchmark....	144 145	end_time = ktime_get(); gup->put_delta_usec = ktime_us_delta(end_time, start_time);
64c349f4a Kirill A. Shutemov mm: add infrastru...	146
f3964599c Jann Horn mm/gup_benchmark:...	147 148 149 150	unlock: if (needs_mmap_lock) mmap_read_unlock(current->mm); free_pages:
64c349f4a Kirill A. Shutemov mm: add infrastru...	151	kvfree(pages);
a7c46c0c0 Navid Emamdoost mm/gup: fix memor...	152	return ret;
64c349f4a Kirill A. Shutemov mm: add infrastru...	153 154 155 156 157 158 159	} static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct gup_benchmark gup; int ret;
714a3a1eb Keith Busch mm/gup_benchmark....	160 161	switch (cmd) { case GUP_FAST_BENCHMARK:
714a3a1eb Keith Busch mm/gup_benchmark....	162	case GUP_BENCHMARK:
41c45d37b John Hubbard mm/gup_benchmark:...	163 164	case PIN_FAST_BENCHMARK: case PIN_BENCHMARK:
657d4f799 Barry Song mm/gup_benchmark:...	165	case PIN_LONGTERM_BENCHMARK:
714a3a1eb Keith Busch mm/gup_benchmark....	166 167	break; default:
64c349f4a Kirill A. Shutemov mm: add infrastru...	168	return -EINVAL;
714a3a1eb Keith Busch mm/gup_benchmark....	169	}
64c349f4a Kirill A. Shutemov mm: add infrastru...	170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190	if (copy_from_user(&gup, (void __user )arg, sizeof(gup))) return -EFAULT; ret = __gup_benchmark_ioctl(cmd, &gup); if (ret) return ret; if (copy_to_user((void __user )arg, &gup, sizeof(gup))) return -EFAULT; return 0; } static const struct file_operations gup_benchmark_fops = { .open = nonseekable_open, .unlocked_ioctl = gup_benchmark_ioctl, }; static int gup_benchmark_init(void) {
d9f7979c9 Greg Kroah-Hartman mm: no need to ch...	191 192	debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL, &gup_benchmark_fops);
64c349f4a Kirill A. Shutemov mm: add infrastru...	193 194 195 196 197	return 0; } late_initcall(gup_benchmark_init);