Blame view
mm/gup_benchmark.c
4.52 KB
64c349f4a mm: add infrastru... |
1 2 3 4 5 6 7 8 |
#include <linux/kernel.h> #include <linux/mm.h> #include <linux/slab.h> #include <linux/uaccess.h> #include <linux/ktime.h> #include <linux/debugfs.h> #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark) |
657d4f799 mm/gup_benchmark:... |
9 10 11 12 |
#define GUP_BENCHMARK _IOWR('g', 2, struct gup_benchmark) #define PIN_FAST_BENCHMARK _IOWR('g', 3, struct gup_benchmark) #define PIN_BENCHMARK _IOWR('g', 4, struct gup_benchmark) #define PIN_LONGTERM_BENCHMARK _IOWR('g', 5, struct gup_benchmark) |
64c349f4a mm: add infrastru... |
13 14 |
struct gup_benchmark { |
26db3d09d mm/gup_benchmark.... |
15 16 |
__u64 get_delta_usec; __u64 put_delta_usec; |
64c349f4a mm: add infrastru... |
17 18 19 20 |
__u64 addr; __u64 size; __u32 nr_pages_per_call; __u32 flags; |
26db3d09d mm/gup_benchmark.... |
21 |
__u64 expansion[10]; /* For future use */ |
64c349f4a mm: add infrastru... |
22 |
}; |
41c45d37b mm/gup_benchmark:... |
23 24 25 26 27 28 29 |
static void put_back_pages(unsigned int cmd, struct page **pages, unsigned long nr_pages) { unsigned long i; switch (cmd) { case GUP_FAST_BENCHMARK: |
41c45d37b mm/gup_benchmark:... |
30 31 32 33 34 35 36 |
case GUP_BENCHMARK: for (i = 0; i < nr_pages; i++) put_page(pages[i]); break; case PIN_FAST_BENCHMARK: case PIN_BENCHMARK: |
657d4f799 mm/gup_benchmark:... |
37 |
case PIN_LONGTERM_BENCHMARK: |
41c45d37b mm/gup_benchmark:... |
38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
unpin_user_pages(pages, nr_pages); break; } } static void verify_dma_pinned(unsigned int cmd, struct page **pages, unsigned long nr_pages) { unsigned long i; struct page *page; switch (cmd) { case PIN_FAST_BENCHMARK: case PIN_BENCHMARK: |
657d4f799 mm/gup_benchmark:... |
52 |
case PIN_LONGTERM_BENCHMARK: |
41c45d37b mm/gup_benchmark:... |
53 54 55 56 57 58 59 60 61 62 63 64 65 |
for (i = 0; i < nr_pages; i++) { page = pages[i]; if (WARN(!page_maybe_dma_pinned(page), "pages[%lu] is NOT dma-pinned ", i)) { dump_page(page, "gup_benchmark failure"); break; } } break; } } |
64c349f4a mm: add infrastru... |
66 67 68 69 |
static int __gup_benchmark_ioctl(unsigned int cmd, struct gup_benchmark *gup) { ktime_t start_time, end_time; |
518968645 mm/gup_benchmark:... |
70 71 |
unsigned long i, nr_pages, addr, next; int nr; |
64c349f4a mm: add infrastru... |
72 |
struct page **pages; |
a7c46c0c0 mm/gup: fix memor... |
73 |
int ret = 0; |
f3964599c mm/gup_benchmark:... |
74 75 |
bool needs_mmap_lock = cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK; |
64c349f4a mm: add infrastru... |
76 |
|
4b408c74e mm/gup_benchmark.... |
77 78 |
if (gup->size > ULONG_MAX) return -EINVAL; |
64c349f4a mm: add infrastru... |
79 |
nr_pages = gup->size / PAGE_SIZE; |
778e1cdd8 treewide: kvzallo... |
80 |
pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL); |
64c349f4a mm: add infrastru... |
81 82 |
if (!pages) return -ENOMEM; |
f3964599c mm/gup_benchmark:... |
83 84 85 86 |
if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) { ret = -EINTR; goto free_pages; } |
64c349f4a mm: add infrastru... |
87 88 89 90 91 92 93 94 95 96 97 98 |
i = 0; nr = gup->nr_pages_per_call; start_time = ktime_get(); for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) { if (nr != gup->nr_pages_per_call) break; next = addr + nr * PAGE_SIZE; if (next > gup->addr + gup->size) { next = gup->addr + gup->size; nr = (next - addr) / PAGE_SIZE; } |
bdffe23ee mm/gup_benchmark:... |
99 100 |
/* Filter out most gup flags: only allow a tiny subset here: */ gup->flags &= FOLL_WRITE; |
714a3a1eb mm/gup_benchmark.... |
101 102 |
switch (cmd) { case GUP_FAST_BENCHMARK: |
bdffe23ee mm/gup_benchmark:... |
103 |
nr = get_user_pages_fast(addr, nr, gup->flags, |
714a3a1eb mm/gup_benchmark.... |
104 105 |
pages + i); break; |
714a3a1eb mm/gup_benchmark.... |
106 |
case GUP_BENCHMARK: |
bdffe23ee mm/gup_benchmark:... |
107 |
nr = get_user_pages(addr, nr, gup->flags, pages + i, |
714a3a1eb mm/gup_benchmark.... |
108 109 |
NULL); break; |
41c45d37b mm/gup_benchmark:... |
110 111 112 113 114 115 116 117 |
case PIN_FAST_BENCHMARK: nr = pin_user_pages_fast(addr, nr, gup->flags, pages + i); break; case PIN_BENCHMARK: nr = pin_user_pages(addr, nr, gup->flags, pages + i, NULL); break; |
657d4f799 mm/gup_benchmark:... |
118 119 120 121 122 |
case PIN_LONGTERM_BENCHMARK: nr = pin_user_pages(addr, nr, gup->flags | FOLL_LONGTERM, pages + i, NULL); break; |
714a3a1eb mm/gup_benchmark.... |
123 |
default: |
a7c46c0c0 mm/gup: fix memor... |
124 |
ret = -EINVAL; |
f3964599c mm/gup_benchmark:... |
125 |
goto unlock; |
714a3a1eb mm/gup_benchmark.... |
126 |
} |
09e35a4a1 mm/gup_benchmark:... |
127 128 |
if (nr <= 0) break; |
64c349f4a mm: add infrastru... |
129 130 131 |
i += nr; } end_time = ktime_get(); |
41c45d37b mm/gup_benchmark:... |
132 133 |
/* Shifting the meaning of nr_pages: now it is actual number pinned: */ nr_pages = i; |
26db3d09d mm/gup_benchmark.... |
134 |
gup->get_delta_usec = ktime_us_delta(end_time, start_time); |
64c349f4a mm: add infrastru... |
135 |
gup->size = addr - gup->addr; |
41c45d37b mm/gup_benchmark:... |
136 137 138 139 140 |
/* * Take an un-benchmark-timed moment to verify DMA pinned * state: print a warning if any non-dma-pinned pages are found: */ verify_dma_pinned(cmd, pages, nr_pages); |
26db3d09d mm/gup_benchmark.... |
141 |
start_time = ktime_get(); |
41c45d37b mm/gup_benchmark:... |
142 143 |
put_back_pages(cmd, pages, nr_pages); |
26db3d09d mm/gup_benchmark.... |
144 145 |
end_time = ktime_get(); gup->put_delta_usec = ktime_us_delta(end_time, start_time); |
64c349f4a mm: add infrastru... |
146 |
|
f3964599c mm/gup_benchmark:... |
147 148 149 150 |
unlock: if (needs_mmap_lock) mmap_read_unlock(current->mm); free_pages: |
64c349f4a mm: add infrastru... |
151 |
kvfree(pages); |
a7c46c0c0 mm/gup: fix memor... |
152 |
return ret; |
64c349f4a mm: add infrastru... |
153 154 155 156 157 158 159 |
} static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct gup_benchmark gup; int ret; |
714a3a1eb mm/gup_benchmark.... |
160 161 |
switch (cmd) { case GUP_FAST_BENCHMARK: |
714a3a1eb mm/gup_benchmark.... |
162 |
case GUP_BENCHMARK: |
41c45d37b mm/gup_benchmark:... |
163 164 |
case PIN_FAST_BENCHMARK: case PIN_BENCHMARK: |
657d4f799 mm/gup_benchmark:... |
165 |
case PIN_LONGTERM_BENCHMARK: |
714a3a1eb mm/gup_benchmark.... |
166 167 |
break; default: |
64c349f4a mm: add infrastru... |
168 |
return -EINVAL; |
714a3a1eb mm/gup_benchmark.... |
169 |
} |
64c349f4a mm: add infrastru... |
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 |
if (copy_from_user(&gup, (void __user *)arg, sizeof(gup))) return -EFAULT; ret = __gup_benchmark_ioctl(cmd, &gup); if (ret) return ret; if (copy_to_user((void __user *)arg, &gup, sizeof(gup))) return -EFAULT; return 0; } static const struct file_operations gup_benchmark_fops = { .open = nonseekable_open, .unlocked_ioctl = gup_benchmark_ioctl, }; static int gup_benchmark_init(void) { |
d9f7979c9 mm: no need to ch... |
191 192 |
debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL, &gup_benchmark_fops); |
64c349f4a mm: add infrastru... |
193 194 195 196 197 |
return 0; } late_initcall(gup_benchmark_init); |