Blame view
mm/maccess.c
9.06 KB
457c89965
|
1 |
// SPDX-License-Identifier: GPL-2.0-only |
c33fa9f56
|
2 |
/* |
3f0acb1eb
|
3 |
* Access kernel or user memory without faulting. |
c33fa9f56
|
4 |
*/ |
b95f1b31b
|
5 |
#include <linux/export.h> |
c33fa9f56
|
6 |
#include <linux/mm.h> |
7c7fcf762
|
7 |
#include <linux/uaccess.h> |
c33fa9f56
|
8 |
|
fe557319a
|
9 10 |
bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size) |
eab0c6089
|
11 12 13 |
{ return true; } |
b58294ead
|
14 |
#ifdef HAVE_GET_KERNEL_NOFAULT |
fe557319a
|
15 |
#define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ |
b58294ead
|
16 17 18 19 20 21 |
while (len >= sizeof(type)) { \ __get_kernel_nofault(dst, src, type, err_label); \ dst += sizeof(type); \ src += sizeof(type); \ len -= sizeof(type); \ } |
fe557319a
|
22 |
long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
b58294ead
|
23 |
{ |
2423de2e6
|
24 25 26 27 |
unsigned long align = 0; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) align = (unsigned long)dst | (unsigned long)src; |
fe557319a
|
28 |
if (!copy_from_kernel_nofault_allowed(src, size)) |
2a71e81d3
|
29 |
return -ERANGE; |
b58294ead
|
30 31 |
pagefault_disable(); |
2423de2e6
|
32 33 34 35 36 37 |
if (!(align & 7)) copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); if (!(align & 3)) copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); if (!(align & 1)) copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); |
fe557319a
|
38 |
copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); |
b58294ead
|
39 40 41 42 43 44 |
pagefault_enable(); return 0; Efault: pagefault_enable(); return -EFAULT; } |
fe557319a
|
45 |
EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
b58294ead
|
46 |
|
fe557319a
|
47 |
#define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ |
b58294ead
|
48 49 50 51 52 53 |
while (len >= sizeof(type)) { \ __put_kernel_nofault(dst, src, type, err_label); \ dst += sizeof(type); \ src += sizeof(type); \ len -= sizeof(type); \ } |
fe557319a
|
54 |
long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
b58294ead
|
55 |
{ |
2423de2e6
|
56 57 58 59 |
unsigned long align = 0; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) align = (unsigned long)dst | (unsigned long)src; |
b58294ead
|
60 |
pagefault_disable(); |
2423de2e6
|
61 62 63 64 65 66 |
if (!(align & 7)) copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); if (!(align & 3)) copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); if (!(align & 1)) copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); |
fe557319a
|
67 |
copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); |
b58294ead
|
68 69 70 71 72 73 74 75 76 77 78 79 80 |
pagefault_enable(); return 0; Efault: pagefault_enable(); return -EFAULT; } long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) { const void *src = unsafe_addr; if (unlikely(count <= 0)) return 0; |
fe557319a
|
81 |
if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) |
2a71e81d3
|
82 |
return -ERANGE; |
b58294ead
|
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
pagefault_disable(); do { __get_kernel_nofault(dst, src, u8, Efault); dst++; src++; } while (dst[-1] && src - unsafe_addr < count); pagefault_enable(); dst[-1] = '\0'; return src - unsafe_addr; Efault: pagefault_enable(); dst[-1] = '\0'; return -EFAULT; } #else /* HAVE_GET_KERNEL_NOFAULT */ |
c33fa9f56
|
100 |
/** |
fe557319a
|
101 |
* copy_from_kernel_nofault(): safely attempt to read from kernel-space |
4f6de12b3
|
102 103 104 105 106 |
* @dst: pointer to the buffer that shall take the data * @src: address to read from * @size: size of the data chunk * * Safely read from kernel address @src to the buffer at @dst. If a kernel |
2a71e81d3
|
107 108 |
* fault happens, handle that and return -EFAULT. If @src is not a valid kernel * address, return -ERANGE. |
0ab32b6f1
|
109 110 |
* * We ensure that the copy_from_user is executed in atomic context so that |
c1e8d7c6a
|
111 |
* do_page_fault() doesn't attempt to take mmap_lock. This makes |
fe557319a
|
112 |
* copy_from_kernel_nofault() suitable for use within regions where the caller |
c1e8d7c6a
|
113 |
* already holds mmap_lock, or other locks which nest inside mmap_lock. |
c33fa9f56
|
114 |
*/ |
fe557319a
|
115 |
long copy_from_kernel_nofault(void *dst, const void *src, size_t size) |
c33fa9f56
|
116 117 |
{ long ret; |
b4b8ac524
|
118 |
mm_segment_t old_fs = get_fs(); |
c33fa9f56
|
119 |
|
fe557319a
|
120 |
if (!copy_from_kernel_nofault_allowed(src, size)) |
2a71e81d3
|
121 |
return -ERANGE; |
eab0c6089
|
122 |
|
b4b8ac524
|
123 |
set_fs(KERNEL_DS); |
cd0309058
|
124 125 126 127 |
pagefault_disable(); ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, size); pagefault_enable(); |
b4b8ac524
|
128 |
set_fs(old_fs); |
c33fa9f56
|
129 |
|
cd0309058
|
130 131 132 |
if (ret) return -EFAULT; return 0; |
c33fa9f56
|
133 |
} |
fe557319a
|
134 |
EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); |
c33fa9f56
|
135 136 |
/** |
fe557319a
|
137 |
* copy_to_kernel_nofault(): safely attempt to write to a location |
c33fa9f56
|
138 139 140 141 142 143 144 |
* @dst: address to write to * @src: pointer to the data that shall be written * @size: size of the data chunk * * Safely write to address @dst from the buffer at @src. If a kernel fault * happens, handle that and return -EFAULT. */ |
fe557319a
|
145 |
long copy_to_kernel_nofault(void *dst, const void *src, size_t size) |
c33fa9f56
|
146 147 |
{ long ret; |
b4b8ac524
|
148 |
mm_segment_t old_fs = get_fs(); |
c33fa9f56
|
149 |
|
b4b8ac524
|
150 |
set_fs(KERNEL_DS); |
cd0309058
|
151 152 153 |
pagefault_disable(); ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); pagefault_enable(); |
b4b8ac524
|
154 |
set_fs(old_fs); |
c33fa9f56
|
155 |
|
cd0309058
|
156 157 158 |
if (ret) return -EFAULT; return 0; |
c33fa9f56
|
159 |
} |
dbb7ee0e4
|
160 |
|
1d1585ca0
|
161 |
/** |
c4cb16442
|
162 |
* strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe |
4f6de12b3
|
163 164 165 166 167 168 169 170 171 172 |
* address. * @dst: Destination address, in kernel space. This buffer must be at * least @count bytes long. * @unsafe_addr: Unsafe address. * @count: Maximum number of bytes to copy, including the trailing NUL. * * Copies a NUL-terminated string from unsafe address to kernel buffer. * * On success, returns the length of the string INCLUDING the trailing NUL. * |
2a71e81d3
|
173 174 175 |
* If access fails, returns -EFAULT (some data may have been copied and the * trailing NUL added). If @unsafe_addr is not a valid kernel address, return * -ERANGE. |
4f6de12b3
|
176 177 178 179 |
* * If @count is smaller than the length of the string, copies @count-1 bytes, * sets the last byte of @dst buffer to NUL and returns @count. */ |
eab0c6089
|
180 181 |
long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) { |
dbb7ee0e4
|
182 183 184 185 186 187 |
mm_segment_t old_fs = get_fs(); const void *src = unsafe_addr; long ret; if (unlikely(count <= 0)) return 0; |
fe557319a
|
188 |
if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) |
2a71e81d3
|
189 |
return -ERANGE; |
dbb7ee0e4
|
190 191 192 193 194 |
set_fs(KERNEL_DS); pagefault_disable(); do { |
bd28b1459
|
195 |
ret = __get_user(*dst++, (const char __user __force *)src++); |
dbb7ee0e4
|
196 197 198 199 200 |
} while (dst[-1] && ret == 0 && src - unsafe_addr < count); dst[-1] = '\0'; pagefault_enable(); set_fs(old_fs); |
9dd861d55
|
201 |
return ret ? -EFAULT : src - unsafe_addr; |
dbb7ee0e4
|
202 |
} |
b58294ead
|
203 |
#endif /* HAVE_GET_KERNEL_NOFAULT */ |
3d7081822
|
204 205 |
/** |
c0ee37e85
|
206 |
* copy_from_user_nofault(): safely attempt to read from a user-space location |
fc3562d79
|
207 208 209 210 211 212 213 |
* @dst: pointer to the buffer that shall take the data * @src: address to read from. This must be a user address. * @size: size of the data chunk * * Safely read from user address @src to the buffer at @dst. If a kernel fault * happens, handle that and return -EFAULT. */ |
c0ee37e85
|
214 |
long copy_from_user_nofault(void *dst, const void __user *src, size_t size) |
fc3562d79
|
215 216 |
{ long ret = -EFAULT; |
3d13f313c
|
217 |
mm_segment_t old_fs = force_uaccess_begin(); |
fc3562d79
|
218 |
|
fc3562d79
|
219 220 221 222 223 |
if (access_ok(src, size)) { pagefault_disable(); ret = __copy_from_user_inatomic(dst, src, size); pagefault_enable(); } |
3d13f313c
|
224 |
force_uaccess_end(old_fs); |
fc3562d79
|
225 226 227 228 229 |
if (ret) return -EFAULT; return 0; } |
c0ee37e85
|
230 |
EXPORT_SYMBOL_GPL(copy_from_user_nofault); |
fc3562d79
|
231 232 |
/** |
c0ee37e85
|
233 |
* copy_to_user_nofault(): safely attempt to write to a user-space location |
fc3562d79
|
234 235 236 237 238 239 240 |
* @dst: address to write to * @src: pointer to the data that shall be written * @size: size of the data chunk * * Safely write to address @dst from the buffer at @src. If a kernel fault * happens, handle that and return -EFAULT. */ |
c0ee37e85
|
241 |
long copy_to_user_nofault(void __user *dst, const void *src, size_t size) |
fc3562d79
|
242 243 |
{ long ret = -EFAULT; |
3d13f313c
|
244 |
mm_segment_t old_fs = force_uaccess_begin(); |
fc3562d79
|
245 |
|
fc3562d79
|
246 247 248 249 250 |
if (access_ok(dst, size)) { pagefault_disable(); ret = __copy_to_user_inatomic(dst, src, size); pagefault_enable(); } |
3d13f313c
|
251 |
force_uaccess_end(old_fs); |
fc3562d79
|
252 253 254 255 256 |
if (ret) return -EFAULT; return 0; } |
c0ee37e85
|
257 |
EXPORT_SYMBOL_GPL(copy_to_user_nofault); |
fc3562d79
|
258 259 |
/** |
bd88bb5d4
|
260 |
* strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user |
3d7081822
|
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 |
* address. * @dst: Destination address, in kernel space. This buffer must be at * least @count bytes long. * @unsafe_addr: Unsafe user address. * @count: Maximum number of bytes to copy, including the trailing NUL. * * Copies a NUL-terminated string from unsafe user address to kernel buffer. * * On success, returns the length of the string INCLUDING the trailing NUL. * * If access fails, returns -EFAULT (some data may have been copied * and the trailing NUL added). * * If @count is smaller than the length of the string, copies @count-1 bytes, * sets the last byte of @dst buffer to NUL and returns @count. */ |
bd88bb5d4
|
277 |
long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, |
3d7081822
|
278 279 |
long count) { |
3d13f313c
|
280 |
mm_segment_t old_fs; |
3d7081822
|
281 282 283 284 |
long ret; if (unlikely(count <= 0)) return 0; |
3d13f313c
|
285 |
old_fs = force_uaccess_begin(); |
3d7081822
|
286 287 288 |
pagefault_disable(); ret = strncpy_from_user(dst, unsafe_addr, count); pagefault_enable(); |
3d13f313c
|
289 |
force_uaccess_end(old_fs); |
3d7081822
|
290 291 292 293 294 295 296 297 298 299 300 301 |
if (ret >= count) { ret = count; dst[ret - 1] = '\0'; } else if (ret > 0) { ret++; } return ret; } /** |
02dddb160
|
302 |
* strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. |
3d7081822
|
303 304 305 306 307 308 309 310 311 312 313 314 315 316 |
* @unsafe_addr: The string to measure. * @count: Maximum count (including NUL) * * Get the size of a NUL-terminated string in user space without pagefault. * * Returns the size of the string INCLUDING the terminating NUL. * * If the string is too long, returns a number larger than @count. User * has to check the return value against "> count". * On exception (or invalid count), returns 0. * * Unlike strnlen_user, this can be used from IRQ handler etc. because * it disables pagefaults. */ |
02dddb160
|
317 |
long strnlen_user_nofault(const void __user *unsafe_addr, long count) |
3d7081822
|
318 |
{ |
3d13f313c
|
319 |
mm_segment_t old_fs; |
3d7081822
|
320 |
int ret; |
3d13f313c
|
321 |
old_fs = force_uaccess_begin(); |
3d7081822
|
322 323 324 |
pagefault_disable(); ret = strnlen_user(unsafe_addr, count); pagefault_enable(); |
3d13f313c
|
325 |
force_uaccess_end(old_fs); |
3d7081822
|
326 327 328 |
return ret; } |