Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 47d632f9f8f3ed62b21f725e98b726d65769b6d7

Authored by Chris Metcalf 2012-03-30 01:39:51 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

arch/tile: optimize get_user/put_user and friends

Use direct load/store for the get_user/put_user.

Previously, we would call out to a helper routine that would do the
appropriate thing and then return, handling the possible exception
internally.  Now we inline the load or store, along with a "we succeeded"
indication in a register; if the load or store faults, we write a
"we failed" indication into the same register and then return to the
following instruction.  This is more efficient and gives us more compact
code, as well as being more in line with what other architectures do.

The special futex assembly source file for TILE-Gx also disappears in
this change; we just use the same inlining idiom there as well, putting
the appropriate atomic operations directly into futex_atomic_op_inuser()
(and thus into the FUTEX_WAIT function).

The underlying atomic copy_from_user, copy_to_user functions were
renamed using the (cryptic) x86 convention as copy_from_user_ll and
copy_to_user_ll.

Signed-off-by: Chris Metcalf <cmetcalf@tilera.com>

Showing 8 changed files with 241 additions and 315 deletions Inline Diff

arch/tile/include/asm/atomic_32.h
arch/tile/include/asm/futex.h
arch/tile/include/asm/uaccess.h
arch/tile/kernel/Makefile
arch/tile/lib/atomic_32.c
arch/tile/lib/exports.c
arch/tile/lib/usercopy_32.S
arch/tile/lib/usercopy_64.S

arch/tile/include/asm/atomic_32.h

Diff comments View file @ 47d632f

 /*
  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  *
  * Do not include directly; use <linux/atomic.h>.
  */
 #ifndef _ASM_TILE_ATOMIC_32_H
 #define _ASM_TILE_ATOMIC_32_H
 #include <asm/barrier.h>
 #include <arch/chip.h>
 #ifndef __ASSEMBLY__
 /* Tile-specific routines to support <linux/atomic.h>. */
 int _atomic_xchg(atomic_t *v, int n);
 int _atomic_xchg_add(atomic_t *v, int i);
 int _atomic_xchg_add_unless(atomic_t *v, int a, int u);
 int _atomic_cmpxchg(atomic_t *v, int o, int n);
 /**
  * atomic_xchg - atomically exchange contents of memory with a new value
  * @v: pointer of type atomic_t
  * @i: integer value to store in memory
  *
  * Atomically sets @v to @i and returns old @v
  */
 static inline int atomic_xchg(atomic_t *v, int n)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic_xchg(v, n);
 }
 /**
  * atomic_cmpxchg - atomically exchange contents of memory if it matches
  * @v: pointer of type atomic_t
  * @o: old value that memory should have
  * @n: new value to write to memory if it matches
  *
  * Atomically checks if @v holds @o and replaces it with @n if so.
  * Returns the old value at @v.
  */
 static inline int atomic_cmpxchg(atomic_t *v, int o, int n)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic_cmpxchg(v, o, n);
 }
 /**
  * atomic_add - add integer to atomic variable
  * @i: integer value to add
  * @v: pointer of type atomic_t
  *
  * Atomically adds @i to @v.
  */
 static inline void atomic_add(int i, atomic_t *v)
 {
 	_atomic_xchg_add(v, i);
 }
 /**
  * atomic_add_return - add integer and return
  * @v: pointer of type atomic_t
  * @i: integer value to add
  *
  * Atomically adds @i to @v and returns @i + @v
  */
 static inline int atomic_add_return(int i, atomic_t *v)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic_xchg_add(v, i) + i;
 }
 /**
  * __atomic_add_unless - add unless the number is already a given value
  * @v: pointer of type atomic_t
  * @a: the amount to add to v...
  * @u: ...unless v is equal to u.
  *
  * Atomically adds @a to @v, so long as @v was not already @u.
  * Returns the old value of @v.
  */
 static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic_xchg_add_unless(v, a, u);
 }
 /**
  * atomic_set - set atomic variable
  * @v: pointer of type atomic_t
  * @i: required value
  *
  * Atomically sets the value of @v to @i.
  *
  * atomic_set() can't be just a raw store, since it would be lost if it
  * fell between the load and store of one of the other atomic ops.
  */
 static inline void atomic_set(atomic_t *v, int n)
 {
 	_atomic_xchg(v, n);
 }
 /* A 64bit atomic type */
 typedef struct {
 	u64 __aligned(8) counter;
 } atomic64_t;
 #define ATOMIC64_INIT(val) { (val) }
 u64 _atomic64_xchg(atomic64_t *v, u64 n);
 u64 _atomic64_xchg_add(atomic64_t *v, u64 i);
 u64 _atomic64_xchg_add_unless(atomic64_t *v, u64 a, u64 u);
 u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n);
 /**
  * atomic64_read - read atomic variable
  * @v: pointer of type atomic64_t
  *
  * Atomically reads the value of @v.
  */
 static inline u64 atomic64_read(const atomic64_t *v)
 {
 	/*
 	 * Requires an atomic op to read both 32-bit parts consistently.
 	 * Casting away const is safe since the atomic support routines
 	 * do not write to memory if the value has not been modified.
 	 */
 	return _atomic64_xchg_add((atomic64_t *)v, 0);
 }
 /**
  * atomic64_xchg - atomically exchange contents of memory with a new value
  * @v: pointer of type atomic64_t
  * @i: integer value to store in memory
  *
  * Atomically sets @v to @i and returns old @v
  */
 static inline u64 atomic64_xchg(atomic64_t *v, u64 n)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic64_xchg(v, n);
 }
 /**
  * atomic64_cmpxchg - atomically exchange contents of memory if it matches
  * @v: pointer of type atomic64_t
  * @o: old value that memory should have
  * @n: new value to write to memory if it matches
  *
  * Atomically checks if @v holds @o and replaces it with @n if so.
  * Returns the old value at @v.
  */
 static inline u64 atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic64_cmpxchg(v, o, n);
 }
 /**
  * atomic64_add - add integer to atomic variable
  * @i: integer value to add
  * @v: pointer of type atomic64_t
  *
  * Atomically adds @i to @v.
  */
 static inline void atomic64_add(u64 i, atomic64_t *v)
 {
 	_atomic64_xchg_add(v, i);
 }
 /**
  * atomic64_add_return - add integer and return
  * @v: pointer of type atomic64_t
  * @i: integer value to add
  *
  * Atomically adds @i to @v and returns @i + @v
  */
 static inline u64 atomic64_add_return(u64 i, atomic64_t *v)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic64_xchg_add(v, i) + i;
 }
 /**
  * atomic64_add_unless - add unless the number is already a given value
  * @v: pointer of type atomic64_t
  * @a: the amount to add to v...
  * @u: ...unless v is equal to u.
  *
  * Atomically adds @a to @v, so long as @v was not already @u.
  * Returns non-zero if @v was not @u, and zero otherwise.
  */
 static inline u64 atomic64_add_unless(atomic64_t *v, u64 a, u64 u)
 {
 	smp_mb();  /* barrier for proper semantics */
 	return _atomic64_xchg_add_unless(v, a, u) != u;
 }
 /**
  * atomic64_set - set atomic variable
  * @v: pointer of type atomic64_t
  * @i: required value
  *
  * Atomically sets the value of @v to @i.
  *
  * atomic64_set() can't be just a raw store, since it would be lost if it
  * fell between the load and store of one of the other atomic ops.
  */
 static inline void atomic64_set(atomic64_t *v, u64 n)
 {
 	_atomic64_xchg(v, n);
 }
 #define atomic64_add_negative(a, v)	(atomic64_add_return((a), (v)) < 0)
 #define atomic64_inc(v)			atomic64_add(1LL, (v))
 #define atomic64_inc_return(v)		atomic64_add_return(1LL, (v))
 #define atomic64_inc_and_test(v)	(atomic64_inc_return(v) == 0)
 #define atomic64_sub_return(i, v)	atomic64_add_return(-(i), (v))
 #define atomic64_sub_and_test(a, v)	(atomic64_sub_return((a), (v)) == 0)
 #define atomic64_sub(i, v)		atomic64_add(-(i), (v))
 #define atomic64_dec(v)			atomic64_sub(1LL, (v))
 #define atomic64_dec_return(v)		atomic64_sub_return(1LL, (v))
 #define atomic64_dec_and_test(v)	(atomic64_dec_return((v)) == 0)
 #define atomic64_inc_not_zero(v)	atomic64_add_unless((v), 1LL, 0LL)
 /*
  * We need to barrier before modifying the word, since the _atomic_xxx()
  * routines just tns the lock and then read/modify/write of the word.
  * But after the word is updated, the routine issues an "mf" before returning,
  * and since it's a function call, we don't even need a compiler barrier.
  */
 #define smp_mb__before_atomic_dec()	smp_mb()
 #define smp_mb__before_atomic_inc()	smp_mb()
 #define smp_mb__after_atomic_dec()	do { } while (0)
 #define smp_mb__after_atomic_inc()	do { } while (0)
 #endif /* !__ASSEMBLY__ */
 /*
  * Internal definitions only beyond this point.
  */
 #define ATOMIC_LOCKS_FOUND_VIA_TABLE() \
   (!CHIP_HAS_CBOX_HOME_MAP() && defined(CONFIG_SMP))
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
 /* Number of entries in atomic_lock_ptr[]. */
 #define ATOMIC_HASH_L1_SHIFT 6
 #define ATOMIC_HASH_L1_SIZE (1 << ATOMIC_HASH_L1_SHIFT)
 /* Number of locks in each struct pointed to by atomic_lock_ptr[]. */
 #define ATOMIC_HASH_L2_SHIFT (CHIP_L2_LOG_LINE_SIZE() - 2)
 #define ATOMIC_HASH_L2_SIZE (1 << ATOMIC_HASH_L2_SHIFT)
 #else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
 /*
  * Number of atomic locks in atomic_locks[]. Must be a power of two.
  * There is no reason for more than PAGE_SIZE / 8 entries, since that
  * is the maximum number of pointer bits we can use to index this.
  * And we cannot have more than PAGE_SIZE / 4, since this has to
  * fit on a single page and each entry takes 4 bytes.
  */
 #define ATOMIC_HASH_SHIFT (PAGE_SHIFT - 3)
 #define ATOMIC_HASH_SIZE (1 << ATOMIC_HASH_SHIFT)
 #ifndef __ASSEMBLY__
 extern int atomic_locks[];
 #endif
 #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
 /*
  * All the code that may fault while holding an atomic lock must
  * place the pointer to the lock in ATOMIC_LOCK_REG so the fault code
  * can correctly release and reacquire the lock.  Note that we
  * mention the register number in a comment in "lib/atomic_asm.S" to help
  * assembly coders from using this register by mistake, so if it
  * is changed here, change that comment as well.
  */
 #define ATOMIC_LOCK_REG 20
 #define ATOMIC_LOCK_REG_NAME r20
 #ifndef __ASSEMBLY__
 /* Called from setup to initialize a hash table to point to per_cpu locks. */
 void __init_atomic_per_cpu(void);
 #ifdef CONFIG_SMP
 /* Support releasing the atomic lock in do_page_fault_ics(). */
 void __atomic_fault_unlock(int *lock_ptr);
 #endif
+/* Return a pointer to the lock for the given address. */
+int *__atomic_hashed_lock(volatile void *v);
 /* Private helper routines in lib/atomic_asm_32.S */
+struct __get_user {
+	unsigned long val;
+	int err;
+};
 extern struct __get_user __atomic_cmpxchg(volatile int *p,
 					  int *lock, int o, int n);
 extern struct __get_user __atomic_xchg(volatile int *p, int *lock, int n);
 extern struct __get_user __atomic_xchg_add(volatile int *p, int *lock, int n);
 extern struct __get_user __atomic_xchg_add_unless(volatile int *p,
 						  int *lock, int o, int n);
 extern struct __get_user __atomic_or(volatile int *p, int *lock, int n);
 extern struct __get_user __atomic_andn(volatile int *p, int *lock, int n);
 extern struct __get_user __atomic_xor(volatile int *p, int *lock, int n);
 extern u64 __atomic64_cmpxchg(volatile u64 *p, int *lock, u64 o, u64 n);
 extern u64 __atomic64_xchg(volatile u64 *p, int *lock, u64 n);
 extern u64 __atomic64_xchg_add(volatile u64 *p, int *lock, u64 n);
 extern u64 __atomic64_xchg_add_unless(volatile u64 *p,
 				      int *lock, u64 o, u64 n);
+/* Return failure from the atomic wrappers. */
+struct __get_user __atomic_bad_address(int __user *addr);
 #endif /* !__ASSEMBLY__ */
 #endif /* _ASM_TILE_ATOMIC_32_H */

arch/tile/include/asm/futex.h

Diff comments View file @ 47d632f

1	/*	1	/*
2	* Copyright 2010 Tilera Corporation. All Rights Reserved.	2	* Copyright 2010 Tilera Corporation. All Rights Reserved.
3	*	3	*
4	* This program is free software; you can redistribute it and/or	4	* This program is free software; you can redistribute it and/or
5	* modify it under the terms of the GNU General Public License	5	* modify it under the terms of the GNU General Public License
6	* as published by the Free Software Foundation, version 2.	6	* as published by the Free Software Foundation, version 2.
7	*	7	*
8	* This program is distributed in the hope that it will be useful, but	8	* This program is distributed in the hope that it will be useful, but
9	* WITHOUT ANY WARRANTY; without even the implied warranty of	9	* WITHOUT ANY WARRANTY; without even the implied warranty of
10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or	10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
11	* NON INFRINGEMENT. See the GNU General Public License for	11	* NON INFRINGEMENT. See the GNU General Public License for
12	* more details.	12	* more details.
13	*	13	*
14	* These routines make two important assumptions:	14	* These routines make two important assumptions:
15	*	15	*
16	* 1. atomic_t is really an int and can be freely cast back and forth	16	* 1. atomic_t is really an int and can be freely cast back and forth
17	* (validated in __init_atomic_per_cpu).	17	* (validated in __init_atomic_per_cpu).
18	*	18	*
19	* 2. userspace uses sys_cmpxchg() for all atomic operations, thus using	19	* 2. userspace uses sys_cmpxchg() for all atomic operations, thus using
20	* the same locking convention that all the kernel atomic routines use.	20	* the same locking convention that all the kernel atomic routines use.
21	*/	21	*/
22		22
23	#ifndef _ASM_TILE_FUTEX_H	23	#ifndef _ASM_TILE_FUTEX_H
24	#define _ASM_TILE_FUTEX_H	24	#define _ASM_TILE_FUTEX_H
25		25
26	#ifndef __ASSEMBLY__	26	#ifndef __ASSEMBLY__
27		27
28	#include <linux/futex.h>	28	#include <linux/futex.h>
29	#include <linux/uaccess.h>	29	#include <linux/uaccess.h>
30	#include <linux/errno.h>	30	#include <linux/errno.h>
		31	#include <asm/atomic.h>
31		32
32	extern struct __get_user futex_set(u32 __user *v, int i);	33	/*
33	extern struct __get_user futex_add(u32 __user *v, int n);	34	* Support macros for futex operations. Do not use these macros directly.
34	extern struct __get_user futex_or(u32 __user *v, int n);	35	* They assume "ret", "val", "oparg", and "uaddr" in the lexical context.
35	extern struct __get_user futex_andn(u32 __user *v, int n);	36	* __futex_cmpxchg() additionally assumes "oldval".
36	extern struct __get_user futex_cmpxchg(u32 __user *v, int o, int n);	37	*/
37		38
38	#ifndef __tilegx__	39	#ifdef __tilegx__
39	extern struct __get_user futex_xor(u32 __user *v, int n);	40
		41	#define __futex_asm(OP) \
		42	asm("1: {" #OP " %1, %3, %4; movei %0, 0 }\n" \
		43	".pushsection .fixup,\"ax\"\n" \
		44	"0: { movei %0, %5; j 9f }\n" \
		45	".section __ex_table,\"a\"\n" \
		46	".quad 1b, 0b\n" \
		47	".popsection\n" \
		48	"9:" \
		49	: "=r" (ret), "=r" (val), "+m" (*(uaddr)) \
		50	: "r" (uaddr), "r" (oparg), "i" (-EFAULT))
		51
		52	#define __futex_set() __futex_asm(exch4)
		53	#define __futex_add() __futex_asm(fetchadd4)
		54	#define __futex_or() __futex_asm(fetchor4)
		55	#define __futex_andn() ({ oparg = ~oparg; __futex_asm(fetchand4); })
		56	#define __futex_cmpxchg() \
		57	({ __insn_mtspr(SPR_CMPEXCH_VALUE, oldval); __futex_asm(cmpexch4); })
		58
		59	#define __futex_xor() \
		60	({ \
		61	u32 oldval, n = oparg; \
		62	if ((ret = __get_user(oldval, uaddr)) == 0) { \
		63	do { \
		64	oparg = oldval ^ n; \
		65	__futex_cmpxchg(); \
		66	} while (ret == 0 && oldval != val); \
		67	} \
		68	})
		69
		70	/* No need to prefetch, since the atomic ops go to the home cache anyway. */
		71	#define __futex_prolog()
		72
40	#else	73	#else
41	static inline struct __get_user futex_xor(u32 __user *uaddr, int n)	74
42	{	75	#define __futex_call(FN) \
43	struct __get_user asm_ret = __get_user_4(uaddr);	76	{ \
44	if (!asm_ret.err) {	77	struct __get_user gu = FN((u32 __force *)uaddr, lock, oparg); \
45	int oldval, newval;	78	val = gu.val; \
46	do {	79	ret = gu.err; \
47	oldval = asm_ret.val;
48	newval = oldval ^ n;
49	asm_ret = futex_cmpxchg(uaddr, oldval, newval);
50	} while (asm_ret.err == 0 && oldval != asm_ret.val);
51	}	80	}
52	return asm_ret;	81
53	}	82	#define __futex_set() __futex_call(__atomic_xchg)
		83	#define __futex_add() __futex_call(__atomic_xchg_add)
		84	#define __futex_or() __futex_call(__atomic_or)
		85	#define __futex_andn() __futex_call(__atomic_andn)
		86	#define __futex_xor() __futex_call(__atomic_xor)
		87
		88	#define __futex_cmpxchg() \
		89	{ \
		90	struct __get_user gu = __atomic_cmpxchg((u32 __force *)uaddr, \
		91	lock, oldval, oparg); \
		92	val = gu.val; \
		93	ret = gu.err; \
		94	}
		95
		96	/*
		97	* Find the lock pointer for the atomic calls to use, and issue a
		98	* prefetch to the user address to bring it into cache. Similar to
		99	* __atomic_setup(), but we can't do a read into the L1 since it might
		100	* fault; instead we do a prefetch into the L2.
		101	*/
		102	#define __futex_prolog() \
		103	int *lock; \
		104	__insn_prefetch(uaddr); \
		105	lock = __atomic_hashed_lock((int __force *)uaddr)
54	#endif	106	#endif
55		107
56	static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)	108	static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr)
57	{	109	{
58	int op = (encoded_op >> 28) & 7;	110	int op = (encoded_op >> 28) & 7;
59	int cmp = (encoded_op >> 24) & 15;	111	int cmp = (encoded_op >> 24) & 15;
60	int oparg = (encoded_op << 8) >> 20;	112	int oparg = (encoded_op << 8) >> 20;
61	int cmparg = (encoded_op << 20) >> 20;	113	int cmparg = (encoded_op << 20) >> 20;
62	int ret;	114	int uninitialized_var(val), ret;
63	struct __get_user asm_ret;
64		115
		116	__futex_prolog();
		117
		118	/* The 32-bit futex code makes this assumption, so validate it here. */
		119	BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));
		120
65	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))	121	if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
66	oparg = 1 << oparg;	122	oparg = 1 << oparg;
67		123
68	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))	124	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
69	return -EFAULT;	125	return -EFAULT;
70		126
71	pagefault_disable();	127	pagefault_disable();
72	switch (op) {	128	switch (op) {
73	case FUTEX_OP_SET:	129	case FUTEX_OP_SET:
74	asm_ret = futex_set(uaddr, oparg);	130	__futex_set();
75	break;	131	break;
76	case FUTEX_OP_ADD:	132	case FUTEX_OP_ADD:
77	asm_ret = futex_add(uaddr, oparg);	133	__futex_add();
78	break;	134	break;
79	case FUTEX_OP_OR:	135	case FUTEX_OP_OR:
80	asm_ret = futex_or(uaddr, oparg);	136	__futex_or();
81	break;	137	break;
82	case FUTEX_OP_ANDN:	138	case FUTEX_OP_ANDN:
83	asm_ret = futex_andn(uaddr, oparg);	139	__futex_andn();
84	break;	140	break;
85	case FUTEX_OP_XOR:	141	case FUTEX_OP_XOR:
86	asm_ret = futex_xor(uaddr, oparg);	142	__futex_xor();
87	break;	143	break;
88	default:	144	default:
89	asm_ret.err = -ENOSYS;	145	ret = -ENOSYS;
		146	break;
90	}	147	}
91	pagefault_enable();	148	pagefault_enable();
92		149
93	ret = asm_ret.err;
94
95	if (!ret) {	150	if (!ret) {
96	switch (cmp) {	151	switch (cmp) {
97	case FUTEX_OP_CMP_EQ:	152	case FUTEX_OP_CMP_EQ:
98	ret = (asm_ret.val == cmparg);	153	ret = (val == cmparg);
99	break;	154	break;
100	case FUTEX_OP_CMP_NE:	155	case FUTEX_OP_CMP_NE:
101	ret = (asm_ret.val != cmparg);	156	ret = (val != cmparg);
102	break;	157	break;
103	case FUTEX_OP_CMP_LT:	158	case FUTEX_OP_CMP_LT:
104	ret = (asm_ret.val < cmparg);	159	ret = (val < cmparg);
105	break;	160	break;
106	case FUTEX_OP_CMP_GE:	161	case FUTEX_OP_CMP_GE:
107	ret = (asm_ret.val >= cmparg);	162	ret = (val >= cmparg);
108	break;	163	break;
109	case FUTEX_OP_CMP_LE:	164	case FUTEX_OP_CMP_LE:
110	ret = (asm_ret.val <= cmparg);	165	ret = (val <= cmparg);
111	break;	166	break;
112	case FUTEX_OP_CMP_GT:	167	case FUTEX_OP_CMP_GT:
113	ret = (asm_ret.val > cmparg);	168	ret = (val > cmparg);
114	break;	169	break;
115	default:	170	default:
116	ret = -ENOSYS;	171	ret = -ENOSYS;
117	}	172	}
118	}	173	}
119	return ret;	174	return ret;
120	}	175	}
121		176
122	static inline int futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,	177	static inline int futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user uaddr,
123	u32 oldval, u32 newval)	178	u32 oldval, u32 oparg)
124	{	179	{
125	struct __get_user asm_ret;	180	int ret, val;
126		181
		182	__futex_prolog();
		183
127	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))	184	if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
128	return -EFAULT;	185	return -EFAULT;
129		186
130	asm_ret = futex_cmpxchg(uaddr, oldval, newval);	187	__futex_cmpxchg();
131	*uval = asm_ret.val;
132	return asm_ret.err;

arch/tile/include/asm/uaccess.h

Diff comments View file @ 47d632f

 /*
  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  */
 #ifndef _ASM_TILE_UACCESS_H
 #define _ASM_TILE_UACCESS_H
 /*
  * User space memory access functions
  */
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <asm-generic/uaccess-unaligned.h>
 #include <asm/processor.h>
 #include <asm/page.h>
 #define VERIFY_READ	0
 #define VERIFY_WRITE	1
 /*
  * The fs value determines whether argument validity checking should be
  * performed or not.  If get_fs() == USER_DS, checking is performed, with
  * get_fs() == KERNEL_DS, checking is bypassed.
  *
  * For historical reasons, these macros are grossly misnamed.
  */
 #define MAKE_MM_SEG(a)  ((mm_segment_t) { (a) })
 #define KERNEL_DS	MAKE_MM_SEG(-1UL)
 #define USER_DS		MAKE_MM_SEG(PAGE_OFFSET)
 #define get_ds()	(KERNEL_DS)
 #define get_fs()	(current_thread_info()->addr_limit)
 #define set_fs(x)	(current_thread_info()->addr_limit = (x))
 #define segment_eq(a, b) ((a).seg == (b).seg)
 #ifndef __tilegx__
 /*
  * We could allow mapping all 16 MB at 0xfc000000, but we set up a
  * special hack in arch_setup_additional_pages() to auto-create a mapping
  * for the first 16 KB, and it would seem strange to have different
  * user-accessible semantics for memory at 0xfc000000 and above 0xfc004000.
  */
 static inline int is_arch_mappable_range(unsigned long addr,
 					 unsigned long size)
 {
 	return (addr >= MEM_USER_INTRPT &&
 		addr < (MEM_USER_INTRPT + INTRPT_SIZE) &&
 		size <= (MEM_USER_INTRPT + INTRPT_SIZE) - addr);
 }
 #define is_arch_mappable_range is_arch_mappable_range
 #else
 #define is_arch_mappable_range(addr, size) 0
 #endif
 /*
  * Test whether a block of memory is a valid user space address.
  * Returns 0 if the range is valid, nonzero otherwise.
  */
 int __range_ok(unsigned long addr, unsigned long size);
 /**
  * access_ok: - Checks if a user space pointer is valid
  * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE.  Note that
  *        %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe
  *        to write to a block, it is always safe to read from it.
  * @addr: User space pointer to start of block to check
  * @size: Size of block to check
  *
  * Context: User context only.  This function may sleep.
  *
  * Checks if a pointer to a block of memory in user space is valid.
  *
  * Returns true (nonzero) if the memory block may be valid, false (zero)
  * if it is definitely invalid.
  *
  * Note that, depending on architecture, this function probably just
  * checks that the pointer is in the user space range - after calling
  * this function, memory access functions may still return -EFAULT.
  */
 #define access_ok(type, addr, size) ({ \
 	__chk_user_ptr(addr); \
 	likely(__range_ok((unsigned long)(addr), (size)) == 0);	\
 })
 /*
  * The exception table consists of pairs of addresses: the first is the
  * address of an instruction that is allowed to fault, and the second is
  * the address at which the program should continue.  No registers are
  * modified, so it is entirely up to the continuation code to figure out
  * what to do.
  *
  * All the routines below use bits of fixup code that are out of line
  * with the main instruction path.  This means when everything is well,
  * we don't even have to jump over them.  Further, they do not intrude
  * on our cache or tlb entries.
  */
 struct exception_table_entry {
 	unsigned long insn, fixup;
 };
 extern int fixup_exception(struct pt_regs *regs);
 /*
- * We return the __get_user_N function results in a structure,
+ * Support macros for __get_user().
- * thus in r0 and r1.  If "err" is zero, "val" is the result
- * of the read; otherwise, "err" is -EFAULT.
  *
- * We rarely need 8-byte values on a 32-bit architecture, but
+ * Implementation note: The "case 8" logic of casting to the type of
- * we size the structure to accommodate.  In practice, for the
+ * the result of subtracting the value from itself is basically a way
- * the smaller reads, we can zero the high word for free, and
+ * of keeping all integer types the same, but casting any pointers to
- * the caller will ignore it by virtue of casting anyway.
+ * ptrdiff_t, i.e. also an integer type.  This way there are no
+ * questionable casts seen by the compiler on an ILP32 platform.
+ *
+ * Note that __get_user() and __put_user() assume proper alignment.
  */
-struct __get_user {
-	unsigned long long val;
-	int err;
-};
-/*
+#ifdef __LP64__
- * FIXME: we should express these as inline extended assembler, since
+#define _ASM_PTR	".quad"
- * they're fundamentally just a variable dereference and some
+#else
- * supporting exception_table gunk.  Note that (a la i386) we can
+#define _ASM_PTR	".long"
- * extend the copy_to_user and copy_from_user routines to call into
+#endif
- * such extended assembler routines, though we will have to use a
- * different return code in that case (1, 2, or 4, rather than -EFAULT).
- */
-extern struct __get_user __get_user_1(const void __user *);
-extern struct __get_user __get_user_2(const void __user *);
-extern struct __get_user __get_user_4(const void __user *);
-extern struct __get_user __get_user_8(const void __user *);
-extern int __put_user_1(long, void __user *);
-extern int __put_user_2(long, void __user *);
-extern int __put_user_4(long, void __user *);
-extern int __put_user_8(long long, void __user *);
-/* Unimplemented routines to cause linker failures */
+#define __get_user_asm(OP, x, ptr, ret)					\
-extern struct __get_user __get_user_bad(void);
+	asm volatile("1: {" #OP " %1, %2; movei %0, 0 }\n"		\
-extern int __put_user_bad(void);
+		     ".pushsection .fixup,\"ax\"\n"			\
+		     "0: { movei %1, 0; movei %0, %3 }\n"		\
+		     "j 9f\n"						\
+		     ".section __ex_table,\"a\"\n"			\
+		     _ASM_PTR " 1b, 0b\n"				\
+		     ".popsection\n"					\
+		     "9:"						\
+		     : "=r" (ret), "=r" (x)				\
+		     : "r" (ptr), "i" (-EFAULT))
-/*
+#ifdef __tilegx__
- * Careful: we have to cast the result to the type of the pointer
+#define __get_user_1(x, ptr, ret) __get_user_asm(ld1u, x, ptr, ret)
- * for sign reasons.
+#define __get_user_2(x, ptr, ret) __get_user_asm(ld2u, x, ptr, ret)
- */
+#define __get_user_4(x, ptr, ret) __get_user_asm(ld4u, x, ptr, ret)
+#define __get_user_8(x, ptr, ret) __get_user_asm(ld, x, ptr, ret)
+#else
+#define __get_user_1(x, ptr, ret) __get_user_asm(lb_u, x, ptr, ret)
+#define __get_user_2(x, ptr, ret) __get_user_asm(lh_u, x, ptr, ret)
+#define __get_user_4(x, ptr, ret) __get_user_asm(lw, x, ptr, ret)
+#ifdef __LITTLE_ENDIAN
+#define __lo32(a, b) a
+#define __hi32(a, b) b
+#else
+#define __lo32(a, b) b
+#define __hi32(a, b) a
+#endif
+#define __get_user_8(x, ptr, ret)					\
+	({								\
+		unsigned int __a, __b;					\
+		asm volatile("1: { lw %1, %3; addi %2, %3, 4 }\n"	\
+			     "2: { lw %2, %2; movei %0, 0 }\n"		\
+			     ".pushsection .fixup,\"ax\"\n"		\
+			     "0: { movei %1, 0; movei %2, 0 }\n"	\
+			     "{ movei %0, %4; j 9f }\n"			\
+			     ".section __ex_table,\"a\"\n"		\
+			     ".word 1b, 0b\n"				\
+			     ".word 2b, 0b\n"				\
+			     ".popsection\n"				\
+			     "9:"					\
+			     : "=r" (ret), "=r" (__a), "=&r" (__b)	\
+			     : "r" (ptr), "i" (-EFAULT));		\
+		(x) = (__typeof(x))(__typeof((x)-(x)))			\
+			(((u64)__hi32(__a, __b) << 32) |		\
+			 __lo32(__a, __b));				\
+	})
+#endif
+extern int __get_user_bad(void)
+  __attribute__((warning("sizeof __get_user argument not 1, 2, 4 or 8")));
 /**
  * __get_user: - Get a simple variable from user space, with less checking.
  * @x:   Variable to store result.
  * @ptr: Source address, in user space.
  *
  * Context: User context only.  This function may sleep.
  *
  * This macro copies a single simple variable from user space to kernel
  * space.  It supports simple types like char and int, but not larger
  * data types like structures or arrays.
  *
  * @ptr must have pointer-to-simple-variable type, and the result of
  * dereferencing @ptr must be assignable to @x without a cast.
  *
  * Returns zero on success, or -EFAULT on error.
  * On error, the variable @x is set to zero.
  *
  * Caller must check the pointer with access_ok() before calling this
  * function.
  */
 #define __get_user(x, ptr)						\
-({	struct __get_user __ret;					\
+	({								\
-	__typeof__(*(ptr)) const __user *__gu_addr = (ptr);		\
+		int __ret;						\
-	__chk_user_ptr(__gu_addr);					\
+		__chk_user_ptr(ptr);					\
-	switch (sizeof(*(__gu_addr))) {					\
+		switch (sizeof(*(ptr))) {				\
-	case 1:								\
+		case 1: __get_user_1(x, ptr, __ret); break;		\
-		__ret = __get_user_1(__gu_addr);			\
+		case 2: __get_user_2(x, ptr, __ret); break;		\
-		break;							\
+		case 4: __get_user_4(x, ptr, __ret); break;		\
-	case 2:								\
+		case 8: __get_user_8(x, ptr, __ret); break;		\
-		__ret = __get_user_2(__gu_addr);			\
+		default: __ret = __get_user_bad(); break;		\
-		break;							\
+		}							\
-	case 4:								\
+		__ret;							\
-		__ret = __get_user_4(__gu_addr);			\
+	})
-		break;							\
-	case 8:								\
-		__ret = __get_user_8(__gu_addr);			\
-		break;							\
-	default:							\
-		__ret = __get_user_bad();				\
-		break;							\
-	}								\
-	(x) = (__typeof__(*__gu_addr)) (__typeof__(*__gu_addr - *__gu_addr)) \
-	  __ret.val;			                                \
-	__ret.err;							\
-})
+/* Support macros for __put_user(). */
+#define __put_user_asm(OP, x, ptr, ret)			\
+	asm volatile("1: {" #OP " %1, %2; movei %0, 0 }\n"		\
+		     ".pushsection .fixup,\"ax\"\n"			\
+		     "0: { movei %0, %3; j 9f }\n"			\
+		     ".section __ex_table,\"a\"\n"			\
+		     _ASM_PTR " 1b, 0b\n"				\
+		     ".popsection\n"					\
+		     "9:"						\
+		     : "=r" (ret)					\
+		     : "r" (ptr), "r" (x), "i" (-EFAULT))
+#ifdef __tilegx__
+#define __put_user_1(x, ptr, ret) __put_user_asm(st1, x, ptr, ret)
+#define __put_user_2(x, ptr, ret) __put_user_asm(st2, x, ptr, ret)
+#define __put_user_4(x, ptr, ret) __put_user_asm(st4, x, ptr, ret)
+#define __put_user_8(x, ptr, ret) __put_user_asm(st, x, ptr, ret)
+#else
+#define __put_user_1(x, ptr, ret) __put_user_asm(sb, x, ptr, ret)
+#define __put_user_2(x, ptr, ret) __put_user_asm(sh, x, ptr, ret)
+#define __put_user_4(x, ptr, ret) __put_user_asm(sw, x, ptr, ret)
+#define __put_user_8(x, ptr, ret)					\
+	({								\
+		u64 __x = (__typeof((x)-(x)))(x);			\
+		int __lo = (int) __x, __hi = (int) (__x >> 32);		\
+		asm volatile("1: { sw %1, %2; addi %0, %1, 4 }\n"	\
+			     "2: { sw %0, %3; movei %0, 0 }\n"		\
+			     ".pushsection .fixup,\"ax\"\n"		\
+			     "0: { movei %0, %4; j 9f }\n"		\
+			     ".section __ex_table,\"a\"\n"		\
+			     ".word 1b, 0b\n"				\
+			     ".word 2b, 0b\n"				\
+			     ".popsection\n"				\
+			     "9:"					\
+			     : "=&r" (ret)				\
+			     : "r" (ptr), "r" (__lo32(__lo, __hi)),	\
+			     "r" (__hi32(__lo, __hi)), "i" (-EFAULT));	\
+	})
+#endif
+extern int __put_user_bad(void)
+  __attribute__((warning("sizeof __put_user argument not 1, 2, 4 or 8")));
 /**
  * __put_user: - Write a simple value into user space, with less checking.
  * @x:   Value to copy to user space.
  * @ptr: Destination address, in user space.
  *
  * Context: User context only.  This function may sleep.
  *
  * This macro copies a single simple value from kernel space to user
  * space.  It supports simple types like char and int, but not larger
  * data types like structures or arrays.
  *
  * @ptr must have pointer-to-simple-variable type, and @x must be assignable
  * to the result of dereferencing @ptr.
  *
  * Caller must check the pointer with access_ok() before calling this
  * function.
  *
  * Returns zero on success, or -EFAULT on error.
- *
- * Implementation note: The "case 8" logic of casting to the type of
- * the result of subtracting the value from itself is basically a way
- * of keeping all integer types the same, but casting any pointers to
- * ptrdiff_t, i.e. also an integer type.  This way there are no
- * questionable casts seen by the compiler on an ILP32 platform.
  */
 #define __put_user(x, ptr)						\
 ({									\
-	int __pu_err = 0;						\
+	int __ret;							\
-	__typeof__(*(ptr)) __user *__pu_addr = (ptr);			\
+	__chk_user_ptr(ptr);						\
-	typeof(*__pu_addr) __pu_val = (x);				\
+	switch (sizeof(*(ptr))) {					\
-	__chk_user_ptr(__pu_addr);					\
+	case 1: __put_user_1(x, ptr, __ret); break;			\
-	switch (sizeof(__pu_val)) {					\
+	case 2: __put_user_2(x, ptr, __ret); break;			\
-	case 1:								\
+	case 4: __put_user_4(x, ptr, __ret); break;			\
-		__pu_err = __put_user_1((long)__pu_val, __pu_addr);	\
+	case 8: __put_user_8(x, ptr, __ret); break;			\
-		break;							\
+	default: __ret = __put_user_bad(); break;			\
-	case 2:								\
-		__pu_err = __put_user_2((long)__pu_val, __pu_addr);	\
-		break;							\
-	case 4:								\
-		__pu_err = __put_user_4((long)__pu_val, __pu_addr);	\
-		break;							\
-	case 8:								\
-		__pu_err =						\
-		  __put_user_8((__typeof__(__pu_val - __pu_val))__pu_val,\
-			__pu_addr);					\
-		break;							\
-	default:							\
-		__pu_err = __put_user_bad();				\
-		break;							\
 	}								\
-	__pu_err;							\
+	__ret;								\
 })
 /*
  * The versions of get_user and put_user without initial underscores
  * check the address of their arguments to make sure they are not
  * in kernel space.
  */
 #define put_user(x, ptr)						\
 ({									\
 	__typeof__(*(ptr)) __user *__Pu_addr = (ptr);			\
 	access_ok(VERIFY_WRITE, (__Pu_addr), sizeof(*(__Pu_addr))) ?	\
 		__put_user((x), (__Pu_addr)) :				\
 		-EFAULT;						\
 })
 #define get_user(x, ptr)						\
 ({									\
 	__typeof__(*(ptr)) const __user *__Gu_addr = (ptr);		\
 	access_ok(VERIFY_READ, (__Gu_addr), sizeof(*(__Gu_addr))) ?	\
 		__get_user((x), (__Gu_addr)) :				\
 		((x) = 0, -EFAULT);					\
 })
 /**
  * __copy_to_user() - copy data into user space, with less checking.
  * @to:   Destination address, in user space.
  * @from: Source address, in kernel space.
  * @n:    Number of bytes to copy.
  *
  * Context: User context only.  This function may sleep.
  *
  * Copy data from kernel space to user space.  Caller must check
  * the specified block with access_ok() before calling this function.
  *
  * Returns number of bytes that could not be copied.
  * On success, this will be zero.
  *
  * An alternate version - __copy_to_user_inatomic() - is designed
  * to be called from atomic context, typically bracketed by calls
  * to pagefault_disable() and pagefault_enable().
  */
 extern unsigned long __must_check __copy_to_user_inatomic(
 	void __user *to, const void *from, unsigned long n);
 static inline unsigned long __must_check
 __copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	might_fault();
 	return __copy_to_user_inatomic(to, from, n);
 }
 static inline unsigned long __must_check
 copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	if (access_ok(VERIFY_WRITE, to, n))
 		n = __copy_to_user(to, from, n);
 	return n;
 }
 /**
  * __copy_from_user() - copy data from user space, with less checking.
  * @to:   Destination address, in kernel space.
  * @from: Source address, in user space.
  * @n:    Number of bytes to copy.
  *
  * Context: User context only.  This function may sleep.
  *
  * Copy data from user space to kernel space.  Caller must check
  * the specified block with access_ok() before calling this function.
  *
  * Returns number of bytes that could not be copied.
  * On success, this will be zero.
  *
  * If some data could not be copied, this function will pad the copied
  * data to the requested size using zero bytes.
  *
  * An alternate version - __copy_from_user_inatomic() - is designed
  * to be called from atomic context, typically bracketed by calls
  * to pagefault_disable() and pagefault_enable().  This version
  * does *NOT* pad with zeros.
  */
 extern unsigned long __must_check __copy_from_user_inatomic(
 	void *to, const void __user *from, unsigned long n);
 extern unsigned long __must_check __copy_from_user_zeroing(
 	void *to, const void __user *from, unsigned long n);
 static inline unsigned long __must_check
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {
        might_fault();
        return __copy_from_user_zeroing(to, from, n);
 }
 static inline unsigned long __must_check
 _copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	if (access_ok(VERIFY_READ, from, n))
 		n = __copy_from_user(to, from, n);
 	else
 		memset(to, 0, n);
 	return n;
 }
 #ifdef CONFIG_DEBUG_COPY_FROM_USER
 extern void copy_from_user_overflow(void)
 	__compiletime_warning("copy_from_user() size is not provably correct");
 static inline unsigned long __must_check copy_from_user(void *to,
 					  const void __user *from,
 					  unsigned long n)
 {
 	int sz = __compiletime_object_size(to);
 	if (likely(sz == -1 || sz >= n))
 		n = _copy_from_user(to, from, n);
 	else
 		copy_from_user_overflow();
 	return n;
 }
 #else
 #define copy_from_user _copy_from_user
 #endif
 #ifdef __tilegx__
 /**
  * __copy_in_user() - copy data within user space, with less checking.
  * @to:   Destination address, in user space.
- * @from: Source address, in kernel space.
+ * @from: Source address, in user space.
  * @n:    Number of bytes to copy.
  *
  * Context: User context only.  This function may sleep.
  *
  * Copy data from user space to user space.  Caller must check
  * the specified blocks with access_ok() before calling this function.
  *
  * Returns number of bytes that could not be copied.
  * On success, this will be zero.
  */
 extern unsigned long __copy_in_user_inatomic(
 	void __user *to, const void __user *from, unsigned long n);
 static inline unsigned long __must_check
 __copy_in_user(void __user *to, const void __user *from, unsigned long n)
 {
 	might_sleep();
 	return __copy_in_user_inatomic(to, from, n);
 }
 static inline unsigned long __must_check
 copy_in_user(void __user *to, const void __user *from, unsigned long n)
 {
 	if (access_ok(VERIFY_WRITE, to, n) && access_ok(VERIFY_READ, from, n))
 		n = __copy_in_user(to, from, n);
 	return n;
 }
 #endif
 /**
  * strlen_user: - Get the size of a string in user space.
  * @str: The string to measure.
  *
  * Context: User context only.  This function may sleep.
  *
  * Get the size of a NUL-terminated string in user space.
  *
  * Returns the size of the string INCLUDING the terminating NUL.
  * On exception, returns 0.
  *
  * If there is a limit on the length of a valid string, you may wish to
  * consider using strnlen_user() instead.
  */
 extern long strnlen_user_asm(const char __user *str, long n);
 static inline long __must_check strnlen_user(const char __user *str, long n)
 {
 	might_fault();
 	return strnlen_user_asm(str, n);
 }
 #define strlen_user(str) strnlen_user(str, LONG_MAX)
 /**
  * strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking.
  * @dst:   Destination address, in kernel space.  This buffer must be at
  *         least @count bytes long.
  * @src:   Source address, in user space.
  * @count: Maximum number of bytes to copy, including the trailing NUL.
  *
  * Copies a NUL-terminated string from userspace to kernel space.
  * Caller must check the specified block with access_ok() before calling
  * this function.
  *
  * On success, returns the length of the string (not including the trailing
  * NUL).
  *
  * If access to userspace fails, returns -EFAULT (some data may have been
  * copied).
  *
  * If @count is smaller than the length of the string, copies @count bytes
  * and returns @count.
  */
 extern long strncpy_from_user_asm(char *dst, const char __user *src, long);
 static inline long __must_check __strncpy_from_user(
 	char *dst, const char __user *src, long count)
 {
 	might_fault();
 	return strncpy_from_user_asm(dst, src, count);
 }
 static inline long __must_check strncpy_from_user(
 	char *dst, const char __user *src, long count)
 {
 	if (access_ok(VERIFY_READ, src, 1))
 		return __strncpy_from_user(dst, src, count);
 	return -EFAULT;
 }
 /**
  * clear_user: - Zero a block of memory in user space.
  * @mem:   Destination address, in user space.
  * @len:   Number of bytes to zero.
  *
  * Zero a block of memory in user space.
  *
  * Returns number of bytes that could not be cleared.
  * On success, this will be zero.
  */
 extern unsigned long clear_user_asm(void __user *mem, unsigned long len);
 static inline unsigned long __must_check __clear_user(
 	void __user *mem, unsigned long len)
 {
 	might_fault();
 	return clear_user_asm(mem, len);
 }
 static inline unsigned long __must_check clear_user(
 	void __user *mem, unsigned long len)
 {
 	if (access_ok(VERIFY_WRITE, mem, len))
 		return __clear_user(mem, len);
 	return len;
 }
 /**
  * flush_user: - Flush a block of memory in user space from cache.
  * @mem:   Destination address, in user space.
  * @len:   Number of bytes to flush.
  *
  * Returns number of bytes that could not be flushed.
  * On success, this will be zero.
  */
 extern unsigned long flush_user_asm(void __user *mem, unsigned long len);
 static inline unsigned long __must_check __flush_user(
 	void __user *mem, unsigned long len)
 {
 	int retval;
 	might_fault();
 	retval = flush_user_asm(mem, len);
 	mb_incoherent();
 	return retval;
 }
 static inline unsigned long __must_check flush_user(
 	void __user *mem, unsigned long len)
 {
 	if (access_ok(VERIFY_WRITE, mem, len))
 		return __flush_user(mem, len);
 	return len;
 }
 /**
  * inv_user: - Invalidate a block of memory in user space from cache.
  * @mem:   Destination address, in user space.
  * @len:   Number of bytes to invalidate.
  *
  * Returns number of bytes that could not be invalidated.
  * On success, this will be zero.
  *
  * Note that on Tile64, the "inv" operation is in fact a
  * "flush and invalidate", so cache write-backs will occur prior
  * to the cache being marked invalid.

arch/tile/kernel/Makefile

Diff comments View file @ 47d632f

 #
 # Makefile for the Linux/TILE kernel.
 #
 extra-y := vmlinux.lds head_$(BITS).o
 obj-y := backtrace.o entry.o init_task.o irq.o messaging.o \
 	pci-dma.o proc.o process.o ptrace.o reboot.o \
 	setup.o signal.o single_step.o stack.o sys.o sysfs.o time.o traps.o \
 	intvec_$(BITS).o regs_$(BITS).o tile-desc_$(BITS).o
 obj-$(CONFIG_HARDWALL)		+= hardwall.o
-obj-$(CONFIG_TILEGX)		+= futex_64.o
 obj-$(CONFIG_COMPAT)		+= compat.o compat_signal.o
 obj-$(CONFIG_SMP)		+= smpboot.o smp.o tlb.o
 obj-$(CONFIG_MODULES)		+= module.o
 obj-$(CONFIG_EARLY_PRINTK)	+= early_printk.o
 obj-$(CONFIG_KEXEC)		+= machine_kexec.o relocate_kernel.o
 obj-$(CONFIG_PCI)		+= pci.o

arch/tile/lib/atomic_32.c

Diff comments View file @ 47d632f

 /*
  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  */
 #include <linux/cache.h>
 #include <linux/delay.h>
 #include <linux/uaccess.h>
 #include <linux/module.h>
 #include <linux/mm.h>
 #include <linux/atomic.h>
-#include <asm/futex.h>
 #include <arch/chip.h>
 /* See <asm/atomic_32.h> */
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
 /*
  * A block of memory containing locks for atomic ops. Each instance of this
  * struct will be homed on a different CPU.
  */
 struct atomic_locks_on_cpu {
 	int lock[ATOMIC_HASH_L2_SIZE];
 } __attribute__((aligned(ATOMIC_HASH_L2_SIZE * 4)));
 static DEFINE_PER_CPU(struct atomic_locks_on_cpu, atomic_lock_pool);
 /* The locks we'll use until __init_atomic_per_cpu is called. */
 static struct atomic_locks_on_cpu __initdata initial_atomic_locks;
 /* Hash into this vector to get a pointer to lock for the given atomic. */
 struct atomic_locks_on_cpu *atomic_lock_ptr[ATOMIC_HASH_L1_SIZE]
 	__write_once = {
 	[0 ... ATOMIC_HASH_L1_SIZE-1] (&initial_atomic_locks)
 };
 #else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
 /* This page is remapped on startup to be hash-for-home. */
 int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss;
 #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
-static inline int *__atomic_hashed_lock(volatile void *v)
+int *__atomic_hashed_lock(volatile void *v)
 {
 	/* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
 	unsigned long i =
 		(unsigned long) v & ((PAGE_SIZE-1) & -sizeof(long long));
 	unsigned long n = __insn_crc32_32(0, i);
 	/* Grab high bits for L1 index. */
 	unsigned long l1_index = n >> ((sizeof(n) * 8) - ATOMIC_HASH_L1_SHIFT);
 	/* Grab low bits for L2 index. */
 	unsigned long l2_index = n & (ATOMIC_HASH_L2_SIZE - 1);
 	return &atomic_lock_ptr[l1_index]->lock[l2_index];
 #else
 	/*
 	 * Use bits [3, 3 + ATOMIC_HASH_SHIFT) as the lock index.
 	 * Using mm works here because atomic_locks is page aligned.
 	 */
 	unsigned long ptr = __insn_mm((unsigned long)v >> 1,
 				      (unsigned long)atomic_locks,
 				      2, (ATOMIC_HASH_SHIFT + 2) - 1);
 	return (int *)ptr;
 #endif
 }
 #ifdef CONFIG_SMP
 /* Return whether the passed pointer is a valid atomic lock pointer. */
 static int is_atomic_lock(int *p)
 {
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
 	int i;
 	for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
 		if (p >= &atomic_lock_ptr[i]->lock[0] &&
 		    p < &atomic_lock_ptr[i]->lock[ATOMIC_HASH_L2_SIZE]) {
 			return 1;
 		}
 	}
 	return 0;
 #else
 	return p >= &atomic_locks[0] && p < &atomic_locks[ATOMIC_HASH_SIZE];
 #endif
 }
 void __atomic_fault_unlock(int *irqlock_word)
 {
 	BUG_ON(!is_atomic_lock(irqlock_word));
 	BUG_ON(*irqlock_word != 1);
 	*irqlock_word = 0;
 }
 #endif /* CONFIG_SMP */
 static inline int *__atomic_setup(volatile void *v)
 {
 	/* Issue a load to the target to bring it into cache. */
 	*(volatile int *)v;
 	return __atomic_hashed_lock(v);
 }
 int _atomic_xchg(atomic_t *v, int n)
 {
 	return __atomic_xchg(&v->counter, __atomic_setup(v), n).val;
 }
 EXPORT_SYMBOL(_atomic_xchg);
 int _atomic_xchg_add(atomic_t *v, int i)
 {
 	return __atomic_xchg_add(&v->counter, __atomic_setup(v), i).val;
 }
 EXPORT_SYMBOL(_atomic_xchg_add);
 int _atomic_xchg_add_unless(atomic_t *v, int a, int u)
 {
 	/*
 	 * Note: argument order is switched here since it is easier
 	 * to use the first argument consistently as the "old value"
 	 * in the assembly, as is done for _atomic_cmpxchg().
 	 */
 	return __atomic_xchg_add_unless(&v->counter, __atomic_setup(v), u, a)
 		.val;
 }
 EXPORT_SYMBOL(_atomic_xchg_add_unless);
 int _atomic_cmpxchg(atomic_t *v, int o, int n)
 {
 	return __atomic_cmpxchg(&v->counter, __atomic_setup(v), o, n).val;
 }
 EXPORT_SYMBOL(_atomic_cmpxchg);
 unsigned long _atomic_or(volatile unsigned long *p, unsigned long mask)
 {
 	return __atomic_or((int *)p, __atomic_setup(p), mask).val;
 }
 EXPORT_SYMBOL(_atomic_or);
 unsigned long _atomic_andn(volatile unsigned long *p, unsigned long mask)
 {
 	return __atomic_andn((int *)p, __atomic_setup(p), mask).val;
 }
 EXPORT_SYMBOL(_atomic_andn);
 unsigned long _atomic_xor(volatile unsigned long *p, unsigned long mask)
 {
 	return __atomic_xor((int *)p, __atomic_setup(p), mask).val;
 }
 EXPORT_SYMBOL(_atomic_xor);
 u64 _atomic64_xchg(atomic64_t *v, u64 n)
 {
 	return __atomic64_xchg(&v->counter, __atomic_setup(v), n);
 }
 EXPORT_SYMBOL(_atomic64_xchg);
 u64 _atomic64_xchg_add(atomic64_t *v, u64 i)
 {
 	return __atomic64_xchg_add(&v->counter, __atomic_setup(v), i);
 }
 EXPORT_SYMBOL(_atomic64_xchg_add);
 u64 _atomic64_xchg_add_unless(atomic64_t *v, u64 a, u64 u)
 {
 	/*
 	 * Note: argument order is switched here since it is easier
 	 * to use the first argument consistently as the "old value"
 	 * in the assembly, as is done for _atomic_cmpxchg().
 	 */
 	return __atomic64_xchg_add_unless(&v->counter, __atomic_setup(v),
 					  u, a);
 }
 EXPORT_SYMBOL(_atomic64_xchg_add_unless);
 u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n)
 {
 	return __atomic64_cmpxchg(&v->counter, __atomic_setup(v), o, n);
 }
 EXPORT_SYMBOL(_atomic64_cmpxchg);
-static inline int *__futex_setup(int __user *v)
-{
-	/*
-	 * Issue a prefetch to the counter to bring it into cache.
-	 * As for __atomic_setup, but we can't do a read into the L1
-	 * since it might fault; instead we do a prefetch into the L2.
-	 */
-	__insn_prefetch(v);
-	return __atomic_hashed_lock((int __force *)v);
-}
-struct __get_user futex_set(u32 __user *v, int i)
-{
-	return __atomic_xchg((int __force *)v, __futex_setup(v), i);
-}
-struct __get_user futex_add(u32 __user *v, int n)
-{
-	return __atomic_xchg_add((int __force *)v, __futex_setup(v), n);
-}
-struct __get_user futex_or(u32 __user *v, int n)
-{
-	return __atomic_or((int __force *)v, __futex_setup(v), n);
-}
-struct __get_user futex_andn(u32 __user *v, int n)
-{
-	return __atomic_andn((int __force *)v, __futex_setup(v), n);
-}
-struct __get_user futex_xor(u32 __user *v, int n)
-{
-	return __atomic_xor((int __force *)v, __futex_setup(v), n);
-}
-struct __get_user futex_cmpxchg(u32 __user *v, int o, int n)
-{
-	return __atomic_cmpxchg((int __force *)v, __futex_setup(v), o, n);
-}
 /*
  * If any of the atomic or futex routines hit a bad address (not in
  * the page tables at kernel PL) this routine is called.  The futex
  * routines are never used on kernel space, and the normal atomics and
  * bitops are never used on user space.  So a fault on kernel space
  * must be fatal, but a fault on userspace is a futex fault and we
  * need to return -EFAULT.  Note that the context this routine is
  * invoked in is the context of the "_atomic_xxx()" routines called
  * by the functions in this file.
  */
 struct __get_user __atomic_bad_address(int __user *addr)
 {
 	if (unlikely(!access_ok(VERIFY_WRITE, addr, sizeof(int))))
 		panic("Bad address used for kernel atomic op: %p\n", addr);
 	return (struct __get_user) { .err = -EFAULT };
 }
 #if CHIP_HAS_CBOX_HOME_MAP()
 static int __init noatomichash(char *str)
 {
 	pr_warning("noatomichash is deprecated.\n");
 	return 1;
 }
 __setup("noatomichash", noatomichash);
 #endif
 void __init __init_atomic_per_cpu(void)
 {
 #if ATOMIC_LOCKS_FOUND_VIA_TABLE()
 	unsigned int i;
 	int actual_cpu;
 	/*
 	 * Before this is called from setup, we just have one lock for
 	 * all atomic objects/operations.  Here we replace the
 	 * elements of atomic_lock_ptr so that they point at per_cpu
 	 * integers.  This seemingly over-complex approach stems from
 	 * the fact that DEFINE_PER_CPU defines an entry for each cpu
 	 * in the grid, not each cpu from 0..ATOMIC_HASH_SIZE-1.  But
 	 * for efficient hashing of atomics to their locks we want a
 	 * compile time constant power of 2 for the size of this
 	 * table, so we use ATOMIC_HASH_SIZE.
 	 *
 	 * Here we populate atomic_lock_ptr from the per cpu
 	 * atomic_lock_pool, interspersing by actual cpu so that
 	 * subsequent elements are homed on consecutive cpus.
 	 */
 	actual_cpu = cpumask_first(cpu_possible_mask);
 	for (i = 0; i < ATOMIC_HASH_L1_SIZE; ++i) {
 		/*
 		 * Preincrement to slightly bias against using cpu 0,
 		 * which has plenty of stuff homed on it already.
 		 */
 		actual_cpu = cpumask_next(actual_cpu, cpu_possible_mask);
 		if (actual_cpu >= nr_cpu_ids)
 			actual_cpu = cpumask_first(cpu_possible_mask);
 		atomic_lock_ptr[i] = &per_cpu(atomic_lock_pool, actual_cpu);
 	}
 #else /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
 	/* Validate power-of-two and "bigger than cpus" assumption */
 	BUILD_BUG_ON(ATOMIC_HASH_SIZE & (ATOMIC_HASH_SIZE-1));
 	BUG_ON(ATOMIC_HASH_SIZE < nr_cpu_ids);
 	/*
 	 * On TILEPro we prefer to use a single hash-for-home
 	 * page, since this means atomic operations are less
 	 * likely to encounter a TLB fault and thus should
 	 * in general perform faster.  You may wish to disable
 	 * this in situations where few hash-for-home tiles
 	 * are configured.
 	 */
 	BUG_ON((unsigned long)atomic_locks % PAGE_SIZE != 0);
 	/* The locks must all fit on one page. */
 	BUILD_BUG_ON(ATOMIC_HASH_SIZE * sizeof(int) > PAGE_SIZE);
 	/*
 	 * We use the page offset of the atomic value's address as
 	 * an index into atomic_locks, excluding the low 3 bits.
 	 * That should not produce more indices than ATOMIC_HASH_SIZE.
 	 */
 	BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE);
 #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */
-	/* The futex code makes this assumption, so we validate it here. */
-	BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int));
 }

arch/tile/lib/exports.c

Diff comments View file @ 47d632f

 /*
  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  *
  * Exports from assembler code and from libtile-cc.
  */
 #include <linux/module.h>
 /* arch/tile/lib/usercopy.S */
 #include <linux/uaccess.h>
-EXPORT_SYMBOL(__get_user_1);
-EXPORT_SYMBOL(__get_user_2);
-EXPORT_SYMBOL(__get_user_4);
-EXPORT_SYMBOL(__get_user_8);
-EXPORT_SYMBOL(__put_user_1);
-EXPORT_SYMBOL(__put_user_2);
-EXPORT_SYMBOL(__put_user_4);
-EXPORT_SYMBOL(__put_user_8);
 EXPORT_SYMBOL(strnlen_user_asm);
 EXPORT_SYMBOL(strncpy_from_user_asm);
 EXPORT_SYMBOL(clear_user_asm);
 EXPORT_SYMBOL(flush_user_asm);
 EXPORT_SYMBOL(inv_user_asm);
 EXPORT_SYMBOL(finv_user_asm);
 /* arch/tile/kernel/entry.S */
 #include <linux/kernel.h>
 #include <asm/processor.h>
 EXPORT_SYMBOL(current_text_addr);
 EXPORT_SYMBOL(dump_stack);
 /* arch/tile/kernel/head.S */
 EXPORT_SYMBOL(empty_zero_page);
 /* arch/tile/lib/, various memcpy files */
 EXPORT_SYMBOL(memcpy);
 EXPORT_SYMBOL(__copy_to_user_inatomic);
 EXPORT_SYMBOL(__copy_from_user_inatomic);
 EXPORT_SYMBOL(__copy_from_user_zeroing);
 #ifdef __tilegx__
 EXPORT_SYMBOL(__copy_in_user_inatomic);
 #endif
 /* hypervisor glue */
 #include <hv/hypervisor.h>
 EXPORT_SYMBOL(hv_dev_open);
 EXPORT_SYMBOL(hv_dev_pread);
 EXPORT_SYMBOL(hv_dev_pwrite);
 EXPORT_SYMBOL(hv_dev_preada);
 EXPORT_SYMBOL(hv_dev_pwritea);
 EXPORT_SYMBOL(hv_dev_poll);
 EXPORT_SYMBOL(hv_dev_poll_cancel);
 EXPORT_SYMBOL(hv_dev_close);
 EXPORT_SYMBOL(hv_sysconf);
 EXPORT_SYMBOL(hv_confstr);
 /* libgcc.a */
 uint32_t __udivsi3(uint32_t dividend, uint32_t divisor);
 EXPORT_SYMBOL(__udivsi3);
 int32_t __divsi3(int32_t dividend, int32_t divisor);
 EXPORT_SYMBOL(__divsi3);
 uint64_t __udivdi3(uint64_t dividend, uint64_t divisor);
 EXPORT_SYMBOL(__udivdi3);
 int64_t __divdi3(int64_t dividend, int64_t divisor);
 EXPORT_SYMBOL(__divdi3);
 uint32_t __umodsi3(uint32_t dividend, uint32_t divisor);
 EXPORT_SYMBOL(__umodsi3);
 int32_t __modsi3(int32_t dividend, int32_t divisor);
 EXPORT_SYMBOL(__modsi3);
 uint64_t __umoddi3(uint64_t dividend, uint64_t divisor);
 EXPORT_SYMBOL(__umoddi3);
 int64_t __moddi3(int64_t dividend, int64_t divisor);
 EXPORT_SYMBOL(__moddi3);
 #ifndef __tilegx__
 int64_t __muldi3(int64_t, int64_t);
 EXPORT_SYMBOL(__muldi3);
 uint64_t __lshrdi3(uint64_t, unsigned int);
 EXPORT_SYMBOL(__lshrdi3);
 uint64_t __ashrdi3(uint64_t, unsigned int);
 EXPORT_SYMBOL(__ashrdi3);
 uint64_t __ashldi3(uint64_t, unsigned int);
 EXPORT_SYMBOL(__ashldi3);
 #endif

arch/tile/lib/usercopy_32.S

Diff comments View file @ 47d632f

 /*
  * Copyright 2010 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  */
 #include <linux/linkage.h>
 #include <asm/errno.h>
 #include <asm/cache.h>
 #include <arch/chip.h>
 /* Access user memory, but use MMU to avoid propagating kernel exceptions. */
-	.pushsection .fixup,"ax"
-get_user_fault:
-	{ move r0, zero; move r1, zero }
-	{ movei r2, -EFAULT; jrp lr }
-	ENDPROC(get_user_fault)
-put_user_fault:
-	{ movei r0, -EFAULT; jrp lr }
-	ENDPROC(put_user_fault)
-	.popsection
-/*
- * __get_user_N functions take a pointer in r0, and return 0 in r2
- * on success, with the value in r0; or else -EFAULT in r2.
- */
-#define __get_user_N(bytes, LOAD) \
-	STD_ENTRY(__get_user_##bytes); \
-1:	{ LOAD r0, r0; move r1, zero; move r2, zero }; \
-	jrp lr; \
-	STD_ENDPROC(__get_user_##bytes); \
-	.pushsection __ex_table,"a"; \
-	.word 1b, get_user_fault; \
-	.popsection
-__get_user_N(1, lb_u)
-__get_user_N(2, lh_u)
-__get_user_N(4, lw)
-/*
- * __get_user_8 takes a pointer in r0, and returns 0 in r2
- * on success, with the value in r0/r1; or else -EFAULT in r2.
- */
-	STD_ENTRY(__get_user_8);
-1:	{ lw r0, r0; addi r1, r0, 4 };
-2:	{ lw r1, r1; move r2, zero };
-	jrp lr;
-	STD_ENDPROC(__get_user_8);
-	.pushsection __ex_table,"a";
-	.word 1b, get_user_fault;
-	.word 2b, get_user_fault;
-	.popsection
-/*
- * __put_user_N functions take a value in r0 and a pointer in r1,
- * and return 0 in r0 on success or -EFAULT on failure.
- */
-#define __put_user_N(bytes, STORE) \
-	STD_ENTRY(__put_user_##bytes); \
-1:	{ STORE r1, r0; move r0, zero }; \
-	jrp lr; \
-	STD_ENDPROC(__put_user_##bytes); \
-	.pushsection __ex_table,"a"; \
-	.word 1b, put_user_fault; \
-	.popsection
-__put_user_N(1, sb)
-__put_user_N(2, sh)
-__put_user_N(4, sw)
-/*
- * __put_user_8 takes a value in r0/r1 and a pointer in r2,
- * and returns 0 in r0 on success or -EFAULT on failure.
- */
-STD_ENTRY(__put_user_8)
-1:      { sw r2, r0; addi r2, r2, 4 }
-2:      { sw r2, r1; move r0, zero }
-	jrp lr
-	STD_ENDPROC(__put_user_8)
-	.pushsection __ex_table,"a"
-	.word 1b, put_user_fault
-	.word 2b, put_user_fault
-	.popsection
 /*
  * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
  * It returns the length, including the terminating NUL, or zero on exception.
  * If length is greater than the bound, returns one plus the bound.
  */
 STD_ENTRY(strnlen_user_asm)
 	{ bz r1, 2f; addi r3, r0, -1 }  /* bias down to include NUL */
 1:      { lb_u r4, r0; addi r1, r1, -1 }
 	bz r4, 2f
 	{ bnzt r1, 1b; addi r0, r0, 1 }
 2:      { sub r0, r0, r3; jrp lr }
 	STD_ENDPROC(strnlen_user_asm)
 	.pushsection .fixup,"ax"
 strnlen_user_fault:
 	{ move r0, zero; jrp lr }
 	ENDPROC(strnlen_user_fault)
 	.section __ex_table,"a"
 	.word 1b, strnlen_user_fault
 	.popsection
 /*
  * strncpy_from_user_asm takes the kernel target pointer in r0,
  * the userspace source pointer in r1, and the length bound (including
  * the trailing NUL) in r2.  On success, it returns the string length
  * (not including the trailing NUL), or -EFAULT on failure.
  */
 STD_ENTRY(strncpy_from_user_asm)
 	{ bz r2, 2f; move r3, r0 }
 1:      { lb_u r4, r1; addi r1, r1, 1; addi r2, r2, -1 }
 	{ sb r0, r4; addi r0, r0, 1 }
 	bz r2, 2f
 	bnzt r4, 1b
 	addi r0, r0, -1   /* don't count the trailing NUL */
 2:      { sub r0, r0, r3; jrp lr }
 	STD_ENDPROC(strncpy_from_user_asm)
 	.pushsection .fixup,"ax"
 strncpy_from_user_fault:
 	{ movei r0, -EFAULT; jrp lr }
 	ENDPROC(strncpy_from_user_fault)
 	.section __ex_table,"a"
 	.word 1b, strncpy_from_user_fault
 	.popsection
 /*
  * clear_user_asm takes the user target address in r0 and the
  * number of bytes to zero in r1.
  * It returns the number of uncopiable bytes (hopefully zero) in r0.
  * Note that we don't use a separate .fixup section here since we fall
  * through into the "fixup" code as the last straight-line bundle anyway.
  */
 STD_ENTRY(clear_user_asm)
 	{ bz r1, 2f; or r2, r0, r1 }
 	andi r2, r2, 3
 	bzt r2, .Lclear_aligned_user_asm
 1:      { sb r0, zero; addi r0, r0, 1; addi r1, r1, -1 }
 	bnzt r1, 1b
 2:      { move r0, r1; jrp lr }
 	.pushsection __ex_table,"a"
 	.word 1b, 2b
 	.popsection
 .Lclear_aligned_user_asm:
 1:      { sw r0, zero; addi r0, r0, 4; addi r1, r1, -4 }
 	bnzt r1, 1b
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(clear_user_asm)
 	.pushsection __ex_table,"a"
 	.word 1b, 2b
 	.popsection
 /*
  * flush_user_asm takes the user target address in r0 and the
  * number of bytes to flush in r1.
  * It returns the number of unflushable bytes (hopefully zero) in r0.
  */
 STD_ENTRY(flush_user_asm)
 	bz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { flush r0; addi r1, r1, -CHIP_FLUSH_STRIDE() }
 	{ addi r0, r0, CHIP_FLUSH_STRIDE(); bnzt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(flush_user_asm)
 	.pushsection __ex_table,"a"
 	.word 1b, 2b
 	.popsection
 /*
  * inv_user_asm takes the user target address in r0 and the
  * number of bytes to invalidate in r1.
  * It returns the number of not inv'able bytes (hopefully zero) in r0.
  */
 STD_ENTRY(inv_user_asm)
 	bz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { inv r0; addi r1, r1, -CHIP_INV_STRIDE() }
 	{ addi r0, r0, CHIP_INV_STRIDE(); bnzt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(inv_user_asm)
 	.pushsection __ex_table,"a"
 	.word 1b, 2b
 	.popsection
 /*
  * finv_user_asm takes the user target address in r0 and the
  * number of bytes to flush-invalidate in r1.
  * It returns the number of not finv'able bytes (hopefully zero) in r0.
  */
 STD_ENTRY(finv_user_asm)
 	bz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { finv r0; addi r1, r1, -CHIP_FINV_STRIDE() }
 	{ addi r0, r0, CHIP_FINV_STRIDE(); bnzt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(finv_user_asm)
 	.pushsection __ex_table,"a"
 	.word 1b, 2b
 	.popsection

arch/tile/lib/usercopy_64.S

Diff comments View file @ 47d632f

 /*
  * Copyright 2011 Tilera Corporation. All Rights Reserved.
  *
  *   This program is free software; you can redistribute it and/or
  *   modify it under the terms of the GNU General Public License
  *   as published by the Free Software Foundation, version 2.
  *
  *   This program is distributed in the hope that it will be useful, but
  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  *   NON INFRINGEMENT.  See the GNU General Public License for
  *   more details.
  */
 #include <linux/linkage.h>
 #include <asm/errno.h>
 #include <asm/cache.h>
 #include <arch/chip.h>
 /* Access user memory, but use MMU to avoid propagating kernel exceptions. */
-	.pushsection .fixup,"ax"
-get_user_fault:
-	{ movei r1, -EFAULT; move r0, zero }
-	jrp lr
-	ENDPROC(get_user_fault)
-put_user_fault:
-	{ movei r0, -EFAULT; jrp lr }
-	ENDPROC(put_user_fault)
-	.popsection
-/*
- * __get_user_N functions take a pointer in r0, and return 0 in r1
- * on success, with the value in r0; or else -EFAULT in r1.
- */
-#define __get_user_N(bytes, LOAD) \
-	STD_ENTRY(__get_user_##bytes); \
-1:	{ LOAD r0, r0; move r1, zero }; \
-	jrp lr; \
-	STD_ENDPROC(__get_user_##bytes); \
-	.pushsection __ex_table,"a"; \
-	.quad 1b, get_user_fault; \
-	.popsection
-__get_user_N(1, ld1u)
-__get_user_N(2, ld2u)
-__get_user_N(4, ld4u)
-__get_user_N(8, ld)
-/*
- * __put_user_N functions take a value in r0 and a pointer in r1,
- * and return 0 in r0 on success or -EFAULT on failure.
- */
-#define __put_user_N(bytes, STORE) \
-	STD_ENTRY(__put_user_##bytes); \
-1:	{ STORE r1, r0; move r0, zero }; \
-	jrp lr; \
-	STD_ENDPROC(__put_user_##bytes); \
-	.pushsection __ex_table,"a"; \
-	.quad 1b, put_user_fault; \
-	.popsection
-__put_user_N(1, st1)
-__put_user_N(2, st2)
-__put_user_N(4, st4)
-__put_user_N(8, st)
 /*
  * strnlen_user_asm takes the pointer in r0, and the length bound in r1.
  * It returns the length, including the terminating NUL, or zero on exception.
  * If length is greater than the bound, returns one plus the bound.
  */
 STD_ENTRY(strnlen_user_asm)
 	{ beqz r1, 2f; addi r3, r0, -1 }  /* bias down to include NUL */
 1:      { ld1u r4, r0; addi r1, r1, -1 }
 	beqz r4, 2f
 	{ bnezt r1, 1b; addi r0, r0, 1 }
 2:      { sub r0, r0, r3; jrp lr }
 	STD_ENDPROC(strnlen_user_asm)
 	.pushsection .fixup,"ax"
 strnlen_user_fault:
 	{ move r0, zero; jrp lr }
 	ENDPROC(strnlen_user_fault)
 	.section __ex_table,"a"
 	.quad 1b, strnlen_user_fault
 	.popsection
 /*
  * strncpy_from_user_asm takes the kernel target pointer in r0,
  * the userspace source pointer in r1, and the length bound (including
  * the trailing NUL) in r2.  On success, it returns the string length
  * (not including the trailing NUL), or -EFAULT on failure.
  */
 STD_ENTRY(strncpy_from_user_asm)
 	{ beqz r2, 2f; move r3, r0 }
 1:      { ld1u r4, r1; addi r1, r1, 1; addi r2, r2, -1 }
 	{ st1 r0, r4; addi r0, r0, 1 }
 	beqz r2, 2f
 	bnezt r4, 1b
 	addi r0, r0, -1   /* don't count the trailing NUL */
 2:      { sub r0, r0, r3; jrp lr }
 	STD_ENDPROC(strncpy_from_user_asm)
 	.pushsection .fixup,"ax"
 strncpy_from_user_fault:
 	{ movei r0, -EFAULT; jrp lr }
 	ENDPROC(strncpy_from_user_fault)
 	.section __ex_table,"a"
 	.quad 1b, strncpy_from_user_fault
 	.popsection
 /*
  * clear_user_asm takes the user target address in r0 and the
  * number of bytes to zero in r1.
  * It returns the number of uncopiable bytes (hopefully zero) in r0.
  * Note that we don't use a separate .fixup section here since we fall
  * through into the "fixup" code as the last straight-line bundle anyway.
  */
 STD_ENTRY(clear_user_asm)
 	{ beqz r1, 2f; or r2, r0, r1 }
 	andi r2, r2, 7
 	beqzt r2, .Lclear_aligned_user_asm
 1:      { st1 r0, zero; addi r0, r0, 1; addi r1, r1, -1 }
 	bnezt r1, 1b
 2:      { move r0, r1; jrp lr }
 	.pushsection __ex_table,"a"
 	.quad 1b, 2b
 	.popsection
 .Lclear_aligned_user_asm:
 1:      { st r0, zero; addi r0, r0, 8; addi r1, r1, -8 }
 	bnezt r1, 1b
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(clear_user_asm)
 	.pushsection __ex_table,"a"
 	.quad 1b, 2b
 	.popsection
 /*
  * flush_user_asm takes the user target address in r0 and the
  * number of bytes to flush in r1.
  * It returns the number of unflushable bytes (hopefully zero) in r0.
  */
 STD_ENTRY(flush_user_asm)
 	beqz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { flush r0; addi r1, r1, -CHIP_FLUSH_STRIDE() }
 	{ addi r0, r0, CHIP_FLUSH_STRIDE(); bnezt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(flush_user_asm)
 	.pushsection __ex_table,"a"
 	.quad 1b, 2b
 	.popsection
 /*
  * inv_user_asm takes the user target address in r0 and the
  * number of bytes to invalidate in r1.
  * It returns the number of not inv'able bytes (hopefully zero) in r0.
  */
 STD_ENTRY(inv_user_asm)
 	beqz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { inv r0; addi r1, r1, -CHIP_INV_STRIDE() }
 	{ addi r0, r0, CHIP_INV_STRIDE(); bnezt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(inv_user_asm)
 	.pushsection __ex_table,"a"
 	.quad 1b, 2b
 	.popsection
 /*
  * finv_user_asm takes the user target address in r0 and the
  * number of bytes to flush-invalidate in r1.
  * It returns the number of not finv'able bytes (hopefully zero) in r0.
  */
 STD_ENTRY(finv_user_asm)
 	beqz r1, 2f
 	{ movei r2, L2_CACHE_BYTES; add r1, r0, r1 }
 	{ sub r2, zero, r2; addi r1, r1, L2_CACHE_BYTES-1 }
 	{ and r0, r0, r2; and r1, r1, r2 }
 	{ sub r1, r1, r0 }
 1:      { finv r0; addi r1, r1, -CHIP_FINV_STRIDE() }
 	{ addi r0, r0, CHIP_FINV_STRIDE(); bnezt r1, 1b }
 2:      { move r0, r1; jrp lr }
 	STD_ENDPROC(finv_user_asm)
 	.pushsection __ex_table,"a"
 	.quad 1b, 2b
 	.popsection