Commit 951f22d5b1f0eaae35dafc669e3774a0c2084d10

Authored by Martin Schwidefsky
Committed by Linus Torvalds
1 parent 8449d003f3

[PATCH] s390: spin lock retry

Split spin lock and r/w lock implementation into a single try which is done
inline and an out of line function that repeatedly tries to get the lock
before doing the cpu_relax().  Add a system control to set the number of
retries before a cpu is yielded.

The reason for the spin lock retry is that the diagnose 0x44 that is used to
give up the virtual cpu is quite expensive.  For spin locks that are held only
for a short period of time the costs of the diagnoses outweights the savings
for spin locks that are held for a longer timer.  The default retry count is
1000.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

Showing 9 changed files with 230 additions and 190 deletions Side-by-side Diff

arch/s390/kernel/head64.S
... ... @@ -658,10 +658,8 @@
658 658 #
659 659 la %r1,0f-.LPG1(%r13) # set program check address
660 660 stg %r1,__LC_PGM_NEW_PSW+8
661   - mvc __LC_DIAG44_OPCODE(8),.Lnop-.LPG1(%r13)
662 661 diag 0,0,0x44 # test diag 0x44
663 662 oi 7(%r12),32 # set diag44 flag
664   - mvc __LC_DIAG44_OPCODE(8),.Ldiag44-.LPG1(%r13)
665 663 0:
666 664  
667 665 #
... ... @@ -702,7 +700,6 @@
702 700 .L4malign:.quad 0xffffffffffc00000
703 701 .Lscan2g:.quad 0x80000000 + 0x20000 - 8 # 2GB + 128K - 8
704 702 .Lnop: .long 0x07000700
705   -.Ldiag44:.long 0x83000044
706 703  
707 704 .org PARMAREA-64
708 705 .Lduct: .long 0,0,0,0,0,0,0,0
arch/s390/kernel/setup.c
... ... @@ -431,12 +431,6 @@
431 431 ctl_set_bit(14, 29);
432 432 }
433 433 #endif
434   -#ifdef CONFIG_ARCH_S390X
435   - if (MACHINE_HAS_DIAG44)
436   - lc->diag44_opcode = 0x83000044;
437   - else
438   - lc->diag44_opcode = 0x07000700;
439   -#endif /* CONFIG_ARCH_S390X */
440 434 set_prefix((u32)(unsigned long) lc);
441 435 }
442 436  
arch/s390/lib/Makefile
... ... @@ -5,6 +5,6 @@
5 5 EXTRA_AFLAGS := -traditional
6 6  
7 7 lib-y += delay.o string.o
8   -lib-$(CONFIG_ARCH_S390_31) += uaccess.o
9   -lib-$(CONFIG_ARCH_S390X) += uaccess64.o
  8 +lib-$(CONFIG_ARCH_S390_31) += uaccess.o spinlock.o
  9 +lib-$(CONFIG_ARCH_S390X) += uaccess64.o spinlock.o
arch/s390/lib/spinlock.c
  1 +/*
  2 + * arch/s390/lib/spinlock.c
  3 + * Out of line spinlock code.
  4 + *
  5 + * S390 version
  6 + * Copyright (C) 2004 IBM Deutschland Entwicklung GmbH, IBM Corporation
  7 + * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
  8 + */
  9 +
  10 +#include <linux/types.h>
  11 +#include <linux/module.h>
  12 +#include <linux/spinlock.h>
  13 +#include <linux/init.h>
  14 +#include <asm/io.h>
  15 +
  16 +atomic_t spin_retry_counter;
  17 +int spin_retry = 1000;
  18 +
  19 +/**
  20 + * spin_retry= parameter
  21 + */
  22 +static int __init spin_retry_setup(char *str)
  23 +{
  24 + spin_retry = simple_strtoul(str, &str, 0);
  25 + return 1;
  26 +}
  27 +__setup("spin_retry=", spin_retry_setup);
  28 +
  29 +static inline void
  30 +_diag44(void)
  31 +{
  32 +#ifdef __s390x__
  33 + if (MACHINE_HAS_DIAG44)
  34 +#endif
  35 + asm volatile("diag 0,0,0x44");
  36 +}
  37 +
  38 +void
  39 +_raw_spin_lock_wait(spinlock_t *lp, unsigned int pc)
  40 +{
  41 + int count = spin_retry;
  42 +
  43 + while (1) {
  44 + if (count-- <= 0) {
  45 + _diag44();
  46 + count = spin_retry;
  47 + }
  48 + atomic_inc(&spin_retry_counter);
  49 + if (_raw_compare_and_swap(&lp->lock, 0, pc) == 0)
  50 + return;
  51 + }
  52 +}
  53 +EXPORT_SYMBOL(_raw_spin_lock_wait);
  54 +
  55 +int
  56 +_raw_spin_trylock_retry(spinlock_t *lp, unsigned int pc)
  57 +{
  58 + int count = spin_retry;
  59 +
  60 + while (count-- > 0) {
  61 + atomic_inc(&spin_retry_counter);
  62 + if (_raw_compare_and_swap(&lp->lock, 0, pc) == 0)
  63 + return 1;
  64 + }
  65 + return 0;
  66 +}
  67 +EXPORT_SYMBOL(_raw_spin_trylock_retry);
  68 +
  69 +void
  70 +_raw_read_lock_wait(rwlock_t *rw)
  71 +{
  72 + unsigned int old;
  73 + int count = spin_retry;
  74 +
  75 + while (1) {
  76 + if (count-- <= 0) {
  77 + _diag44();
  78 + count = spin_retry;
  79 + }
  80 + atomic_inc(&spin_retry_counter);
  81 + old = rw->lock & 0x7fffffffU;
  82 + if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
  83 + return;
  84 + }
  85 +}
  86 +EXPORT_SYMBOL(_raw_read_lock_wait);
  87 +
  88 +int
  89 +_raw_read_trylock_retry(rwlock_t *rw)
  90 +{
  91 + unsigned int old;
  92 + int count = spin_retry;
  93 +
  94 + while (count-- > 0) {
  95 + atomic_inc(&spin_retry_counter);
  96 + old = rw->lock & 0x7fffffffU;
  97 + if (_raw_compare_and_swap(&rw->lock, old, old + 1) == old)
  98 + return 1;
  99 + }
  100 + return 0;
  101 +}
  102 +EXPORT_SYMBOL(_raw_read_trylock_retry);
  103 +
  104 +void
  105 +_raw_write_lock_wait(rwlock_t *rw)
  106 +{
  107 + int count = spin_retry;
  108 +
  109 + while (1) {
  110 + if (count-- <= 0) {
  111 + _diag44();
  112 + count = spin_retry;
  113 + }
  114 + atomic_inc(&spin_retry_counter);
  115 + if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
  116 + return;
  117 + }
  118 +}
  119 +EXPORT_SYMBOL(_raw_write_lock_wait);
  120 +
  121 +int
  122 +_raw_write_trylock_retry(rwlock_t *rw)
  123 +{
  124 + int count = spin_retry;
  125 +
  126 + while (count-- > 0) {
  127 + atomic_inc(&spin_retry_counter);
  128 + if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0)
  129 + return 1;
  130 + }
  131 + return 0;
  132 +}
  133 +EXPORT_SYMBOL(_raw_write_trylock_retry);
include/asm-s390/lowcore.h
... ... @@ -90,7 +90,6 @@
90 90 #define __LC_SYSTEM_TIMER 0x278
91 91 #define __LC_LAST_UPDATE_CLOCK 0x280
92 92 #define __LC_STEAL_CLOCK 0x288
93   -#define __LC_DIAG44_OPCODE 0x290
94 93 #define __LC_KERNEL_STACK 0xD40
95 94 #define __LC_THREAD_INFO 0xD48
96 95 #define __LC_ASYNC_STACK 0xD50
... ... @@ -286,8 +285,7 @@
286 285 __u64 system_timer; /* 0x278 */
287 286 __u64 last_update_clock; /* 0x280 */
288 287 __u64 steal_clock; /* 0x288 */
289   - __u32 diag44_opcode; /* 0x290 */
290   - __u8 pad8[0xc00-0x294]; /* 0x294 */
  288 + __u8 pad8[0xc00-0x290]; /* 0x290 */
291 289 /* System info area */
292 290 __u64 save_area[16]; /* 0xc00 */
293 291 __u8 pad9[0xd40-0xc80]; /* 0xc80 */
include/asm-s390/processor.h
... ... @@ -203,7 +203,10 @@
203 203 # define cpu_relax() asm volatile ("diag 0,0,68" : : : "memory")
204 204 #else /* __s390x__ */
205 205 # define cpu_relax() \
206   - asm volatile ("ex 0,%0" : : "i" (__LC_DIAG44_OPCODE) : "memory")
  206 + do { \
  207 + if (MACHINE_HAS_DIAG44) \
  208 + asm volatile ("diag 0,0,68" : : : "memory"); \
  209 + } while (0)
207 210 #endif /* __s390x__ */
208 211  
209 212 /*
include/asm-s390/spinlock.h
... ... @@ -11,21 +11,16 @@
11 11 #ifndef __ASM_SPINLOCK_H
12 12 #define __ASM_SPINLOCK_H
13 13  
14   -#ifdef __s390x__
15   -/*
16   - * Grmph, take care of %&#! user space programs that include
17   - * asm/spinlock.h. The diagnose is only available in kernel
18   - * context.
19   - */
20   -#ifdef __KERNEL__
21   -#include <asm/lowcore.h>
22   -#define __DIAG44_INSN "ex"
23   -#define __DIAG44_OPERAND __LC_DIAG44_OPCODE
24   -#else
25   -#define __DIAG44_INSN "#"
26   -#define __DIAG44_OPERAND 0
27   -#endif
28   -#endif /* __s390x__ */
  14 +static inline int
  15 +_raw_compare_and_swap(volatile unsigned int *lock,
  16 + unsigned int old, unsigned int new)
  17 +{
  18 + asm volatile ("cs %0,%3,0(%4)"
  19 + : "=d" (old), "=m" (*lock)
  20 + : "0" (old), "d" (new), "a" (lock), "m" (*lock)
  21 + : "cc", "memory" );
  22 + return old;
  23 +}
29 24  
30 25 /*
31 26 * Simple spin lock operations. There are two variants, one clears IRQ's
32 27  
33 28  
34 29  
35 30  
36 31  
37 32  
38 33  
39 34  
... ... @@ -41,58 +36,35 @@
41 36 #endif
42 37 } __attribute__ ((aligned (4))) spinlock_t;
43 38  
44   -#define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
45   -#define spin_lock_init(lp) do { (lp)->lock = 0; } while(0)
  39 +#define SPIN_LOCK_UNLOCKED (spinlock_t) { 0 }
  40 +#define spin_lock_init(lp) do { (lp)->lock = 0; } while(0)
46 41 #define spin_unlock_wait(lp) do { barrier(); } while(((volatile spinlock_t *)(lp))->lock)
47   -#define spin_is_locked(x) ((x)->lock != 0)
  42 +#define spin_is_locked(x) ((x)->lock != 0)
48 43 #define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
49 44  
50   -extern inline void _raw_spin_lock(spinlock_t *lp)
  45 +extern void _raw_spin_lock_wait(spinlock_t *lp, unsigned int pc);
  46 +extern int _raw_spin_trylock_retry(spinlock_t *lp, unsigned int pc);
  47 +
  48 +static inline void _raw_spin_lock(spinlock_t *lp)
51 49 {
52   -#ifndef __s390x__
53   - unsigned int reg1, reg2;
54   - __asm__ __volatile__(" bras %0,1f\n"
55   - "0: diag 0,0,68\n"
56   - "1: slr %1,%1\n"
57   - " cs %1,%0,0(%3)\n"
58   - " jl 0b\n"
59   - : "=&d" (reg1), "=&d" (reg2), "=m" (lp->lock)
60   - : "a" (&lp->lock), "m" (lp->lock)
61   - : "cc", "memory" );
62   -#else /* __s390x__ */
63   - unsigned long reg1, reg2;
64   - __asm__ __volatile__(" bras %1,1f\n"
65   - "0: " __DIAG44_INSN " 0,%4\n"
66   - "1: slr %0,%0\n"
67   - " cs %0,%1,0(%3)\n"
68   - " jl 0b\n"
69   - : "=&d" (reg1), "=&d" (reg2), "=m" (lp->lock)
70   - : "a" (&lp->lock), "i" (__DIAG44_OPERAND),
71   - "m" (lp->lock) : "cc", "memory" );
72   -#endif /* __s390x__ */
  50 + unsigned long pc = (unsigned long) __builtin_return_address(0);
  51 +
  52 + if (unlikely(_raw_compare_and_swap(&lp->lock, 0, pc) != 0))
  53 + _raw_spin_lock_wait(lp, pc);
73 54 }
74 55  
75   -extern inline int _raw_spin_trylock(spinlock_t *lp)
  56 +static inline int _raw_spin_trylock(spinlock_t *lp)
76 57 {
77   - unsigned long reg;
78   - unsigned int result;
  58 + unsigned long pc = (unsigned long) __builtin_return_address(0);
79 59  
80   - __asm__ __volatile__(" basr %1,0\n"
81   - "0: cs %0,%1,0(%3)"
82   - : "=d" (result), "=&d" (reg), "=m" (lp->lock)
83   - : "a" (&lp->lock), "m" (lp->lock), "0" (0)
84   - : "cc", "memory" );
85   - return !result;
  60 + if (likely(_raw_compare_and_swap(&lp->lock, 0, pc) == 0))
  61 + return 1;
  62 + return _raw_spin_trylock_retry(lp, pc);
86 63 }
87 64  
88   -extern inline void _raw_spin_unlock(spinlock_t *lp)
  65 +static inline void _raw_spin_unlock(spinlock_t *lp)
89 66 {
90   - unsigned int old;
91   -
92   - __asm__ __volatile__("cs %0,%3,0(%4)"
93   - : "=d" (old), "=m" (lp->lock)
94   - : "0" (lp->lock), "d" (0), "a" (lp)
95   - : "cc", "memory" );
  67 + _raw_compare_and_swap(&lp->lock, lp->lock, 0);
96 68 }
97 69  
98 70 /*
... ... @@ -106,7 +78,7 @@
106 78 * read-locks.
107 79 */
108 80 typedef struct {
109   - volatile unsigned long lock;
  81 + volatile unsigned int lock;
110 82 volatile unsigned long owner_pc;
111 83 #ifdef CONFIG_PREEMPT
112 84 unsigned int break_lock;
113 85  
114 86  
115 87  
116 88  
117 89  
118 90  
... ... @@ -129,123 +101,55 @@
129 101 */
130 102 #define write_can_lock(x) ((x)->lock == 0)
131 103  
132   -#ifndef __s390x__
133   -#define _raw_read_lock(rw) \
134   - asm volatile(" l 2,0(%1)\n" \
135   - " j 1f\n" \
136   - "0: diag 0,0,68\n" \
137   - "1: la 2,0(2)\n" /* clear high (=write) bit */ \
138   - " la 3,1(2)\n" /* one more reader */ \
139   - " cs 2,3,0(%1)\n" /* try to write new value */ \
140   - " jl 0b" \
141   - : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
142   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
143   -#else /* __s390x__ */
144   -#define _raw_read_lock(rw) \
145   - asm volatile(" lg 2,0(%1)\n" \
146   - " j 1f\n" \
147   - "0: " __DIAG44_INSN " 0,%2\n" \
148   - "1: nihh 2,0x7fff\n" /* clear high (=write) bit */ \
149   - " la 3,1(2)\n" /* one more reader */ \
150   - " csg 2,3,0(%1)\n" /* try to write new value */ \
151   - " jl 0b" \
152   - : "=m" ((rw)->lock) \
153   - : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
154   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
155   -#endif /* __s390x__ */
  104 +extern void _raw_read_lock_wait(rwlock_t *lp);
  105 +extern int _raw_read_trylock_retry(rwlock_t *lp);
  106 +extern void _raw_write_lock_wait(rwlock_t *lp);
  107 +extern int _raw_write_trylock_retry(rwlock_t *lp);
156 108  
157   -#ifndef __s390x__
158   -#define _raw_read_unlock(rw) \
159   - asm volatile(" l 2,0(%1)\n" \
160   - " j 1f\n" \
161   - "0: diag 0,0,68\n" \
162   - "1: lr 3,2\n" \
163   - " ahi 3,-1\n" /* one less reader */ \
164   - " cs 2,3,0(%1)\n" \
165   - " jl 0b" \
166   - : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
167   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
168   -#else /* __s390x__ */
169   -#define _raw_read_unlock(rw) \
170   - asm volatile(" lg 2,0(%1)\n" \
171   - " j 1f\n" \
172   - "0: " __DIAG44_INSN " 0,%2\n" \
173   - "1: lgr 3,2\n" \
174   - " bctgr 3,0\n" /* one less reader */ \
175   - " csg 2,3,0(%1)\n" \
176   - " jl 0b" \
177   - : "=m" ((rw)->lock) \
178   - : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
179   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
180   -#endif /* __s390x__ */
  109 +static inline void _raw_read_lock(rwlock_t *rw)
  110 +{
  111 + unsigned int old;
  112 + old = rw->lock & 0x7fffffffU;
  113 + if (_raw_compare_and_swap(&rw->lock, old, old + 1) != old)
  114 + _raw_read_lock_wait(rw);
  115 +}
181 116  
182   -#ifndef __s390x__
183   -#define _raw_write_lock(rw) \
184   - asm volatile(" lhi 3,1\n" \
185   - " sll 3,31\n" /* new lock value = 0x80000000 */ \
186   - " j 1f\n" \
187   - "0: diag 0,0,68\n" \
188   - "1: slr 2,2\n" /* old lock value must be 0 */ \
189   - " cs 2,3,0(%1)\n" \
190   - " jl 0b" \
191   - : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
192   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
193   -#else /* __s390x__ */
194   -#define _raw_write_lock(rw) \
195   - asm volatile(" llihh 3,0x8000\n" /* new lock value = 0x80...0 */ \
196   - " j 1f\n" \
197   - "0: " __DIAG44_INSN " 0,%2\n" \
198   - "1: slgr 2,2\n" /* old lock value must be 0 */ \
199   - " csg 2,3,0(%1)\n" \
200   - " jl 0b" \
201   - : "=m" ((rw)->lock) \
202   - : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
203   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
204   -#endif /* __s390x__ */
  117 +static inline void _raw_read_unlock(rwlock_t *rw)
  118 +{
  119 + unsigned int old, cmp;
205 120  
206   -#ifndef __s390x__
207   -#define _raw_write_unlock(rw) \
208   - asm volatile(" slr 3,3\n" /* new lock value = 0 */ \
209   - " j 1f\n" \
210   - "0: diag 0,0,68\n" \
211   - "1: lhi 2,1\n" \
212   - " sll 2,31\n" /* old lock value must be 0x80000000 */ \
213   - " cs 2,3,0(%1)\n" \
214   - " jl 0b" \
215   - : "=m" ((rw)->lock) : "a" (&(rw)->lock), \
216   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
217   -#else /* __s390x__ */
218   -#define _raw_write_unlock(rw) \
219   - asm volatile(" slgr 3,3\n" /* new lock value = 0 */ \
220   - " j 1f\n" \
221   - "0: " __DIAG44_INSN " 0,%2\n" \
222   - "1: llihh 2,0x8000\n" /* old lock value must be 0x8..0 */\
223   - " csg 2,3,0(%1)\n" \
224   - " jl 0b" \
225   - : "=m" ((rw)->lock) \
226   - : "a" (&(rw)->lock), "i" (__DIAG44_OPERAND), \
227   - "m" ((rw)->lock) : "2", "3", "cc", "memory" )
228   -#endif /* __s390x__ */
  121 + old = rw->lock;
  122 + do {
  123 + cmp = old;
  124 + old = _raw_compare_and_swap(&rw->lock, old, old - 1);
  125 + } while (cmp != old);
  126 +}
229 127  
230   -#define _raw_read_trylock(lock) generic_raw_read_trylock(lock)
  128 +static inline void _raw_write_lock(rwlock_t *rw)
  129 +{
  130 + if (unlikely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) != 0))
  131 + _raw_write_lock_wait(rw);
  132 +}
231 133  
232   -extern inline int _raw_write_trylock(rwlock_t *rw)
  134 +static inline void _raw_write_unlock(rwlock_t *rw)
233 135 {
234   - unsigned long result, reg;
235   -
236   - __asm__ __volatile__(
237   -#ifndef __s390x__
238   - " lhi %1,1\n"
239   - " sll %1,31\n"
240   - " cs %0,%1,0(%3)"
241   -#else /* __s390x__ */
242   - " llihh %1,0x8000\n"
243   - "0: csg %0,%1,0(%3)\n"
244   -#endif /* __s390x__ */
245   - : "=d" (result), "=&d" (reg), "=m" (rw->lock)
246   - : "a" (&rw->lock), "m" (rw->lock), "0" (0UL)
247   - : "cc", "memory" );
248   - return result == 0;
  136 + _raw_compare_and_swap(&rw->lock, 0x80000000, 0);
  137 +}
  138 +
  139 +static inline int _raw_read_trylock(rwlock_t *rw)
  140 +{
  141 + unsigned int old;
  142 + old = rw->lock & 0x7fffffffU;
  143 + if (likely(_raw_compare_and_swap(&rw->lock, old, old + 1) == old))
  144 + return 1;
  145 + return _raw_read_trylock_retry(rw);
  146 +}
  147 +
  148 +static inline int _raw_write_trylock(rwlock_t *rw)
  149 +{
  150 + if (likely(_raw_compare_and_swap(&rw->lock, 0, 0x80000000) == 0))
  151 + return 1;
  152 + return _raw_write_trylock_retry(rw);
249 153 }
250 154  
251 155 #endif /* __ASM_SPINLOCK_H */
include/linux/sysctl.h
... ... @@ -145,6 +145,7 @@
145 145 KERN_BOOTLOADER_TYPE=67, /* int: boot loader type */
146 146 KERN_RANDOMIZE=68, /* int: randomize virtual address space */
147 147 KERN_SETUID_DUMPABLE=69, /* int: behaviour of dumps for setuid core */
  148 + KERN_SPIN_RETRY=70, /* int: number of spinlock retries */
148 149 };
149 150  
150 151  
... ... @@ -114,6 +114,7 @@
114 114 extern int sysctl_ieee_emulation_warnings;
115 115 #endif
116 116 extern int sysctl_userprocess_debug;
  117 +extern int spin_retry;
117 118 #endif
118 119  
119 120 extern int sysctl_hz_timer;
... ... @@ -647,7 +648,16 @@
647 648 .mode = 0644,
648 649 .proc_handler = &proc_dointvec,
649 650 },
650   -
  651 +#if defined(CONFIG_ARCH_S390)
  652 + {
  653 + .ctl_name = KERN_SPIN_RETRY,
  654 + .procname = "spin_retry",
  655 + .data = &spin_retry,
  656 + .maxlen = sizeof (int),
  657 + .mode = 0644,
  658 + .proc_handler = &proc_dointvec,
  659 + },
  660 +#endif
651 661 { .ctl_name = 0 }
652 662 };
653 663