Eric Lee / smarc-ti-linux-kernel | Embedian Git Server

Commit d9e9e8e2fe832180f5c8f659a63def2e8fcaea4a

Authored by Linus Torvalds 2014-04-28 02:21:03 +0800

Exists in master and in 13 other branches

Merge branch 'irq-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull irq fixes from Thomas Gleixner:
 "A slighlty large fix for a subtle issue in the CPU hotplug code of
  certain ARM SoCs, where the not yet online cpu needs to setup the cpu
  local timer and needs to set the interrupt affinity to itself.
  Setting interrupt affinity to a not online cpu is prohibited and
  therefor the timer interrupt ends up on the wrong cpu, which leads to
  nasty complications.

  The SoC folks tried to hack around that in the SoC code in some more
  than nasty ways.  The proper solution is to have a way to enforce the
  affinity setting to a not online cpu.  The core patch to the genirq
  code provides that facility and the follow up patches make use of it
  in the GIC interrupt controller and the exynos timer driver.

  The change to the core code has no implications to existing users,
  except for the rename of the locked function and therefor the
  necessary fixup in mips/cavium.  Aside of that, no runtime impact is
  possible, as none of the existing interrupt chips implements anything
  which depends on the force argument of the irq_set_affinity()
  callback"

* 'irq-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  clocksource: Exynos_mct: Register clock event after request_irq()
  clocksource: Exynos_mct: Use irq_force_affinity() in cpu bringup
  irqchip: Gic: Support forced affinity setting
  genirq: Allow forcing cpu affinity of interrupts

Showing 6 changed files Inline Diff

arch/mips/cavium-octeon/octeon-irq.c
drivers/clocksource/exynos_mct.c
drivers/irqchip/irq-gic.c
include/linux/interrupt.h
include/linux/irq.h
kernel/irq/manage.c

arch/mips/cavium-octeon/octeon-irq.c

Diff comments View file @ d9e9e8e

1	/*	1	/*
2	* This file is subject to the terms and conditions of the GNU General Public	2	* This file is subject to the terms and conditions of the GNU General Public
3	* License. See the file "COPYING" in the main directory of this archive	3	* License. See the file "COPYING" in the main directory of this archive
4	* for more details.	4	* for more details.
5	*	5	*
6	* Copyright (C) 2004-2012 Cavium, Inc.	6	* Copyright (C) 2004-2012 Cavium, Inc.
7	*/	7	*/
8		8
9	#include <linux/interrupt.h>	9	#include <linux/interrupt.h>
10	#include <linux/irqdomain.h>	10	#include <linux/irqdomain.h>
11	#include <linux/bitops.h>	11	#include <linux/bitops.h>
12	#include <linux/percpu.h>	12	#include <linux/percpu.h>
13	#include <linux/slab.h>	13	#include <linux/slab.h>
14	#include <linux/irq.h>	14	#include <linux/irq.h>
15	#include <linux/smp.h>	15	#include <linux/smp.h>
16	#include <linux/of.h>	16	#include <linux/of.h>
17		17
18	#include <asm/octeon/octeon.h>	18	#include <asm/octeon/octeon.h>
19	#include <asm/octeon/cvmx-ciu2-defs.h>	19	#include <asm/octeon/cvmx-ciu2-defs.h>
20		20
21	static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);	21	static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
22	static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);	22	static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
23	static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);	23	static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);
24		24
25	static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];	25	static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
26		26
27	union octeon_ciu_chip_data {	27	union octeon_ciu_chip_data {
28	void *p;	28	void *p;
29	unsigned long l;	29	unsigned long l;
30	struct {	30	struct {
31	unsigned long line:6;	31	unsigned long line:6;
32	unsigned long bit:6;	32	unsigned long bit:6;
33	unsigned long gpio_line:6;	33	unsigned long gpio_line:6;
34	} s;	34	} s;
35	};	35	};
36		36
37	struct octeon_core_chip_data {	37	struct octeon_core_chip_data {
38	struct mutex core_irq_mutex;	38	struct mutex core_irq_mutex;
39	bool current_en;	39	bool current_en;
40	bool desired_en;	40	bool desired_en;
41	u8 bit;	41	u8 bit;
42	};	42	};
43		43
44	#define MIPS_CORE_IRQ_LINES 8	44	#define MIPS_CORE_IRQ_LINES 8
45		45
46	static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];	46	static struct octeon_core_chip_data octeon_irq_core_chip_data[MIPS_CORE_IRQ_LINES];
47		47
48	static void octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line,	48	static void octeon_irq_set_ciu_mapping(int irq, int line, int bit, int gpio_line,
49	struct irq_chip *chip,	49	struct irq_chip *chip,
50	irq_flow_handler_t handler)	50	irq_flow_handler_t handler)
51	{	51	{
52	union octeon_ciu_chip_data cd;	52	union octeon_ciu_chip_data cd;
53		53
54	irq_set_chip_and_handler(irq, chip, handler);	54	irq_set_chip_and_handler(irq, chip, handler);
55		55
56	cd.l = 0;	56	cd.l = 0;
57	cd.s.line = line;	57	cd.s.line = line;
58	cd.s.bit = bit;	58	cd.s.bit = bit;
59	cd.s.gpio_line = gpio_line;	59	cd.s.gpio_line = gpio_line;
60		60
61	irq_set_chip_data(irq, cd.p);	61	irq_set_chip_data(irq, cd.p);
62	octeon_irq_ciu_to_irq[line][bit] = irq;	62	octeon_irq_ciu_to_irq[line][bit] = irq;
63	}	63	}
64		64
65	static void octeon_irq_force_ciu_mapping(struct irq_domain *domain,	65	static void octeon_irq_force_ciu_mapping(struct irq_domain *domain,
66	int irq, int line, int bit)	66	int irq, int line, int bit)
67	{	67	{
68	irq_domain_associate(domain, irq, line << 6 \| bit);	68	irq_domain_associate(domain, irq, line << 6 \| bit);
69	}	69	}
70		70
71	static int octeon_coreid_for_cpu(int cpu)	71	static int octeon_coreid_for_cpu(int cpu)
72	{	72	{
73	#ifdef CONFIG_SMP	73	#ifdef CONFIG_SMP
74	return cpu_logical_map(cpu);	74	return cpu_logical_map(cpu);
75	#else	75	#else
76	return cvmx_get_core_num();	76	return cvmx_get_core_num();
77	#endif	77	#endif
78	}	78	}
79		79
80	static int octeon_cpu_for_coreid(int coreid)	80	static int octeon_cpu_for_coreid(int coreid)
81	{	81	{
82	#ifdef CONFIG_SMP	82	#ifdef CONFIG_SMP
83	return cpu_number_map(coreid);	83	return cpu_number_map(coreid);
84	#else	84	#else
85	return smp_processor_id();	85	return smp_processor_id();
86	#endif	86	#endif
87	}	87	}
88		88
89	static void octeon_irq_core_ack(struct irq_data *data)	89	static void octeon_irq_core_ack(struct irq_data *data)
90	{	90	{
91	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	91	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
92	unsigned int bit = cd->bit;	92	unsigned int bit = cd->bit;
93		93
94	/*	94	/*
95	* We don't need to disable IRQs to make these atomic since	95	* We don't need to disable IRQs to make these atomic since
96	* they are already disabled earlier in the low level	96	* they are already disabled earlier in the low level
97	* interrupt code.	97	* interrupt code.
98	*/	98	*/
99	clear_c0_status(0x100 << bit);	99	clear_c0_status(0x100 << bit);
100	/* The two user interrupts must be cleared manually. */	100	/* The two user interrupts must be cleared manually. */
101	if (bit < 2)	101	if (bit < 2)
102	clear_c0_cause(0x100 << bit);	102	clear_c0_cause(0x100 << bit);
103	}	103	}
104		104
105	static void octeon_irq_core_eoi(struct irq_data *data)	105	static void octeon_irq_core_eoi(struct irq_data *data)
106	{	106	{
107	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	107	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
108		108
109	/*	109	/*
110	* We don't need to disable IRQs to make these atomic since	110	* We don't need to disable IRQs to make these atomic since
111	* they are already disabled earlier in the low level	111	* they are already disabled earlier in the low level
112	* interrupt code.	112	* interrupt code.
113	*/	113	*/
114	set_c0_status(0x100 << cd->bit);	114	set_c0_status(0x100 << cd->bit);
115	}	115	}
116		116
117	static void octeon_irq_core_set_enable_local(void *arg)	117	static void octeon_irq_core_set_enable_local(void *arg)
118	{	118	{
119	struct irq_data *data = arg;	119	struct irq_data *data = arg;
120	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	120	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
121	unsigned int mask = 0x100 << cd->bit;	121	unsigned int mask = 0x100 << cd->bit;
122		122
123	/*	123	/*
124	* Interrupts are already disabled, so these are atomic.	124	* Interrupts are already disabled, so these are atomic.
125	*/	125	*/
126	if (cd->desired_en)	126	if (cd->desired_en)
127	set_c0_status(mask);	127	set_c0_status(mask);
128	else	128	else
129	clear_c0_status(mask);	129	clear_c0_status(mask);
130		130
131	}	131	}
132		132
133	static void octeon_irq_core_disable(struct irq_data *data)	133	static void octeon_irq_core_disable(struct irq_data *data)
134	{	134	{
135	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	135	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
136	cd->desired_en = false;	136	cd->desired_en = false;
137	}	137	}
138		138
139	static void octeon_irq_core_enable(struct irq_data *data)	139	static void octeon_irq_core_enable(struct irq_data *data)
140	{	140	{
141	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	141	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
142	cd->desired_en = true;	142	cd->desired_en = true;
143	}	143	}
144		144
145	static void octeon_irq_core_bus_lock(struct irq_data *data)	145	static void octeon_irq_core_bus_lock(struct irq_data *data)
146	{	146	{
147	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	147	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
148		148
149	mutex_lock(&cd->core_irq_mutex);	149	mutex_lock(&cd->core_irq_mutex);
150	}	150	}
151		151
152	static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)	152	static void octeon_irq_core_bus_sync_unlock(struct irq_data *data)
153	{	153	{
154	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);	154	struct octeon_core_chip_data *cd = irq_data_get_irq_chip_data(data);
155		155
156	if (cd->desired_en != cd->current_en) {	156	if (cd->desired_en != cd->current_en) {
157	on_each_cpu(octeon_irq_core_set_enable_local, data, 1);	157	on_each_cpu(octeon_irq_core_set_enable_local, data, 1);
158		158
159	cd->current_en = cd->desired_en;	159	cd->current_en = cd->desired_en;
160	}	160	}
161		161
162	mutex_unlock(&cd->core_irq_mutex);	162	mutex_unlock(&cd->core_irq_mutex);
163	}	163	}
164		164
165	static struct irq_chip octeon_irq_chip_core = {	165	static struct irq_chip octeon_irq_chip_core = {
166	.name = "Core",	166	.name = "Core",
167	.irq_enable = octeon_irq_core_enable,	167	.irq_enable = octeon_irq_core_enable,
168	.irq_disable = octeon_irq_core_disable,	168	.irq_disable = octeon_irq_core_disable,
169	.irq_ack = octeon_irq_core_ack,	169	.irq_ack = octeon_irq_core_ack,
170	.irq_eoi = octeon_irq_core_eoi,	170	.irq_eoi = octeon_irq_core_eoi,
171	.irq_bus_lock = octeon_irq_core_bus_lock,	171	.irq_bus_lock = octeon_irq_core_bus_lock,
172	.irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,	172	.irq_bus_sync_unlock = octeon_irq_core_bus_sync_unlock,
173		173
174	.irq_cpu_online = octeon_irq_core_eoi,	174	.irq_cpu_online = octeon_irq_core_eoi,
175	.irq_cpu_offline = octeon_irq_core_ack,	175	.irq_cpu_offline = octeon_irq_core_ack,
176	.flags = IRQCHIP_ONOFFLINE_ENABLED,	176	.flags = IRQCHIP_ONOFFLINE_ENABLED,
177	};	177	};
178		178
179	static void __init octeon_irq_init_core(void)	179	static void __init octeon_irq_init_core(void)
180	{	180	{
181	int i;	181	int i;
182	int irq;	182	int irq;
183	struct octeon_core_chip_data *cd;	183	struct octeon_core_chip_data *cd;
184		184
185	for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {	185	for (i = 0; i < MIPS_CORE_IRQ_LINES; i++) {
186	cd = &octeon_irq_core_chip_data[i];	186	cd = &octeon_irq_core_chip_data[i];
187	cd->current_en = false;	187	cd->current_en = false;
188	cd->desired_en = false;	188	cd->desired_en = false;
189	cd->bit = i;	189	cd->bit = i;
190	mutex_init(&cd->core_irq_mutex);	190	mutex_init(&cd->core_irq_mutex);
191		191
192	irq = OCTEON_IRQ_SW0 + i;	192	irq = OCTEON_IRQ_SW0 + i;
193	irq_set_chip_data(irq, cd);	193	irq_set_chip_data(irq, cd);
194	irq_set_chip_and_handler(irq, &octeon_irq_chip_core,	194	irq_set_chip_and_handler(irq, &octeon_irq_chip_core,
195	handle_percpu_irq);	195	handle_percpu_irq);
196	}	196	}
197	}	197	}
198		198
199	static int next_cpu_for_irq(struct irq_data *data)	199	static int next_cpu_for_irq(struct irq_data *data)
200	{	200	{
201		201
202	#ifdef CONFIG_SMP	202	#ifdef CONFIG_SMP
203	int cpu;	203	int cpu;
204	int weight = cpumask_weight(data->affinity);	204	int weight = cpumask_weight(data->affinity);
205		205
206	if (weight > 1) {	206	if (weight > 1) {
207	cpu = smp_processor_id();	207	cpu = smp_processor_id();
208	for (;;) {	208	for (;;) {
209	cpu = cpumask_next(cpu, data->affinity);	209	cpu = cpumask_next(cpu, data->affinity);
210	if (cpu >= nr_cpu_ids) {	210	if (cpu >= nr_cpu_ids) {
211	cpu = -1;	211	cpu = -1;
212	continue;	212	continue;
213	} else if (cpumask_test_cpu(cpu, cpu_online_mask)) {	213	} else if (cpumask_test_cpu(cpu, cpu_online_mask)) {
214	break;	214	break;
215	}	215	}
216	}	216	}
217	} else if (weight == 1) {	217	} else if (weight == 1) {
218	cpu = cpumask_first(data->affinity);	218	cpu = cpumask_first(data->affinity);
219	} else {	219	} else {
220	cpu = smp_processor_id();	220	cpu = smp_processor_id();
221	}	221	}
222	return cpu;	222	return cpu;
223	#else	223	#else
224	return smp_processor_id();	224	return smp_processor_id();
225	#endif	225	#endif
226	}	226	}
227		227
228	static void octeon_irq_ciu_enable(struct irq_data *data)	228	static void octeon_irq_ciu_enable(struct irq_data *data)
229	{	229	{
230	int cpu = next_cpu_for_irq(data);	230	int cpu = next_cpu_for_irq(data);
231	int coreid = octeon_coreid_for_cpu(cpu);	231	int coreid = octeon_coreid_for_cpu(cpu);
232	unsigned long *pen;	232	unsigned long *pen;
233	unsigned long flags;	233	unsigned long flags;
234	union octeon_ciu_chip_data cd;	234	union octeon_ciu_chip_data cd;
235	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);	235	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
236		236
237	cd.p = irq_data_get_irq_chip_data(data);	237	cd.p = irq_data_get_irq_chip_data(data);
238		238
239	raw_spin_lock_irqsave(lock, flags);	239	raw_spin_lock_irqsave(lock, flags);
240	if (cd.s.line == 0) {	240	if (cd.s.line == 0) {
241	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);	241	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
242	__set_bit(cd.s.bit, pen);	242	__set_bit(cd.s.bit, pen);
243	/*	243	/*
244	* Must be visible to octeon_irq_ip{2,3}_ciu() before	244	* Must be visible to octeon_irq_ip{2,3}_ciu() before
245	* enabling the irq.	245	* enabling the irq.
246	*/	246	*/
247	wmb();	247	wmb();
248	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);	248	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
249	} else {	249	} else {
250	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	250	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
251	__set_bit(cd.s.bit, pen);	251	__set_bit(cd.s.bit, pen);
252	/*	252	/*
253	* Must be visible to octeon_irq_ip{2,3}_ciu() before	253	* Must be visible to octeon_irq_ip{2,3}_ciu() before
254	* enabling the irq.	254	* enabling the irq.
255	*/	255	*/
256	wmb();	256	wmb();
257	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);	257	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
258	}	258	}
259	raw_spin_unlock_irqrestore(lock, flags);	259	raw_spin_unlock_irqrestore(lock, flags);
260	}	260	}
261		261
262	static void octeon_irq_ciu_enable_local(struct irq_data *data)	262	static void octeon_irq_ciu_enable_local(struct irq_data *data)
263	{	263	{
264	unsigned long *pen;	264	unsigned long *pen;
265	unsigned long flags;	265	unsigned long flags;
266	union octeon_ciu_chip_data cd;	266	union octeon_ciu_chip_data cd;
267	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);	267	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);
268		268
269	cd.p = irq_data_get_irq_chip_data(data);	269	cd.p = irq_data_get_irq_chip_data(data);
270		270
271	raw_spin_lock_irqsave(lock, flags);	271	raw_spin_lock_irqsave(lock, flags);
272	if (cd.s.line == 0) {	272	if (cd.s.line == 0) {
273	pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);	273	pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
274	__set_bit(cd.s.bit, pen);	274	__set_bit(cd.s.bit, pen);
275	/*	275	/*
276	* Must be visible to octeon_irq_ip{2,3}_ciu() before	276	* Must be visible to octeon_irq_ip{2,3}_ciu() before
277	* enabling the irq.	277	* enabling the irq.
278	*/	278	*/
279	wmb();	279	wmb();
280	cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);	280	cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
281	} else {	281	} else {
282	pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);	282	pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
283	__set_bit(cd.s.bit, pen);	283	__set_bit(cd.s.bit, pen);
284	/*	284	/*
285	* Must be visible to octeon_irq_ip{2,3}_ciu() before	285	* Must be visible to octeon_irq_ip{2,3}_ciu() before
286	* enabling the irq.	286	* enabling the irq.
287	*/	287	*/
288	wmb();	288	wmb();
289	cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);	289	cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
290	}	290	}
291	raw_spin_unlock_irqrestore(lock, flags);	291	raw_spin_unlock_irqrestore(lock, flags);
292	}	292	}
293		293
294	static void octeon_irq_ciu_disable_local(struct irq_data *data)	294	static void octeon_irq_ciu_disable_local(struct irq_data *data)
295	{	295	{
296	unsigned long *pen;	296	unsigned long *pen;
297	unsigned long flags;	297	unsigned long flags;
298	union octeon_ciu_chip_data cd;	298	union octeon_ciu_chip_data cd;
299	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);	299	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);
300		300
301	cd.p = irq_data_get_irq_chip_data(data);	301	cd.p = irq_data_get_irq_chip_data(data);
302		302
303	raw_spin_lock_irqsave(lock, flags);	303	raw_spin_lock_irqsave(lock, flags);
304	if (cd.s.line == 0) {	304	if (cd.s.line == 0) {
305	pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);	305	pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
306	__clear_bit(cd.s.bit, pen);	306	__clear_bit(cd.s.bit, pen);
307	/*	307	/*
308	* Must be visible to octeon_irq_ip{2,3}_ciu() before	308	* Must be visible to octeon_irq_ip{2,3}_ciu() before
309	* enabling the irq.	309	* enabling the irq.
310	*/	310	*/
311	wmb();	311	wmb();
312	cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);	312	cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
313	} else {	313	} else {
314	pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);	314	pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
315	__clear_bit(cd.s.bit, pen);	315	__clear_bit(cd.s.bit, pen);
316	/*	316	/*
317	* Must be visible to octeon_irq_ip{2,3}_ciu() before	317	* Must be visible to octeon_irq_ip{2,3}_ciu() before
318	* enabling the irq.	318	* enabling the irq.
319	*/	319	*/
320	wmb();	320	wmb();
321	cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);	321	cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
322	}	322	}
323	raw_spin_unlock_irqrestore(lock, flags);	323	raw_spin_unlock_irqrestore(lock, flags);
324	}	324	}
325		325
326	static void octeon_irq_ciu_disable_all(struct irq_data *data)	326	static void octeon_irq_ciu_disable_all(struct irq_data *data)
327	{	327	{
328	unsigned long flags;	328	unsigned long flags;
329	unsigned long *pen;	329	unsigned long *pen;
330	int cpu;	330	int cpu;
331	union octeon_ciu_chip_data cd;	331	union octeon_ciu_chip_data cd;
332	raw_spinlock_t *lock;	332	raw_spinlock_t *lock;
333		333
334	cd.p = irq_data_get_irq_chip_data(data);	334	cd.p = irq_data_get_irq_chip_data(data);
335		335
336	for_each_online_cpu(cpu) {	336	for_each_online_cpu(cpu) {
337	int coreid = octeon_coreid_for_cpu(cpu);	337	int coreid = octeon_coreid_for_cpu(cpu);
338	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);	338	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
339	if (cd.s.line == 0)	339	if (cd.s.line == 0)
340	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);	340	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
341	else	341	else
342	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	342	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
343		343
344	raw_spin_lock_irqsave(lock, flags);	344	raw_spin_lock_irqsave(lock, flags);
345	__clear_bit(cd.s.bit, pen);	345	__clear_bit(cd.s.bit, pen);
346	/*	346	/*
347	* Must be visible to octeon_irq_ip{2,3}_ciu() before	347	* Must be visible to octeon_irq_ip{2,3}_ciu() before
348	* enabling the irq.	348	* enabling the irq.
349	*/	349	*/
350	wmb();	350	wmb();
351	if (cd.s.line == 0)	351	if (cd.s.line == 0)
352	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);	352	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
353	else	353	else
354	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);	354	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
355	raw_spin_unlock_irqrestore(lock, flags);	355	raw_spin_unlock_irqrestore(lock, flags);
356	}	356	}
357	}	357	}
358		358
359	static void octeon_irq_ciu_enable_all(struct irq_data *data)	359	static void octeon_irq_ciu_enable_all(struct irq_data *data)
360	{	360	{
361	unsigned long flags;	361	unsigned long flags;
362	unsigned long *pen;	362	unsigned long *pen;
363	int cpu;	363	int cpu;
364	union octeon_ciu_chip_data cd;	364	union octeon_ciu_chip_data cd;
365	raw_spinlock_t *lock;	365	raw_spinlock_t *lock;
366		366
367	cd.p = irq_data_get_irq_chip_data(data);	367	cd.p = irq_data_get_irq_chip_data(data);
368		368
369	for_each_online_cpu(cpu) {	369	for_each_online_cpu(cpu) {
370	int coreid = octeon_coreid_for_cpu(cpu);	370	int coreid = octeon_coreid_for_cpu(cpu);
371	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);	371	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
372	if (cd.s.line == 0)	372	if (cd.s.line == 0)
373	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);	373	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
374	else	374	else
375	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	375	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
376		376
377	raw_spin_lock_irqsave(lock, flags);	377	raw_spin_lock_irqsave(lock, flags);
378	__set_bit(cd.s.bit, pen);	378	__set_bit(cd.s.bit, pen);
379	/*	379	/*
380	* Must be visible to octeon_irq_ip{2,3}_ciu() before	380	* Must be visible to octeon_irq_ip{2,3}_ciu() before
381	* enabling the irq.	381	* enabling the irq.
382	*/	382	*/
383	wmb();	383	wmb();
384	if (cd.s.line == 0)	384	if (cd.s.line == 0)
385	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);	385	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
386	else	386	else
387	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);	387	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
388	raw_spin_unlock_irqrestore(lock, flags);	388	raw_spin_unlock_irqrestore(lock, flags);
389	}	389	}
390	}	390	}
391		391
392	/*	392	/*
393	* Enable the irq on the next core in the affinity set for chips that	393	* Enable the irq on the next core in the affinity set for chips that
394	* have the EN*_W1{S,C} registers.	394	* have the EN*_W1{S,C} registers.
395	*/	395	*/
396	static void octeon_irq_ciu_enable_v2(struct irq_data *data)	396	static void octeon_irq_ciu_enable_v2(struct irq_data *data)
397	{	397	{
398	u64 mask;	398	u64 mask;
399	int cpu = next_cpu_for_irq(data);	399	int cpu = next_cpu_for_irq(data);
400	union octeon_ciu_chip_data cd;	400	union octeon_ciu_chip_data cd;
401		401
402	cd.p = irq_data_get_irq_chip_data(data);	402	cd.p = irq_data_get_irq_chip_data(data);
403	mask = 1ull << (cd.s.bit);	403	mask = 1ull << (cd.s.bit);
404		404
405	/*	405	/*
406	* Called under the desc lock, so these should never get out	406	* Called under the desc lock, so these should never get out
407	* of sync.	407	* of sync.
408	*/	408	*/
409	if (cd.s.line == 0) {	409	if (cd.s.line == 0) {
410	int index = octeon_coreid_for_cpu(cpu) * 2;	410	int index = octeon_coreid_for_cpu(cpu) * 2;
411	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));	411	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
412	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);	412	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
413	} else {	413	} else {
414	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;	414	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
415	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));	415	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
416	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);	416	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
417	}	417	}
418	}	418	}
419		419
420	/*	420	/*
421	* Enable the irq on the current CPU for chips that	421	* Enable the irq on the current CPU for chips that
422	* have the EN*_W1{S,C} registers.	422	* have the EN*_W1{S,C} registers.
423	*/	423	*/
424	static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)	424	static void octeon_irq_ciu_enable_local_v2(struct irq_data *data)
425	{	425	{
426	u64 mask;	426	u64 mask;
427	union octeon_ciu_chip_data cd;	427	union octeon_ciu_chip_data cd;
428		428
429	cd.p = irq_data_get_irq_chip_data(data);	429	cd.p = irq_data_get_irq_chip_data(data);
430	mask = 1ull << (cd.s.bit);	430	mask = 1ull << (cd.s.bit);
431		431
432	if (cd.s.line == 0) {	432	if (cd.s.line == 0) {
433	int index = cvmx_get_core_num() * 2;	433	int index = cvmx_get_core_num() * 2;
434	set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));	434	set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
435	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);	435	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
436	} else {	436	} else {
437	int index = cvmx_get_core_num() * 2 + 1;	437	int index = cvmx_get_core_num() * 2 + 1;
438	set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));	438	set_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
439	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);	439	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
440	}	440	}
441	}	441	}
442		442
443	static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)	443	static void octeon_irq_ciu_disable_local_v2(struct irq_data *data)
444	{	444	{
445	u64 mask;	445	u64 mask;
446	union octeon_ciu_chip_data cd;	446	union octeon_ciu_chip_data cd;
447		447
448	cd.p = irq_data_get_irq_chip_data(data);	448	cd.p = irq_data_get_irq_chip_data(data);
449	mask = 1ull << (cd.s.bit);	449	mask = 1ull << (cd.s.bit);
450		450
451	if (cd.s.line == 0) {	451	if (cd.s.line == 0) {
452	int index = cvmx_get_core_num() * 2;	452	int index = cvmx_get_core_num() * 2;
453	clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));	453	clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu0_en_mirror));
454	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);	454	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
455	} else {	455	} else {
456	int index = cvmx_get_core_num() * 2 + 1;	456	int index = cvmx_get_core_num() * 2 + 1;
457	clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));	457	clear_bit(cd.s.bit, &__get_cpu_var(octeon_irq_ciu1_en_mirror));
458	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);	458	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
459	}	459	}
460	}	460	}
461		461
462	/*	462	/*
463	* Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.	463	* Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
464	*/	464	*/
465	static void octeon_irq_ciu_ack(struct irq_data *data)	465	static void octeon_irq_ciu_ack(struct irq_data *data)
466	{	466	{
467	u64 mask;	467	u64 mask;
468	union octeon_ciu_chip_data cd;	468	union octeon_ciu_chip_data cd;
469		469
470	cd.p = irq_data_get_irq_chip_data(data);	470	cd.p = irq_data_get_irq_chip_data(data);
471	mask = 1ull << (cd.s.bit);	471	mask = 1ull << (cd.s.bit);
472		472
473	if (cd.s.line == 0) {	473	if (cd.s.line == 0) {
474	int index = cvmx_get_core_num() * 2;	474	int index = cvmx_get_core_num() * 2;
475	cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);	475	cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
476	} else {	476	} else {
477	cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);	477	cvmx_write_csr(CVMX_CIU_INT_SUM1, mask);
478	}	478	}
479	}	479	}
480		480
481	/*	481	/*
482	* Disable the irq on the all cores for chips that have the EN*_W1{S,C}	482	* Disable the irq on the all cores for chips that have the EN*_W1{S,C}
483	* registers.	483	* registers.
484	*/	484	*/
485	static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)	485	static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
486	{	486	{
487	int cpu;	487	int cpu;
488	u64 mask;	488	u64 mask;
489	union octeon_ciu_chip_data cd;	489	union octeon_ciu_chip_data cd;
490		490
491	cd.p = irq_data_get_irq_chip_data(data);	491	cd.p = irq_data_get_irq_chip_data(data);
492	mask = 1ull << (cd.s.bit);	492	mask = 1ull << (cd.s.bit);
493		493
494	if (cd.s.line == 0) {	494	if (cd.s.line == 0) {
495	for_each_online_cpu(cpu) {	495	for_each_online_cpu(cpu) {
496	int index = octeon_coreid_for_cpu(cpu) * 2;	496	int index = octeon_coreid_for_cpu(cpu) * 2;
497	clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));	497	clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
498	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);	498	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
499	}	499	}
500	} else {	500	} else {
501	for_each_online_cpu(cpu) {	501	for_each_online_cpu(cpu) {
502	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;	502	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
503	clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));	503	clear_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
504	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);	504	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
505	}	505	}
506	}	506	}
507	}	507	}
508		508
509	/*	509	/*
510	* Enable the irq on the all cores for chips that have the EN*_W1{S,C}	510	* Enable the irq on the all cores for chips that have the EN*_W1{S,C}
511	* registers.	511	* registers.
512	*/	512	*/
513	static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)	513	static void octeon_irq_ciu_enable_all_v2(struct irq_data *data)
514	{	514	{
515	int cpu;	515	int cpu;
516	u64 mask;	516	u64 mask;
517	union octeon_ciu_chip_data cd;	517	union octeon_ciu_chip_data cd;
518		518
519	cd.p = irq_data_get_irq_chip_data(data);	519	cd.p = irq_data_get_irq_chip_data(data);
520	mask = 1ull << (cd.s.bit);	520	mask = 1ull << (cd.s.bit);
521		521
522	if (cd.s.line == 0) {	522	if (cd.s.line == 0) {
523	for_each_online_cpu(cpu) {	523	for_each_online_cpu(cpu) {
524	int index = octeon_coreid_for_cpu(cpu) * 2;	524	int index = octeon_coreid_for_cpu(cpu) * 2;
525	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));	525	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu0_en_mirror, cpu));
526	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);	526	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
527	}	527	}
528	} else {	528	} else {
529	for_each_online_cpu(cpu) {	529	for_each_online_cpu(cpu) {
530	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;	530	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
531	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));	531	set_bit(cd.s.bit, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
532	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);	532	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
533	}	533	}
534	}	534	}
535	}	535	}
536		536
537	static void octeon_irq_gpio_setup(struct irq_data *data)	537	static void octeon_irq_gpio_setup(struct irq_data *data)
538	{	538	{
539	union cvmx_gpio_bit_cfgx cfg;	539	union cvmx_gpio_bit_cfgx cfg;
540	union octeon_ciu_chip_data cd;	540	union octeon_ciu_chip_data cd;
541	u32 t = irqd_get_trigger_type(data);	541	u32 t = irqd_get_trigger_type(data);
542		542
543	cd.p = irq_data_get_irq_chip_data(data);	543	cd.p = irq_data_get_irq_chip_data(data);
544		544
545	cfg.u64 = 0;	545	cfg.u64 = 0;
546	cfg.s.int_en = 1;	546	cfg.s.int_en = 1;
547	cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0;	547	cfg.s.int_type = (t & IRQ_TYPE_EDGE_BOTH) != 0;
548	cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW \| IRQ_TYPE_EDGE_FALLING)) != 0;	548	cfg.s.rx_xor = (t & (IRQ_TYPE_LEVEL_LOW \| IRQ_TYPE_EDGE_FALLING)) != 0;
549		549
550	/* 140 nS glitch filter*/	550	/* 140 nS glitch filter*/
551	cfg.s.fil_cnt = 7;	551	cfg.s.fil_cnt = 7;
552	cfg.s.fil_sel = 3;	552	cfg.s.fil_sel = 3;
553		553
554	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), cfg.u64);	554	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), cfg.u64);
555	}	555	}
556		556
557	static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data)	557	static void octeon_irq_ciu_enable_gpio_v2(struct irq_data *data)
558	{	558	{
559	octeon_irq_gpio_setup(data);	559	octeon_irq_gpio_setup(data);
560	octeon_irq_ciu_enable_v2(data);	560	octeon_irq_ciu_enable_v2(data);
561	}	561	}
562		562
563	static void octeon_irq_ciu_enable_gpio(struct irq_data *data)	563	static void octeon_irq_ciu_enable_gpio(struct irq_data *data)
564	{	564	{
565	octeon_irq_gpio_setup(data);	565	octeon_irq_gpio_setup(data);
566	octeon_irq_ciu_enable(data);	566	octeon_irq_ciu_enable(data);
567	}	567	}
568		568
569	static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t)	569	static int octeon_irq_ciu_gpio_set_type(struct irq_data *data, unsigned int t)
570	{	570	{
571	irqd_set_trigger_type(data, t);	571	irqd_set_trigger_type(data, t);
572	octeon_irq_gpio_setup(data);	572	octeon_irq_gpio_setup(data);
573		573
574	return IRQ_SET_MASK_OK;	574	return IRQ_SET_MASK_OK;
575	}	575	}
576		576
577	static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data)	577	static void octeon_irq_ciu_disable_gpio_v2(struct irq_data *data)
578	{	578	{
579	union octeon_ciu_chip_data cd;	579	union octeon_ciu_chip_data cd;
580		580
581	cd.p = irq_data_get_irq_chip_data(data);	581	cd.p = irq_data_get_irq_chip_data(data);
582	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);	582	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);
583		583
584	octeon_irq_ciu_disable_all_v2(data);	584	octeon_irq_ciu_disable_all_v2(data);
585	}	585	}
586		586
587	static void octeon_irq_ciu_disable_gpio(struct irq_data *data)	587	static void octeon_irq_ciu_disable_gpio(struct irq_data *data)
588	{	588	{
589	union octeon_ciu_chip_data cd;	589	union octeon_ciu_chip_data cd;
590		590
591	cd.p = irq_data_get_irq_chip_data(data);	591	cd.p = irq_data_get_irq_chip_data(data);
592	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);	592	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);
593		593
594	octeon_irq_ciu_disable_all(data);	594	octeon_irq_ciu_disable_all(data);
595	}	595	}
596		596
597	static void octeon_irq_ciu_gpio_ack(struct irq_data *data)	597	static void octeon_irq_ciu_gpio_ack(struct irq_data *data)
598	{	598	{
599	union octeon_ciu_chip_data cd;	599	union octeon_ciu_chip_data cd;
600	u64 mask;	600	u64 mask;
601		601
602	cd.p = irq_data_get_irq_chip_data(data);	602	cd.p = irq_data_get_irq_chip_data(data);
603	mask = 1ull << (cd.s.gpio_line);	603	mask = 1ull << (cd.s.gpio_line);
604		604
605	cvmx_write_csr(CVMX_GPIO_INT_CLR, mask);	605	cvmx_write_csr(CVMX_GPIO_INT_CLR, mask);
606	}	606	}
607		607
608	static void octeon_irq_handle_gpio(unsigned int irq, struct irq_desc *desc)	608	static void octeon_irq_handle_gpio(unsigned int irq, struct irq_desc *desc)
609	{	609	{
610	if (irq_get_trigger_type(irq) & IRQ_TYPE_EDGE_BOTH)	610	if (irq_get_trigger_type(irq) & IRQ_TYPE_EDGE_BOTH)
611	handle_edge_irq(irq, desc);	611	handle_edge_irq(irq, desc);
612	else	612	else
613	handle_level_irq(irq, desc);	613	handle_level_irq(irq, desc);
614	}	614	}
615		615
616	#ifdef CONFIG_SMP	616	#ifdef CONFIG_SMP
617		617
618	static void octeon_irq_cpu_offline_ciu(struct irq_data *data)	618	static void octeon_irq_cpu_offline_ciu(struct irq_data *data)
619	{	619	{
620	int cpu = smp_processor_id();	620	int cpu = smp_processor_id();
621	cpumask_t new_affinity;	621	cpumask_t new_affinity;
622		622
623	if (!cpumask_test_cpu(cpu, data->affinity))	623	if (!cpumask_test_cpu(cpu, data->affinity))
624	return;	624	return;
625		625
626	if (cpumask_weight(data->affinity) > 1) {	626	if (cpumask_weight(data->affinity) > 1) {
627	/*	627	/*
628	* It has multi CPU affinity, just remove this CPU	628	* It has multi CPU affinity, just remove this CPU
629	* from the affinity set.	629	* from the affinity set.
630	*/	630	*/
631	cpumask_copy(&new_affinity, data->affinity);	631	cpumask_copy(&new_affinity, data->affinity);
632	cpumask_clear_cpu(cpu, &new_affinity);	632	cpumask_clear_cpu(cpu, &new_affinity);
633	} else {	633	} else {
634	/* Otherwise, put it on lowest numbered online CPU. */	634	/* Otherwise, put it on lowest numbered online CPU. */
635	cpumask_clear(&new_affinity);	635	cpumask_clear(&new_affinity);
636	cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);	636	cpumask_set_cpu(cpumask_first(cpu_online_mask), &new_affinity);
637	}	637	}
638	__irq_set_affinity_locked(data, &new_affinity);	638	irq_set_affinity_locked(data, &new_affinity, false);
639	}	639	}
640		640
641	static int octeon_irq_ciu_set_affinity(struct irq_data *data,	641	static int octeon_irq_ciu_set_affinity(struct irq_data *data,
642	const struct cpumask *dest, bool force)	642	const struct cpumask *dest, bool force)
643	{	643	{
644	int cpu;	644	int cpu;
645	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);	645	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
646	unsigned long flags;	646	unsigned long flags;
647	union octeon_ciu_chip_data cd;	647	union octeon_ciu_chip_data cd;
648	unsigned long *pen;	648	unsigned long *pen;
649	raw_spinlock_t *lock;	649	raw_spinlock_t *lock;
650		650
651	cd.p = irq_data_get_irq_chip_data(data);	651	cd.p = irq_data_get_irq_chip_data(data);
652		652
653	/*	653	/*
654	* For non-v2 CIU, we will allow only single CPU affinity.	654	* For non-v2 CIU, we will allow only single CPU affinity.
655	* This removes the need to do locking in the .ack/.eoi	655	* This removes the need to do locking in the .ack/.eoi
656	* functions.	656	* functions.
657	*/	657	*/
658	if (cpumask_weight(dest) != 1)	658	if (cpumask_weight(dest) != 1)
659	return -EINVAL;	659	return -EINVAL;
660		660
661	if (!enable_one)	661	if (!enable_one)
662	return 0;	662	return 0;
663		663
664		664
665	for_each_online_cpu(cpu) {	665	for_each_online_cpu(cpu) {
666	int coreid = octeon_coreid_for_cpu(cpu);	666	int coreid = octeon_coreid_for_cpu(cpu);
667		667
668	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);	668	lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
669	raw_spin_lock_irqsave(lock, flags);	669	raw_spin_lock_irqsave(lock, flags);
670		670
671	if (cd.s.line == 0)	671	if (cd.s.line == 0)
672	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);	672	pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
673	else	673	else
674	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	674	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
675		675
676	if (cpumask_test_cpu(cpu, dest) && enable_one) {	676	if (cpumask_test_cpu(cpu, dest) && enable_one) {
677	enable_one = 0;	677	enable_one = 0;
678	__set_bit(cd.s.bit, pen);	678	__set_bit(cd.s.bit, pen);
679	} else {	679	} else {
680	__clear_bit(cd.s.bit, pen);	680	__clear_bit(cd.s.bit, pen);
681	}	681	}
682	/*	682	/*
683	* Must be visible to octeon_irq_ip{2,3}_ciu() before	683	* Must be visible to octeon_irq_ip{2,3}_ciu() before
684	* enabling the irq.	684	* enabling the irq.
685	*/	685	*/
686	wmb();	686	wmb();
687		687
688	if (cd.s.line == 0)	688	if (cd.s.line == 0)
689	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);	689	cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
690	else	690	else
691	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);	691	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
692		692
693	raw_spin_unlock_irqrestore(lock, flags);	693	raw_spin_unlock_irqrestore(lock, flags);
694	}	694	}
695	return 0;	695	return 0;
696	}	696	}
697		697
698	/*	698	/*
699	* Set affinity for the irq for chips that have the EN*_W1{S,C}	699	* Set affinity for the irq for chips that have the EN*_W1{S,C}
700	* registers.	700	* registers.
701	*/	701	*/
702	static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,	702	static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
703	const struct cpumask *dest,	703	const struct cpumask *dest,
704	bool force)	704	bool force)
705	{	705	{
706	int cpu;	706	int cpu;
707	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);	707	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
708	u64 mask;	708	u64 mask;
709	union octeon_ciu_chip_data cd;	709	union octeon_ciu_chip_data cd;
710		710
711	if (!enable_one)	711	if (!enable_one)
712	return 0;	712	return 0;
713		713
714	cd.p = irq_data_get_irq_chip_data(data);	714	cd.p = irq_data_get_irq_chip_data(data);
715	mask = 1ull << cd.s.bit;	715	mask = 1ull << cd.s.bit;
716		716
717	if (cd.s.line == 0) {	717	if (cd.s.line == 0) {
718	for_each_online_cpu(cpu) {	718	for_each_online_cpu(cpu) {
719	unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);	719	unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
720	int index = octeon_coreid_for_cpu(cpu) * 2;	720	int index = octeon_coreid_for_cpu(cpu) * 2;
721	if (cpumask_test_cpu(cpu, dest) && enable_one) {	721	if (cpumask_test_cpu(cpu, dest) && enable_one) {
722	enable_one = false;	722	enable_one = false;
723	set_bit(cd.s.bit, pen);	723	set_bit(cd.s.bit, pen);
724	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);	724	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
725	} else {	725	} else {
726	clear_bit(cd.s.bit, pen);	726	clear_bit(cd.s.bit, pen);
727	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);	727	cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
728	}	728	}
729	}	729	}
730	} else {	730	} else {
731	for_each_online_cpu(cpu) {	731	for_each_online_cpu(cpu) {
732	unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	732	unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
733	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;	733	int index = octeon_coreid_for_cpu(cpu) * 2 + 1;
734	if (cpumask_test_cpu(cpu, dest) && enable_one) {	734	if (cpumask_test_cpu(cpu, dest) && enable_one) {
735	enable_one = false;	735	enable_one = false;
736	set_bit(cd.s.bit, pen);	736	set_bit(cd.s.bit, pen);
737	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);	737	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
738	} else {	738	} else {
739	clear_bit(cd.s.bit, pen);	739	clear_bit(cd.s.bit, pen);
740	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);	740	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
741	}	741	}
742	}	742	}
743	}	743	}
744	return 0;	744	return 0;
745	}	745	}
746	#endif	746	#endif
747		747
748	/*	748	/*
749	* Newer octeon chips have support for lockless CIU operation.	749	* Newer octeon chips have support for lockless CIU operation.
750	*/	750	*/
751	static struct irq_chip octeon_irq_chip_ciu_v2 = {	751	static struct irq_chip octeon_irq_chip_ciu_v2 = {
752	.name = "CIU",	752	.name = "CIU",
753	.irq_enable = octeon_irq_ciu_enable_v2,	753	.irq_enable = octeon_irq_ciu_enable_v2,
754	.irq_disable = octeon_irq_ciu_disable_all_v2,	754	.irq_disable = octeon_irq_ciu_disable_all_v2,
755	.irq_ack = octeon_irq_ciu_ack,	755	.irq_ack = octeon_irq_ciu_ack,
756	.irq_mask = octeon_irq_ciu_disable_local_v2,	756	.irq_mask = octeon_irq_ciu_disable_local_v2,
757	.irq_unmask = octeon_irq_ciu_enable_v2,	757	.irq_unmask = octeon_irq_ciu_enable_v2,
758	#ifdef CONFIG_SMP	758	#ifdef CONFIG_SMP
759	.irq_set_affinity = octeon_irq_ciu_set_affinity_v2,	759	.irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
760	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,	760	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,
761	#endif	761	#endif
762	};	762	};
763		763
764	static struct irq_chip octeon_irq_chip_ciu = {	764	static struct irq_chip octeon_irq_chip_ciu = {
765	.name = "CIU",	765	.name = "CIU",
766	.irq_enable = octeon_irq_ciu_enable,	766	.irq_enable = octeon_irq_ciu_enable,
767	.irq_disable = octeon_irq_ciu_disable_all,	767	.irq_disable = octeon_irq_ciu_disable_all,
768	.irq_ack = octeon_irq_ciu_ack,	768	.irq_ack = octeon_irq_ciu_ack,
769	.irq_mask = octeon_irq_ciu_disable_local,	769	.irq_mask = octeon_irq_ciu_disable_local,
770	.irq_unmask = octeon_irq_ciu_enable,	770	.irq_unmask = octeon_irq_ciu_enable,
771	#ifdef CONFIG_SMP	771	#ifdef CONFIG_SMP
772	.irq_set_affinity = octeon_irq_ciu_set_affinity,	772	.irq_set_affinity = octeon_irq_ciu_set_affinity,
773	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,	773	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,
774	#endif	774	#endif
775	};	775	};
776		776
777	/* The mbox versions don't do any affinity or round-robin. */	777	/* The mbox versions don't do any affinity or round-robin. */
778	static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {	778	static struct irq_chip octeon_irq_chip_ciu_mbox_v2 = {
779	.name = "CIU-M",	779	.name = "CIU-M",
780	.irq_enable = octeon_irq_ciu_enable_all_v2,	780	.irq_enable = octeon_irq_ciu_enable_all_v2,
781	.irq_disable = octeon_irq_ciu_disable_all_v2,	781	.irq_disable = octeon_irq_ciu_disable_all_v2,
782	.irq_ack = octeon_irq_ciu_disable_local_v2,	782	.irq_ack = octeon_irq_ciu_disable_local_v2,
783	.irq_eoi = octeon_irq_ciu_enable_local_v2,	783	.irq_eoi = octeon_irq_ciu_enable_local_v2,
784		784
785	.irq_cpu_online = octeon_irq_ciu_enable_local_v2,	785	.irq_cpu_online = octeon_irq_ciu_enable_local_v2,
786	.irq_cpu_offline = octeon_irq_ciu_disable_local_v2,	786	.irq_cpu_offline = octeon_irq_ciu_disable_local_v2,
787	.flags = IRQCHIP_ONOFFLINE_ENABLED,	787	.flags = IRQCHIP_ONOFFLINE_ENABLED,
788	};	788	};
789		789
790	static struct irq_chip octeon_irq_chip_ciu_mbox = {	790	static struct irq_chip octeon_irq_chip_ciu_mbox = {
791	.name = "CIU-M",	791	.name = "CIU-M",
792	.irq_enable = octeon_irq_ciu_enable_all,	792	.irq_enable = octeon_irq_ciu_enable_all,
793	.irq_disable = octeon_irq_ciu_disable_all,	793	.irq_disable = octeon_irq_ciu_disable_all,
794	.irq_ack = octeon_irq_ciu_disable_local,	794	.irq_ack = octeon_irq_ciu_disable_local,
795	.irq_eoi = octeon_irq_ciu_enable_local,	795	.irq_eoi = octeon_irq_ciu_enable_local,
796		796
797	.irq_cpu_online = octeon_irq_ciu_enable_local,	797	.irq_cpu_online = octeon_irq_ciu_enable_local,
798	.irq_cpu_offline = octeon_irq_ciu_disable_local,	798	.irq_cpu_offline = octeon_irq_ciu_disable_local,
799	.flags = IRQCHIP_ONOFFLINE_ENABLED,	799	.flags = IRQCHIP_ONOFFLINE_ENABLED,
800	};	800	};
801		801
802	static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = {	802	static struct irq_chip octeon_irq_chip_ciu_gpio_v2 = {
803	.name = "CIU-GPIO",	803	.name = "CIU-GPIO",
804	.irq_enable = octeon_irq_ciu_enable_gpio_v2,	804	.irq_enable = octeon_irq_ciu_enable_gpio_v2,
805	.irq_disable = octeon_irq_ciu_disable_gpio_v2,	805	.irq_disable = octeon_irq_ciu_disable_gpio_v2,
806	.irq_ack = octeon_irq_ciu_gpio_ack,	806	.irq_ack = octeon_irq_ciu_gpio_ack,
807	.irq_mask = octeon_irq_ciu_disable_local_v2,	807	.irq_mask = octeon_irq_ciu_disable_local_v2,
808	.irq_unmask = octeon_irq_ciu_enable_v2,	808	.irq_unmask = octeon_irq_ciu_enable_v2,
809	.irq_set_type = octeon_irq_ciu_gpio_set_type,	809	.irq_set_type = octeon_irq_ciu_gpio_set_type,
810	#ifdef CONFIG_SMP	810	#ifdef CONFIG_SMP
811	.irq_set_affinity = octeon_irq_ciu_set_affinity_v2,	811	.irq_set_affinity = octeon_irq_ciu_set_affinity_v2,
812	#endif	812	#endif
813	.flags = IRQCHIP_SET_TYPE_MASKED,	813	.flags = IRQCHIP_SET_TYPE_MASKED,
814	};	814	};
815		815
816	static struct irq_chip octeon_irq_chip_ciu_gpio = {	816	static struct irq_chip octeon_irq_chip_ciu_gpio = {
817	.name = "CIU-GPIO",	817	.name = "CIU-GPIO",
818	.irq_enable = octeon_irq_ciu_enable_gpio,	818	.irq_enable = octeon_irq_ciu_enable_gpio,
819	.irq_disable = octeon_irq_ciu_disable_gpio,	819	.irq_disable = octeon_irq_ciu_disable_gpio,
820	.irq_mask = octeon_irq_ciu_disable_local,	820	.irq_mask = octeon_irq_ciu_disable_local,
821	.irq_unmask = octeon_irq_ciu_enable,	821	.irq_unmask = octeon_irq_ciu_enable,
822	.irq_ack = octeon_irq_ciu_gpio_ack,	822	.irq_ack = octeon_irq_ciu_gpio_ack,
823	.irq_set_type = octeon_irq_ciu_gpio_set_type,	823	.irq_set_type = octeon_irq_ciu_gpio_set_type,
824	#ifdef CONFIG_SMP	824	#ifdef CONFIG_SMP
825	.irq_set_affinity = octeon_irq_ciu_set_affinity,	825	.irq_set_affinity = octeon_irq_ciu_set_affinity,
826	#endif	826	#endif
827	.flags = IRQCHIP_SET_TYPE_MASKED,	827	.flags = IRQCHIP_SET_TYPE_MASKED,
828	};	828	};
829		829
830	/*	830	/*
831	* Watchdog interrupts are special. They are associated with a single	831	* Watchdog interrupts are special. They are associated with a single
832	* core, so we hardwire the affinity to that core.	832	* core, so we hardwire the affinity to that core.
833	*/	833	*/
834	static void octeon_irq_ciu_wd_enable(struct irq_data *data)	834	static void octeon_irq_ciu_wd_enable(struct irq_data *data)
835	{	835	{
836	unsigned long flags;	836	unsigned long flags;
837	unsigned long *pen;	837	unsigned long *pen;
838	int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */	838	int coreid = data->irq - OCTEON_IRQ_WDOG0; /* Bit 0-63 of EN1 */
839	int cpu = octeon_cpu_for_coreid(coreid);	839	int cpu = octeon_cpu_for_coreid(coreid);
840	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);	840	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
841		841
842	raw_spin_lock_irqsave(lock, flags);	842	raw_spin_lock_irqsave(lock, flags);
843	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);	843	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
844	__set_bit(coreid, pen);	844	__set_bit(coreid, pen);
845	/*	845	/*
846	* Must be visible to octeon_irq_ip{2,3}_ciu() before enabling	846	* Must be visible to octeon_irq_ip{2,3}_ciu() before enabling
847	* the irq.	847	* the irq.
848	*/	848	*/
849	wmb();	849	wmb();
850	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);	850	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
851	raw_spin_unlock_irqrestore(lock, flags);	851	raw_spin_unlock_irqrestore(lock, flags);
852	}	852	}
853		853
854	/*	854	/*
855	* Watchdog interrupts are special. They are associated with a single	855	* Watchdog interrupts are special. They are associated with a single
856	* core, so we hardwire the affinity to that core.	856	* core, so we hardwire the affinity to that core.
857	*/	857	*/
858	static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)	858	static void octeon_irq_ciu1_wd_enable_v2(struct irq_data *data)
859	{	859	{
860	int coreid = data->irq - OCTEON_IRQ_WDOG0;	860	int coreid = data->irq - OCTEON_IRQ_WDOG0;
861	int cpu = octeon_cpu_for_coreid(coreid);	861	int cpu = octeon_cpu_for_coreid(coreid);
862		862
863	set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));	863	set_bit(coreid, &per_cpu(octeon_irq_ciu1_en_mirror, cpu));
864	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);	864	cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid * 2 + 1), 1ull << coreid);
865	}	865	}
866		866
867		867
868	static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {	868	static struct irq_chip octeon_irq_chip_ciu_wd_v2 = {
869	.name = "CIU-W",	869	.name = "CIU-W",
870	.irq_enable = octeon_irq_ciu1_wd_enable_v2,	870	.irq_enable = octeon_irq_ciu1_wd_enable_v2,
871	.irq_disable = octeon_irq_ciu_disable_all_v2,	871	.irq_disable = octeon_irq_ciu_disable_all_v2,
872	.irq_mask = octeon_irq_ciu_disable_local_v2,	872	.irq_mask = octeon_irq_ciu_disable_local_v2,
873	.irq_unmask = octeon_irq_ciu_enable_local_v2,	873	.irq_unmask = octeon_irq_ciu_enable_local_v2,
874	};	874	};
875		875
876	static struct irq_chip octeon_irq_chip_ciu_wd = {	876	static struct irq_chip octeon_irq_chip_ciu_wd = {
877	.name = "CIU-W",	877	.name = "CIU-W",
878	.irq_enable = octeon_irq_ciu_wd_enable,	878	.irq_enable = octeon_irq_ciu_wd_enable,
879	.irq_disable = octeon_irq_ciu_disable_all,	879	.irq_disable = octeon_irq_ciu_disable_all,
880	.irq_mask = octeon_irq_ciu_disable_local,	880	.irq_mask = octeon_irq_ciu_disable_local,
881	.irq_unmask = octeon_irq_ciu_enable_local,	881	.irq_unmask = octeon_irq_ciu_enable_local,
882	};	882	};
883		883
884	static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)	884	static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)
885	{	885	{
886	bool edge = false;	886	bool edge = false;
887		887
888	if (line == 0)	888	if (line == 0)
889	switch (bit) {	889	switch (bit) {
890	case 48 ... 49: /* GMX DRP */	890	case 48 ... 49: /* GMX DRP */
891	case 50: /* IPD_DRP */	891	case 50: /* IPD_DRP */
892	case 52 ... 55: /* Timers */	892	case 52 ... 55: /* Timers */
893	case 58: /* MPI */	893	case 58: /* MPI */
894	edge = true;	894	edge = true;
895	break;	895	break;
896	default:	896	default:
897	break;	897	break;
898	}	898	}
899	else /* line == 1 */	899	else /* line == 1 */
900	switch (bit) {	900	switch (bit) {
901	case 47: /* PTP */	901	case 47: /* PTP */
902	edge = true;	902	edge = true;
903	break;	903	break;
904	default:	904	default:
905	break;	905	break;
906	}	906	}
907	return edge;	907	return edge;
908	}	908	}
909		909
910	struct octeon_irq_gpio_domain_data {	910	struct octeon_irq_gpio_domain_data {
911	unsigned int base_hwirq;	911	unsigned int base_hwirq;
912	};	912	};
913		913
914	static int octeon_irq_gpio_xlat(struct irq_domain *d,	914	static int octeon_irq_gpio_xlat(struct irq_domain *d,
915	struct device_node *node,	915	struct device_node *node,
916	const u32 *intspec,	916	const u32 *intspec,
917	unsigned int intsize,	917	unsigned int intsize,
918	unsigned long *out_hwirq,	918	unsigned long *out_hwirq,
919	unsigned int *out_type)	919	unsigned int *out_type)
920	{	920	{
921	unsigned int type;	921	unsigned int type;
922	unsigned int pin;	922	unsigned int pin;
923	unsigned int trigger;	923	unsigned int trigger;
924		924
925	if (d->of_node != node)	925	if (d->of_node != node)
926	return -EINVAL;	926	return -EINVAL;
927		927
928	if (intsize < 2)	928	if (intsize < 2)
929	return -EINVAL;	929	return -EINVAL;
930		930
931	pin = intspec[0];	931	pin = intspec[0];
932	if (pin >= 16)	932	if (pin >= 16)
933	return -EINVAL;	933	return -EINVAL;
934		934
935	trigger = intspec[1];	935	trigger = intspec[1];
936		936
937	switch (trigger) {	937	switch (trigger) {
938	case 1:	938	case 1:
939	type = IRQ_TYPE_EDGE_RISING;	939	type = IRQ_TYPE_EDGE_RISING;
940	break;	940	break;
941	case 2:	941	case 2:
942	type = IRQ_TYPE_EDGE_FALLING;	942	type = IRQ_TYPE_EDGE_FALLING;
943	break;	943	break;
944	case 4:	944	case 4:
945	type = IRQ_TYPE_LEVEL_HIGH;	945	type = IRQ_TYPE_LEVEL_HIGH;
946	break;	946	break;
947	case 8:	947	case 8:
948	type = IRQ_TYPE_LEVEL_LOW;	948	type = IRQ_TYPE_LEVEL_LOW;
949	break;	949	break;
950	default:	950	default:
951	pr_err("Error: (%s) Invalid irq trigger specification: %x\n",	951	pr_err("Error: (%s) Invalid irq trigger specification: %x\n",
952	node->name,	952	node->name,
953	trigger);	953	trigger);
954	type = IRQ_TYPE_LEVEL_LOW;	954	type = IRQ_TYPE_LEVEL_LOW;
955	break;	955	break;
956	}	956	}
957	*out_type = type;	957	*out_type = type;
958	*out_hwirq = pin;	958	*out_hwirq = pin;
959		959
960	return 0;	960	return 0;
961	}	961	}
962		962
963	static int octeon_irq_ciu_xlat(struct irq_domain *d,	963	static int octeon_irq_ciu_xlat(struct irq_domain *d,
964	struct device_node *node,	964	struct device_node *node,
965	const u32 *intspec,	965	const u32 *intspec,
966	unsigned int intsize,	966	unsigned int intsize,
967	unsigned long *out_hwirq,	967	unsigned long *out_hwirq,
968	unsigned int *out_type)	968	unsigned int *out_type)
969	{	969	{
970	unsigned int ciu, bit;	970	unsigned int ciu, bit;
971		971
972	ciu = intspec[0];	972	ciu = intspec[0];
973	bit = intspec[1];	973	bit = intspec[1];
974		974
975	if (ciu > 1 \|\| bit > 63)	975	if (ciu > 1 \|\| bit > 63)
976	return -EINVAL;	976	return -EINVAL;
977		977
978	*out_hwirq = (ciu << 6) \| bit;	978	*out_hwirq = (ciu << 6) \| bit;
979	*out_type = 0;	979	*out_type = 0;
980		980
981	return 0;	981	return 0;
982	}	982	}
983		983
984	static struct irq_chip *octeon_irq_ciu_chip;	984	static struct irq_chip *octeon_irq_ciu_chip;
985	static struct irq_chip *octeon_irq_gpio_chip;	985	static struct irq_chip *octeon_irq_gpio_chip;
986		986
987	static bool octeon_irq_virq_in_range(unsigned int virq)	987	static bool octeon_irq_virq_in_range(unsigned int virq)
988	{	988	{
989	/* We cannot let it overflow the mapping array. */	989	/* We cannot let it overflow the mapping array. */
990	if (virq < (1ul << 8 * sizeof(octeon_irq_ciu_to_irq[0][0])))	990	if (virq < (1ul << 8 * sizeof(octeon_irq_ciu_to_irq[0][0])))
991	return true;	991	return true;
992		992
993	WARN_ONCE(true, "virq out of range %u.\n", virq);	993	WARN_ONCE(true, "virq out of range %u.\n", virq);
994	return false;	994	return false;
995	}	995	}
996		996
997	static int octeon_irq_ciu_map(struct irq_domain *d,	997	static int octeon_irq_ciu_map(struct irq_domain *d,
998	unsigned int virq, irq_hw_number_t hw)	998	unsigned int virq, irq_hw_number_t hw)
999	{	999	{
1000	unsigned int line = hw >> 6;	1000	unsigned int line = hw >> 6;
1001	unsigned int bit = hw & 63;	1001	unsigned int bit = hw & 63;
1002		1002
1003	if (!octeon_irq_virq_in_range(virq))	1003	if (!octeon_irq_virq_in_range(virq))
1004	return -EINVAL;	1004	return -EINVAL;
1005		1005
1006	/* Don't map irq if it is reserved for GPIO. */	1006	/* Don't map irq if it is reserved for GPIO. */
1007	if (line == 0 && bit >= 16 && bit <32)	1007	if (line == 0 && bit >= 16 && bit <32)
1008	return 0;	1008	return 0;
1009		1009
1010	if (line > 1 \|\| octeon_irq_ciu_to_irq[line][bit] != 0)	1010	if (line > 1 \|\| octeon_irq_ciu_to_irq[line][bit] != 0)
1011	return -EINVAL;	1011	return -EINVAL;
1012		1012
1013	if (octeon_irq_ciu_is_edge(line, bit))	1013	if (octeon_irq_ciu_is_edge(line, bit))
1014	octeon_irq_set_ciu_mapping(virq, line, bit, 0,	1014	octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1015	octeon_irq_ciu_chip,	1015	octeon_irq_ciu_chip,
1016	handle_edge_irq);	1016	handle_edge_irq);
1017	else	1017	else
1018	octeon_irq_set_ciu_mapping(virq, line, bit, 0,	1018	octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1019	octeon_irq_ciu_chip,	1019	octeon_irq_ciu_chip,
1020	handle_level_irq);	1020	handle_level_irq);
1021		1021
1022	return 0;	1022	return 0;
1023	}	1023	}
1024		1024
1025	static int octeon_irq_gpio_map_common(struct irq_domain *d,	1025	static int octeon_irq_gpio_map_common(struct irq_domain *d,
1026	unsigned int virq, irq_hw_number_t hw,	1026	unsigned int virq, irq_hw_number_t hw,
1027	int line_limit, struct irq_chip *chip)	1027	int line_limit, struct irq_chip *chip)
1028	{	1028	{
1029	struct octeon_irq_gpio_domain_data *gpiod = d->host_data;	1029	struct octeon_irq_gpio_domain_data *gpiod = d->host_data;
1030	unsigned int line, bit;	1030	unsigned int line, bit;
1031		1031
1032	if (!octeon_irq_virq_in_range(virq))	1032	if (!octeon_irq_virq_in_range(virq))
1033	return -EINVAL;	1033	return -EINVAL;
1034		1034
1035	line = (hw + gpiod->base_hwirq) >> 6;	1035	line = (hw + gpiod->base_hwirq) >> 6;
1036	bit = (hw + gpiod->base_hwirq) & 63;	1036	bit = (hw + gpiod->base_hwirq) & 63;
1037	if (line > line_limit \|\| octeon_irq_ciu_to_irq[line][bit] != 0)	1037	if (line > line_limit \|\| octeon_irq_ciu_to_irq[line][bit] != 0)
1038	return -EINVAL;	1038	return -EINVAL;
1039		1039
1040	octeon_irq_set_ciu_mapping(virq, line, bit, hw,	1040	octeon_irq_set_ciu_mapping(virq, line, bit, hw,
1041	chip, octeon_irq_handle_gpio);	1041	chip, octeon_irq_handle_gpio);
1042	return 0;	1042	return 0;
1043	}	1043	}
1044		1044
1045	static int octeon_irq_gpio_map(struct irq_domain *d,	1045	static int octeon_irq_gpio_map(struct irq_domain *d,
1046	unsigned int virq, irq_hw_number_t hw)	1046	unsigned int virq, irq_hw_number_t hw)
1047	{	1047	{
1048	return octeon_irq_gpio_map_common(d, virq, hw, 1, octeon_irq_gpio_chip);	1048	return octeon_irq_gpio_map_common(d, virq, hw, 1, octeon_irq_gpio_chip);
1049	}	1049	}
1050		1050
1051	static struct irq_domain_ops octeon_irq_domain_ciu_ops = {	1051	static struct irq_domain_ops octeon_irq_domain_ciu_ops = {
1052	.map = octeon_irq_ciu_map,	1052	.map = octeon_irq_ciu_map,
1053	.xlate = octeon_irq_ciu_xlat,	1053	.xlate = octeon_irq_ciu_xlat,
1054	};	1054	};
1055		1055
1056	static struct irq_domain_ops octeon_irq_domain_gpio_ops = {	1056	static struct irq_domain_ops octeon_irq_domain_gpio_ops = {
1057	.map = octeon_irq_gpio_map,	1057	.map = octeon_irq_gpio_map,
1058	.xlate = octeon_irq_gpio_xlat,	1058	.xlate = octeon_irq_gpio_xlat,
1059	};	1059	};
1060		1060
1061	static void octeon_irq_ip2_ciu(void)	1061	static void octeon_irq_ip2_ciu(void)
1062	{	1062	{
1063	const unsigned long core_id = cvmx_get_core_num();	1063	const unsigned long core_id = cvmx_get_core_num();
1064	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));	1064	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
1065		1065
1066	ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);	1066	ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
1067	if (likely(ciu_sum)) {	1067	if (likely(ciu_sum)) {
1068	int bit = fls64(ciu_sum) - 1;	1068	int bit = fls64(ciu_sum) - 1;
1069	int irq = octeon_irq_ciu_to_irq[0][bit];	1069	int irq = octeon_irq_ciu_to_irq[0][bit];
1070	if (likely(irq))	1070	if (likely(irq))
1071	do_IRQ(irq);	1071	do_IRQ(irq);
1072	else	1072	else
1073	spurious_interrupt();	1073	spurious_interrupt();
1074	} else {	1074	} else {
1075	spurious_interrupt();	1075	spurious_interrupt();
1076	}	1076	}
1077	}	1077	}
1078		1078
1079	static void octeon_irq_ip3_ciu(void)	1079	static void octeon_irq_ip3_ciu(void)
1080	{	1080	{
1081	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);	1081	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
1082		1082
1083	ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);	1083	ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
1084	if (likely(ciu_sum)) {	1084	if (likely(ciu_sum)) {
1085	int bit = fls64(ciu_sum) - 1;	1085	int bit = fls64(ciu_sum) - 1;
1086	int irq = octeon_irq_ciu_to_irq[1][bit];	1086	int irq = octeon_irq_ciu_to_irq[1][bit];
1087	if (likely(irq))	1087	if (likely(irq))
1088	do_IRQ(irq);	1088	do_IRQ(irq);
1089	else	1089	else
1090	spurious_interrupt();	1090	spurious_interrupt();
1091	} else {	1091	} else {
1092	spurious_interrupt();	1092	spurious_interrupt();
1093	}	1093	}
1094	}	1094	}
1095		1095
1096	static bool octeon_irq_use_ip4;	1096	static bool octeon_irq_use_ip4;
1097		1097
1098	static void octeon_irq_local_enable_ip4(void *arg)	1098	static void octeon_irq_local_enable_ip4(void *arg)
1099	{	1099	{
1100	set_c0_status(STATUSF_IP4);	1100	set_c0_status(STATUSF_IP4);
1101	}	1101	}
1102		1102
1103	static void octeon_irq_ip4_mask(void)	1103	static void octeon_irq_ip4_mask(void)
1104	{	1104	{
1105	clear_c0_status(STATUSF_IP4);	1105	clear_c0_status(STATUSF_IP4);
1106	spurious_interrupt();	1106	spurious_interrupt();
1107	}	1107	}
1108		1108
1109	static void (*octeon_irq_ip2)(void);	1109	static void (*octeon_irq_ip2)(void);
1110	static void (*octeon_irq_ip3)(void);	1110	static void (*octeon_irq_ip3)(void);
1111	static void (*octeon_irq_ip4)(void);	1111	static void (*octeon_irq_ip4)(void);
1112		1112
1113	void (*octeon_irq_setup_secondary)(void);	1113	void (*octeon_irq_setup_secondary)(void);
1114		1114
1115	void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h)	1115	void octeon_irq_set_ip4_handler(octeon_irq_ip4_handler_t h)
1116	{	1116	{
1117	octeon_irq_ip4 = h;	1117	octeon_irq_ip4 = h;
1118	octeon_irq_use_ip4 = true;	1118	octeon_irq_use_ip4 = true;
1119	on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1);	1119	on_each_cpu(octeon_irq_local_enable_ip4, NULL, 1);
1120	}	1120	}
1121		1121
1122	static void octeon_irq_percpu_enable(void)	1122	static void octeon_irq_percpu_enable(void)
1123	{	1123	{
1124	irq_cpu_online();	1124	irq_cpu_online();
1125	}	1125	}
1126		1126
1127	static void octeon_irq_init_ciu_percpu(void)	1127	static void octeon_irq_init_ciu_percpu(void)
1128	{	1128	{
1129	int coreid = cvmx_get_core_num();	1129	int coreid = cvmx_get_core_num();
1130		1130
1131		1131
1132	__get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;	1132	__get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;
1133	__get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;	1133	__get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;
1134	wmb();	1134	wmb();
1135	raw_spin_lock_init(&__get_cpu_var(octeon_irq_ciu_spinlock));	1135	raw_spin_lock_init(&__get_cpu_var(octeon_irq_ciu_spinlock));
1136	/*	1136	/*
1137	* Disable All CIU Interrupts. The ones we need will be	1137	* Disable All CIU Interrupts. The ones we need will be
1138	* enabled later. Read the SUM register so we know the write	1138	* enabled later. Read the SUM register so we know the write
1139	* completed.	1139	* completed.
1140	*/	1140	*/
1141	cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);	1141	cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2)), 0);
1142	cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);	1142	cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid * 2 + 1)), 0);
1143	cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);	1143	cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2)), 0);
1144	cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);	1144	cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid * 2 + 1)), 0);
1145	cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));	1145	cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid * 2)));
1146	}	1146	}
1147		1147
1148	static void octeon_irq_init_ciu2_percpu(void)	1148	static void octeon_irq_init_ciu2_percpu(void)
1149	{	1149	{
1150	u64 regx, ipx;	1150	u64 regx, ipx;
1151	int coreid = cvmx_get_core_num();	1151	int coreid = cvmx_get_core_num();
1152	u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid);	1152	u64 base = CVMX_CIU2_EN_PPX_IP2_WRKQ(coreid);
1153		1153
1154	/*	1154	/*
1155	* Disable All CIU2 Interrupts. The ones we need will be	1155	* Disable All CIU2 Interrupts. The ones we need will be
1156	* enabled later. Read the SUM register so we know the write	1156	* enabled later. Read the SUM register so we know the write
1157	* completed.	1157	* completed.
1158	*	1158	*
1159	* There are 9 registers and 3 IPX levels with strides 0x1000	1159	* There are 9 registers and 3 IPX levels with strides 0x1000
1160	* and 0x200 respectivly. Use loops to clear them.	1160	* and 0x200 respectivly. Use loops to clear them.
1161	*/	1161	*/
1162	for (regx = 0; regx <= 0x8000; regx += 0x1000) {	1162	for (regx = 0; regx <= 0x8000; regx += 0x1000) {
1163	for (ipx = 0; ipx <= 0x400; ipx += 0x200)	1163	for (ipx = 0; ipx <= 0x400; ipx += 0x200)
1164	cvmx_write_csr(base + regx + ipx, 0);	1164	cvmx_write_csr(base + regx + ipx, 0);
1165	}	1165	}
1166		1166
1167	cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid));	1167	cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(coreid));
1168	}	1168	}
1169		1169
1170	static void octeon_irq_setup_secondary_ciu(void)	1170	static void octeon_irq_setup_secondary_ciu(void)
1171	{	1171	{
1172	octeon_irq_init_ciu_percpu();	1172	octeon_irq_init_ciu_percpu();
1173	octeon_irq_percpu_enable();	1173	octeon_irq_percpu_enable();
1174		1174
1175	/* Enable the CIU lines */	1175	/* Enable the CIU lines */
1176	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);	1176	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);
1177	clear_c0_status(STATUSF_IP4);	1177	clear_c0_status(STATUSF_IP4);
1178	}	1178	}
1179		1179
1180	static void octeon_irq_setup_secondary_ciu2(void)	1180	static void octeon_irq_setup_secondary_ciu2(void)
1181	{	1181	{
1182	octeon_irq_init_ciu2_percpu();	1182	octeon_irq_init_ciu2_percpu();
1183	octeon_irq_percpu_enable();	1183	octeon_irq_percpu_enable();
1184		1184
1185	/* Enable the CIU lines */	1185	/* Enable the CIU lines */
1186	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);	1186	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);
1187	if (octeon_irq_use_ip4)	1187	if (octeon_irq_use_ip4)
1188	set_c0_status(STATUSF_IP4);	1188	set_c0_status(STATUSF_IP4);
1189	else	1189	else
1190	clear_c0_status(STATUSF_IP4);	1190	clear_c0_status(STATUSF_IP4);
1191	}	1191	}
1192		1192
1193	static void __init octeon_irq_init_ciu(void)	1193	static void __init octeon_irq_init_ciu(void)
1194	{	1194	{
1195	unsigned int i;	1195	unsigned int i;
1196	struct irq_chip *chip;	1196	struct irq_chip *chip;
1197	struct irq_chip *chip_mbox;	1197	struct irq_chip *chip_mbox;
1198	struct irq_chip *chip_wd;	1198	struct irq_chip *chip_wd;
1199	struct device_node *gpio_node;	1199	struct device_node *gpio_node;
1200	struct device_node *ciu_node;	1200	struct device_node *ciu_node;
1201	struct irq_domain *ciu_domain = NULL;	1201	struct irq_domain *ciu_domain = NULL;
1202		1202
1203	octeon_irq_init_ciu_percpu();	1203	octeon_irq_init_ciu_percpu();
1204	octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;	1204	octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
1205		1205
1206	octeon_irq_ip2 = octeon_irq_ip2_ciu;	1206	octeon_irq_ip2 = octeon_irq_ip2_ciu;
1207	octeon_irq_ip3 = octeon_irq_ip3_ciu;	1207	octeon_irq_ip3 = octeon_irq_ip3_ciu;
1208	if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) \|\|	1208	if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) \|\|
1209	OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) \|\|	1209	OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) \|\|
1210	OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) \|\|	1210	OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) \|\|
1211	OCTEON_IS_MODEL(OCTEON_CN6XXX)) {	1211	OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
1212	chip = &octeon_irq_chip_ciu_v2;	1212	chip = &octeon_irq_chip_ciu_v2;
1213	chip_mbox = &octeon_irq_chip_ciu_mbox_v2;	1213	chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
1214	chip_wd = &octeon_irq_chip_ciu_wd_v2;	1214	chip_wd = &octeon_irq_chip_ciu_wd_v2;
1215	octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;	1215	octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;
1216	} else {	1216	} else {
1217	chip = &octeon_irq_chip_ciu;	1217	chip = &octeon_irq_chip_ciu;
1218	chip_mbox = &octeon_irq_chip_ciu_mbox;	1218	chip_mbox = &octeon_irq_chip_ciu_mbox;
1219	chip_wd = &octeon_irq_chip_ciu_wd;	1219	chip_wd = &octeon_irq_chip_ciu_wd;
1220	octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio;	1220	octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio;
1221	}	1221	}
1222	octeon_irq_ciu_chip = chip;	1222	octeon_irq_ciu_chip = chip;
1223	octeon_irq_ip4 = octeon_irq_ip4_mask;	1223	octeon_irq_ip4 = octeon_irq_ip4_mask;
1224		1224
1225	/* Mips internal */	1225	/* Mips internal */
1226	octeon_irq_init_core();	1226	octeon_irq_init_core();
1227		1227
1228	gpio_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-gpio");	1228	gpio_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-gpio");
1229	if (gpio_node) {	1229	if (gpio_node) {
1230	struct octeon_irq_gpio_domain_data *gpiod;	1230	struct octeon_irq_gpio_domain_data *gpiod;
1231		1231
1232	gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);	1232	gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);
1233	if (gpiod) {	1233	if (gpiod) {
1234	/* gpio domain host_data is the base hwirq number. */	1234	/* gpio domain host_data is the base hwirq number. */
1235	gpiod->base_hwirq = 16;	1235	gpiod->base_hwirq = 16;
1236	irq_domain_add_linear(gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod);	1236	irq_domain_add_linear(gpio_node, 16, &octeon_irq_domain_gpio_ops, gpiod);
1237	of_node_put(gpio_node);	1237	of_node_put(gpio_node);
1238	} else	1238	} else
1239	pr_warn("Cannot allocate memory for GPIO irq_domain.\n");	1239	pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1240	} else	1240	} else
1241	pr_warn("Cannot find device node for cavium,octeon-3860-gpio.\n");	1241	pr_warn("Cannot find device node for cavium,octeon-3860-gpio.\n");
1242		1242
1243	ciu_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-ciu");	1243	ciu_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-ciu");
1244	if (ciu_node) {	1244	if (ciu_node) {
1245	ciu_domain = irq_domain_add_tree(ciu_node, &octeon_irq_domain_ciu_ops, NULL);	1245	ciu_domain = irq_domain_add_tree(ciu_node, &octeon_irq_domain_ciu_ops, NULL);
1246	irq_set_default_host(ciu_domain);	1246	irq_set_default_host(ciu_domain);
1247	of_node_put(ciu_node);	1247	of_node_put(ciu_node);
1248	} else	1248	} else
1249	panic("Cannot find device node for cavium,octeon-3860-ciu.");	1249	panic("Cannot find device node for cavium,octeon-3860-ciu.");
1250		1250
1251	/* CIU_0 */	1251	/* CIU_0 */
1252	for (i = 0; i < 16; i++)	1252	for (i = 0; i < 16; i++)
1253	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0);	1253	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i + 0);
1254		1254
1255	octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq);	1255	octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0, 0, 32, 0, chip_mbox, handle_percpu_irq);
1256	octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq);	1256	octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1, 0, 33, 0, chip_mbox, handle_percpu_irq);
1257		1257
1258	for (i = 0; i < 4; i++)	1258	for (i = 0; i < 4; i++)
1259	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36);	1259	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_INT0, 0, i + 36);
1260	for (i = 0; i < 4; i++)	1260	for (i = 0; i < 4; i++)
1261	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40);	1261	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 0, i + 40);
1262		1262
1263	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46);	1263	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_RML, 0, 46);
1264	for (i = 0; i < 4; i++)	1264	for (i = 0; i < 4; i++)
1265	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52);	1265	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_TIMER0, 0, i + 52);
1266		1266
1267	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56);	1267	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 0, 56);
1268		1268
1269	/* CIU_1 */	1269	/* CIU_1 */
1270	for (i = 0; i < 16; i++)	1270	for (i = 0; i < 16; i++)
1271	octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, handle_level_irq);	1271	octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i + 0, 0, chip_wd, handle_level_irq);
1272		1272
1273	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17);	1273	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB1, 1, 17);
1274		1274
1275	/* Enable the CIU lines */	1275	/* Enable the CIU lines */
1276	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);	1276	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);
1277	clear_c0_status(STATUSF_IP4);	1277	clear_c0_status(STATUSF_IP4);
1278	}	1278	}
1279		1279
1280	/*	1280	/*
1281	* Watchdog interrupts are special. They are associated with a single	1281	* Watchdog interrupts are special. They are associated with a single
1282	* core, so we hardwire the affinity to that core.	1282	* core, so we hardwire the affinity to that core.
1283	*/	1283	*/
1284	static void octeon_irq_ciu2_wd_enable(struct irq_data *data)	1284	static void octeon_irq_ciu2_wd_enable(struct irq_data *data)
1285	{	1285	{
1286	u64 mask;	1286	u64 mask;
1287	u64 en_addr;	1287	u64 en_addr;
1288	int coreid = data->irq - OCTEON_IRQ_WDOG0;	1288	int coreid = data->irq - OCTEON_IRQ_WDOG0;
1289	union octeon_ciu_chip_data cd;	1289	union octeon_ciu_chip_data cd;
1290		1290
1291	cd.p = irq_data_get_irq_chip_data(data);	1291	cd.p = irq_data_get_irq_chip_data(data);
1292	mask = 1ull << (cd.s.bit);	1292	mask = 1ull << (cd.s.bit);
1293		1293
1294	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);	1294	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);
1295	cvmx_write_csr(en_addr, mask);	1295	cvmx_write_csr(en_addr, mask);
1296		1296
1297	}	1297	}
1298		1298
1299	static void octeon_irq_ciu2_enable(struct irq_data *data)	1299	static void octeon_irq_ciu2_enable(struct irq_data *data)
1300	{	1300	{
1301	u64 mask;	1301	u64 mask;
1302	u64 en_addr;	1302	u64 en_addr;
1303	int cpu = next_cpu_for_irq(data);	1303	int cpu = next_cpu_for_irq(data);
1304	int coreid = octeon_coreid_for_cpu(cpu);	1304	int coreid = octeon_coreid_for_cpu(cpu);
1305	union octeon_ciu_chip_data cd;	1305	union octeon_ciu_chip_data cd;
1306		1306
1307	cd.p = irq_data_get_irq_chip_data(data);	1307	cd.p = irq_data_get_irq_chip_data(data);
1308	mask = 1ull << (cd.s.bit);	1308	mask = 1ull << (cd.s.bit);
1309		1309
1310	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);	1310	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);
1311	cvmx_write_csr(en_addr, mask);	1311	cvmx_write_csr(en_addr, mask);
1312	}	1312	}
1313		1313
1314	static void octeon_irq_ciu2_enable_local(struct irq_data *data)	1314	static void octeon_irq_ciu2_enable_local(struct irq_data *data)
1315	{	1315	{
1316	u64 mask;	1316	u64 mask;
1317	u64 en_addr;	1317	u64 en_addr;
1318	int coreid = cvmx_get_core_num();	1318	int coreid = cvmx_get_core_num();
1319	union octeon_ciu_chip_data cd;	1319	union octeon_ciu_chip_data cd;
1320		1320
1321	cd.p = irq_data_get_irq_chip_data(data);	1321	cd.p = irq_data_get_irq_chip_data(data);
1322	mask = 1ull << (cd.s.bit);	1322	mask = 1ull << (cd.s.bit);
1323		1323
1324	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);	1324	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(coreid) + (0x1000ull * cd.s.line);
1325	cvmx_write_csr(en_addr, mask);	1325	cvmx_write_csr(en_addr, mask);
1326		1326
1327	}	1327	}
1328		1328
1329	static void octeon_irq_ciu2_disable_local(struct irq_data *data)	1329	static void octeon_irq_ciu2_disable_local(struct irq_data *data)
1330	{	1330	{
1331	u64 mask;	1331	u64 mask;
1332	u64 en_addr;	1332	u64 en_addr;
1333	int coreid = cvmx_get_core_num();	1333	int coreid = cvmx_get_core_num();
1334	union octeon_ciu_chip_data cd;	1334	union octeon_ciu_chip_data cd;
1335		1335
1336	cd.p = irq_data_get_irq_chip_data(data);	1336	cd.p = irq_data_get_irq_chip_data(data);
1337	mask = 1ull << (cd.s.bit);	1337	mask = 1ull << (cd.s.bit);
1338		1338
1339	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + (0x1000ull * cd.s.line);	1339	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(coreid) + (0x1000ull * cd.s.line);
1340	cvmx_write_csr(en_addr, mask);	1340	cvmx_write_csr(en_addr, mask);
1341		1341
1342	}	1342	}
1343		1343
1344	static void octeon_irq_ciu2_ack(struct irq_data *data)	1344	static void octeon_irq_ciu2_ack(struct irq_data *data)
1345	{	1345	{
1346	u64 mask;	1346	u64 mask;
1347	u64 en_addr;	1347	u64 en_addr;
1348	int coreid = cvmx_get_core_num();	1348	int coreid = cvmx_get_core_num();
1349	union octeon_ciu_chip_data cd;	1349	union octeon_ciu_chip_data cd;
1350		1350
1351	cd.p = irq_data_get_irq_chip_data(data);	1351	cd.p = irq_data_get_irq_chip_data(data);
1352	mask = 1ull << (cd.s.bit);	1352	mask = 1ull << (cd.s.bit);
1353		1353
1354	en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd.s.line);	1354	en_addr = CVMX_CIU2_RAW_PPX_IP2_WRKQ(coreid) + (0x1000ull * cd.s.line);
1355	cvmx_write_csr(en_addr, mask);	1355	cvmx_write_csr(en_addr, mask);
1356		1356
1357	}	1357	}
1358		1358
1359	static void octeon_irq_ciu2_disable_all(struct irq_data *data)	1359	static void octeon_irq_ciu2_disable_all(struct irq_data *data)
1360	{	1360	{
1361	int cpu;	1361	int cpu;
1362	u64 mask;	1362	u64 mask;
1363	union octeon_ciu_chip_data cd;	1363	union octeon_ciu_chip_data cd;
1364		1364
1365	cd.p = irq_data_get_irq_chip_data(data);	1365	cd.p = irq_data_get_irq_chip_data(data);
1366	mask = 1ull << (cd.s.bit);	1366	mask = 1ull << (cd.s.bit);
1367		1367
1368	for_each_online_cpu(cpu) {	1368	for_each_online_cpu(cpu) {
1369	u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);	1369	u64 en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);
1370	cvmx_write_csr(en_addr, mask);	1370	cvmx_write_csr(en_addr, mask);
1371	}	1371	}
1372	}	1372	}
1373		1373
1374	static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data)	1374	static void octeon_irq_ciu2_mbox_enable_all(struct irq_data *data)
1375	{	1375	{
1376	int cpu;	1376	int cpu;
1377	u64 mask;	1377	u64 mask;
1378		1378
1379	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);	1379	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1380		1380
1381	for_each_online_cpu(cpu) {	1381	for_each_online_cpu(cpu) {
1382	u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(octeon_coreid_for_cpu(cpu));	1382	u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(octeon_coreid_for_cpu(cpu));
1383	cvmx_write_csr(en_addr, mask);	1383	cvmx_write_csr(en_addr, mask);
1384	}	1384	}
1385	}	1385	}
1386		1386
1387	static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data)	1387	static void octeon_irq_ciu2_mbox_disable_all(struct irq_data *data)
1388	{	1388	{
1389	int cpu;	1389	int cpu;
1390	u64 mask;	1390	u64 mask;
1391		1391
1392	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);	1392	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1393		1393
1394	for_each_online_cpu(cpu) {	1394	for_each_online_cpu(cpu) {
1395	u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(octeon_coreid_for_cpu(cpu));	1395	u64 en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(octeon_coreid_for_cpu(cpu));
1396	cvmx_write_csr(en_addr, mask);	1396	cvmx_write_csr(en_addr, mask);
1397	}	1397	}
1398	}	1398	}
1399		1399
1400	static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data)	1400	static void octeon_irq_ciu2_mbox_enable_local(struct irq_data *data)
1401	{	1401	{
1402	u64 mask;	1402	u64 mask;
1403	u64 en_addr;	1403	u64 en_addr;
1404	int coreid = cvmx_get_core_num();	1404	int coreid = cvmx_get_core_num();
1405		1405
1406	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);	1406	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1407	en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid);	1407	en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1S(coreid);
1408	cvmx_write_csr(en_addr, mask);	1408	cvmx_write_csr(en_addr, mask);
1409	}	1409	}
1410		1410
1411	static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data)	1411	static void octeon_irq_ciu2_mbox_disable_local(struct irq_data *data)
1412	{	1412	{
1413	u64 mask;	1413	u64 mask;
1414	u64 en_addr;	1414	u64 en_addr;
1415	int coreid = cvmx_get_core_num();	1415	int coreid = cvmx_get_core_num();
1416		1416
1417	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);	1417	mask = 1ull << (data->irq - OCTEON_IRQ_MBOX0);
1418	en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid);	1418	en_addr = CVMX_CIU2_EN_PPX_IP3_MBOX_W1C(coreid);
1419	cvmx_write_csr(en_addr, mask);	1419	cvmx_write_csr(en_addr, mask);
1420	}	1420	}
1421		1421
1422	#ifdef CONFIG_SMP	1422	#ifdef CONFIG_SMP
1423	static int octeon_irq_ciu2_set_affinity(struct irq_data *data,	1423	static int octeon_irq_ciu2_set_affinity(struct irq_data *data,
1424	const struct cpumask *dest, bool force)	1424	const struct cpumask *dest, bool force)
1425	{	1425	{
1426	int cpu;	1426	int cpu;
1427	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);	1427	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
1428	u64 mask;	1428	u64 mask;
1429	union octeon_ciu_chip_data cd;	1429	union octeon_ciu_chip_data cd;
1430		1430
1431	if (!enable_one)	1431	if (!enable_one)
1432	return 0;	1432	return 0;
1433		1433
1434	cd.p = irq_data_get_irq_chip_data(data);	1434	cd.p = irq_data_get_irq_chip_data(data);
1435	mask = 1ull << cd.s.bit;	1435	mask = 1ull << cd.s.bit;
1436		1436
1437	for_each_online_cpu(cpu) {	1437	for_each_online_cpu(cpu) {
1438	u64 en_addr;	1438	u64 en_addr;
1439	if (cpumask_test_cpu(cpu, dest) && enable_one) {	1439	if (cpumask_test_cpu(cpu, dest) && enable_one) {
1440	enable_one = false;	1440	enable_one = false;
1441	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);	1441	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1S(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);
1442	} else {	1442	} else {
1443	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);	1443	en_addr = CVMX_CIU2_EN_PPX_IP2_WRKQ_W1C(octeon_coreid_for_cpu(cpu)) + (0x1000ull * cd.s.line);
1444	}	1444	}
1445	cvmx_write_csr(en_addr, mask);	1445	cvmx_write_csr(en_addr, mask);
1446	}	1446	}
1447		1447
1448	return 0;	1448	return 0;
1449	}	1449	}
1450	#endif	1450	#endif
1451		1451
1452	static void octeon_irq_ciu2_enable_gpio(struct irq_data *data)	1452	static void octeon_irq_ciu2_enable_gpio(struct irq_data *data)
1453	{	1453	{
1454	octeon_irq_gpio_setup(data);	1454	octeon_irq_gpio_setup(data);
1455	octeon_irq_ciu2_enable(data);	1455	octeon_irq_ciu2_enable(data);
1456	}	1456	}
1457		1457
1458	static void octeon_irq_ciu2_disable_gpio(struct irq_data *data)	1458	static void octeon_irq_ciu2_disable_gpio(struct irq_data *data)
1459	{	1459	{
1460	union octeon_ciu_chip_data cd;	1460	union octeon_ciu_chip_data cd;
1461	cd.p = irq_data_get_irq_chip_data(data);	1461	cd.p = irq_data_get_irq_chip_data(data);
1462		1462
1463	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);	1463	cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd.s.gpio_line), 0);
1464		1464
1465	octeon_irq_ciu2_disable_all(data);	1465	octeon_irq_ciu2_disable_all(data);
1466	}	1466	}
1467		1467
1468	static struct irq_chip octeon_irq_chip_ciu2 = {	1468	static struct irq_chip octeon_irq_chip_ciu2 = {
1469	.name = "CIU2-E",	1469	.name = "CIU2-E",
1470	.irq_enable = octeon_irq_ciu2_enable,	1470	.irq_enable = octeon_irq_ciu2_enable,
1471	.irq_disable = octeon_irq_ciu2_disable_all,	1471	.irq_disable = octeon_irq_ciu2_disable_all,
1472	.irq_ack = octeon_irq_ciu2_ack,	1472	.irq_ack = octeon_irq_ciu2_ack,
1473	.irq_mask = octeon_irq_ciu2_disable_local,	1473	.irq_mask = octeon_irq_ciu2_disable_local,
1474	.irq_unmask = octeon_irq_ciu2_enable,	1474	.irq_unmask = octeon_irq_ciu2_enable,
1475	#ifdef CONFIG_SMP	1475	#ifdef CONFIG_SMP
1476	.irq_set_affinity = octeon_irq_ciu2_set_affinity,	1476	.irq_set_affinity = octeon_irq_ciu2_set_affinity,
1477	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,	1477	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1478	#endif	1478	#endif
1479	};	1479	};
1480		1480
1481	static struct irq_chip octeon_irq_chip_ciu2_mbox = {	1481	static struct irq_chip octeon_irq_chip_ciu2_mbox = {
1482	.name = "CIU2-M",	1482	.name = "CIU2-M",
1483	.irq_enable = octeon_irq_ciu2_mbox_enable_all,	1483	.irq_enable = octeon_irq_ciu2_mbox_enable_all,
1484	.irq_disable = octeon_irq_ciu2_mbox_disable_all,	1484	.irq_disable = octeon_irq_ciu2_mbox_disable_all,
1485	.irq_ack = octeon_irq_ciu2_mbox_disable_local,	1485	.irq_ack = octeon_irq_ciu2_mbox_disable_local,
1486	.irq_eoi = octeon_irq_ciu2_mbox_enable_local,	1486	.irq_eoi = octeon_irq_ciu2_mbox_enable_local,
1487		1487
1488	.irq_cpu_online = octeon_irq_ciu2_mbox_enable_local,	1488	.irq_cpu_online = octeon_irq_ciu2_mbox_enable_local,
1489	.irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local,	1489	.irq_cpu_offline = octeon_irq_ciu2_mbox_disable_local,
1490	.flags = IRQCHIP_ONOFFLINE_ENABLED,	1490	.flags = IRQCHIP_ONOFFLINE_ENABLED,
1491	};	1491	};
1492		1492
1493	static struct irq_chip octeon_irq_chip_ciu2_wd = {	1493	static struct irq_chip octeon_irq_chip_ciu2_wd = {
1494	.name = "CIU2-W",	1494	.name = "CIU2-W",
1495	.irq_enable = octeon_irq_ciu2_wd_enable,	1495	.irq_enable = octeon_irq_ciu2_wd_enable,
1496	.irq_disable = octeon_irq_ciu2_disable_all,	1496	.irq_disable = octeon_irq_ciu2_disable_all,
1497	.irq_mask = octeon_irq_ciu2_disable_local,	1497	.irq_mask = octeon_irq_ciu2_disable_local,
1498	.irq_unmask = octeon_irq_ciu2_enable_local,	1498	.irq_unmask = octeon_irq_ciu2_enable_local,
1499	};	1499	};
1500		1500
1501	static struct irq_chip octeon_irq_chip_ciu2_gpio = {	1501	static struct irq_chip octeon_irq_chip_ciu2_gpio = {
1502	.name = "CIU-GPIO",	1502	.name = "CIU-GPIO",
1503	.irq_enable = octeon_irq_ciu2_enable_gpio,	1503	.irq_enable = octeon_irq_ciu2_enable_gpio,
1504	.irq_disable = octeon_irq_ciu2_disable_gpio,	1504	.irq_disable = octeon_irq_ciu2_disable_gpio,
1505	.irq_ack = octeon_irq_ciu_gpio_ack,	1505	.irq_ack = octeon_irq_ciu_gpio_ack,
1506	.irq_mask = octeon_irq_ciu2_disable_local,	1506	.irq_mask = octeon_irq_ciu2_disable_local,
1507	.irq_unmask = octeon_irq_ciu2_enable,	1507	.irq_unmask = octeon_irq_ciu2_enable,
1508	.irq_set_type = octeon_irq_ciu_gpio_set_type,	1508	.irq_set_type = octeon_irq_ciu_gpio_set_type,
1509	#ifdef CONFIG_SMP	1509	#ifdef CONFIG_SMP
1510	.irq_set_affinity = octeon_irq_ciu2_set_affinity,	1510	.irq_set_affinity = octeon_irq_ciu2_set_affinity,
1511	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,	1511	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,
1512	#endif	1512	#endif
1513	.flags = IRQCHIP_SET_TYPE_MASKED,	1513	.flags = IRQCHIP_SET_TYPE_MASKED,
1514	};	1514	};
1515		1515
1516	static int octeon_irq_ciu2_xlat(struct irq_domain *d,	1516	static int octeon_irq_ciu2_xlat(struct irq_domain *d,
1517	struct device_node *node,	1517	struct device_node *node,
1518	const u32 *intspec,	1518	const u32 *intspec,
1519	unsigned int intsize,	1519	unsigned int intsize,
1520	unsigned long *out_hwirq,	1520	unsigned long *out_hwirq,
1521	unsigned int *out_type)	1521	unsigned int *out_type)
1522	{	1522	{
1523	unsigned int ciu, bit;	1523	unsigned int ciu, bit;
1524		1524
1525	ciu = intspec[0];	1525	ciu = intspec[0];
1526	bit = intspec[1];	1526	bit = intspec[1];
1527		1527
1528	*out_hwirq = (ciu << 6) \| bit;	1528	*out_hwirq = (ciu << 6) \| bit;
1529	*out_type = 0;	1529	*out_type = 0;
1530		1530
1531	return 0;	1531	return 0;
1532	}	1532	}
1533		1533
1534	static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit)	1534	static bool octeon_irq_ciu2_is_edge(unsigned int line, unsigned int bit)
1535	{	1535	{
1536	bool edge = false;	1536	bool edge = false;
1537		1537
1538	if (line == 3) /* MIO */	1538	if (line == 3) /* MIO */
1539	switch (bit) {	1539	switch (bit) {
1540	case 2: /* IPD_DRP */	1540	case 2: /* IPD_DRP */
1541	case 8 ... 11: /* Timers */	1541	case 8 ... 11: /* Timers */
1542	case 48: /* PTP */	1542	case 48: /* PTP */
1543	edge = true;	1543	edge = true;
1544	break;	1544	break;
1545	default:	1545	default:
1546	break;	1546	break;
1547	}	1547	}
1548	else if (line == 6) /* PKT */	1548	else if (line == 6) /* PKT */
1549	switch (bit) {	1549	switch (bit) {
1550	case 52 ... 53: /* ILK_DRP */	1550	case 52 ... 53: /* ILK_DRP */
1551	case 8 ... 12: /* GMX_DRP */	1551	case 8 ... 12: /* GMX_DRP */
1552	edge = true;	1552	edge = true;
1553	break;	1553	break;
1554	default:	1554	default:
1555	break;	1555	break;
1556	}	1556	}
1557	return edge;	1557	return edge;
1558	}	1558	}
1559		1559
1560	static int octeon_irq_ciu2_map(struct irq_domain *d,	1560	static int octeon_irq_ciu2_map(struct irq_domain *d,
1561	unsigned int virq, irq_hw_number_t hw)	1561	unsigned int virq, irq_hw_number_t hw)
1562	{	1562	{
1563	unsigned int line = hw >> 6;	1563	unsigned int line = hw >> 6;
1564	unsigned int bit = hw & 63;	1564	unsigned int bit = hw & 63;
1565		1565
1566	if (!octeon_irq_virq_in_range(virq))	1566	if (!octeon_irq_virq_in_range(virq))
1567	return -EINVAL;	1567	return -EINVAL;
1568		1568
1569	/*	1569	/*
1570	* Don't map irq if it is reserved for GPIO.	1570	* Don't map irq if it is reserved for GPIO.
1571	* (Line 7 are the GPIO lines.)	1571	* (Line 7 are the GPIO lines.)
1572	*/	1572	*/
1573	if (line == 7)	1573	if (line == 7)
1574	return 0;	1574	return 0;
1575		1575
1576	if (line > 7 \|\| octeon_irq_ciu_to_irq[line][bit] != 0)	1576	if (line > 7 \|\| octeon_irq_ciu_to_irq[line][bit] != 0)
1577	return -EINVAL;	1577	return -EINVAL;
1578		1578
1579	if (octeon_irq_ciu2_is_edge(line, bit))	1579	if (octeon_irq_ciu2_is_edge(line, bit))
1580	octeon_irq_set_ciu_mapping(virq, line, bit, 0,	1580	octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1581	&octeon_irq_chip_ciu2,	1581	&octeon_irq_chip_ciu2,
1582	handle_edge_irq);	1582	handle_edge_irq);
1583	else	1583	else
1584	octeon_irq_set_ciu_mapping(virq, line, bit, 0,	1584	octeon_irq_set_ciu_mapping(virq, line, bit, 0,
1585	&octeon_irq_chip_ciu2,	1585	&octeon_irq_chip_ciu2,
1586	handle_level_irq);	1586	handle_level_irq);
1587		1587
1588	return 0;	1588	return 0;
1589	}	1589	}
1590	static int octeon_irq_ciu2_gpio_map(struct irq_domain *d,	1590	static int octeon_irq_ciu2_gpio_map(struct irq_domain *d,
1591	unsigned int virq, irq_hw_number_t hw)	1591	unsigned int virq, irq_hw_number_t hw)
1592	{	1592	{
1593	return octeon_irq_gpio_map_common(d, virq, hw, 7, &octeon_irq_chip_ciu2_gpio);	1593	return octeon_irq_gpio_map_common(d, virq, hw, 7, &octeon_irq_chip_ciu2_gpio);
1594	}	1594	}
1595		1595
1596	static struct irq_domain_ops octeon_irq_domain_ciu2_ops = {	1596	static struct irq_domain_ops octeon_irq_domain_ciu2_ops = {
1597	.map = octeon_irq_ciu2_map,	1597	.map = octeon_irq_ciu2_map,
1598	.xlate = octeon_irq_ciu2_xlat,	1598	.xlate = octeon_irq_ciu2_xlat,
1599	};	1599	};
1600		1600
1601	static struct irq_domain_ops octeon_irq_domain_ciu2_gpio_ops = {	1601	static struct irq_domain_ops octeon_irq_domain_ciu2_gpio_ops = {
1602	.map = octeon_irq_ciu2_gpio_map,	1602	.map = octeon_irq_ciu2_gpio_map,
1603	.xlate = octeon_irq_gpio_xlat,	1603	.xlate = octeon_irq_gpio_xlat,
1604	};	1604	};
1605		1605
1606	static void octeon_irq_ciu2(void)	1606	static void octeon_irq_ciu2(void)
1607	{	1607	{
1608	int line;	1608	int line;
1609	int bit;	1609	int bit;
1610	int irq;	1610	int irq;
1611	u64 src_reg, src, sum;	1611	u64 src_reg, src, sum;
1612	const unsigned long core_id = cvmx_get_core_num();	1612	const unsigned long core_id = cvmx_get_core_num();
1613		1613
1614	sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful;	1614	sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP2(core_id)) & 0xfful;
1615		1615
1616	if (unlikely(!sum))	1616	if (unlikely(!sum))
1617	goto spurious;	1617	goto spurious;
1618		1618
1619	line = fls64(sum) - 1;	1619	line = fls64(sum) - 1;
1620	src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line);	1620	src_reg = CVMX_CIU2_SRC_PPX_IP2_WRKQ(core_id) + (0x1000 * line);
1621	src = cvmx_read_csr(src_reg);	1621	src = cvmx_read_csr(src_reg);
1622		1622
1623	if (unlikely(!src))	1623	if (unlikely(!src))
1624	goto spurious;	1624	goto spurious;
1625		1625
1626	bit = fls64(src) - 1;	1626	bit = fls64(src) - 1;
1627	irq = octeon_irq_ciu_to_irq[line][bit];	1627	irq = octeon_irq_ciu_to_irq[line][bit];
1628	if (unlikely(!irq))	1628	if (unlikely(!irq))
1629	goto spurious;	1629	goto spurious;
1630		1630
1631	do_IRQ(irq);	1631	do_IRQ(irq);
1632	goto out;	1632	goto out;
1633		1633
1634	spurious:	1634	spurious:
1635	spurious_interrupt();	1635	spurious_interrupt();
1636	out:	1636	out:
1637	/* CN68XX pass 1.x has an errata that accessing the ACK registers	1637	/* CN68XX pass 1.x has an errata that accessing the ACK registers
1638	can stop interrupts from propagating */	1638	can stop interrupts from propagating */
1639	if (OCTEON_IS_MODEL(OCTEON_CN68XX))	1639	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1640	cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);	1640	cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1641	else	1641	else
1642	cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id));	1642	cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP2(core_id));
1643	return;	1643	return;
1644	}	1644	}
1645		1645
1646	static void octeon_irq_ciu2_mbox(void)	1646	static void octeon_irq_ciu2_mbox(void)
1647	{	1647	{
1648	int line;	1648	int line;
1649		1649
1650	const unsigned long core_id = cvmx_get_core_num();	1650	const unsigned long core_id = cvmx_get_core_num();
1651	u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60;	1651	u64 sum = cvmx_read_csr(CVMX_CIU2_SUM_PPX_IP3(core_id)) >> 60;
1652		1652
1653	if (unlikely(!sum))	1653	if (unlikely(!sum))
1654	goto spurious;	1654	goto spurious;
1655		1655
1656	line = fls64(sum) - 1;	1656	line = fls64(sum) - 1;
1657		1657
1658	do_IRQ(OCTEON_IRQ_MBOX0 + line);	1658	do_IRQ(OCTEON_IRQ_MBOX0 + line);
1659	goto out;	1659	goto out;
1660		1660
1661	spurious:	1661	spurious:
1662	spurious_interrupt();	1662	spurious_interrupt();
1663	out:	1663	out:
1664	/* CN68XX pass 1.x has an errata that accessing the ACK registers	1664	/* CN68XX pass 1.x has an errata that accessing the ACK registers
1665	can stop interrupts from propagating */	1665	can stop interrupts from propagating */
1666	if (OCTEON_IS_MODEL(OCTEON_CN68XX))	1666	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1667	cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);	1667	cvmx_read_csr(CVMX_CIU2_INTR_CIU_READY);
1668	else	1668	else
1669	cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id));	1669	cvmx_read_csr(CVMX_CIU2_ACK_PPX_IP3(core_id));
1670	return;	1670	return;
1671	}	1671	}
1672		1672
1673	static void __init octeon_irq_init_ciu2(void)	1673	static void __init octeon_irq_init_ciu2(void)
1674	{	1674	{
1675	unsigned int i;	1675	unsigned int i;
1676	struct device_node *gpio_node;	1676	struct device_node *gpio_node;
1677	struct device_node *ciu_node;	1677	struct device_node *ciu_node;
1678	struct irq_domain *ciu_domain = NULL;	1678	struct irq_domain *ciu_domain = NULL;
1679		1679
1680	octeon_irq_init_ciu2_percpu();	1680	octeon_irq_init_ciu2_percpu();
1681	octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2;	1681	octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu2;
1682		1682
1683	octeon_irq_ip2 = octeon_irq_ciu2;	1683	octeon_irq_ip2 = octeon_irq_ciu2;
1684	octeon_irq_ip3 = octeon_irq_ciu2_mbox;	1684	octeon_irq_ip3 = octeon_irq_ciu2_mbox;
1685	octeon_irq_ip4 = octeon_irq_ip4_mask;	1685	octeon_irq_ip4 = octeon_irq_ip4_mask;
1686		1686
1687	/* Mips internal */	1687	/* Mips internal */
1688	octeon_irq_init_core();	1688	octeon_irq_init_core();
1689		1689
1690	gpio_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-gpio");	1690	gpio_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-3860-gpio");
1691	if (gpio_node) {	1691	if (gpio_node) {
1692	struct octeon_irq_gpio_domain_data *gpiod;	1692	struct octeon_irq_gpio_domain_data *gpiod;
1693		1693
1694	gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);	1694	gpiod = kzalloc(sizeof(*gpiod), GFP_KERNEL);
1695	if (gpiod) {	1695	if (gpiod) {
1696	/* gpio domain host_data is the base hwirq number. */	1696	/* gpio domain host_data is the base hwirq number. */
1697	gpiod->base_hwirq = 7 << 6;	1697	gpiod->base_hwirq = 7 << 6;
1698	irq_domain_add_linear(gpio_node, 16, &octeon_irq_domain_ciu2_gpio_ops, gpiod);	1698	irq_domain_add_linear(gpio_node, 16, &octeon_irq_domain_ciu2_gpio_ops, gpiod);
1699	of_node_put(gpio_node);	1699	of_node_put(gpio_node);
1700	} else	1700	} else
1701	pr_warn("Cannot allocate memory for GPIO irq_domain.\n");	1701	pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1702	} else	1702	} else
1703	pr_warn("Cannot find device node for cavium,octeon-3860-gpio.\n");	1703	pr_warn("Cannot find device node for cavium,octeon-3860-gpio.\n");
1704		1704
1705	ciu_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-6880-ciu2");	1705	ciu_node = of_find_compatible_node(NULL, NULL, "cavium,octeon-6880-ciu2");
1706	if (ciu_node) {	1706	if (ciu_node) {
1707	ciu_domain = irq_domain_add_tree(ciu_node, &octeon_irq_domain_ciu2_ops, NULL);	1707	ciu_domain = irq_domain_add_tree(ciu_node, &octeon_irq_domain_ciu2_ops, NULL);
1708	irq_set_default_host(ciu_domain);	1708	irq_set_default_host(ciu_domain);
1709	of_node_put(ciu_node);	1709	of_node_put(ciu_node);
1710	} else	1710	} else
1711	panic("Cannot find device node for cavium,octeon-6880-ciu2.");	1711	panic("Cannot find device node for cavium,octeon-6880-ciu2.");
1712		1712
1713	/* CUI2 */	1713	/* CUI2 */
1714	for (i = 0; i < 64; i++)	1714	for (i = 0; i < 64; i++)
1715	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i);	1715	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_WORKQ0, 0, i);
1716		1716
1717	for (i = 0; i < 32; i++)	1717	for (i = 0; i < 32; i++)
1718	octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0,	1718	octeon_irq_set_ciu_mapping(i + OCTEON_IRQ_WDOG0, 1, i, 0,
1719	&octeon_irq_chip_ciu2_wd, handle_level_irq);	1719	&octeon_irq_chip_ciu2_wd, handle_level_irq);
1720		1720
1721	for (i = 0; i < 4; i++)	1721	for (i = 0; i < 4; i++)
1722	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8);	1722	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_TIMER0, 3, i + 8);
1723		1723
1724	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44);	1724	octeon_irq_force_ciu_mapping(ciu_domain, OCTEON_IRQ_USB0, 3, 44);
1725		1725
1726	for (i = 0; i < 4; i++)	1726	for (i = 0; i < 4; i++)
1727	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i);	1727	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_INT0, 4, i);
1728		1728
1729	for (i = 0; i < 4; i++)	1729	for (i = 0; i < 4; i++)
1730	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8);	1730	octeon_irq_force_ciu_mapping(ciu_domain, i + OCTEON_IRQ_PCI_MSI0, 4, i + 8);
1731		1731
1732	irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);	1732	irq_set_chip_and_handler(OCTEON_IRQ_MBOX0, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
1733	irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);	1733	irq_set_chip_and_handler(OCTEON_IRQ_MBOX1, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
1734	irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);	1734	irq_set_chip_and_handler(OCTEON_IRQ_MBOX2, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
1735	irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);	1735	irq_set_chip_and_handler(OCTEON_IRQ_MBOX3, &octeon_irq_chip_ciu2_mbox, handle_percpu_irq);
1736		1736
1737	/* Enable the CIU lines */	1737	/* Enable the CIU lines */
1738	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);	1738	set_c0_status(STATUSF_IP3 \| STATUSF_IP2);
1739	clear_c0_status(STATUSF_IP4);	1739	clear_c0_status(STATUSF_IP4);
1740	}	1740	}
1741		1741
1742	void __init arch_init_irq(void)	1742	void __init arch_init_irq(void)
1743	{	1743	{
1744	#ifdef CONFIG_SMP	1744	#ifdef CONFIG_SMP
1745	/* Set the default affinity to the boot cpu. */	1745	/* Set the default affinity to the boot cpu. */
1746	cpumask_clear(irq_default_affinity);	1746	cpumask_clear(irq_default_affinity);
1747	cpumask_set_cpu(smp_processor_id(), irq_default_affinity);	1747	cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
1748	#endif	1748	#endif
1749	if (OCTEON_IS_MODEL(OCTEON_CN68XX))	1749	if (OCTEON_IS_MODEL(OCTEON_CN68XX))
1750	octeon_irq_init_ciu2();	1750	octeon_irq_init_ciu2();
1751	else	1751	else
1752	octeon_irq_init_ciu();	1752	octeon_irq_init_ciu();
1753	}	1753	}
1754		1754
1755	asmlinkage void plat_irq_dispatch(void)	1755	asmlinkage void plat_irq_dispatch(void)
1756	{	1756	{
1757	unsigned long cop0_cause;	1757	unsigned long cop0_cause;
1758	unsigned long cop0_status;	1758	unsigned long cop0_status;
1759		1759
1760	while (1) {	1760	while (1) {
1761	cop0_cause = read_c0_cause();	1761	cop0_cause = read_c0_cause();
1762	cop0_status = read_c0_status();	1762	cop0_status = read_c0_status();
1763	cop0_cause &= cop0_status;	1763	cop0_cause &= cop0_status;
1764	cop0_cause &= ST0_IM;	1764	cop0_cause &= ST0_IM;
1765		1765
1766	if (unlikely(cop0_cause & STATUSF_IP2))	1766	if (unlikely(cop0_cause & STATUSF_IP2))
1767	octeon_irq_ip2();	1767	octeon_irq_ip2();
1768	else if (unlikely(cop0_cause & STATUSF_IP3))	1768	else if (unlikely(cop0_cause & STATUSF_IP3))
1769	octeon_irq_ip3();	1769	octeon_irq_ip3();
1770	else if (unlikely(cop0_cause & STATUSF_IP4))	1770	else if (unlikely(cop0_cause & STATUSF_IP4))
1771	octeon_irq_ip4();	1771	octeon_irq_ip4();
1772	else if (likely(cop0_cause))	1772	else if (likely(cop0_cause))
1773	do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);	1773	do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
1774	else	1774	else
1775	break;	1775	break;
1776	}	1776	}
1777	}	1777	}
1778		1778
1779	#ifdef CONFIG_HOTPLUG_CPU	1779	#ifdef CONFIG_HOTPLUG_CPU
1780		1780
1781	void octeon_fixup_irqs(void)	1781	void octeon_fixup_irqs(void)
1782	{	1782	{
1783	irq_cpu_offline();	1783	irq_cpu_offline();
1784	}	1784	}
1785		1785
1786	#endif /* CONFIG_HOTPLUG_CPU */	1786	#endif /* CONFIG_HOTPLUG_CPU */
1787		1787

drivers/clocksource/exynos_mct.c

Diff comments View file @ d9e9e8e

 /* linux/arch/arm/mach-exynos4/mct.c
  *
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *		http://www.samsung.com
  *
  * EXYNOS4 MCT(Multi-Core Timer) support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
 */
 #include <linux/sched.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/err.h>
 #include <linux/clk.h>
 #include <linux/clockchips.h>
 #include <linux/cpu.h>
 #include <linux/platform_device.h>
 #include <linux/delay.h>
 #include <linux/percpu.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/of_address.h>
 #include <linux/clocksource.h>
 #define EXYNOS4_MCTREG(x)		(x)
 #define EXYNOS4_MCT_G_CNT_L		EXYNOS4_MCTREG(0x100)
 #define EXYNOS4_MCT_G_CNT_U		EXYNOS4_MCTREG(0x104)
 #define EXYNOS4_MCT_G_CNT_WSTAT		EXYNOS4_MCTREG(0x110)
 #define EXYNOS4_MCT_G_COMP0_L		EXYNOS4_MCTREG(0x200)
 #define EXYNOS4_MCT_G_COMP0_U		EXYNOS4_MCTREG(0x204)
 #define EXYNOS4_MCT_G_COMP0_ADD_INCR	EXYNOS4_MCTREG(0x208)
 #define EXYNOS4_MCT_G_TCON		EXYNOS4_MCTREG(0x240)
 #define EXYNOS4_MCT_G_INT_CSTAT		EXYNOS4_MCTREG(0x244)
 #define EXYNOS4_MCT_G_INT_ENB		EXYNOS4_MCTREG(0x248)
 #define EXYNOS4_MCT_G_WSTAT		EXYNOS4_MCTREG(0x24C)
 #define _EXYNOS4_MCT_L_BASE		EXYNOS4_MCTREG(0x300)
 #define EXYNOS4_MCT_L_BASE(x)		(_EXYNOS4_MCT_L_BASE + (0x100 * x))
 #define EXYNOS4_MCT_L_MASK		(0xffffff00)
 #define MCT_L_TCNTB_OFFSET		(0x00)
 #define MCT_L_ICNTB_OFFSET		(0x08)
 #define MCT_L_TCON_OFFSET		(0x20)
 #define MCT_L_INT_CSTAT_OFFSET		(0x30)
 #define MCT_L_INT_ENB_OFFSET		(0x34)
 #define MCT_L_WSTAT_OFFSET		(0x40)
 #define MCT_G_TCON_START		(1 << 8)
 #define MCT_G_TCON_COMP0_AUTO_INC	(1 << 1)
 #define MCT_G_TCON_COMP0_ENABLE		(1 << 0)
 #define MCT_L_TCON_INTERVAL_MODE	(1 << 2)
 #define MCT_L_TCON_INT_START		(1 << 1)
 #define MCT_L_TCON_TIMER_START		(1 << 0)
 #define TICK_BASE_CNT	1
 enum {
 	MCT_INT_SPI,
 	MCT_INT_PPI
 };
 enum {
 	MCT_G0_IRQ,
 	MCT_G1_IRQ,
 	MCT_G2_IRQ,
 	MCT_G3_IRQ,
 	MCT_L0_IRQ,
 	MCT_L1_IRQ,
 	MCT_L2_IRQ,
 	MCT_L3_IRQ,
 	MCT_L4_IRQ,
 	MCT_L5_IRQ,
 	MCT_L6_IRQ,
 	MCT_L7_IRQ,
 	MCT_NR_IRQS,
 };
 static void __iomem *reg_base;
 static unsigned long clk_rate;
 static unsigned int mct_int_type;
 static int mct_irqs[MCT_NR_IRQS];
 struct mct_clock_event_device {
 	struct clock_event_device evt;
 	unsigned long base;
 	char name[10];
 };
 static void exynos4_mct_write(unsigned int value, unsigned long offset)
 {
 	unsigned long stat_addr;
 	u32 mask;
 	u32 i;
 	__raw_writel(value, reg_base + offset);
 	if (likely(offset >= EXYNOS4_MCT_L_BASE(0))) {
 		stat_addr = (offset & ~EXYNOS4_MCT_L_MASK) + MCT_L_WSTAT_OFFSET;
 		switch (offset & EXYNOS4_MCT_L_MASK) {
 		case MCT_L_TCON_OFFSET:
 			mask = 1 << 3;		/* L_TCON write status */
 			break;
 		case MCT_L_ICNTB_OFFSET:
 			mask = 1 << 1;		/* L_ICNTB write status */
 			break;
 		case MCT_L_TCNTB_OFFSET:
 			mask = 1 << 0;		/* L_TCNTB write status */
 			break;
 		default:
 			return;
 		}
 	} else {
 		switch (offset) {
 		case EXYNOS4_MCT_G_TCON:
 			stat_addr = EXYNOS4_MCT_G_WSTAT;
 			mask = 1 << 16;		/* G_TCON write status */
 			break;
 		case EXYNOS4_MCT_G_COMP0_L:
 			stat_addr = EXYNOS4_MCT_G_WSTAT;
 			mask = 1 << 0;		/* G_COMP0_L write status */
 			break;
 		case EXYNOS4_MCT_G_COMP0_U:
 			stat_addr = EXYNOS4_MCT_G_WSTAT;
 			mask = 1 << 1;		/* G_COMP0_U write status */
 			break;
 		case EXYNOS4_MCT_G_COMP0_ADD_INCR:
 			stat_addr = EXYNOS4_MCT_G_WSTAT;
 			mask = 1 << 2;		/* G_COMP0_ADD_INCR w status */
 			break;
 		case EXYNOS4_MCT_G_CNT_L:
 			stat_addr = EXYNOS4_MCT_G_CNT_WSTAT;
 			mask = 1 << 0;		/* G_CNT_L write status */
 			break;
 		case EXYNOS4_MCT_G_CNT_U:
 			stat_addr = EXYNOS4_MCT_G_CNT_WSTAT;
 			mask = 1 << 1;		/* G_CNT_U write status */
 			break;
 		default:
 			return;
 		}
 	}
 	/* Wait maximum 1 ms until written values are applied */
 	for (i = 0; i < loops_per_jiffy / 1000 * HZ; i++)
 		if (__raw_readl(reg_base + stat_addr) & mask) {
 			__raw_writel(mask, reg_base + stat_addr);
 			return;
 		}
 	panic("MCT hangs after writing %d (offset:0x%lx)\n", value, offset);
 }
 /* Clocksource handling */
 static void exynos4_mct_frc_start(u32 hi, u32 lo)
 {
 	u32 reg;
 	exynos4_mct_write(lo, EXYNOS4_MCT_G_CNT_L);
 	exynos4_mct_write(hi, EXYNOS4_MCT_G_CNT_U);
 	reg = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON);
 	reg |= MCT_G_TCON_START;
 	exynos4_mct_write(reg, EXYNOS4_MCT_G_TCON);
 }
 static cycle_t exynos4_frc_read(struct clocksource *cs)
 {
 	unsigned int lo, hi;
 	u32 hi2 = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_U);
 	do {
 		hi = hi2;
 		lo = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_L);
 		hi2 = __raw_readl(reg_base + EXYNOS4_MCT_G_CNT_U);
 	} while (hi != hi2);
 	return ((cycle_t)hi << 32) | lo;
 }
 static void exynos4_frc_resume(struct clocksource *cs)
 {
 	exynos4_mct_frc_start(0, 0);
 }
 struct clocksource mct_frc = {
 	.name		= "mct-frc",
 	.rating		= 400,
 	.read		= exynos4_frc_read,
 	.mask		= CLOCKSOURCE_MASK(64),
 	.flags		= CLOCK_SOURCE_IS_CONTINUOUS,
 	.resume		= exynos4_frc_resume,
 };
 static void __init exynos4_clocksource_init(void)
 {
 	exynos4_mct_frc_start(0, 0);
 	if (clocksource_register_hz(&mct_frc, clk_rate))
 		panic("%s: can't register clocksource\n", mct_frc.name);
 }
 static void exynos4_mct_comp0_stop(void)
 {
 	unsigned int tcon;
 	tcon = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON);
 	tcon &= ~(MCT_G_TCON_COMP0_ENABLE | MCT_G_TCON_COMP0_AUTO_INC);
 	exynos4_mct_write(tcon, EXYNOS4_MCT_G_TCON);
 	exynos4_mct_write(0, EXYNOS4_MCT_G_INT_ENB);
 }
 static void exynos4_mct_comp0_start(enum clock_event_mode mode,
 				    unsigned long cycles)
 {
 	unsigned int tcon;
 	cycle_t comp_cycle;
 	tcon = __raw_readl(reg_base + EXYNOS4_MCT_G_TCON);
 	if (mode == CLOCK_EVT_MODE_PERIODIC) {
 		tcon |= MCT_G_TCON_COMP0_AUTO_INC;
 		exynos4_mct_write(cycles, EXYNOS4_MCT_G_COMP0_ADD_INCR);
 	}
 	comp_cycle = exynos4_frc_read(&mct_frc) + cycles;
 	exynos4_mct_write((u32)comp_cycle, EXYNOS4_MCT_G_COMP0_L);
 	exynos4_mct_write((u32)(comp_cycle >> 32), EXYNOS4_MCT_G_COMP0_U);
 	exynos4_mct_write(0x1, EXYNOS4_MCT_G_INT_ENB);
 	tcon |= MCT_G_TCON_COMP0_ENABLE;
 	exynos4_mct_write(tcon , EXYNOS4_MCT_G_TCON);
 }
 static int exynos4_comp_set_next_event(unsigned long cycles,
 				       struct clock_event_device *evt)
 {
 	exynos4_mct_comp0_start(evt->mode, cycles);
 	return 0;
 }
 static void exynos4_comp_set_mode(enum clock_event_mode mode,
 				  struct clock_event_device *evt)
 {
 	unsigned long cycles_per_jiffy;
 	exynos4_mct_comp0_stop();
 	switch (mode) {
 	case CLOCK_EVT_MODE_PERIODIC:
 		cycles_per_jiffy =
 			(((unsigned long long) NSEC_PER_SEC / HZ * evt->mult) >> evt->shift);
 		exynos4_mct_comp0_start(mode, cycles_per_jiffy);
 		break;
 	case CLOCK_EVT_MODE_ONESHOT:
 	case CLOCK_EVT_MODE_UNUSED:
 	case CLOCK_EVT_MODE_SHUTDOWN:
 	case CLOCK_EVT_MODE_RESUME:
 		break;
 	}
 }
 static struct clock_event_device mct_comp_device = {
 	.name		= "mct-comp",
 	.features       = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT,
 	.rating		= 250,
 	.set_next_event	= exynos4_comp_set_next_event,
 	.set_mode	= exynos4_comp_set_mode,
 };
 static irqreturn_t exynos4_mct_comp_isr(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = dev_id;
 	exynos4_mct_write(0x1, EXYNOS4_MCT_G_INT_CSTAT);
 	evt->event_handler(evt);
 	return IRQ_HANDLED;
 }
 static struct irqaction mct_comp_event_irq = {
 	.name		= "mct_comp_irq",
 	.flags		= IRQF_TIMER | IRQF_IRQPOLL,
 	.handler	= exynos4_mct_comp_isr,
 	.dev_id		= &mct_comp_device,
 };
 static void exynos4_clockevent_init(void)
 {
 	mct_comp_device.cpumask = cpumask_of(0);
 	clockevents_config_and_register(&mct_comp_device, clk_rate,
 					0xf, 0xffffffff);
 	setup_irq(mct_irqs[MCT_G0_IRQ], &mct_comp_event_irq);
 }
 static DEFINE_PER_CPU(struct mct_clock_event_device, percpu_mct_tick);
 /* Clock event handling */
 static void exynos4_mct_tick_stop(struct mct_clock_event_device *mevt)
 {
 	unsigned long tmp;
 	unsigned long mask = MCT_L_TCON_INT_START | MCT_L_TCON_TIMER_START;
 	unsigned long offset = mevt->base + MCT_L_TCON_OFFSET;
 	tmp = __raw_readl(reg_base + offset);
 	if (tmp & mask) {
 		tmp &= ~mask;
 		exynos4_mct_write(tmp, offset);
 	}
 }
 static void exynos4_mct_tick_start(unsigned long cycles,
 				   struct mct_clock_event_device *mevt)
 {
 	unsigned long tmp;
 	exynos4_mct_tick_stop(mevt);
 	tmp = (1 << 31) | cycles;	/* MCT_L_UPDATE_ICNTB */
 	/* update interrupt count buffer */
 	exynos4_mct_write(tmp, mevt->base + MCT_L_ICNTB_OFFSET);
 	/* enable MCT tick interrupt */
 	exynos4_mct_write(0x1, mevt->base + MCT_L_INT_ENB_OFFSET);
 	tmp = __raw_readl(reg_base + mevt->base + MCT_L_TCON_OFFSET);
 	tmp |= MCT_L_TCON_INT_START | MCT_L_TCON_TIMER_START |
 	       MCT_L_TCON_INTERVAL_MODE;
 	exynos4_mct_write(tmp, mevt->base + MCT_L_TCON_OFFSET);
 }
 static int exynos4_tick_set_next_event(unsigned long cycles,
 				       struct clock_event_device *evt)
 {
 	struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick);
 	exynos4_mct_tick_start(cycles, mevt);
 	return 0;
 }
 static inline void exynos4_tick_set_mode(enum clock_event_mode mode,
 					 struct clock_event_device *evt)
 {
 	struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick);
 	unsigned long cycles_per_jiffy;
 	exynos4_mct_tick_stop(mevt);
 	switch (mode) {
 	case CLOCK_EVT_MODE_PERIODIC:
 		cycles_per_jiffy =
 			(((unsigned long long) NSEC_PER_SEC / HZ * evt->mult) >> evt->shift);
 		exynos4_mct_tick_start(cycles_per_jiffy, mevt);
 		break;
 	case CLOCK_EVT_MODE_ONESHOT:
 	case CLOCK_EVT_MODE_UNUSED:
 	case CLOCK_EVT_MODE_SHUTDOWN:
 	case CLOCK_EVT_MODE_RESUME:
 		break;
 	}
 }
 static int exynos4_mct_tick_clear(struct mct_clock_event_device *mevt)
 {
 	struct clock_event_device *evt = &mevt->evt;
 	/*
 	 * This is for supporting oneshot mode.
 	 * Mct would generate interrupt periodically
 	 * without explicit stopping.
 	 */
 	if (evt->mode != CLOCK_EVT_MODE_PERIODIC)
 		exynos4_mct_tick_stop(mevt);
 	/* Clear the MCT tick interrupt */
 	if (__raw_readl(reg_base + mevt->base + MCT_L_INT_CSTAT_OFFSET) & 1) {
 		exynos4_mct_write(0x1, mevt->base + MCT_L_INT_CSTAT_OFFSET);
 		return 1;
 	} else {
 		return 0;
 	}
 }
 static irqreturn_t exynos4_mct_tick_isr(int irq, void *dev_id)
 {
 	struct mct_clock_event_device *mevt = dev_id;
 	struct clock_event_device *evt = &mevt->evt;
 	exynos4_mct_tick_clear(mevt);
 	evt->event_handler(evt);
 	return IRQ_HANDLED;
 }
 static int exynos4_local_timer_setup(struct clock_event_device *evt)
 {
 	struct mct_clock_event_device *mevt;
 	unsigned int cpu = smp_processor_id();
 	mevt = container_of(evt, struct mct_clock_event_device, evt);
 	mevt->base = EXYNOS4_MCT_L_BASE(cpu);
 	snprintf(mevt->name, sizeof(mevt->name), "mct_tick%d", cpu);
 	evt->name = mevt->name;
 	evt->cpumask = cpumask_of(cpu);
 	evt->set_next_event = exynos4_tick_set_next_event;
 	evt->set_mode = exynos4_tick_set_mode;
 	evt->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT;
 	evt->rating = 450;
-	clockevents_config_and_register(evt, clk_rate / (TICK_BASE_CNT + 1),
-					0xf, 0x7fffffff);
 	exynos4_mct_write(TICK_BASE_CNT, mevt->base + MCT_L_TCNTB_OFFSET);
 	if (mct_int_type == MCT_INT_SPI) {
 		evt->irq = mct_irqs[MCT_L0_IRQ + cpu];
 		if (request_irq(evt->irq, exynos4_mct_tick_isr,
 				IRQF_TIMER | IRQF_NOBALANCING,
 				evt->name, mevt)) {
 			pr_err("exynos-mct: cannot register IRQ %d\n",
 				evt->irq);
 			return -EIO;
 		}
+		irq_force_affinity(mct_irqs[MCT_L0_IRQ + cpu], cpumask_of(cpu));
 	} else {
 		enable_percpu_irq(mct_irqs[MCT_L0_IRQ], 0);
 	}
+	clockevents_config_and_register(evt, clk_rate / (TICK_BASE_CNT + 1),
+					0xf, 0x7fffffff);
 	return 0;
 }
 static void exynos4_local_timer_stop(struct clock_event_device *evt)
 {
 	evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt);
 	if (mct_int_type == MCT_INT_SPI)
 		free_irq(evt->irq, this_cpu_ptr(&percpu_mct_tick));
 	else
 		disable_percpu_irq(mct_irqs[MCT_L0_IRQ]);
 }
 static int exynos4_mct_cpu_notify(struct notifier_block *self,
 					   unsigned long action, void *hcpu)
 {
 	struct mct_clock_event_device *mevt;
-	unsigned int cpu;
 	/*
 	 * Grab cpu pointer in each case to avoid spurious
 	 * preemptible warnings
 	 */
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_STARTING:
 		mevt = this_cpu_ptr(&percpu_mct_tick);
 		exynos4_local_timer_setup(&mevt->evt);
-		break;
-	case CPU_ONLINE:
-		cpu = (unsigned long)hcpu;
-		if (mct_int_type == MCT_INT_SPI)
-			irq_set_affinity(mct_irqs[MCT_L0_IRQ + cpu],
-						cpumask_of(cpu));
 		break;
 	case CPU_DYING:
 		mevt = this_cpu_ptr(&percpu_mct_tick);
 		exynos4_local_timer_stop(&mevt->evt);
 		break;
 	}
 	return NOTIFY_OK;
 }
 static struct notifier_block exynos4_mct_cpu_nb = {
 	.notifier_call = exynos4_mct_cpu_notify,
 };
 static void __init exynos4_timer_resources(struct device_node *np, void __iomem *base)
 {
 	int err;
 	struct mct_clock_event_device *mevt = this_cpu_ptr(&percpu_mct_tick);
 	struct clk *mct_clk, *tick_clk;
 	tick_clk = np ? of_clk_get_by_name(np, "fin_pll") :
 				clk_get(NULL, "fin_pll");
 	if (IS_ERR(tick_clk))
 		panic("%s: unable to determine tick clock rate\n", __func__);
 	clk_rate = clk_get_rate(tick_clk);
 	mct_clk = np ? of_clk_get_by_name(np, "mct") : clk_get(NULL, "mct");
 	if (IS_ERR(mct_clk))
 		panic("%s: unable to retrieve mct clock instance\n", __func__);
 	clk_prepare_enable(mct_clk);
 	reg_base = base;
 	if (!reg_base)
 		panic("%s: unable to ioremap mct address space\n", __func__);
 	if (mct_int_type == MCT_INT_PPI) {
 		err = request_percpu_irq(mct_irqs[MCT_L0_IRQ],
 					 exynos4_mct_tick_isr, "MCT",
 					 &percpu_mct_tick);
 		WARN(err, "MCT: can't request IRQ %d (%d)\n",
 		     mct_irqs[MCT_L0_IRQ], err);
 	} else {
 		irq_set_affinity(mct_irqs[MCT_L0_IRQ], cpumask_of(0));
 	}
 	err = register_cpu_notifier(&exynos4_mct_cpu_nb);
 	if (err)
 		goto out_irq;
 	/* Immediately configure the timer on the boot CPU */
 	exynos4_local_timer_setup(&mevt->evt);
 	return;
 out_irq:
 	free_percpu_irq(mct_irqs[MCT_L0_IRQ], &percpu_mct_tick);
 }
 void __init mct_init(void __iomem *base, int irq_g0, int irq_l0, int irq_l1)
 {
 	mct_irqs[MCT_G0_IRQ] = irq_g0;
 	mct_irqs[MCT_L0_IRQ] = irq_l0;
 	mct_irqs[MCT_L1_IRQ] = irq_l1;
 	mct_int_type = MCT_INT_SPI;
 	exynos4_timer_resources(NULL, base);
 	exynos4_clocksource_init();
 	exynos4_clockevent_init();
 }
 static void __init mct_init_dt(struct device_node *np, unsigned int int_type)
 {
 	u32 nr_irqs, i;
 	mct_int_type = int_type;
 	/* This driver uses only one global timer interrupt */
 	mct_irqs[MCT_G0_IRQ] = irq_of_parse_and_map(np, MCT_G0_IRQ);
 	/*
 	 * Find out the number of local irqs specified. The local
 	 * timer irqs are specified after the four global timer
 	 * irqs are specified.
 	 */
 #ifdef CONFIG_OF
 	nr_irqs = of_irq_count(np);
 #else
 	nr_irqs = 0;
 #endif
 	for (i = MCT_L0_IRQ; i < nr_irqs; i++)
 		mct_irqs[i] = irq_of_parse_and_map(np, i);
 	exynos4_timer_resources(np, of_iomap(np, 0));
 	exynos4_clocksource_init();
 	exynos4_clockevent_init();
 }
 static void __init mct_init_spi(struct device_node *np)
 {
 	return mct_init_dt(np, MCT_INT_SPI);
 }
 static void __init mct_init_ppi(struct device_node *np)
 {
 	return mct_init_dt(np, MCT_INT_PPI);
 }

drivers/irqchip/irq-gic.c

Diff comments View file @ d9e9e8e

 /*
  *  linux/arch/arm/common/gic.c
  *
  *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * Interrupt architecture for the GIC:
  *
  * o There is one Interrupt Distributor, which receives interrupts
  *   from system devices and sends them to the Interrupt Controllers.
  *
  * o There is one CPU Interface per CPU, which sends interrupts sent
  *   by the Distributor, and interrupts generated locally, to the
  *   associated CPU. The base address of the CPU interface is usually
  *   aliased so that the same address points to different chips depending
  *   on the CPU it is accessed from.
  *
  * Note that IRQs 0-31 are special - they are local to each CPU.
  * As such, the enable set/clear, pending set/clear and active bit
  * registers are banked per-cpu for these sources.
  */
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/err.h>
 #include <linux/module.h>
 #include <linux/list.h>
 #include <linux/smp.h>
 #include <linux/cpu.h>
 #include <linux/cpu_pm.h>
 #include <linux/cpumask.h>
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/irqdomain.h>
 #include <linux/interrupt.h>
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <linux/irqchip/chained_irq.h>
 #include <linux/irqchip/arm-gic.h>
 #include <asm/irq.h>
 #include <asm/exception.h>
 #include <asm/smp_plat.h>
 #include "irqchip.h"
 union gic_base {
 	void __iomem *common_base;
 	void __percpu * __iomem *percpu_base;
 };
 struct gic_chip_data {
 	union gic_base dist_base;
 	union gic_base cpu_base;
 #ifdef CONFIG_CPU_PM
 	u32 saved_spi_enable[DIV_ROUND_UP(1020, 32)];
 	u32 saved_spi_conf[DIV_ROUND_UP(1020, 16)];
 	u32 saved_spi_target[DIV_ROUND_UP(1020, 4)];
 	u32 __percpu *saved_ppi_enable;
 	u32 __percpu *saved_ppi_conf;
 #endif
 	struct irq_domain *domain;
 	unsigned int gic_irqs;
 #ifdef CONFIG_GIC_NON_BANKED
 	void __iomem *(*get_base)(union gic_base *);
 #endif
 };
 static DEFINE_RAW_SPINLOCK(irq_controller_lock);
 /*
  * The GIC mapping of CPU interfaces does not necessarily match
  * the logical CPU numbering.  Let's use a mapping as returned
  * by the GIC itself.
  */
 #define NR_GIC_CPU_IF 8
 static u8 gic_cpu_map[NR_GIC_CPU_IF] __read_mostly;
 /*
  * Supported arch specific GIC irq extension.
  * Default make them NULL.
  */
 struct irq_chip gic_arch_extn = {
 	.irq_eoi	= NULL,
 	.irq_mask	= NULL,
 	.irq_unmask	= NULL,
 	.irq_retrigger	= NULL,
 	.irq_set_type	= NULL,
 	.irq_set_wake	= NULL,
 };
 #ifndef MAX_GIC_NR
 #define MAX_GIC_NR	1
 #endif
 static struct gic_chip_data gic_data[MAX_GIC_NR] __read_mostly;
 #ifdef CONFIG_GIC_NON_BANKED
 static void __iomem *gic_get_percpu_base(union gic_base *base)
 {
 	return *__this_cpu_ptr(base->percpu_base);
 }
 static void __iomem *gic_get_common_base(union gic_base *base)
 {
 	return base->common_base;
 }
 static inline void __iomem *gic_data_dist_base(struct gic_chip_data *data)
 {
 	return data->get_base(&data->dist_base);
 }
 static inline void __iomem *gic_data_cpu_base(struct gic_chip_data *data)
 {
 	return data->get_base(&data->cpu_base);
 }
 static inline void gic_set_base_accessor(struct gic_chip_data *data,
 					 void __iomem *(*f)(union gic_base *))
 {
 	data->get_base = f;
 }
 #else
 #define gic_data_dist_base(d)	((d)->dist_base.common_base)
 #define gic_data_cpu_base(d)	((d)->cpu_base.common_base)
 #define gic_set_base_accessor(d, f)
 #endif
 static inline void __iomem *gic_dist_base(struct irq_data *d)
 {
 	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
 	return gic_data_dist_base(gic_data);
 }
 static inline void __iomem *gic_cpu_base(struct irq_data *d)
 {
 	struct gic_chip_data *gic_data = irq_data_get_irq_chip_data(d);
 	return gic_data_cpu_base(gic_data);
 }
 static inline unsigned int gic_irq(struct irq_data *d)
 {
 	return d->hwirq;
 }
 /*
  * Routines to acknowledge, disable and enable interrupts
  */
 static void gic_mask_irq(struct irq_data *d)
 {
 	u32 mask = 1 << (gic_irq(d) % 32);
 	raw_spin_lock(&irq_controller_lock);
 	writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_CLEAR + (gic_irq(d) / 32) * 4);
 	if (gic_arch_extn.irq_mask)
 		gic_arch_extn.irq_mask(d);
 	raw_spin_unlock(&irq_controller_lock);
 }
 static void gic_unmask_irq(struct irq_data *d)
 {
 	u32 mask = 1 << (gic_irq(d) % 32);
 	raw_spin_lock(&irq_controller_lock);
 	if (gic_arch_extn.irq_unmask)
 		gic_arch_extn.irq_unmask(d);
 	writel_relaxed(mask, gic_dist_base(d) + GIC_DIST_ENABLE_SET + (gic_irq(d) / 32) * 4);
 	raw_spin_unlock(&irq_controller_lock);
 }
 static void gic_eoi_irq(struct irq_data *d)
 {
 	if (gic_arch_extn.irq_eoi) {
 		raw_spin_lock(&irq_controller_lock);
 		gic_arch_extn.irq_eoi(d);
 		raw_spin_unlock(&irq_controller_lock);
 	}
 	writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI);
 }
 static int gic_set_type(struct irq_data *d, unsigned int type)
 {
 	void __iomem *base = gic_dist_base(d);
 	unsigned int gicirq = gic_irq(d);
 	u32 enablemask = 1 << (gicirq % 32);
 	u32 enableoff = (gicirq / 32) * 4;
 	u32 confmask = 0x2 << ((gicirq % 16) * 2);
 	u32 confoff = (gicirq / 16) * 4;
 	bool enabled = false;
 	u32 val;
 	/* Interrupt configuration for SGIs can't be changed */
 	if (gicirq < 16)
 		return -EINVAL;
 	if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
 		return -EINVAL;
 	raw_spin_lock(&irq_controller_lock);
 	if (gic_arch_extn.irq_set_type)
 		gic_arch_extn.irq_set_type(d, type);
 	val = readl_relaxed(base + GIC_DIST_CONFIG + confoff);
 	if (type == IRQ_TYPE_LEVEL_HIGH)
 		val &= ~confmask;
 	else if (type == IRQ_TYPE_EDGE_RISING)
 		val |= confmask;
 	/*
 	 * As recommended by the spec, disable the interrupt before changing
 	 * the configuration
 	 */
 	if (readl_relaxed(base + GIC_DIST_ENABLE_SET + enableoff) & enablemask) {
 		writel_relaxed(enablemask, base + GIC_DIST_ENABLE_CLEAR + enableoff);
 		enabled = true;
 	}
 	writel_relaxed(val, base + GIC_DIST_CONFIG + confoff);
 	if (enabled)
 		writel_relaxed(enablemask, base + GIC_DIST_ENABLE_SET + enableoff);
 	raw_spin_unlock(&irq_controller_lock);
 	return 0;
 }
 static int gic_retrigger(struct irq_data *d)
 {
 	if (gic_arch_extn.irq_retrigger)
 		return gic_arch_extn.irq_retrigger(d);
 	/* the genirq layer expects 0 if we can't retrigger in hardware */
 	return 0;
 }
 #ifdef CONFIG_SMP
 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
 			    bool force)
 {
 	void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + (gic_irq(d) & ~3);
-	unsigned int shift = (gic_irq(d) % 4) * 8;
+	unsigned int cpu, shift = (gic_irq(d) % 4) * 8;
-	unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
 	u32 val, mask, bit;
+	if (!force)
+		cpu = cpumask_any_and(mask_val, cpu_online_mask);
+	else
+		cpu = cpumask_first(mask_val);
 	if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids)
 		return -EINVAL;
 	raw_spin_lock(&irq_controller_lock);
 	mask = 0xff << shift;
 	bit = gic_cpu_map[cpu] << shift;
 	val = readl_relaxed(reg) & ~mask;
 	writel_relaxed(val | bit, reg);
 	raw_spin_unlock(&irq_controller_lock);
 	return IRQ_SET_MASK_OK;
 }
 #endif
 #ifdef CONFIG_PM
 static int gic_set_wake(struct irq_data *d, unsigned int on)
 {
 	int ret = -ENXIO;
 	if (gic_arch_extn.irq_set_wake)
 		ret = gic_arch_extn.irq_set_wake(d, on);
 	return ret;
 }
 #else
 #define gic_set_wake	NULL
 #endif
 static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
 {
 	u32 irqstat, irqnr;
 	struct gic_chip_data *gic = &gic_data[0];
 	void __iomem *cpu_base = gic_data_cpu_base(gic);
 	do {
 		irqstat = readl_relaxed(cpu_base + GIC_CPU_INTACK);
 		irqnr = irqstat & ~0x1c00;
 		if (likely(irqnr > 15 && irqnr < 1021)) {
 			irqnr = irq_find_mapping(gic->domain, irqnr);
 			handle_IRQ(irqnr, regs);
 			continue;
 		}
 		if (irqnr < 16) {
 			writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI);
 #ifdef CONFIG_SMP
 			handle_IPI(irqnr, regs);
 #endif
 			continue;
 		}
 		break;
 	} while (1);
 }
 static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc)
 {
 	struct gic_chip_data *chip_data = irq_get_handler_data(irq);
 	struct irq_chip *chip = irq_get_chip(irq);
 	unsigned int cascade_irq, gic_irq;
 	unsigned long status;
 	chained_irq_enter(chip, desc);
 	raw_spin_lock(&irq_controller_lock);
 	status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK);
 	raw_spin_unlock(&irq_controller_lock);
 	gic_irq = (status & 0x3ff);
 	if (gic_irq == 1023)
 		goto out;
 	cascade_irq = irq_find_mapping(chip_data->domain, gic_irq);
 	if (unlikely(gic_irq < 32 || gic_irq > 1020))
 		handle_bad_irq(cascade_irq, desc);
 	else
 		generic_handle_irq(cascade_irq);
  out:
 	chained_irq_exit(chip, desc);
 }
 static struct irq_chip gic_chip = {
 	.name			= "GIC",
 	.irq_mask		= gic_mask_irq,
 	.irq_unmask		= gic_unmask_irq,
 	.irq_eoi		= gic_eoi_irq,
 	.irq_set_type		= gic_set_type,
 	.irq_retrigger		= gic_retrigger,
 #ifdef CONFIG_SMP
 	.irq_set_affinity	= gic_set_affinity,
 #endif
 	.irq_set_wake		= gic_set_wake,
 };
 void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq)
 {
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	if (irq_set_handler_data(irq, &gic_data[gic_nr]) != 0)
 		BUG();
 	irq_set_chained_handler(irq, gic_handle_cascade_irq);
 }
 static u8 gic_get_cpumask(struct gic_chip_data *gic)
 {
 	void __iomem *base = gic_data_dist_base(gic);
 	u32 mask, i;
 	for (i = mask = 0; i < 32; i += 4) {
 		mask = readl_relaxed(base + GIC_DIST_TARGET + i);
 		mask |= mask >> 16;
 		mask |= mask >> 8;
 		if (mask)
 			break;
 	}
 	if (!mask)
 		pr_crit("GIC CPU mask not found - kernel will fail to boot.\n");
 	return mask;
 }
 static void __init gic_dist_init(struct gic_chip_data *gic)
 {
 	unsigned int i;
 	u32 cpumask;
 	unsigned int gic_irqs = gic->gic_irqs;
 	void __iomem *base = gic_data_dist_base(gic);
 	writel_relaxed(0, base + GIC_DIST_CTRL);
 	/*
 	 * Set all global interrupts to be level triggered, active low.
 	 */
 	for (i = 32; i < gic_irqs; i += 16)
 		writel_relaxed(0, base + GIC_DIST_CONFIG + i * 4 / 16);
 	/*
 	 * Set all global interrupts to this CPU only.
 	 */
 	cpumask = gic_get_cpumask(gic);
 	cpumask |= cpumask << 8;
 	cpumask |= cpumask << 16;
 	for (i = 32; i < gic_irqs; i += 4)
 		writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4);
 	/*
 	 * Set priority on all global interrupts.
 	 */
 	for (i = 32; i < gic_irqs; i += 4)
 		writel_relaxed(0xa0a0a0a0, base + GIC_DIST_PRI + i * 4 / 4);
 	/*
 	 * Disable all interrupts.  Leave the PPI and SGIs alone
 	 * as these enables are banked registers.
 	 */
 	for (i = 32; i < gic_irqs; i += 32)
 		writel_relaxed(0xffffffff, base + GIC_DIST_ENABLE_CLEAR + i * 4 / 32);
 	writel_relaxed(1, base + GIC_DIST_CTRL);
 }
 static void gic_cpu_init(struct gic_chip_data *gic)
 {
 	void __iomem *dist_base = gic_data_dist_base(gic);
 	void __iomem *base = gic_data_cpu_base(gic);
 	unsigned int cpu_mask, cpu = smp_processor_id();
 	int i;
 	/*
 	 * Get what the GIC says our CPU mask is.
 	 */
 	BUG_ON(cpu >= NR_GIC_CPU_IF);
 	cpu_mask = gic_get_cpumask(gic);
 	gic_cpu_map[cpu] = cpu_mask;
 	/*
 	 * Clear our mask from the other map entries in case they're
 	 * still undefined.
 	 */
 	for (i = 0; i < NR_GIC_CPU_IF; i++)
 		if (i != cpu)
 			gic_cpu_map[i] &= ~cpu_mask;
 	/*
 	 * Deal with the banked PPI and SGI interrupts - disable all
 	 * PPI interrupts, ensure all SGI interrupts are enabled.
 	 */
 	writel_relaxed(0xffff0000, dist_base + GIC_DIST_ENABLE_CLEAR);
 	writel_relaxed(0x0000ffff, dist_base + GIC_DIST_ENABLE_SET);
 	/*
 	 * Set priority on PPI and SGI interrupts
 	 */
 	for (i = 0; i < 32; i += 4)
 		writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4 / 4);
 	writel_relaxed(0xf0, base + GIC_CPU_PRIMASK);
 	writel_relaxed(1, base + GIC_CPU_CTRL);
 }
 void gic_cpu_if_down(void)
 {
 	void __iomem *cpu_base = gic_data_cpu_base(&gic_data[0]);
 	writel_relaxed(0, cpu_base + GIC_CPU_CTRL);
 }
 #ifdef CONFIG_CPU_PM
 /*
  * Saves the GIC distributor registers during suspend or idle.  Must be called
  * with interrupts disabled but before powering down the GIC.  After calling
  * this function, no interrupts will be delivered by the GIC, and another
  * platform-specific wakeup source must be enabled.
  */
 static void gic_dist_save(unsigned int gic_nr)
 {
 	unsigned int gic_irqs;
 	void __iomem *dist_base;
 	int i;
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	gic_irqs = gic_data[gic_nr].gic_irqs;
 	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
 	if (!dist_base)
 		return;
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
 		gic_data[gic_nr].saved_spi_conf[i] =
 			readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
 		gic_data[gic_nr].saved_spi_target[i] =
 			readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
 		gic_data[gic_nr].saved_spi_enable[i] =
 			readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
 }
 /*
  * Restores the GIC distributor registers during resume or when coming out of
  * idle.  Must be called before enabling interrupts.  If a level interrupt
  * that occured while the GIC was suspended is still present, it will be
  * handled normally, but any edge interrupts that occured will not be seen by
  * the GIC and need to be handled by the platform-specific wakeup source.
  */
 static void gic_dist_restore(unsigned int gic_nr)
 {
 	unsigned int gic_irqs;
 	unsigned int i;
 	void __iomem *dist_base;
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	gic_irqs = gic_data[gic_nr].gic_irqs;
 	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
 	if (!dist_base)
 		return;
 	writel_relaxed(0, dist_base + GIC_DIST_CTRL);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 16); i++)
 		writel_relaxed(gic_data[gic_nr].saved_spi_conf[i],
 			dist_base + GIC_DIST_CONFIG + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
 		writel_relaxed(0xa0a0a0a0,
 			dist_base + GIC_DIST_PRI + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++)
 		writel_relaxed(gic_data[gic_nr].saved_spi_target[i],
 			dist_base + GIC_DIST_TARGET + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(gic_irqs, 32); i++)
 		writel_relaxed(gic_data[gic_nr].saved_spi_enable[i],
 			dist_base + GIC_DIST_ENABLE_SET + i * 4);
 	writel_relaxed(1, dist_base + GIC_DIST_CTRL);
 }
 static void gic_cpu_save(unsigned int gic_nr)
 {
 	int i;
 	u32 *ptr;
 	void __iomem *dist_base;
 	void __iomem *cpu_base;
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
 	cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
 	if (!dist_base || !cpu_base)
 		return;
 	ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);
 	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
 		ptr[i] = readl_relaxed(dist_base + GIC_DIST_ENABLE_SET + i * 4);
 	ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_conf);
 	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
 		ptr[i] = readl_relaxed(dist_base + GIC_DIST_CONFIG + i * 4);
 }
 static void gic_cpu_restore(unsigned int gic_nr)
 {
 	int i;
 	u32 *ptr;
 	void __iomem *dist_base;
 	void __iomem *cpu_base;
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
 	cpu_base = gic_data_cpu_base(&gic_data[gic_nr]);
 	if (!dist_base || !cpu_base)
 		return;
 	ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_enable);
 	for (i = 0; i < DIV_ROUND_UP(32, 32); i++)
 		writel_relaxed(ptr[i], dist_base + GIC_DIST_ENABLE_SET + i * 4);
 	ptr = __this_cpu_ptr(gic_data[gic_nr].saved_ppi_conf);
 	for (i = 0; i < DIV_ROUND_UP(32, 16); i++)
 		writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4);
 	for (i = 0; i < DIV_ROUND_UP(32, 4); i++)
 		writel_relaxed(0xa0a0a0a0, dist_base + GIC_DIST_PRI + i * 4);
 	writel_relaxed(0xf0, cpu_base + GIC_CPU_PRIMASK);
 	writel_relaxed(1, cpu_base + GIC_CPU_CTRL);
 }
 static int gic_notifier(struct notifier_block *self, unsigned long cmd,	void *v)
 {
 	int i;
 	for (i = 0; i < MAX_GIC_NR; i++) {
 #ifdef CONFIG_GIC_NON_BANKED
 		/* Skip over unused GICs */
 		if (!gic_data[i].get_base)
 			continue;
 #endif
 		switch (cmd) {
 		case CPU_PM_ENTER:
 			gic_cpu_save(i);
 			break;
 		case CPU_PM_ENTER_FAILED:
 		case CPU_PM_EXIT:
 			gic_cpu_restore(i);
 			break;
 		case CPU_CLUSTER_PM_ENTER:
 			gic_dist_save(i);
 			break;
 		case CPU_CLUSTER_PM_ENTER_FAILED:
 		case CPU_CLUSTER_PM_EXIT:
 			gic_dist_restore(i);
 			break;
 		}
 	}
 	return NOTIFY_OK;
 }
 static struct notifier_block gic_notifier_block = {
 	.notifier_call = gic_notifier,
 };
 static void __init gic_pm_init(struct gic_chip_data *gic)
 {
 	gic->saved_ppi_enable = __alloc_percpu(DIV_ROUND_UP(32, 32) * 4,
 		sizeof(u32));
 	BUG_ON(!gic->saved_ppi_enable);
 	gic->saved_ppi_conf = __alloc_percpu(DIV_ROUND_UP(32, 16) * 4,
 		sizeof(u32));
 	BUG_ON(!gic->saved_ppi_conf);
 	if (gic == &gic_data[0])
 		cpu_pm_register_notifier(&gic_notifier_block);
 }
 #else
 static void __init gic_pm_init(struct gic_chip_data *gic)
 {
 }
 #endif
 #ifdef CONFIG_SMP
 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
 {
 	int cpu;
 	unsigned long flags, map = 0;
 	raw_spin_lock_irqsave(&irq_controller_lock, flags);
 	/* Convert our logical CPU mask into a physical one. */
 	for_each_cpu(cpu, mask)
 		map |= gic_cpu_map[cpu];
 	/*
 	 * Ensure that stores to Normal memory are visible to the
 	 * other CPUs before they observe us issuing the IPI.
 	 */
 	dmb(ishst);
 	/* this always happens on GIC0 */
 	writel_relaxed(map << 16 | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
 	raw_spin_unlock_irqrestore(&irq_controller_lock, flags);
 }
 #endif
 #ifdef CONFIG_BL_SWITCHER
 /*
  * gic_send_sgi - send a SGI directly to given CPU interface number
  *
  * cpu_id: the ID for the destination CPU interface
  * irq: the IPI number to send a SGI for
  */
 void gic_send_sgi(unsigned int cpu_id, unsigned int irq)
 {
 	BUG_ON(cpu_id >= NR_GIC_CPU_IF);
 	cpu_id = 1 << cpu_id;
 	/* this always happens on GIC0 */
 	writel_relaxed((cpu_id << 16) | irq, gic_data_dist_base(&gic_data[0]) + GIC_DIST_SOFTINT);
 }
 /*
  * gic_get_cpu_id - get the CPU interface ID for the specified CPU
  *
  * @cpu: the logical CPU number to get the GIC ID for.
  *
  * Return the CPU interface ID for the given logical CPU number,
  * or -1 if the CPU number is too large or the interface ID is
  * unknown (more than one bit set).
  */
 int gic_get_cpu_id(unsigned int cpu)
 {
 	unsigned int cpu_bit;
 	if (cpu >= NR_GIC_CPU_IF)
 		return -1;
 	cpu_bit = gic_cpu_map[cpu];
 	if (cpu_bit & (cpu_bit - 1))
 		return -1;
 	return __ffs(cpu_bit);
 }
 /*
  * gic_migrate_target - migrate IRQs to another CPU interface
  *
  * @new_cpu_id: the CPU target ID to migrate IRQs to
  *
  * Migrate all peripheral interrupts with a target matching the current CPU
  * to the interface corresponding to @new_cpu_id.  The CPU interface mapping
  * is also updated.  Targets to other CPU interfaces are unchanged.
  * This must be called with IRQs locally disabled.
  */
 void gic_migrate_target(unsigned int new_cpu_id)
 {
 	unsigned int cur_cpu_id, gic_irqs, gic_nr = 0;
 	void __iomem *dist_base;
 	int i, ror_val, cpu = smp_processor_id();
 	u32 val, cur_target_mask, active_mask;
 	if (gic_nr >= MAX_GIC_NR)
 		BUG();
 	dist_base = gic_data_dist_base(&gic_data[gic_nr]);
 	if (!dist_base)
 		return;
 	gic_irqs = gic_data[gic_nr].gic_irqs;
 	cur_cpu_id = __ffs(gic_cpu_map[cpu]);
 	cur_target_mask = 0x01010101 << cur_cpu_id;
 	ror_val = (cur_cpu_id - new_cpu_id) & 31;
 	raw_spin_lock(&irq_controller_lock);
 	/* Update the target interface for this logical CPU */
 	gic_cpu_map[cpu] = 1 << new_cpu_id;
 	/*
 	 * Find all the peripheral interrupts targetting the current
 	 * CPU interface and migrate them to the new CPU interface.
 	 * We skip DIST_TARGET 0 to 7 as they are read-only.
 	 */
 	for (i = 8; i < DIV_ROUND_UP(gic_irqs, 4); i++) {
 		val = readl_relaxed(dist_base + GIC_DIST_TARGET + i * 4);
 		active_mask = val & cur_target_mask;
 		if (active_mask) {
 			val &= ~active_mask;
 			val |= ror32(active_mask, ror_val);
 			writel_relaxed(val, dist_base + GIC_DIST_TARGET + i*4);
 		}
 	}
 	raw_spin_unlock(&irq_controller_lock);
 	/*
 	 * Now let's migrate and clear any potential SGIs that might be
 	 * pending for us (cur_cpu_id).  Since GIC_DIST_SGI_PENDING_SET
 	 * is a banked register, we can only forward the SGI using
 	 * GIC_DIST_SOFTINT.  The original SGI source is lost but Linux
 	 * doesn't use that information anyway.
 	 *
 	 * For the same reason we do not adjust SGI source information
 	 * for previously sent SGIs by us to other CPUs either.
 	 */
 	for (i = 0; i < 16; i += 4) {
 		int j;
 		val = readl_relaxed(dist_base + GIC_DIST_SGI_PENDING_SET + i);
 		if (!val)
 			continue;
 		writel_relaxed(val, dist_base + GIC_DIST_SGI_PENDING_CLEAR + i);
 		for (j = i; j < i + 4; j++) {
 			if (val & 0xff)
 				writel_relaxed((1 << (new_cpu_id + 16)) | j,
 						dist_base + GIC_DIST_SOFTINT);
 			val >>= 8;
 		}
 	}
 }
 /*
  * gic_get_sgir_physaddr - get the physical address for the SGI register
  *
  * REturn the physical address of the SGI register to be used
  * by some early assembly code when the kernel is not yet available.
  */
 static unsigned long gic_dist_physaddr;
 unsigned long gic_get_sgir_physaddr(void)
 {
 	if (!gic_dist_physaddr)
 		return 0;
 	return gic_dist_physaddr + GIC_DIST_SOFTINT;
 }
 void __init gic_init_physaddr(struct device_node *node)
 {
 	struct resource res;
 	if (of_address_to_resource(node, 0, &res) == 0) {
 		gic_dist_physaddr = res.start;
 		pr_info("GIC physical location is %#lx\n", gic_dist_physaddr);
 	}
 }
 #else
 #define gic_init_physaddr(node)  do { } while (0)
 #endif
 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
 				irq_hw_number_t hw)
 {
 	if (hw < 32) {
 		irq_set_percpu_devid(irq);
 		irq_set_chip_and_handler(irq, &gic_chip,
 					 handle_percpu_devid_irq);
 		set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
 	} else {
 		irq_set_chip_and_handler(irq, &gic_chip,
 					 handle_fasteoi_irq);
 		set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
 		gic_routable_irq_domain_ops->map(d, irq, hw);
 	}
 	irq_set_chip_data(irq, d->host_data);
 	return 0;
 }
 static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq)
 {
 	gic_routable_irq_domain_ops->unmap(d, irq);
 }
 static int gic_irq_domain_xlate(struct irq_domain *d,
 				struct device_node *controller,
 				const u32 *intspec, unsigned int intsize,
 				unsigned long *out_hwirq, unsigned int *out_type)
 {
 	unsigned long ret = 0;
 	if (d->of_node != controller)
 		return -EINVAL;
 	if (intsize < 3)
 		return -EINVAL;
 	/* Get the interrupt number and add 16 to skip over SGIs */
 	*out_hwirq = intspec[1] + 16;
 	/* For SPIs, we need to add 16 more to get the GIC irq ID number */
 	if (!intspec[0]) {
 		ret = gic_routable_irq_domain_ops->xlate(d, controller,
 							 intspec,
 							 intsize,
 							 out_hwirq,
 							 out_type);
 		if (IS_ERR_VALUE(ret))
 			return ret;
 	}
 	*out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
 	return ret;
 }
 #ifdef CONFIG_SMP
 static int gic_secondary_init(struct notifier_block *nfb, unsigned long action,
 			      void *hcpu)
 {
 	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
 		gic_cpu_init(&gic_data[0]);
 	return NOTIFY_OK;
 }
 /*
  * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
  * priority because the GIC needs to be up before the ARM generic timers.
  */
 static struct notifier_block gic_cpu_notifier = {
 	.notifier_call = gic_secondary_init,
 	.priority = 100,
 };
 #endif
 static const struct irq_domain_ops gic_irq_domain_ops = {
 	.map = gic_irq_domain_map,
 	.unmap = gic_irq_domain_unmap,
 	.xlate = gic_irq_domain_xlate,
 };
 /* Default functions for routable irq domain */
 static int gic_routable_irq_domain_map(struct irq_domain *d, unsigned int irq,
 			      irq_hw_number_t hw)
 {
 	return 0;
 }
 static void gic_routable_irq_domain_unmap(struct irq_domain *d,
 					  unsigned int irq)
 {
 }
 static int gic_routable_irq_domain_xlate(struct irq_domain *d,
 				struct device_node *controller,
 				const u32 *intspec, unsigned int intsize,
 				unsigned long *out_hwirq,
 				unsigned int *out_type)
 {
 	*out_hwirq += 16;
 	return 0;
 }
 const struct irq_domain_ops gic_default_routable_irq_domain_ops = {
 	.map = gic_routable_irq_domain_map,
 	.unmap = gic_routable_irq_domain_unmap,
 	.xlate = gic_routable_irq_domain_xlate,
 };
 const struct irq_domain_ops *gic_routable_irq_domain_ops =
 					&gic_default_routable_irq_domain_ops;
 void __init gic_init_bases(unsigned int gic_nr, int irq_start,
 			   void __iomem *dist_base, void __iomem *cpu_base,
 			   u32 percpu_offset, struct device_node *node)
 {
 	irq_hw_number_t hwirq_base;
 	struct gic_chip_data *gic;
 	int gic_irqs, irq_base, i;
 	int nr_routable_irqs;
 	BUG_ON(gic_nr >= MAX_GIC_NR);
 	gic = &gic_data[gic_nr];
 #ifdef CONFIG_GIC_NON_BANKED
 	if (percpu_offset) { /* Frankein-GIC without banked registers... */
 		unsigned int cpu;
 		gic->dist_base.percpu_base = alloc_percpu(void __iomem *);
 		gic->cpu_base.percpu_base = alloc_percpu(void __iomem *);
 		if (WARN_ON(!gic->dist_base.percpu_base ||
 			    !gic->cpu_base.percpu_base)) {
 			free_percpu(gic->dist_base.percpu_base);
 			free_percpu(gic->cpu_base.percpu_base);
 			return;
 		}
 		for_each_possible_cpu(cpu) {
 			unsigned long offset = percpu_offset * cpu_logical_map(cpu);
 			*per_cpu_ptr(gic->dist_base.percpu_base, cpu) = dist_base + offset;
 			*per_cpu_ptr(gic->cpu_base.percpu_base, cpu) = cpu_base + offset;
 		}
 		gic_set_base_accessor(gic, gic_get_percpu_base);
 	} else
 #endif
 	{			/* Normal, sane GIC... */
 		WARN(percpu_offset,
 		     "GIC_NON_BANKED not enabled, ignoring %08x offset!",
 		     percpu_offset);
 		gic->dist_base.common_base = dist_base;
 		gic->cpu_base.common_base = cpu_base;
 		gic_set_base_accessor(gic, gic_get_common_base);
 	}
 	/*
 	 * Initialize the CPU interface map to all CPUs.
 	 * It will be refined as each CPU probes its ID.
 	 */
 	for (i = 0; i < NR_GIC_CPU_IF; i++)
 		gic_cpu_map[i] = 0xff;
 	/*
 	 * For primary GICs, skip over SGIs.
 	 * For secondary GICs, skip over PPIs, too.
 	 */
 	if (gic_nr == 0 && (irq_start & 31) > 0) {
 		hwirq_base = 16;
 		if (irq_start != -1)
 			irq_start = (irq_start & ~31) + 16;
 	} else {
 		hwirq_base = 32;
 	}
 	/*
 	 * Find out how many interrupts are supported.
 	 * The GIC only supports up to 1020 interrupt sources.
 	 */
 	gic_irqs = readl_relaxed(gic_data_dist_base(gic) + GIC_DIST_CTR) & 0x1f;
 	gic_irqs = (gic_irqs + 1) * 32;
 	if (gic_irqs > 1020)
 		gic_irqs = 1020;
 	gic->gic_irqs = gic_irqs;
 	gic_irqs -= hwirq_base; /* calculate # of irqs to allocate */
 	if (of_property_read_u32(node, "arm,routable-irqs",
 				 &nr_routable_irqs)) {
 		irq_base = irq_alloc_descs(irq_start, 16, gic_irqs,
 					   numa_node_id());
 		if (IS_ERR_VALUE(irq_base)) {
 			WARN(1, "Cannot allocate irq_descs @ IRQ%d, assuming pre-allocated\n",
 			     irq_start);
 			irq_base = irq_start;
 		}
 		gic->domain = irq_domain_add_legacy(node, gic_irqs, irq_base,
 					hwirq_base, &gic_irq_domain_ops, gic);
 	} else {
 		gic->domain = irq_domain_add_linear(node, nr_routable_irqs,
 						    &gic_irq_domain_ops,
 						    gic);
 	}
 	if (WARN_ON(!gic->domain))
 		return;
 	if (gic_nr == 0) {
 #ifdef CONFIG_SMP
 		set_smp_cross_call(gic_raise_softirq);
 		register_cpu_notifier(&gic_cpu_notifier);
 #endif
 		set_handle_irq(gic_handle_irq);
 	}
 	gic_chip.flags |= gic_arch_extn.flags;
 	gic_dist_init(gic);
 	gic_cpu_init(gic);
 	gic_pm_init(gic);
 }
 #ifdef CONFIG_OF
 static int gic_cnt __initdata;
 static int __init
 gic_of_init(struct device_node *node, struct device_node *parent)
 {
 	void __iomem *cpu_base;
 	void __iomem *dist_base;
 	u32 percpu_offset;
 	int irq;
 	if (WARN_ON(!node))
 		return -ENODEV;
 	dist_base = of_iomap(node, 0);
 	WARN(!dist_base, "unable to map gic dist registers\n");
 	cpu_base = of_iomap(node, 1);
 	WARN(!cpu_base, "unable to map gic cpu registers\n");
 	if (of_property_read_u32(node, "cpu-offset", &percpu_offset))
 		percpu_offset = 0;
 	gic_init_bases(gic_cnt, -1, dist_base, cpu_base, percpu_offset, node);
 	if (!gic_cnt)
 		gic_init_physaddr(node);
 	if (parent) {
 		irq = irq_of_parse_and_map(node, 0);
 		gic_cascade_irq(gic_cnt, irq);
 	}
 	gic_cnt++;
 	return 0;
 }
 IRQCHIP_DECLARE(cortex_a15_gic, "arm,cortex-a15-gic", gic_of_init);
 IRQCHIP_DECLARE(cortex_a9_gic, "arm,cortex-a9-gic", gic_of_init);
 IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init);
 IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init);
 #endif

include/linux/interrupt.h

Diff comments View file @ d9e9e8e

 /* interrupt.h */
 #ifndef _LINUX_INTERRUPT_H
 #define _LINUX_INTERRUPT_H
 #include <linux/kernel.h>
 #include <linux/linkage.h>
 #include <linux/bitops.h>
 #include <linux/preempt.h>
 #include <linux/cpumask.h>
 #include <linux/irqreturn.h>
 #include <linux/irqnr.h>
 #include <linux/hardirq.h>
 #include <linux/irqflags.h>
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
 #include <linux/atomic.h>
 #include <asm/ptrace.h>
 #include <asm/irq.h>
 /*
  * These correspond to the IORESOURCE_IRQ_* defines in
  * linux/ioport.h to select the interrupt line behaviour.  When
  * requesting an interrupt without specifying a IRQF_TRIGGER, the
  * setting should be assumed to be "as already configured", which
  * may be as per machine or firmware initialisation.
  */
 #define IRQF_TRIGGER_NONE	0x00000000
 #define IRQF_TRIGGER_RISING	0x00000001
 #define IRQF_TRIGGER_FALLING	0x00000002
 #define IRQF_TRIGGER_HIGH	0x00000004
 #define IRQF_TRIGGER_LOW	0x00000008
 #define IRQF_TRIGGER_MASK	(IRQF_TRIGGER_HIGH | IRQF_TRIGGER_LOW | \
 				 IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)
 #define IRQF_TRIGGER_PROBE	0x00000010
 /*
  * These flags used only by the kernel as part of the
  * irq handling routines.
  *
  * IRQF_DISABLED - keep irqs disabled when calling the action handler.
  *                 DEPRECATED. This flag is a NOOP and scheduled to be removed
  * IRQF_SHARED - allow sharing the irq among several devices
  * IRQF_PROBE_SHARED - set by callers when they expect sharing mismatches to occur
  * IRQF_TIMER - Flag to mark this interrupt as timer interrupt
  * IRQF_PERCPU - Interrupt is per cpu
  * IRQF_NOBALANCING - Flag to exclude this interrupt from irq balancing
  * IRQF_IRQPOLL - Interrupt is used for polling (only the interrupt that is
  *                registered first in an shared interrupt is considered for
  *                performance reasons)
  * IRQF_ONESHOT - Interrupt is not reenabled after the hardirq handler finished.
  *                Used by threaded interrupts which need to keep the
  *                irq line disabled until the threaded handler has been run.
  * IRQF_NO_SUSPEND - Do not disable this IRQ during suspend
  * IRQF_FORCE_RESUME - Force enable it on resume even if IRQF_NO_SUSPEND is set
  * IRQF_NO_THREAD - Interrupt cannot be threaded
  * IRQF_EARLY_RESUME - Resume IRQ early during syscore instead of at device
  *                resume time.
  */
 #define IRQF_DISABLED		0x00000020
 #define IRQF_SHARED		0x00000080
 #define IRQF_PROBE_SHARED	0x00000100
 #define __IRQF_TIMER		0x00000200
 #define IRQF_PERCPU		0x00000400
 #define IRQF_NOBALANCING	0x00000800
 #define IRQF_IRQPOLL		0x00001000
 #define IRQF_ONESHOT		0x00002000
 #define IRQF_NO_SUSPEND		0x00004000
 #define IRQF_FORCE_RESUME	0x00008000
 #define IRQF_NO_THREAD		0x00010000
 #define IRQF_EARLY_RESUME	0x00020000
 #define IRQF_TIMER		(__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD)
 /*
  * These values can be returned by request_any_context_irq() and
  * describe the context the interrupt will be run in.
  *
  * IRQC_IS_HARDIRQ - interrupt runs in hardirq context
  * IRQC_IS_NESTED - interrupt runs in a nested threaded context
  */
 enum {
 	IRQC_IS_HARDIRQ	= 0,
 	IRQC_IS_NESTED,
 };
 typedef irqreturn_t (*irq_handler_t)(int, void *);
 /**
  * struct irqaction - per interrupt action descriptor
  * @handler:	interrupt handler function
  * @name:	name of the device
  * @dev_id:	cookie to identify the device
  * @percpu_dev_id:	cookie to identify the device
  * @next:	pointer to the next irqaction for shared interrupts
  * @irq:	interrupt number
  * @flags:	flags (see IRQF_* above)
  * @thread_fn:	interrupt handler function for threaded interrupts
  * @thread:	thread pointer for threaded interrupts
  * @thread_flags:	flags related to @thread
  * @thread_mask:	bitmask for keeping track of @thread activity
  * @dir:	pointer to the proc/irq/NN/name entry
  */
 struct irqaction {
 	irq_handler_t		handler;
 	void			*dev_id;
 	void __percpu		*percpu_dev_id;
 	struct irqaction	*next;
 	irq_handler_t		thread_fn;
 	struct task_struct	*thread;
 	unsigned int		irq;
 	unsigned int		flags;
 	unsigned long		thread_flags;
 	unsigned long		thread_mask;
 	const char		*name;
 	struct proc_dir_entry	*dir;
 } ____cacheline_internodealigned_in_smp;
 extern irqreturn_t no_action(int cpl, void *dev_id);
 extern int __must_check
 request_threaded_irq(unsigned int irq, irq_handler_t handler,
 		     irq_handler_t thread_fn,
 		     unsigned long flags, const char *name, void *dev);
 static inline int __must_check
 request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags,
 	    const char *name, void *dev)
 {
 	return request_threaded_irq(irq, handler, NULL, flags, name, dev);
 }
 extern int __must_check
 request_any_context_irq(unsigned int irq, irq_handler_t handler,
 			unsigned long flags, const char *name, void *dev_id);
 extern int __must_check
 request_percpu_irq(unsigned int irq, irq_handler_t handler,
 		   const char *devname, void __percpu *percpu_dev_id);
 extern void free_irq(unsigned int, void *);
 extern void free_percpu_irq(unsigned int, void __percpu *);
 struct device;
 extern int __must_check
 devm_request_threaded_irq(struct device *dev, unsigned int irq,
 			  irq_handler_t handler, irq_handler_t thread_fn,
 			  unsigned long irqflags, const char *devname,
 			  void *dev_id);
 static inline int __must_check
 devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler,
 		 unsigned long irqflags, const char *devname, void *dev_id)
 {
 	return devm_request_threaded_irq(dev, irq, handler, NULL, irqflags,
 					 devname, dev_id);
 }
 extern int __must_check
 devm_request_any_context_irq(struct device *dev, unsigned int irq,
 		 irq_handler_t handler, unsigned long irqflags,
 		 const char *devname, void *dev_id);
 extern void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id);
 /*
  * On lockdep we dont want to enable hardirqs in hardirq
  * context. Use local_irq_enable_in_hardirq() to annotate
  * kernel code that has to do this nevertheless (pretty much
  * the only valid case is for old/broken hardware that is
  * insanely slow).
  *
  * NOTE: in theory this might break fragile code that relies
  * on hardirq delivery - in practice we dont seem to have such
  * places left. So the only effect should be slightly increased
  * irqs-off latencies.
  */
 #ifdef CONFIG_LOCKDEP
 # define local_irq_enable_in_hardirq()	do { } while (0)
 #else
 # define local_irq_enable_in_hardirq()	local_irq_enable()
 #endif
 extern void disable_irq_nosync(unsigned int irq);
 extern void disable_irq(unsigned int irq);
 extern void disable_percpu_irq(unsigned int irq);
 extern void enable_irq(unsigned int irq);
 extern void enable_percpu_irq(unsigned int irq, unsigned int type);
 extern void irq_wake_thread(unsigned int irq, void *dev_id);
 /* The following three functions are for the core kernel use only. */
 extern void suspend_device_irqs(void);
 extern void resume_device_irqs(void);
 #ifdef CONFIG_PM_SLEEP
 extern int check_wakeup_irqs(void);
 #else
 static inline int check_wakeup_irqs(void) { return 0; }
 #endif
 #if defined(CONFIG_SMP)
 extern cpumask_var_t irq_default_affinity;
-extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask);
+/* Internal implementation. Use the helpers below */
+extern int __irq_set_affinity(unsigned int irq, const struct cpumask *cpumask,
+			      bool force);
+/**
+ * irq_set_affinity - Set the irq affinity of a given irq
+ * @irq:	Interrupt to set affinity
+ * @mask:	cpumask
+ *
+ * Fails if cpumask does not contain an online CPU
+ */
+static inline int
+irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
+{
+	return __irq_set_affinity(irq, cpumask, false);
+}
+/**
+ * irq_force_affinity - Force the irq affinity of a given irq
+ * @irq:	Interrupt to set affinity
+ * @mask:	cpumask
+ *
+ * Same as irq_set_affinity, but without checking the mask against
+ * online cpus.
+ *
+ * Solely for low level cpu hotplug code, where we need to make per
+ * cpu interrupts affine before the cpu becomes online.
+ */
+static inline int
+irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
+{
+	return __irq_set_affinity(irq, cpumask, true);
+}
 extern int irq_can_set_affinity(unsigned int irq);
 extern int irq_select_affinity(unsigned int irq);
 extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m);
 /**
  * struct irq_affinity_notify - context for notification of IRQ affinity changes
  * @irq:		Interrupt to which notification applies
  * @kref:		Reference count, for internal use
  * @work:		Work item, for internal use
  * @notify:		Function to be called on change.  This will be
  *			called in process context.
  * @release:		Function to be called on release.  This will be
  *			called in process context.  Once registered, the
  *			structure must only be freed when this function is
  *			called or later.
  */
 struct irq_affinity_notify {
 	unsigned int irq;
 	struct kref kref;
 	struct work_struct work;
 	void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask);
 	void (*release)(struct kref *ref);
 };
 extern int
 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify);
 #else /* CONFIG_SMP */
 static inline int irq_set_affinity(unsigned int irq, const struct cpumask *m)
 {
 	return -EINVAL;
 }
 static inline int irq_can_set_affinity(unsigned int irq)
 {
 	return 0;
 }
 static inline int irq_select_affinity(unsigned int irq)  { return 0; }
 static inline int irq_set_affinity_hint(unsigned int irq,
 					const struct cpumask *m)
 {
 	return -EINVAL;
 }
 #endif /* CONFIG_SMP */
 /*
  * Special lockdep variants of irq disabling/enabling.
  * These should be used for locking constructs that
  * know that a particular irq context which is disabled,
  * and which is the only irq-context user of a lock,
  * that it's safe to take the lock in the irq-disabled
  * section without disabling hardirqs.
  *
  * On !CONFIG_LOCKDEP they are equivalent to the normal
  * irq disable/enable methods.
  */
 static inline void disable_irq_nosync_lockdep(unsigned int irq)
 {
 	disable_irq_nosync(irq);
 #ifdef CONFIG_LOCKDEP
 	local_irq_disable();
 #endif
 }
 static inline void disable_irq_nosync_lockdep_irqsave(unsigned int irq, unsigned long *flags)
 {
 	disable_irq_nosync(irq);
 #ifdef CONFIG_LOCKDEP
 	local_irq_save(*flags);
 #endif
 }
 static inline void disable_irq_lockdep(unsigned int irq)
 {
 	disable_irq(irq);
 #ifdef CONFIG_LOCKDEP
 	local_irq_disable();
 #endif
 }
 static inline void enable_irq_lockdep(unsigned int irq)
 {
 #ifdef CONFIG_LOCKDEP
 	local_irq_enable();
 #endif
 	enable_irq(irq);
 }
 static inline void enable_irq_lockdep_irqrestore(unsigned int irq, unsigned long *flags)
 {
 #ifdef CONFIG_LOCKDEP
 	local_irq_restore(*flags);
 #endif
 	enable_irq(irq);
 }
 /* IRQ wakeup (PM) control: */
 extern int irq_set_irq_wake(unsigned int irq, unsigned int on);
 static inline int enable_irq_wake(unsigned int irq)
 {
 	return irq_set_irq_wake(irq, 1);
 }
 static inline int disable_irq_wake(unsigned int irq)
 {
 	return irq_set_irq_wake(irq, 0);
 }
 #ifdef CONFIG_IRQ_FORCED_THREADING
 extern bool force_irqthreads;
 #else
 #define force_irqthreads	(0)
 #endif
 #ifndef __ARCH_SET_SOFTIRQ_PENDING
 #define set_softirq_pending(x) (local_softirq_pending() = (x))
 #define or_softirq_pending(x)  (local_softirq_pending() |= (x))
 #endif
 /* Some architectures might implement lazy enabling/disabling of
  * interrupts. In some cases, such as stop_machine, we might want
  * to ensure that after a local_irq_disable(), interrupts have
  * really been disabled in hardware. Such architectures need to
  * implement the following hook.
  */
 #ifndef hard_irq_disable
 #define hard_irq_disable()	do { } while(0)
 #endif
 /* PLEASE, avoid to allocate new softirqs, if you need not _really_ high
    frequency threaded job scheduling. For almost all the purposes
    tasklets are more than enough. F.e. all serial device BHs et
    al. should be converted to tasklets, not to softirqs.
  */
 enum
 {
 	HI_SOFTIRQ=0,
 	TIMER_SOFTIRQ,
 	NET_TX_SOFTIRQ,
 	NET_RX_SOFTIRQ,
 	BLOCK_SOFTIRQ,
 	BLOCK_IOPOLL_SOFTIRQ,
 	TASKLET_SOFTIRQ,
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ,
 	RCU_SOFTIRQ,    /* Preferable RCU should always be the last softirq */
 	NR_SOFTIRQS
 };
 #define SOFTIRQ_STOP_IDLE_MASK (~(1 << RCU_SOFTIRQ))
 /* map softirq index to softirq name. update 'softirq_to_name' in
  * kernel/softirq.c when adding a new softirq.
  */
 extern const char * const softirq_to_name[NR_SOFTIRQS];
 /* softirq mask and active fields moved to irq_cpustat_t in
  * asm/hardirq.h to get better cache usage.  KAO
  */
 struct softirq_action
 {
 	void	(*action)(struct softirq_action *);
 };
 asmlinkage void do_softirq(void);
 asmlinkage void __do_softirq(void);
 #ifdef __ARCH_HAS_DO_SOFTIRQ
 void do_softirq_own_stack(void);
 #else
 static inline void do_softirq_own_stack(void)
 {
 	__do_softirq();
 }
 #endif
 extern void open_softirq(int nr, void (*action)(struct softirq_action *));
 extern void softirq_init(void);
 extern void __raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq(unsigned int nr);
 DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
 static inline struct task_struct *this_cpu_ksoftirqd(void)
 {
 	return this_cpu_read(ksoftirqd);
 }
 /* Tasklets --- multithreaded analogue of BHs.
    Main feature differing them of generic softirqs: tasklet
    is running only on one CPU simultaneously.
    Main feature differing them of BHs: different tasklets
    may be run simultaneously on different CPUs.
    Properties:
    * If tasklet_schedule() is called, then tasklet is guaranteed
      to be executed on some cpu at least once after this.
    * If the tasklet is already scheduled, but its execution is still not
      started, it will be executed only once.
    * If this tasklet is already running on another CPU (or schedule is called
      from tasklet itself), it is rescheduled for later.
    * Tasklet is strictly serialized wrt itself, but not
      wrt another tasklets. If client needs some intertask synchronization,
      he makes it with spinlocks.
  */
 struct tasklet_struct
 {
 	struct tasklet_struct *next;
 	unsigned long state;
 	atomic_t count;
 	void (*func)(unsigned long);
 	unsigned long data;
 };
 #define DECLARE_TASKLET(name, func, data) \
 struct tasklet_struct name = { NULL, 0, ATOMIC_INIT(0), func, data }
 #define DECLARE_TASKLET_DISABLED(name, func, data) \
 struct tasklet_struct name = { NULL, 0, ATOMIC_INIT(1), func, data }
 enum
 {
 	TASKLET_STATE_SCHED,	/* Tasklet is scheduled for execution */
 	TASKLET_STATE_RUN	/* Tasklet is running (SMP only) */
 };
 #ifdef CONFIG_SMP
 static inline int tasklet_trylock(struct tasklet_struct *t)
 {
 	return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state);
 }
 static inline void tasklet_unlock(struct tasklet_struct *t)
 {
 	smp_mb__before_clear_bit();
 	clear_bit(TASKLET_STATE_RUN, &(t)->state);
 }
 static inline void tasklet_unlock_wait(struct tasklet_struct *t)
 {
 	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); }
 }
 #else
 #define tasklet_trylock(t) 1
 #define tasklet_unlock_wait(t) do { } while (0)
 #define tasklet_unlock(t) do { } while (0)
 #endif
 extern void __tasklet_schedule(struct tasklet_struct *t);
 static inline void tasklet_schedule(struct tasklet_struct *t)
 {
 	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 		__tasklet_schedule(t);
 }
 extern void __tasklet_hi_schedule(struct tasklet_struct *t);
 static inline void tasklet_hi_schedule(struct tasklet_struct *t)
 {
 	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 		__tasklet_hi_schedule(t);
 }
 extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
 /*
  * This version avoids touching any other tasklets. Needed for kmemcheck
  * in order not to take any page faults while enqueueing this tasklet;
  * consider VERY carefully whether you really need this or
  * tasklet_hi_schedule()...
  */
 static inline void tasklet_hi_schedule_first(struct tasklet_struct *t)
 {
 	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
 		__tasklet_hi_schedule_first(t);
 }
 static inline void tasklet_disable_nosync(struct tasklet_struct *t)
 {
 	atomic_inc(&t->count);
 	smp_mb__after_atomic_inc();
 }
 static inline void tasklet_disable(struct tasklet_struct *t)
 {
 	tasklet_disable_nosync(t);
 	tasklet_unlock_wait(t);
 	smp_mb();
 }
 static inline void tasklet_enable(struct tasklet_struct *t)
 {
 	smp_mb__before_atomic_dec();
 	atomic_dec(&t->count);
 }
 static inline void tasklet_hi_enable(struct tasklet_struct *t)
 {
 	smp_mb__before_atomic_dec();
 	atomic_dec(&t->count);
 }
 extern void tasklet_kill(struct tasklet_struct *t);
 extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu);
 extern void tasklet_init(struct tasklet_struct *t,
 			 void (*func)(unsigned long), unsigned long data);
 struct tasklet_hrtimer {
 	struct hrtimer		timer;
 	struct tasklet_struct	tasklet;
 	enum hrtimer_restart	(*function)(struct hrtimer *);
 };
 extern void
 tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 		     enum hrtimer_restart (*function)(struct hrtimer *),
 		     clockid_t which_clock, enum hrtimer_mode mode);
 static inline
 int tasklet_hrtimer_start(struct tasklet_hrtimer *ttimer, ktime_t time,
 			  const enum hrtimer_mode mode)
 {
 	return hrtimer_start(&ttimer->timer, time, mode);
 }
 static inline
 void tasklet_hrtimer_cancel(struct tasklet_hrtimer *ttimer)
 {
 	hrtimer_cancel(&ttimer->timer);
 	tasklet_kill(&ttimer->tasklet);
 }
 /*
  * Autoprobing for irqs:
  *
  * probe_irq_on() and probe_irq_off() provide robust primitives
  * for accurate IRQ probing during kernel initialization.  They are
  * reasonably simple to use, are not "fooled" by spurious interrupts,
  * and, unlike other attempts at IRQ probing, they do not get hung on
  * stuck interrupts (such as unused PS2 mouse interfaces on ASUS boards).
  *
  * For reasonably foolproof probing, use them as follows:
  *
  * 1. clear and/or mask the device's internal interrupt.
  * 2. sti();
  * 3. irqs = probe_irq_on();      // "take over" all unassigned idle IRQs
  * 4. enable the device and cause it to trigger an interrupt.
  * 5. wait for the device to interrupt, using non-intrusive polling or a delay.
  * 6. irq = probe_irq_off(irqs);  // get IRQ number, 0=none, negative=multiple
  * 7. service the device to clear its pending interrupt.
  * 8. loop again if paranoia is required.
  *
  * probe_irq_on() returns a mask of allocated irq's.
  *
  * probe_irq_off() takes the mask as a parameter,
  * and returns the irq number which occurred,
  * or zero if none occurred, or a negative irq number
  * if more than one irq occurred.
  */
 #if !defined(CONFIG_GENERIC_IRQ_PROBE)
 static inline unsigned long probe_irq_on(void)
 {
 	return 0;
 }
 static inline int probe_irq_off(unsigned long val)
 {
 	return 0;
 }
 static inline unsigned int probe_irq_mask(unsigned long val)
 {
 	return 0;
 }
 #else
 extern unsigned long probe_irq_on(void);	/* returns 0 on failure */
 extern int probe_irq_off(unsigned long);	/* returns 0 or negative on failure */
 extern unsigned int probe_irq_mask(unsigned long);	/* returns mask of ISA interrupts */
 #endif
 #ifdef CONFIG_PROC_FS
 /* Initialize /proc/irq/ */
 extern void init_irq_proc(void);
 #else
 static inline void init_irq_proc(void)
 {
 }
 #endif
 struct seq_file;
 int show_interrupts(struct seq_file *p, void *v);
 int arch_show_interrupts(struct seq_file *p, int prec);
 extern int early_irq_init(void);
 extern int arch_probe_nr_irqs(void);
 extern int arch_early_irq_init(void);
 #endif

include/linux/irq.h

Diff comments View file @ d9e9e8e

 #ifndef _LINUX_IRQ_H
 #define _LINUX_IRQ_H
 /*
  * Please do not include this file in generic code.  There is currently
  * no requirement for any architecture to implement anything held
  * within this file.
  *
  * Thanks. --rmk
  */
 #include <linux/smp.h>
 #include <linux/linkage.h>
 #include <linux/cache.h>
 #include <linux/spinlock.h>
 #include <linux/cpumask.h>
 #include <linux/gfp.h>
 #include <linux/irqreturn.h>
 #include <linux/irqnr.h>
 #include <linux/errno.h>
 #include <linux/topology.h>
 #include <linux/wait.h>
 #include <asm/irq.h>
 #include <asm/ptrace.h>
 #include <asm/irq_regs.h>
 struct seq_file;
 struct module;
 struct irq_desc;
 struct irq_data;
 typedef	void (*irq_flow_handler_t)(unsigned int irq,
 					    struct irq_desc *desc);
 typedef	void (*irq_preflow_handler_t)(struct irq_data *data);
 /*
  * IRQ line status.
  *
  * Bits 0-7 are the same as the IRQF_* bits in linux/interrupt.h
  *
  * IRQ_TYPE_NONE		- default, unspecified type
  * IRQ_TYPE_EDGE_RISING		- rising edge triggered
  * IRQ_TYPE_EDGE_FALLING	- falling edge triggered
  * IRQ_TYPE_EDGE_BOTH		- rising and falling edge triggered
  * IRQ_TYPE_LEVEL_HIGH		- high level triggered
  * IRQ_TYPE_LEVEL_LOW		- low level triggered
  * IRQ_TYPE_LEVEL_MASK		- Mask to filter out the level bits
  * IRQ_TYPE_SENSE_MASK		- Mask for all the above bits
  * IRQ_TYPE_DEFAULT		- For use by some PICs to ask irq_set_type
  *				  to setup the HW to a sane default (used
  *                                by irqdomain map() callbacks to synchronize
  *                                the HW state and SW flags for a newly
  *                                allocated descriptor).
  *
  * IRQ_TYPE_PROBE		- Special flag for probing in progress
  *
  * Bits which can be modified via irq_set/clear/modify_status_flags()
  * IRQ_LEVEL			- Interrupt is level type. Will be also
  *				  updated in the code when the above trigger
  *				  bits are modified via irq_set_irq_type()
  * IRQ_PER_CPU			- Mark an interrupt PER_CPU. Will protect
  *				  it from affinity setting
  * IRQ_NOPROBE			- Interrupt cannot be probed by autoprobing
  * IRQ_NOREQUEST		- Interrupt cannot be requested via
  *				  request_irq()
  * IRQ_NOTHREAD			- Interrupt cannot be threaded
  * IRQ_NOAUTOEN			- Interrupt is not automatically enabled in
  *				  request/setup_irq()
  * IRQ_NO_BALANCING		- Interrupt cannot be balanced (affinity set)
  * IRQ_MOVE_PCNTXT		- Interrupt can be migrated from process context
  * IRQ_NESTED_TRHEAD		- Interrupt nests into another thread
  * IRQ_PER_CPU_DEVID		- Dev_id is a per-cpu variable
  * IRQ_IS_POLLED		- Always polled by another interrupt. Exclude
  *				  it from the spurious interrupt detection
  *				  mechanism and from core side polling.
  */
 enum {
 	IRQ_TYPE_NONE		= 0x00000000,
 	IRQ_TYPE_EDGE_RISING	= 0x00000001,
 	IRQ_TYPE_EDGE_FALLING	= 0x00000002,
 	IRQ_TYPE_EDGE_BOTH	= (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING),
 	IRQ_TYPE_LEVEL_HIGH	= 0x00000004,
 	IRQ_TYPE_LEVEL_LOW	= 0x00000008,
 	IRQ_TYPE_LEVEL_MASK	= (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH),
 	IRQ_TYPE_SENSE_MASK	= 0x0000000f,
 	IRQ_TYPE_DEFAULT	= IRQ_TYPE_SENSE_MASK,
 	IRQ_TYPE_PROBE		= 0x00000010,
 	IRQ_LEVEL		= (1 <<  8),
 	IRQ_PER_CPU		= (1 <<  9),
 	IRQ_NOPROBE		= (1 << 10),
 	IRQ_NOREQUEST		= (1 << 11),
 	IRQ_NOAUTOEN		= (1 << 12),
 	IRQ_NO_BALANCING	= (1 << 13),
 	IRQ_MOVE_PCNTXT		= (1 << 14),
 	IRQ_NESTED_THREAD	= (1 << 15),
 	IRQ_NOTHREAD		= (1 << 16),
 	IRQ_PER_CPU_DEVID	= (1 << 17),
 	IRQ_IS_POLLED		= (1 << 18),
 };
 #define IRQF_MODIFY_MASK	\
 	(IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \
 	 IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \
 	 IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \
 	 IRQ_IS_POLLED)
 #define IRQ_NO_BALANCING_MASK	(IRQ_PER_CPU | IRQ_NO_BALANCING)
 /*
  * Return value for chip->irq_set_affinity()
  *
  * IRQ_SET_MASK_OK	- OK, core updates irq_data.affinity
  * IRQ_SET_MASK_NOCPY	- OK, chip did update irq_data.affinity
  */
 enum {
 	IRQ_SET_MASK_OK = 0,
 	IRQ_SET_MASK_OK_NOCOPY,
 };
 struct msi_desc;
 struct irq_domain;
 /**
  * struct irq_data - per irq and irq chip data passed down to chip functions
  * @mask:		precomputed bitmask for accessing the chip registers
  * @irq:		interrupt number
  * @hwirq:		hardware interrupt number, local to the interrupt domain
  * @node:		node index useful for balancing
  * @state_use_accessors: status information for irq chip functions.
  *			Use accessor functions to deal with it
  * @chip:		low level interrupt hardware access
  * @domain:		Interrupt translation domain; responsible for mapping
  *			between hwirq number and linux irq number.
  * @handler_data:	per-IRQ data for the irq_chip methods
  * @chip_data:		platform-specific per-chip private data for the chip
  *			methods, to allow shared chip implementations
  * @msi_desc:		MSI descriptor
  * @affinity:		IRQ affinity on SMP
  *
  * The fields here need to overlay the ones in irq_desc until we
  * cleaned up the direct references and switched everything over to
  * irq_data.
  */
 struct irq_data {
 	u32			mask;
 	unsigned int		irq;
 	unsigned long		hwirq;
 	unsigned int		node;
 	unsigned int		state_use_accessors;
 	struct irq_chip		*chip;
 	struct irq_domain	*domain;
 	void			*handler_data;
 	void			*chip_data;
 	struct msi_desc		*msi_desc;
 	cpumask_var_t		affinity;
 };
 /*
  * Bit masks for irq_data.state
  *
  * IRQD_TRIGGER_MASK		- Mask for the trigger type bits
  * IRQD_SETAFFINITY_PENDING	- Affinity setting is pending
  * IRQD_NO_BALANCING		- Balancing disabled for this IRQ
  * IRQD_PER_CPU			- Interrupt is per cpu
  * IRQD_AFFINITY_SET		- Interrupt affinity was set
  * IRQD_LEVEL			- Interrupt is level triggered
  * IRQD_WAKEUP_STATE		- Interrupt is configured for wakeup
  *				  from suspend
  * IRDQ_MOVE_PCNTXT		- Interrupt can be moved in process
  *				  context
  * IRQD_IRQ_DISABLED		- Disabled state of the interrupt
  * IRQD_IRQ_MASKED		- Masked state of the interrupt
  * IRQD_IRQ_INPROGRESS		- In progress state of the interrupt
  */
 enum {
 	IRQD_TRIGGER_MASK		= 0xf,
 	IRQD_SETAFFINITY_PENDING	= (1 <<  8),
 	IRQD_NO_BALANCING		= (1 << 10),
 	IRQD_PER_CPU			= (1 << 11),
 	IRQD_AFFINITY_SET		= (1 << 12),
 	IRQD_LEVEL			= (1 << 13),
 	IRQD_WAKEUP_STATE		= (1 << 14),
 	IRQD_MOVE_PCNTXT		= (1 << 15),
 	IRQD_IRQ_DISABLED		= (1 << 16),
 	IRQD_IRQ_MASKED			= (1 << 17),
 	IRQD_IRQ_INPROGRESS		= (1 << 18),
 };
 static inline bool irqd_is_setaffinity_pending(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_SETAFFINITY_PENDING;
 }
 static inline bool irqd_is_per_cpu(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_PER_CPU;
 }
 static inline bool irqd_can_balance(struct irq_data *d)
 {
 	return !(d->state_use_accessors & (IRQD_PER_CPU | IRQD_NO_BALANCING));
 }
 static inline bool irqd_affinity_was_set(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_AFFINITY_SET;
 }
 static inline void irqd_mark_affinity_was_set(struct irq_data *d)
 {
 	d->state_use_accessors |= IRQD_AFFINITY_SET;
 }
 static inline u32 irqd_get_trigger_type(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_TRIGGER_MASK;
 }
 /*
  * Must only be called inside irq_chip.irq_set_type() functions.
  */
 static inline void irqd_set_trigger_type(struct irq_data *d, u32 type)
 {
 	d->state_use_accessors &= ~IRQD_TRIGGER_MASK;
 	d->state_use_accessors |= type & IRQD_TRIGGER_MASK;
 }
 static inline bool irqd_is_level_type(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_LEVEL;
 }
 static inline bool irqd_is_wakeup_set(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_WAKEUP_STATE;
 }
 static inline bool irqd_can_move_in_process_context(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_MOVE_PCNTXT;
 }
 static inline bool irqd_irq_disabled(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_IRQ_DISABLED;
 }
 static inline bool irqd_irq_masked(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_IRQ_MASKED;
 }
 static inline bool irqd_irq_inprogress(struct irq_data *d)
 {
 	return d->state_use_accessors & IRQD_IRQ_INPROGRESS;
 }
 /*
  * Functions for chained handlers which can be enabled/disabled by the
  * standard disable_irq/enable_irq calls. Must be called with
  * irq_desc->lock held.
  */
 static inline void irqd_set_chained_irq_inprogress(struct irq_data *d)
 {
 	d->state_use_accessors |= IRQD_IRQ_INPROGRESS;
 }
 static inline void irqd_clr_chained_irq_inprogress(struct irq_data *d)
 {
 	d->state_use_accessors &= ~IRQD_IRQ_INPROGRESS;
 }
 static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d)
 {
 	return d->hwirq;
 }
 /**
  * struct irq_chip - hardware interrupt chip descriptor
  *
  * @name:		name for /proc/interrupts
  * @irq_startup:	start up the interrupt (defaults to ->enable if NULL)
  * @irq_shutdown:	shut down the interrupt (defaults to ->disable if NULL)
  * @irq_enable:		enable the interrupt (defaults to chip->unmask if NULL)
  * @irq_disable:	disable the interrupt
  * @irq_ack:		start of a new interrupt
  * @irq_mask:		mask an interrupt source
  * @irq_mask_ack:	ack and mask an interrupt source
  * @irq_unmask:		unmask an interrupt source
  * @irq_eoi:		end of interrupt
  * @irq_set_affinity:	set the CPU affinity on SMP machines
  * @irq_retrigger:	resend an IRQ to the CPU
  * @irq_set_type:	set the flow type (IRQ_TYPE_LEVEL/etc.) of an IRQ
  * @irq_set_wake:	enable/disable power-management wake-on of an IRQ
  * @irq_bus_lock:	function to lock access to slow bus (i2c) chips
  * @irq_bus_sync_unlock:function to sync and unlock slow bus (i2c) chips
  * @irq_cpu_online:	configure an interrupt source for a secondary CPU
  * @irq_cpu_offline:	un-configure an interrupt source for a secondary CPU
  * @irq_suspend:	function called from core code on suspend once per chip
  * @irq_resume:		function called from core code on resume once per chip
  * @irq_pm_shutdown:	function called from core code on shutdown once per chip
  * @irq_calc_mask:	Optional function to set irq_data.mask for special cases
  * @irq_print_chip:	optional to print special chip info in show_interrupts
  * @irq_request_resources:	optional to request resources before calling
  *				any other callback related to this irq
  * @irq_release_resources:	optional to release resources acquired with
  *				irq_request_resources
  * @flags:		chip specific flags
  */
 struct irq_chip {
 	const char	*name;
 	unsigned int	(*irq_startup)(struct irq_data *data);
 	void		(*irq_shutdown)(struct irq_data *data);
 	void		(*irq_enable)(struct irq_data *data);
 	void		(*irq_disable)(struct irq_data *data);
 	void		(*irq_ack)(struct irq_data *data);
 	void		(*irq_mask)(struct irq_data *data);
 	void		(*irq_mask_ack)(struct irq_data *data);
 	void		(*irq_unmask)(struct irq_data *data);
 	void		(*irq_eoi)(struct irq_data *data);
 	int		(*irq_set_affinity)(struct irq_data *data, const struct cpumask *dest, bool force);
 	int		(*irq_retrigger)(struct irq_data *data);
 	int		(*irq_set_type)(struct irq_data *data, unsigned int flow_type);
 	int		(*irq_set_wake)(struct irq_data *data, unsigned int on);
 	void		(*irq_bus_lock)(struct irq_data *data);
 	void		(*irq_bus_sync_unlock)(struct irq_data *data);
 	void		(*irq_cpu_online)(struct irq_data *data);
 	void		(*irq_cpu_offline)(struct irq_data *data);
 	void		(*irq_suspend)(struct irq_data *data);
 	void		(*irq_resume)(struct irq_data *data);
 	void		(*irq_pm_shutdown)(struct irq_data *data);
 	void		(*irq_calc_mask)(struct irq_data *data);
 	void		(*irq_print_chip)(struct irq_data *data, struct seq_file *p);
 	int		(*irq_request_resources)(struct irq_data *data);
 	void		(*irq_release_resources)(struct irq_data *data);
 	unsigned long	flags;
 };
 /*
  * irq_chip specific flags
  *
  * IRQCHIP_SET_TYPE_MASKED:	Mask before calling chip.irq_set_type()
  * IRQCHIP_EOI_IF_HANDLED:	Only issue irq_eoi() when irq was handled
  * IRQCHIP_MASK_ON_SUSPEND:	Mask non wake irqs in the suspend path
  * IRQCHIP_ONOFFLINE_ENABLED:	Only call irq_on/off_line callbacks
  *				when irq enabled
  * IRQCHIP_SKIP_SET_WAKE:	Skip chip.irq_set_wake(), for this irq chip
  * IRQCHIP_ONESHOT_SAFE:	One shot does not require mask/unmask
  * IRQCHIP_EOI_THREADED:	Chip requires eoi() on unmask in threaded mode
  */
 enum {
 	IRQCHIP_SET_TYPE_MASKED		= (1 <<  0),
 	IRQCHIP_EOI_IF_HANDLED		= (1 <<  1),
 	IRQCHIP_MASK_ON_SUSPEND		= (1 <<  2),
 	IRQCHIP_ONOFFLINE_ENABLED	= (1 <<  3),
 	IRQCHIP_SKIP_SET_WAKE		= (1 <<  4),
 	IRQCHIP_ONESHOT_SAFE		= (1 <<  5),
 	IRQCHIP_EOI_THREADED		= (1 <<  6),
 };
 /* This include will go away once we isolated irq_desc usage to core code */
 #include <linux/irqdesc.h>
 /*
  * Pick up the arch-dependent methods:
  */
 #include <asm/hw_irq.h>
 #ifndef NR_IRQS_LEGACY
 # define NR_IRQS_LEGACY 0
 #endif
 #ifndef ARCH_IRQ_INIT_FLAGS
 # define ARCH_IRQ_INIT_FLAGS	0
 #endif
 #define IRQ_DEFAULT_INIT_FLAGS	ARCH_IRQ_INIT_FLAGS
 struct irqaction;
 extern int setup_irq(unsigned int irq, struct irqaction *new);
 extern void remove_irq(unsigned int irq, struct irqaction *act);
 extern int setup_percpu_irq(unsigned int irq, struct irqaction *new);
 extern void remove_percpu_irq(unsigned int irq, struct irqaction *act);
 extern void irq_cpu_online(void);
 extern void irq_cpu_offline(void);
-extern int __irq_set_affinity_locked(struct irq_data *data,  const struct cpumask *cpumask);
+extern int irq_set_affinity_locked(struct irq_data *data,
+				   const struct cpumask *cpumask, bool force);
 #if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_PENDING_IRQ)
 void irq_move_irq(struct irq_data *data);
 void irq_move_masked_irq(struct irq_data *data);
 #else
 static inline void irq_move_irq(struct irq_data *data) { }
 static inline void irq_move_masked_irq(struct irq_data *data) { }
 #endif
 extern int no_irq_affinity;
 #ifdef CONFIG_HARDIRQS_SW_RESEND
 int irq_set_parent(int irq, int parent_irq);
 #else
 static inline int irq_set_parent(int irq, int parent_irq)
 {
 	return 0;
 }
 #endif
 /*
  * Built-in IRQ handlers for various IRQ types,
  * callable via desc->handle_irq()
  */
 extern void handle_level_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_edge_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_simple_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_percpu_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_bad_irq(unsigned int irq, struct irq_desc *desc);
 extern void handle_nested_irq(unsigned int irq);
 /* Handling of unhandled and spurious interrupts: */
 extern void note_interrupt(unsigned int irq, struct irq_desc *desc,
 			   irqreturn_t action_ret);
 /* Enable/disable irq debugging output: */
 extern int noirqdebug_setup(char *str);
 /* Checks whether the interrupt can be requested by request_irq(): */
 extern int can_request_irq(unsigned int irq, unsigned long irqflags);
 /* Dummy irq-chip implementations: */
 extern struct irq_chip no_irq_chip;
 extern struct irq_chip dummy_irq_chip;
 extern void
 irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip,
 			      irq_flow_handler_t handle, const char *name);
 static inline void irq_set_chip_and_handler(unsigned int irq, struct irq_chip *chip,
 					    irq_flow_handler_t handle)
 {
 	irq_set_chip_and_handler_name(irq, chip, handle, NULL);
 }
 extern int irq_set_percpu_devid(unsigned int irq);
 extern void
 __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
 		  const char *name);
 static inline void
 irq_set_handler(unsigned int irq, irq_flow_handler_t handle)
 {
 	__irq_set_handler(irq, handle, 0, NULL);
 }
 /*
  * Set a highlevel chained flow handler for a given IRQ.
  * (a chained handler is automatically enabled and set to
  *  IRQ_NOREQUEST, IRQ_NOPROBE, and IRQ_NOTHREAD)
  */
 static inline void
 irq_set_chained_handler(unsigned int irq, irq_flow_handler_t handle)
 {
 	__irq_set_handler(irq, handle, 1, NULL);
 }
 void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set);
 static inline void irq_set_status_flags(unsigned int irq, unsigned long set)
 {
 	irq_modify_status(irq, 0, set);
 }
 static inline void irq_clear_status_flags(unsigned int irq, unsigned long clr)
 {
 	irq_modify_status(irq, clr, 0);
 }
 static inline void irq_set_noprobe(unsigned int irq)
 {
 	irq_modify_status(irq, 0, IRQ_NOPROBE);
 }
 static inline void irq_set_probe(unsigned int irq)
 {
 	irq_modify_status(irq, IRQ_NOPROBE, 0);
 }
 static inline void irq_set_nothread(unsigned int irq)
 {
 	irq_modify_status(irq, 0, IRQ_NOTHREAD);
 }
 static inline void irq_set_thread(unsigned int irq)
 {
 	irq_modify_status(irq, IRQ_NOTHREAD, 0);
 }
 static inline void irq_set_nested_thread(unsigned int irq, bool nest)
 {
 	if (nest)
 		irq_set_status_flags(irq, IRQ_NESTED_THREAD);
 	else
 		irq_clear_status_flags(irq, IRQ_NESTED_THREAD);
 }
 static inline void irq_set_percpu_devid_flags(unsigned int irq)
 {
 	irq_set_status_flags(irq,
 			     IRQ_NOAUTOEN | IRQ_PER_CPU | IRQ_NOTHREAD |
 			     IRQ_NOPROBE | IRQ_PER_CPU_DEVID);
 }
 /* Handle dynamic irq creation and destruction */
 extern unsigned int create_irq_nr(unsigned int irq_want, int node);
 extern unsigned int __create_irqs(unsigned int from, unsigned int count,
 				  int node);
 extern int create_irq(void);
 extern void destroy_irq(unsigned int irq);
 extern void destroy_irqs(unsigned int irq, unsigned int count);
 /*
  * Dynamic irq helper functions. Obsolete. Use irq_alloc_desc* and
  * irq_free_desc instead.
  */
 extern void dynamic_irq_cleanup(unsigned int irq);
 static inline void dynamic_irq_init(unsigned int irq)
 {
 	dynamic_irq_cleanup(irq);
 }
 /* Set/get chip/data for an IRQ: */
 extern int irq_set_chip(unsigned int irq, struct irq_chip *chip);
 extern int irq_set_handler_data(unsigned int irq, void *data);
 extern int irq_set_chip_data(unsigned int irq, void *data);
 extern int irq_set_irq_type(unsigned int irq, unsigned int type);
 extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry);
 extern int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset,
 				struct msi_desc *entry);
 extern struct irq_data *irq_get_irq_data(unsigned int irq);
 static inline struct irq_chip *irq_get_chip(unsigned int irq)
 {
 	struct irq_data *d = irq_get_irq_data(irq);
 	return d ? d->chip : NULL;
 }
 static inline struct irq_chip *irq_data_get_irq_chip(struct irq_data *d)
 {
 	return d->chip;
 }
 static inline void *irq_get_chip_data(unsigned int irq)
 {
 	struct irq_data *d = irq_get_irq_data(irq);
 	return d ? d->chip_data : NULL;
 }
 static inline void *irq_data_get_irq_chip_data(struct irq_data *d)
 {
 	return d->chip_data;
 }
 static inline void *irq_get_handler_data(unsigned int irq)
 {
 	struct irq_data *d = irq_get_irq_data(irq);
 	return d ? d->handler_data : NULL;
 }
 static inline void *irq_data_get_irq_handler_data(struct irq_data *d)
 {
 	return d->handler_data;
 }
 static inline struct msi_desc *irq_get_msi_desc(unsigned int irq)
 {
 	struct irq_data *d = irq_get_irq_data(irq);
 	return d ? d->msi_desc : NULL;
 }
 static inline struct msi_desc *irq_data_get_msi(struct irq_data *d)
 {
 	return d->msi_desc;
 }
 static inline u32 irq_get_trigger_type(unsigned int irq)
 {
 	struct irq_data *d = irq_get_irq_data(irq);
 	return d ? irqd_get_trigger_type(d) : 0;
 }
 int __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
 		struct module *owner);
 /* use macros to avoid needing export.h for THIS_MODULE */
 #define irq_alloc_descs(irq, from, cnt, node)	\
 	__irq_alloc_descs(irq, from, cnt, node, THIS_MODULE)
 #define irq_alloc_desc(node)			\
 	irq_alloc_descs(-1, 0, 1, node)
 #define irq_alloc_desc_at(at, node)		\
 	irq_alloc_descs(at, at, 1, node)
 #define irq_alloc_desc_from(from, node)		\
 	irq_alloc_descs(-1, from, 1, node)
 #define irq_alloc_descs_from(from, cnt, node)	\
 	irq_alloc_descs(-1, from, cnt, node)
 void irq_free_descs(unsigned int irq, unsigned int cnt);
 int irq_reserve_irqs(unsigned int from, unsigned int cnt);
 static inline void irq_free_desc(unsigned int irq)
 {
 	irq_free_descs(irq, 1);
 }
 static inline int irq_reserve_irq(unsigned int irq)
 {
 	return irq_reserve_irqs(irq, 1);
 }
 #ifndef irq_reg_writel
 # define irq_reg_writel(val, addr)	writel(val, addr)
 #endif
 #ifndef irq_reg_readl
 # define irq_reg_readl(addr)		readl(addr)
 #endif
 /**
  * struct irq_chip_regs - register offsets for struct irq_gci
  * @enable:	Enable register offset to reg_base
  * @disable:	Disable register offset to reg_base
  * @mask:	Mask register offset to reg_base
  * @ack:	Ack register offset to reg_base
  * @eoi:	Eoi register offset to reg_base
  * @type:	Type configuration register offset to reg_base
  * @polarity:	Polarity configuration register offset to reg_base
  */
 struct irq_chip_regs {
 	unsigned long		enable;
 	unsigned long		disable;
 	unsigned long		mask;
 	unsigned long		ack;
 	unsigned long		eoi;
 	unsigned long		type;
 	unsigned long		polarity;
 };
 /**
  * struct irq_chip_type - Generic interrupt chip instance for a flow type
  * @chip:		The real interrupt chip which provides the callbacks
  * @regs:		Register offsets for this chip
  * @handler:		Flow handler associated with this chip
  * @type:		Chip can handle these flow types
  * @mask_cache_priv:	Cached mask register private to the chip type
  * @mask_cache:		Pointer to cached mask register
  *
  * A irq_generic_chip can have several instances of irq_chip_type when
  * it requires different functions and register offsets for different
  * flow types.
  */
 struct irq_chip_type {
 	struct irq_chip		chip;
 	struct irq_chip_regs	regs;
 	irq_flow_handler_t	handler;
 	u32			type;
 	u32			mask_cache_priv;
 	u32			*mask_cache;
 };
 /**
  * struct irq_chip_generic - Generic irq chip data structure
  * @lock:		Lock to protect register and cache data access
  * @reg_base:		Register base address (virtual)
  * @irq_base:		Interrupt base nr for this chip
  * @irq_cnt:		Number of interrupts handled by this chip
  * @mask_cache:		Cached mask register shared between all chip types
  * @type_cache:		Cached type register
  * @polarity_cache:	Cached polarity register
  * @wake_enabled:	Interrupt can wakeup from suspend
  * @wake_active:	Interrupt is marked as an wakeup from suspend source
  * @num_ct:		Number of available irq_chip_type instances (usually 1)
  * @private:		Private data for non generic chip callbacks
  * @installed:		bitfield to denote installed interrupts
  * @unused:		bitfield to denote unused interrupts
  * @domain:		irq domain pointer
  * @list:		List head for keeping track of instances
  * @chip_types:		Array of interrupt irq_chip_types
  *
  * Note, that irq_chip_generic can have multiple irq_chip_type
  * implementations which can be associated to a particular irq line of
  * an irq_chip_generic instance. That allows to share and protect
  * state in an irq_chip_generic instance when we need to implement
  * different flow mechanisms (level/edge) for it.
  */
 struct irq_chip_generic {
 	raw_spinlock_t		lock;
 	void __iomem		*reg_base;
 	unsigned int		irq_base;
 	unsigned int		irq_cnt;
 	u32			mask_cache;
 	u32			type_cache;
 	u32			polarity_cache;
 	u32			wake_enabled;
 	u32			wake_active;
 	unsigned int		num_ct;
 	void			*private;
 	unsigned long		installed;
 	unsigned long		unused;
 	struct irq_domain	*domain;
 	struct list_head	list;
 	struct irq_chip_type	chip_types[0];
 };
 /**
  * enum irq_gc_flags - Initialization flags for generic irq chips
  * @IRQ_GC_INIT_MASK_CACHE:	Initialize the mask_cache by reading mask reg
  * @IRQ_GC_INIT_NESTED_LOCK:	Set the lock class of the irqs to nested for
  *				irq chips which need to call irq_set_wake() on
  *				the parent irq. Usually GPIO implementations
  * @IRQ_GC_MASK_CACHE_PER_TYPE:	Mask cache is chip type private
  * @IRQ_GC_NO_MASK:		Do not calculate irq_data->mask
  */
 enum irq_gc_flags {
 	IRQ_GC_INIT_MASK_CACHE		= 1 << 0,
 	IRQ_GC_INIT_NESTED_LOCK		= 1 << 1,
 	IRQ_GC_MASK_CACHE_PER_TYPE	= 1 << 2,
 	IRQ_GC_NO_MASK			= 1 << 3,
 };
 /*
  * struct irq_domain_chip_generic - Generic irq chip data structure for irq domains
  * @irqs_per_chip:	Number of interrupts per chip
  * @num_chips:		Number of chips
  * @irq_flags_to_set:	IRQ* flags to set on irq setup
  * @irq_flags_to_clear:	IRQ* flags to clear on irq setup
  * @gc_flags:		Generic chip specific setup flags
  * @gc:			Array of pointers to generic interrupt chips
  */
 struct irq_domain_chip_generic {
 	unsigned int		irqs_per_chip;
 	unsigned int		num_chips;
 	unsigned int		irq_flags_to_clear;
 	unsigned int		irq_flags_to_set;
 	enum irq_gc_flags	gc_flags;
 	struct irq_chip_generic	*gc[0];
 };
 /* Generic chip callback functions */
 void irq_gc_noop(struct irq_data *d);
 void irq_gc_mask_disable_reg(struct irq_data *d);
 void irq_gc_mask_set_bit(struct irq_data *d);
 void irq_gc_mask_clr_bit(struct irq_data *d);
 void irq_gc_unmask_enable_reg(struct irq_data *d);
 void irq_gc_ack_set_bit(struct irq_data *d);
 void irq_gc_ack_clr_bit(struct irq_data *d);
 void irq_gc_mask_disable_reg_and_ack(struct irq_data *d);
 void irq_gc_eoi(struct irq_data *d);
 int irq_gc_set_wake(struct irq_data *d, unsigned int on);
 /* Setup functions for irq_chip_generic */
 struct irq_chip_generic *
 irq_alloc_generic_chip(const char *name, int nr_ct, unsigned int irq_base,
 		       void __iomem *reg_base, irq_flow_handler_t handler);
 void irq_setup_generic_chip(struct irq_chip_generic *gc, u32 msk,
 			    enum irq_gc_flags flags, unsigned int clr,
 			    unsigned int set);
 int irq_setup_alt_chip(struct irq_data *d, unsigned int type);
 void irq_remove_generic_chip(struct irq_chip_generic *gc, u32 msk,
 			     unsigned int clr, unsigned int set);
 struct irq_chip_generic *irq_get_domain_generic_chip(struct irq_domain *d, unsigned int hw_irq);
 int irq_alloc_domain_generic_chips(struct irq_domain *d, int irqs_per_chip,
 				   int num_ct, const char *name,
 				   irq_flow_handler_t handler,
 				   unsigned int clr, unsigned int set,
 				   enum irq_gc_flags flags);
 static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d)
 {
 	return container_of(d->chip, struct irq_chip_type, chip);
 }
 #define IRQ_MSK(n) (u32)((n) < 32 ? ((1 << (n)) - 1) : UINT_MAX)
 #ifdef CONFIG_SMP
 static inline void irq_gc_lock(struct irq_chip_generic *gc)
 {
 	raw_spin_lock(&gc->lock);
 }
 static inline void irq_gc_unlock(struct irq_chip_generic *gc)
 {
 	raw_spin_unlock(&gc->lock);
 }
 #else
 static inline void irq_gc_lock(struct irq_chip_generic *gc) { }
 static inline void irq_gc_unlock(struct irq_chip_generic *gc) { }
 #endif
 #endif /* _LINUX_IRQ_H */

kernel/irq/manage.c

Diff comments View file @ d9e9e8e

 /*
  * linux/kernel/irq/manage.c
  *
  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
  * Copyright (C) 2005-2006 Thomas Gleixner
  *
  * This file contains driver APIs to the irq subsystem.
  */
 #define pr_fmt(fmt) "genirq: " fmt
 #include <linux/irq.h>
 #include <linux/kthread.h>
 #include <linux/module.h>
 #include <linux/random.h>
 #include <linux/interrupt.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/sched/rt.h>
 #include <linux/task_work.h>
 #include "internals.h"
 #ifdef CONFIG_IRQ_FORCED_THREADING
 __read_mostly bool force_irqthreads;
 static int __init setup_forced_irqthreads(char *arg)
 {
 	force_irqthreads = true;
 	return 0;
 }
 early_param("threadirqs", setup_forced_irqthreads);
 #endif
 static void __synchronize_hardirq(struct irq_desc *desc)
 {
 	bool inprogress;
 	do {
 		unsigned long flags;
 		/*
 		 * Wait until we're out of the critical section.  This might
 		 * give the wrong answer due to the lack of memory barriers.
 		 */
 		while (irqd_irq_inprogress(&desc->irq_data))
 			cpu_relax();
 		/* Ok, that indicated we're done: double-check carefully. */
 		raw_spin_lock_irqsave(&desc->lock, flags);
 		inprogress = irqd_irq_inprogress(&desc->irq_data);
 		raw_spin_unlock_irqrestore(&desc->lock, flags);
 		/* Oops, that failed? */
 	} while (inprogress);
 }
 /**
  *	synchronize_hardirq - wait for pending hard IRQ handlers (on other CPUs)
  *	@irq: interrupt number to wait for
  *
  *	This function waits for any pending hard IRQ handlers for this
  *	interrupt to complete before returning. If you use this
  *	function while holding a resource the IRQ handler may need you
  *	will deadlock. It does not take associated threaded handlers
  *	into account.
  *
  *	Do not use this for shutdown scenarios where you must be sure
  *	that all parts (hardirq and threaded handler) have completed.
  *
  *	This function may be called - with care - from IRQ context.
  */
 void synchronize_hardirq(unsigned int irq)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (desc)
 		__synchronize_hardirq(desc);
 }
 EXPORT_SYMBOL(synchronize_hardirq);
 /**
  *	synchronize_irq - wait for pending IRQ handlers (on other CPUs)
  *	@irq: interrupt number to wait for
  *
  *	This function waits for any pending IRQ handlers for this interrupt
  *	to complete before returning. If you use this function while
  *	holding a resource the IRQ handler may need you will deadlock.
  *
  *	This function may be called - with care - from IRQ context.
  */
 void synchronize_irq(unsigned int irq)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (desc) {
 		__synchronize_hardirq(desc);
 		/*
 		 * We made sure that no hardirq handler is
 		 * running. Now verify that no threaded handlers are
 		 * active.
 		 */
 		wait_event(desc->wait_for_threads,
 			   !atomic_read(&desc->threads_active));
 	}
 }
 EXPORT_SYMBOL(synchronize_irq);
 #ifdef CONFIG_SMP
 cpumask_var_t irq_default_affinity;
 /**
  *	irq_can_set_affinity - Check if the affinity of a given irq can be set
  *	@irq:		Interrupt to check
  *
  */
 int irq_can_set_affinity(unsigned int irq)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (!desc || !irqd_can_balance(&desc->irq_data) ||
 	    !desc->irq_data.chip || !desc->irq_data.chip->irq_set_affinity)
 		return 0;
 	return 1;
 }
 /**
  *	irq_set_thread_affinity - Notify irq threads to adjust affinity
  *	@desc:		irq descriptor which has affitnity changed
  *
  *	We just set IRQTF_AFFINITY and delegate the affinity setting
  *	to the interrupt thread itself. We can not call
  *	set_cpus_allowed_ptr() here as we hold desc->lock and this
  *	code can be called from hard interrupt context.
  */
 void irq_set_thread_affinity(struct irq_desc *desc)
 {
 	struct irqaction *action = desc->action;
 	while (action) {
 		if (action->thread)
 			set_bit(IRQTF_AFFINITY, &action->thread_flags);
 		action = action->next;
 	}
 }
 #ifdef CONFIG_GENERIC_PENDING_IRQ
 static inline bool irq_can_move_pcntxt(struct irq_data *data)
 {
 	return irqd_can_move_in_process_context(data);
 }
 static inline bool irq_move_pending(struct irq_data *data)
 {
 	return irqd_is_setaffinity_pending(data);
 }
 static inline void
 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
 {
 	cpumask_copy(desc->pending_mask, mask);
 }
 static inline void
 irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
 {
 	cpumask_copy(mask, desc->pending_mask);
 }
 #else
 static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
 static inline bool irq_move_pending(struct irq_data *data) { return false; }
 static inline void
 irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
 static inline void
 irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
 #endif
 int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
 			bool force)
 {
 	struct irq_desc *desc = irq_data_to_desc(data);
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	int ret;
-	ret = chip->irq_set_affinity(data, mask, false);
+	ret = chip->irq_set_affinity(data, mask, force);
 	switch (ret) {
 	case IRQ_SET_MASK_OK:
 		cpumask_copy(data->affinity, mask);
 	case IRQ_SET_MASK_OK_NOCOPY:
 		irq_set_thread_affinity(desc);
 		ret = 0;
 	}
 	return ret;
 }
-int __irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask)
+int irq_set_affinity_locked(struct irq_data *data, const struct cpumask *mask,
+			    bool force)
 {
 	struct irq_chip *chip = irq_data_get_irq_chip(data);
 	struct irq_desc *desc = irq_data_to_desc(data);
 	int ret = 0;
 	if (!chip || !chip->irq_set_affinity)
 		return -EINVAL;
 	if (irq_can_move_pcntxt(data)) {
-		ret = irq_do_set_affinity(data, mask, false);
+		ret = irq_do_set_affinity(data, mask, force);
 	} else {
 		irqd_set_move_pending(data);
 		irq_copy_pending(desc, mask);
 	}
 	if (desc->affinity_notify) {
 		kref_get(&desc->affinity_notify->kref);
 		schedule_work(&desc->affinity_notify->work);
 	}
 	irqd_set(data, IRQD_AFFINITY_SET);
 	return ret;
 }
-/**
+int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
- *	irq_set_affinity - Set the irq affinity of a given irq
- *	@irq:		Interrupt to set affinity
- *	@mask:		cpumask
- *
- */
-int irq_set_affinity(unsigned int irq, const struct cpumask *mask)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	unsigned long flags;
 	int ret;
 	if (!desc)
 		return -EINVAL;
 	raw_spin_lock_irqsave(&desc->lock, flags);
-	ret =  __irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask);
+	ret = irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask, force);
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	return ret;
 }
 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 	if (!desc)
 		return -EINVAL;
 	desc->affinity_hint = m;
 	irq_put_desc_unlock(desc, flags);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(irq_set_affinity_hint);
 static void irq_affinity_notify(struct work_struct *work)
 {
 	struct irq_affinity_notify *notify =
 		container_of(work, struct irq_affinity_notify, work);
 	struct irq_desc *desc = irq_to_desc(notify->irq);
 	cpumask_var_t cpumask;
 	unsigned long flags;
 	if (!desc || !alloc_cpumask_var(&cpumask, GFP_KERNEL))
 		goto out;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	if (irq_move_pending(&desc->irq_data))
 		irq_get_pending(cpumask, desc);
 	else
 		cpumask_copy(cpumask, desc->irq_data.affinity);
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	notify->notify(notify, cpumask);
 	free_cpumask_var(cpumask);
 out:
 	kref_put(&notify->kref, notify->release);
 }
 /**
  *	irq_set_affinity_notifier - control notification of IRQ affinity changes
  *	@irq:		Interrupt for which to enable/disable notification
  *	@notify:	Context for notification, or %NULL to disable
  *			notification.  Function pointers must be initialised;
  *			the other fields will be initialised by this function.
  *
  *	Must be called in process context.  Notification may only be enabled
  *	after the IRQ is allocated and must be disabled before the IRQ is
  *	freed using free_irq().
  */
 int
 irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	struct irq_affinity_notify *old_notify;
 	unsigned long flags;
 	/* The release function is promised process context */
 	might_sleep();
 	if (!desc)
 		return -EINVAL;
 	/* Complete initialisation of *notify */
 	if (notify) {
 		notify->irq = irq;
 		kref_init(&notify->kref);
 		INIT_WORK(&notify->work, irq_affinity_notify);
 	}
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	old_notify = desc->affinity_notify;
 	desc->affinity_notify = notify;
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	if (old_notify)
 		kref_put(&old_notify->kref, old_notify->release);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
 #ifndef CONFIG_AUTO_IRQ_AFFINITY
 /*
  * Generic version of the affinity autoselector.
  */
 static int
 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 {
 	struct cpumask *set = irq_default_affinity;
 	int node = desc->irq_data.node;
 	/* Excludes PER_CPU and NO_BALANCE interrupts */
 	if (!irq_can_set_affinity(irq))
 		return 0;
 	/*
 	 * Preserve an userspace affinity setup, but make sure that
 	 * one of the targets is online.
 	 */
 	if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) {
 		if (cpumask_intersects(desc->irq_data.affinity,
 				       cpu_online_mask))
 			set = desc->irq_data.affinity;
 		else
 			irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
 	}
 	cpumask_and(mask, cpu_online_mask, set);
 	if (node != NUMA_NO_NODE) {
 		const struct cpumask *nodemask = cpumask_of_node(node);
 		/* make sure at least one of the cpus in nodemask is online */
 		if (cpumask_intersects(mask, nodemask))
 			cpumask_and(mask, mask, nodemask);
 	}
 	irq_do_set_affinity(&desc->irq_data, mask, false);
 	return 0;
 }
 #else
 static inline int
 setup_affinity(unsigned int irq, struct irq_desc *d, struct cpumask *mask)
 {
 	return irq_select_affinity(irq);
 }
 #endif
 /*
  * Called when affinity is set via /proc/irq
  */
 int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	unsigned long flags;
 	int ret;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	ret = setup_affinity(irq, desc, mask);
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	return ret;
 }
 #else
 static inline int
 setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask)
 {
 	return 0;
 }
 #endif
 void __disable_irq(struct irq_desc *desc, unsigned int irq, bool suspend)
 {
 	if (suspend) {
 		if (!desc->action || (desc->action->flags & IRQF_NO_SUSPEND))
 			return;
 		desc->istate |= IRQS_SUSPENDED;
 	}
 	if (!desc->depth++)
 		irq_disable(desc);
 }
 static int __disable_irq_nosync(unsigned int irq)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 	if (!desc)
 		return -EINVAL;
 	__disable_irq(desc, irq, false);
 	irq_put_desc_busunlock(desc, flags);
 	return 0;
 }
 /**
  *	disable_irq_nosync - disable an irq without waiting
  *	@irq: Interrupt to disable
  *
  *	Disable the selected interrupt line.  Disables and Enables are
  *	nested.
  *	Unlike disable_irq(), this function does not ensure existing
  *	instances of the IRQ handler have completed before returning.
  *
  *	This function may be called from IRQ context.
  */
 void disable_irq_nosync(unsigned int irq)
 {
 	__disable_irq_nosync(irq);
 }
 EXPORT_SYMBOL(disable_irq_nosync);
 /**
  *	disable_irq - disable an irq and wait for completion
  *	@irq: Interrupt to disable
  *
  *	Disable the selected interrupt line.  Enables and Disables are
  *	nested.
  *	This function waits for any pending IRQ handlers for this interrupt
  *	to complete before returning. If you use this function while
  *	holding a resource the IRQ handler may need you will deadlock.
  *
  *	This function may be called - with care - from IRQ context.
  */
 void disable_irq(unsigned int irq)
 {
 	if (!__disable_irq_nosync(irq))
 		synchronize_irq(irq);
 }
 EXPORT_SYMBOL(disable_irq);
 void __enable_irq(struct irq_desc *desc, unsigned int irq, bool resume)
 {
 	if (resume) {
 		if (!(desc->istate & IRQS_SUSPENDED)) {
 			if (!desc->action)
 				return;
 			if (!(desc->action->flags & IRQF_FORCE_RESUME))
 				return;
 			/* Pretend that it got disabled ! */
 			desc->depth++;
 		}
 		desc->istate &= ~IRQS_SUSPENDED;
 	}
 	switch (desc->depth) {
 	case 0:
  err_out:
 		WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
 		break;
 	case 1: {
 		if (desc->istate & IRQS_SUSPENDED)
 			goto err_out;
 		/* Prevent probing on this irq: */
 		irq_settings_set_noprobe(desc);
 		irq_enable(desc);
 		check_irq_resend(desc, irq);
 		/* fall-through */
 	}
 	default:
 		desc->depth--;
 	}
 }
 /**
  *	enable_irq - enable handling of an irq
  *	@irq: Interrupt to enable
  *
  *	Undoes the effect of one call to disable_irq().  If this
  *	matches the last disable, processing of interrupts on this
  *	IRQ line is re-enabled.
  *
  *	This function may be called from IRQ context only when
  *	desc->irq_data.chip->bus_lock and desc->chip->bus_sync_unlock are NULL !
  */
 void enable_irq(unsigned int irq)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 	if (!desc)
 		return;
 	if (WARN(!desc->irq_data.chip,
 		 KERN_ERR "enable_irq before setup/request_irq: irq %u\n", irq))
 		goto out;
 	__enable_irq(desc, irq, false);
 out:
 	irq_put_desc_busunlock(desc, flags);
 }
 EXPORT_SYMBOL(enable_irq);
 static int set_irq_wake_real(unsigned int irq, unsigned int on)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	int ret = -ENXIO;
 	if (irq_desc_get_chip(desc)->flags &  IRQCHIP_SKIP_SET_WAKE)
 		return 0;
 	if (desc->irq_data.chip->irq_set_wake)
 		ret = desc->irq_data.chip->irq_set_wake(&desc->irq_data, on);
 	return ret;
 }
 /**
  *	irq_set_irq_wake - control irq power management wakeup
  *	@irq:	interrupt to control
  *	@on:	enable/disable power management wakeup
  *
  *	Enable/disable power management wakeup mode, which is
  *	disabled by default.  Enables and disables must match,
  *	just as they match for non-wakeup mode support.
  *
  *	Wakeup mode lets this IRQ wake the system from sleep
  *	states like "suspend to RAM".
  */
 int irq_set_irq_wake(unsigned int irq, unsigned int on)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_buslock(irq, &flags, IRQ_GET_DESC_CHECK_GLOBAL);
 	int ret = 0;
 	if (!desc)
 		return -EINVAL;
 	/* wakeup-capable irqs can be shared between drivers that
 	 * don't need to have the same sleep mode behaviors.
 	 */
 	if (on) {
 		if (desc->wake_depth++ == 0) {
 			ret = set_irq_wake_real(irq, on);
 			if (ret)
 				desc->wake_depth = 0;
 			else
 				irqd_set(&desc->irq_data, IRQD_WAKEUP_STATE);
 		}
 	} else {
 		if (desc->wake_depth == 0) {
 			WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
 		} else if (--desc->wake_depth == 0) {
 			ret = set_irq_wake_real(irq, on);
 			if (ret)
 				desc->wake_depth = 1;
 			else
 				irqd_clear(&desc->irq_data, IRQD_WAKEUP_STATE);
 		}
 	}
 	irq_put_desc_busunlock(desc, flags);
 	return ret;
 }
 EXPORT_SYMBOL(irq_set_irq_wake);
 /*
  * Internal function that tells the architecture code whether a
  * particular irq has been exclusively allocated or is available
  * for driver use.
  */
 int can_request_irq(unsigned int irq, unsigned long irqflags)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 	int canrequest = 0;
 	if (!desc)
 		return 0;
 	if (irq_settings_can_request(desc)) {
 		if (!desc->action ||
 		    irqflags & desc->action->flags & IRQF_SHARED)
 			canrequest = 1;
 	}
 	irq_put_desc_unlock(desc, flags);
 	return canrequest;
 }
 int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
 		      unsigned long flags)
 {
 	struct irq_chip *chip = desc->irq_data.chip;
 	int ret, unmask = 0;
 	if (!chip || !chip->irq_set_type) {
 		/*
 		 * IRQF_TRIGGER_* but the PIC does not support multiple
 		 * flow-types?
 		 */
 		pr_debug("No set_type function for IRQ %d (%s)\n", irq,
 			 chip ? (chip->name ? : "unknown") : "unknown");
 		return 0;
 	}
 	flags &= IRQ_TYPE_SENSE_MASK;
 	if (chip->flags & IRQCHIP_SET_TYPE_MASKED) {
 		if (!irqd_irq_masked(&desc->irq_data))
 			mask_irq(desc);
 		if (!irqd_irq_disabled(&desc->irq_data))
 			unmask = 1;
 	}
 	/* caller masked out all except trigger mode flags */
 	ret = chip->irq_set_type(&desc->irq_data, flags);
 	switch (ret) {
 	case IRQ_SET_MASK_OK:
 		irqd_clear(&desc->irq_data, IRQD_TRIGGER_MASK);
 		irqd_set(&desc->irq_data, flags);
 	case IRQ_SET_MASK_OK_NOCOPY:
 		flags = irqd_get_trigger_type(&desc->irq_data);
 		irq_settings_set_trigger_mask(desc, flags);
 		irqd_clear(&desc->irq_data, IRQD_LEVEL);
 		irq_settings_clr_level(desc);
 		if (flags & IRQ_TYPE_LEVEL_MASK) {
 			irq_settings_set_level(desc);
 			irqd_set(&desc->irq_data, IRQD_LEVEL);
 		}
 		ret = 0;
 		break;
 	default:
 		pr_err("Setting trigger mode %lu for irq %u failed (%pF)\n",
 		       flags, irq, chip->irq_set_type);
 	}
 	if (unmask)
 		unmask_irq(desc);
 	return ret;
 }
 #ifdef CONFIG_HARDIRQS_SW_RESEND
 int irq_set_parent(int irq, int parent_irq)
 {
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, 0);
 	if (!desc)
 		return -EINVAL;
 	desc->parent_irq = parent_irq;
 	irq_put_desc_unlock(desc, flags);
 	return 0;
 }
 #endif
 /*
  * Default primary interrupt handler for threaded interrupts. Is
  * assigned as primary handler when request_threaded_irq is called
  * with handler == NULL. Useful for oneshot interrupts.
  */
 static irqreturn_t irq_default_primary_handler(int irq, void *dev_id)
 {
 	return IRQ_WAKE_THREAD;
 }
 /*
  * Primary handler for nested threaded interrupts. Should never be
  * called.
  */
 static irqreturn_t irq_nested_primary_handler(int irq, void *dev_id)
 {
 	WARN(1, "Primary handler called for nested irq %d\n", irq);
 	return IRQ_NONE;
 }
 static int irq_wait_for_interrupt(struct irqaction *action)
 {
 	set_current_state(TASK_INTERRUPTIBLE);
 	while (!kthread_should_stop()) {
 		if (test_and_clear_bit(IRQTF_RUNTHREAD,
 				       &action->thread_flags)) {
 			__set_current_state(TASK_RUNNING);
 			return 0;
 		}
 		schedule();
 		set_current_state(TASK_INTERRUPTIBLE);
 	}
 	__set_current_state(TASK_RUNNING);
 	return -1;
 }
 /*
  * Oneshot interrupts keep the irq line masked until the threaded
  * handler finished. unmask if the interrupt has not been disabled and
  * is marked MASKED.
  */
 static void irq_finalize_oneshot(struct irq_desc *desc,
 				 struct irqaction *action)
 {
 	if (!(desc->istate & IRQS_ONESHOT))
 		return;
 again:
 	chip_bus_lock(desc);
 	raw_spin_lock_irq(&desc->lock);
 	/*
 	 * Implausible though it may be we need to protect us against
 	 * the following scenario:
 	 *
 	 * The thread is faster done than the hard interrupt handler
 	 * on the other CPU. If we unmask the irq line then the
 	 * interrupt can come in again and masks the line, leaves due
 	 * to IRQS_INPROGRESS and the irq line is masked forever.
 	 *
 	 * This also serializes the state of shared oneshot handlers
 	 * versus "desc->threads_onehsot |= action->thread_mask;" in
 	 * irq_wake_thread(). See the comment there which explains the
 	 * serialization.
 	 */
 	if (unlikely(irqd_irq_inprogress(&desc->irq_data))) {
 		raw_spin_unlock_irq(&desc->lock);
 		chip_bus_sync_unlock(desc);
 		cpu_relax();
 		goto again;
 	}
 	/*
 	 * Now check again, whether the thread should run. Otherwise
 	 * we would clear the threads_oneshot bit of this thread which
 	 * was just set.
 	 */
 	if (test_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 		goto out_unlock;
 	desc->threads_oneshot &= ~action->thread_mask;
 	if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) &&
 	    irqd_irq_masked(&desc->irq_data))
 		unmask_threaded_irq(desc);
 out_unlock:
 	raw_spin_unlock_irq(&desc->lock);
 	chip_bus_sync_unlock(desc);
 }
 #ifdef CONFIG_SMP
 /*
  * Check whether we need to change the affinity of the interrupt thread.
  */
 static void
 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action)
 {
 	cpumask_var_t mask;
 	bool valid = true;
 	if (!test_and_clear_bit(IRQTF_AFFINITY, &action->thread_flags))
 		return;
 	/*
 	 * In case we are out of memory we set IRQTF_AFFINITY again and
 	 * try again next time
 	 */
 	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
 		set_bit(IRQTF_AFFINITY, &action->thread_flags);
 		return;
 	}
 	raw_spin_lock_irq(&desc->lock);
 	/*
 	 * This code is triggered unconditionally. Check the affinity
 	 * mask pointer. For CPU_MASK_OFFSTACK=n this is optimized out.
 	 */
 	if (desc->irq_data.affinity)
 		cpumask_copy(mask, desc->irq_data.affinity);
 	else
 		valid = false;
 	raw_spin_unlock_irq(&desc->lock);
 	if (valid)
 		set_cpus_allowed_ptr(current, mask);
 	free_cpumask_var(mask);
 }
 #else
 static inline void
 irq_thread_check_affinity(struct irq_desc *desc, struct irqaction *action) { }
 #endif
 /*
  * Interrupts which are not explicitely requested as threaded
  * interrupts rely on the implicit bh/preempt disable of the hard irq
  * context. So we need to disable bh here to avoid deadlocks and other
  * side effects.
  */
 static irqreturn_t
 irq_forced_thread_fn(struct irq_desc *desc, struct irqaction *action)
 {
 	irqreturn_t ret;
 	local_bh_disable();
 	ret = action->thread_fn(action->irq, action->dev_id);
 	irq_finalize_oneshot(desc, action);
 	local_bh_enable();
 	return ret;
 }
 /*
  * Interrupts explicitly requested as threaded interrupts want to be
  * preemtible - many of them need to sleep and wait for slow busses to
  * complete.
  */
 static irqreturn_t irq_thread_fn(struct irq_desc *desc,
 		struct irqaction *action)
 {
 	irqreturn_t ret;
 	ret = action->thread_fn(action->irq, action->dev_id);
 	irq_finalize_oneshot(desc, action);
 	return ret;
 }
 static void wake_threads_waitq(struct irq_desc *desc)
 {
 	if (atomic_dec_and_test(&desc->threads_active))
 		wake_up(&desc->wait_for_threads);
 }
 static void irq_thread_dtor(struct callback_head *unused)
 {
 	struct task_struct *tsk = current;
 	struct irq_desc *desc;
 	struct irqaction *action;
 	if (WARN_ON_ONCE(!(current->flags & PF_EXITING)))
 		return;
 	action = kthread_data(tsk);
 	pr_err("exiting task \"%s\" (%d) is an active IRQ thread (irq %d)\n",
 	       tsk->comm, tsk->pid, action->irq);
 	desc = irq_to_desc(action->irq);
 	/*
 	 * If IRQTF_RUNTHREAD is set, we need to decrement
 	 * desc->threads_active and wake possible waiters.
 	 */
 	if (test_and_clear_bit(IRQTF_RUNTHREAD, &action->thread_flags))
 		wake_threads_waitq(desc);
 	/* Prevent a stale desc->threads_oneshot */
 	irq_finalize_oneshot(desc, action);
 }
 /*
  * Interrupt handler thread
  */
 static int irq_thread(void *data)
 {
 	struct callback_head on_exit_work;
 	struct irqaction *action = data;
 	struct irq_desc *desc = irq_to_desc(action->irq);
 	irqreturn_t (*handler_fn)(struct irq_desc *desc,
 			struct irqaction *action);
 	if (force_irqthreads && test_bit(IRQTF_FORCED_THREAD,
 					&action->thread_flags))
 		handler_fn = irq_forced_thread_fn;
 	else
 		handler_fn = irq_thread_fn;
 	init_task_work(&on_exit_work, irq_thread_dtor);
 	task_work_add(current, &on_exit_work, false);
 	irq_thread_check_affinity(desc, action);
 	while (!irq_wait_for_interrupt(action)) {
 		irqreturn_t action_ret;
 		irq_thread_check_affinity(desc, action);
 		action_ret = handler_fn(desc, action);
 		if (!noirqdebug)
 			note_interrupt(action->irq, desc, action_ret);
 		wake_threads_waitq(desc);
 	}
 	/*
 	 * This is the regular exit path. __free_irq() is stopping the
 	 * thread via kthread_stop() after calling
 	 * synchronize_irq(). So neither IRQTF_RUNTHREAD nor the
 	 * oneshot mask bit can be set. We cannot verify that as we
 	 * cannot touch the oneshot mask at this point anymore as
 	 * __setup_irq() might have given out currents thread_mask
 	 * again.
 	 */
 	task_work_cancel(current, irq_thread_dtor);
 	return 0;
 }
 /**
  *	irq_wake_thread - wake the irq thread for the action identified by dev_id
  *	@irq:		Interrupt line
  *	@dev_id:	Device identity for which the thread should be woken
  *
  */
 void irq_wake_thread(unsigned int irq, void *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	struct irqaction *action;
 	unsigned long flags;
 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 		return;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	for (action = desc->action; action; action = action->next) {
 		if (action->dev_id == dev_id) {
 			if (action->thread)
 				__irq_wake_thread(desc, action);
 			break;
 		}
 	}
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 }
 EXPORT_SYMBOL_GPL(irq_wake_thread);
 static void irq_setup_forced_threading(struct irqaction *new)
 {
 	if (!force_irqthreads)
 		return;
 	if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT))
 		return;
 	new->flags |= IRQF_ONESHOT;
 	if (!new->thread_fn) {
 		set_bit(IRQTF_FORCED_THREAD, &new->thread_flags);
 		new->thread_fn = new->handler;
 		new->handler = irq_default_primary_handler;
 	}
 }
 static int irq_request_resources(struct irq_desc *desc)
 {
 	struct irq_data *d = &desc->irq_data;
 	struct irq_chip *c = d->chip;
 	return c->irq_request_resources ? c->irq_request_resources(d) : 0;
 }
 static void irq_release_resources(struct irq_desc *desc)
 {
 	struct irq_data *d = &desc->irq_data;
 	struct irq_chip *c = d->chip;
 	if (c->irq_release_resources)
 		c->irq_release_resources(d);
 }
 /*
  * Internal function to register an irqaction - typically used to
  * allocate special interrupts that are part of the architecture.
  */
 static int
 __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
 {
 	struct irqaction *old, **old_ptr;
 	unsigned long flags, thread_mask = 0;
 	int ret, nested, shared = 0;
 	cpumask_var_t mask;
 	if (!desc)
 		return -EINVAL;
 	if (desc->irq_data.chip == &no_irq_chip)
 		return -ENOSYS;
 	if (!try_module_get(desc->owner))
 		return -ENODEV;
 	/*
 	 * Check whether the interrupt nests into another interrupt
 	 * thread.
 	 */
 	nested = irq_settings_is_nested_thread(desc);
 	if (nested) {
 		if (!new->thread_fn) {
 			ret = -EINVAL;
 			goto out_mput;
 		}
 		/*
 		 * Replace the primary handler which was provided from
 		 * the driver for non nested interrupt handling by the
 		 * dummy function which warns when called.
 		 */
 		new->handler = irq_nested_primary_handler;
 	} else {
 		if (irq_settings_can_thread(desc))
 			irq_setup_forced_threading(new);
 	}
 	/*
 	 * Create a handler thread when a thread function is supplied
 	 * and the interrupt does not nest into another interrupt
 	 * thread.
 	 */
 	if (new->thread_fn && !nested) {
 		struct task_struct *t;
 		static const struct sched_param param = {
 			.sched_priority = MAX_USER_RT_PRIO/2,
 		};
 		t = kthread_create(irq_thread, new, "irq/%d-%s", irq,
 				   new->name);
 		if (IS_ERR(t)) {
 			ret = PTR_ERR(t);
 			goto out_mput;
 		}
 		sched_setscheduler_nocheck(t, SCHED_FIFO, &param);
 		/*
 		 * We keep the reference to the task struct even if
 		 * the thread dies to avoid that the interrupt code
 		 * references an already freed task_struct.
 		 */
 		get_task_struct(t);
 		new->thread = t;
 		/*
 		 * Tell the thread to set its affinity. This is
 		 * important for shared interrupt handlers as we do
 		 * not invoke setup_affinity() for the secondary
 		 * handlers as everything is already set up. Even for
 		 * interrupts marked with IRQF_NO_BALANCE this is
 		 * correct as we want the thread to move to the cpu(s)
 		 * on which the requesting code placed the interrupt.
 		 */
 		set_bit(IRQTF_AFFINITY, &new->thread_flags);
 	}
 	if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
 		ret = -ENOMEM;
 		goto out_thread;
 	}
 	/*
 	 * Drivers are often written to work w/o knowledge about the
 	 * underlying irq chip implementation, so a request for a
 	 * threaded irq without a primary hard irq context handler
 	 * requires the ONESHOT flag to be set. Some irq chips like
 	 * MSI based interrupts are per se one shot safe. Check the
 	 * chip flags, so we can avoid the unmask dance at the end of
 	 * the threaded handler for those.
 	 */
 	if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)
 		new->flags &= ~IRQF_ONESHOT;
 	/*
 	 * The following block of code has to be executed atomically
 	 */
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	old_ptr = &desc->action;
 	old = *old_ptr;
 	if (old) {
 		/*
 		 * Can't share interrupts unless both agree to and are
 		 * the same type (level, edge, polarity). So both flag
 		 * fields must have IRQF_SHARED set and the bits which
 		 * set the trigger type must match. Also all must
 		 * agree on ONESHOT.
 		 */
 		if (!((old->flags & new->flags) & IRQF_SHARED) ||
 		    ((old->flags ^ new->flags) & IRQF_TRIGGER_MASK) ||
 		    ((old->flags ^ new->flags) & IRQF_ONESHOT))
 			goto mismatch;
 		/* All handlers must agree on per-cpuness */
 		if ((old->flags & IRQF_PERCPU) !=
 		    (new->flags & IRQF_PERCPU))
 			goto mismatch;
 		/* add new interrupt at end of irq queue */
 		do {
 			/*
 			 * Or all existing action->thread_mask bits,
 			 * so we can find the next zero bit for this
 			 * new action.
 			 */
 			thread_mask |= old->thread_mask;
 			old_ptr = &old->next;
 			old = *old_ptr;
 		} while (old);
 		shared = 1;
 	}
 	/*
 	 * Setup the thread mask for this irqaction for ONESHOT. For
 	 * !ONESHOT irqs the thread mask is 0 so we can avoid a
 	 * conditional in irq_wake_thread().
 	 */
 	if (new->flags & IRQF_ONESHOT) {
 		/*
 		 * Unlikely to have 32 resp 64 irqs sharing one line,
 		 * but who knows.
 		 */
 		if (thread_mask == ~0UL) {
 			ret = -EBUSY;
 			goto out_mask;
 		}
 		/*
 		 * The thread_mask for the action is or'ed to
 		 * desc->thread_active to indicate that the
 		 * IRQF_ONESHOT thread handler has been woken, but not
 		 * yet finished. The bit is cleared when a thread
 		 * completes. When all threads of a shared interrupt
 		 * line have completed desc->threads_active becomes
 		 * zero and the interrupt line is unmasked. See
 		 * handle.c:irq_wake_thread() for further information.
 		 *
 		 * If no thread is woken by primary (hard irq context)
 		 * interrupt handlers, then desc->threads_active is
 		 * also checked for zero to unmask the irq line in the
 		 * affected hard irq flow handlers
 		 * (handle_[fasteoi|level]_irq).
 		 *
 		 * The new action gets the first zero bit of
 		 * thread_mask assigned. See the loop above which or's
 		 * all existing action->thread_mask bits.
 		 */
 		new->thread_mask = 1 << ffz(thread_mask);
 	} else if (new->handler == irq_default_primary_handler &&
 		   !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) {
 		/*
 		 * The interrupt was requested with handler = NULL, so
 		 * we use the default primary handler for it. But it
 		 * does not have the oneshot flag set. In combination
 		 * with level interrupts this is deadly, because the
 		 * default primary handler just wakes the thread, then
 		 * the irq lines is reenabled, but the device still
 		 * has the level irq asserted. Rinse and repeat....
 		 *
 		 * While this works for edge type interrupts, we play
 		 * it safe and reject unconditionally because we can't
 		 * say for sure which type this interrupt really
 		 * has. The type flags are unreliable as the
 		 * underlying chip implementation can override them.
 		 */
 		pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
 		       irq);
 		ret = -EINVAL;
 		goto out_mask;
 	}
 	if (!shared) {
 		ret = irq_request_resources(desc);
 		if (ret) {
 			pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
 			       new->name, irq, desc->irq_data.chip->name);
 			goto out_mask;
 		}
 		init_waitqueue_head(&desc->wait_for_threads);
 		/* Setup the type (level, edge polarity) if configured: */
 		if (new->flags & IRQF_TRIGGER_MASK) {
 			ret = __irq_set_trigger(desc, irq,
 					new->flags & IRQF_TRIGGER_MASK);
 			if (ret)
 				goto out_mask;
 		}
 		desc->istate &= ~(IRQS_AUTODETECT | IRQS_SPURIOUS_DISABLED | \
 				  IRQS_ONESHOT | IRQS_WAITING);
 		irqd_clear(&desc->irq_data, IRQD_IRQ_INPROGRESS);
 		if (new->flags & IRQF_PERCPU) {
 			irqd_set(&desc->irq_data, IRQD_PER_CPU);
 			irq_settings_set_per_cpu(desc);
 		}
 		if (new->flags & IRQF_ONESHOT)
 			desc->istate |= IRQS_ONESHOT;
 		if (irq_settings_can_autoenable(desc))
 			irq_startup(desc, true);
 		else
 			/* Undo nested disables: */
 			desc->depth = 1;
 		/* Exclude IRQ from balancing if requested */
 		if (new->flags & IRQF_NOBALANCING) {
 			irq_settings_set_no_balancing(desc);
 			irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
 		}
 		/* Set default affinity mask once everything is setup */
 		setup_affinity(irq, desc, mask);
 	} else if (new->flags & IRQF_TRIGGER_MASK) {
 		unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
 		unsigned int omsk = irq_settings_get_trigger_mask(desc);
 		if (nmsk != omsk)
 			/* hope the handler works with current  trigger mode */
 			pr_warning("irq %d uses trigger mode %u; requested %u\n",
 				   irq, nmsk, omsk);
 	}
 	new->irq = irq;
 	*old_ptr = new;
 	/* Reset broken irq detection when installing new handler */
 	desc->irq_count = 0;
 	desc->irqs_unhandled = 0;
 	/*
 	 * Check whether we disabled the irq via the spurious handler
 	 * before. Reenable it and give it another chance.
 	 */
 	if (shared && (desc->istate & IRQS_SPURIOUS_DISABLED)) {
 		desc->istate &= ~IRQS_SPURIOUS_DISABLED;
 		__enable_irq(desc, irq, false);
 	}
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	/*
 	 * Strictly no need to wake it up, but hung_task complains
 	 * when no hard interrupt wakes the thread up.
 	 */
 	if (new->thread)
 		wake_up_process(new->thread);
 	register_irq_proc(irq, desc);
 	new->dir = NULL;
 	register_handler_proc(irq, new);
 	free_cpumask_var(mask);
 	return 0;
 mismatch:
 	if (!(new->flags & IRQF_PROBE_SHARED)) {
 		pr_err("Flags mismatch irq %d. %08x (%s) vs. %08x (%s)\n",
 		       irq, new->flags, new->name, old->flags, old->name);
 #ifdef CONFIG_DEBUG_SHIRQ
 		dump_stack();
 #endif
 	}
 	ret = -EBUSY;
 out_mask:
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	free_cpumask_var(mask);
 out_thread:
 	if (new->thread) {
 		struct task_struct *t = new->thread;
 		new->thread = NULL;
 		kthread_stop(t);
 		put_task_struct(t);
 	}
 out_mput:
 	module_put(desc->owner);
 	return ret;
 }
 /**
  *	setup_irq - setup an interrupt
  *	@irq: Interrupt line to setup
  *	@act: irqaction for the interrupt
  *
  * Used to statically setup interrupts in the early boot process.
  */
 int setup_irq(unsigned int irq, struct irqaction *act)
 {
 	int retval;
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 		return -EINVAL;
 	chip_bus_lock(desc);
 	retval = __setup_irq(irq, desc, act);
 	chip_bus_sync_unlock(desc);
 	return retval;
 }
 EXPORT_SYMBOL_GPL(setup_irq);
 /*
  * Internal function to unregister an irqaction - used to free
  * regular and special interrupts that are part of the architecture.
  */
 static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	struct irqaction *action, **action_ptr;
 	unsigned long flags;
 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
 	if (!desc)
 		return NULL;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	/*
 	 * There can be multiple actions per IRQ descriptor, find the right
 	 * one based on the dev_id:
 	 */
 	action_ptr = &desc->action;
 	for (;;) {
 		action = *action_ptr;
 		if (!action) {
 			WARN(1, "Trying to free already-free IRQ %d\n", irq);
 			raw_spin_unlock_irqrestore(&desc->lock, flags);
 			return NULL;
 		}
 		if (action->dev_id == dev_id)
 			break;
 		action_ptr = &action->next;
 	}
 	/* Found it - now remove it from the list of entries: */
 	*action_ptr = action->next;
 	/* If this was the last handler, shut down the IRQ line: */
 	if (!desc->action) {
 		irq_shutdown(desc);
 		irq_release_resources(desc);
 	}
 #ifdef CONFIG_SMP
 	/* make sure affinity_hint is cleaned up */
 	if (WARN_ON_ONCE(desc->affinity_hint))
 		desc->affinity_hint = NULL;
 #endif
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	unregister_handler_proc(irq, action);
 	/* Make sure it's not being used on another CPU: */
 	synchronize_irq(irq);
 #ifdef CONFIG_DEBUG_SHIRQ
 	/*
 	 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
 	 * event to happen even now it's being freed, so let's make sure that
 	 * is so by doing an extra call to the handler ....
 	 *
 	 * ( We do this after actually deregistering it, to make sure that a
 	 *   'real' IRQ doesn't run in * parallel with our fake. )
 	 */
 	if (action->flags & IRQF_SHARED) {
 		local_irq_save(flags);
 		action->handler(irq, dev_id);
 		local_irq_restore(flags);
 	}
 #endif
 	if (action->thread) {
 		kthread_stop(action->thread);
 		put_task_struct(action->thread);
 	}
 	module_put(desc->owner);
 	return action;
 }
 /**
  *	remove_irq - free an interrupt
  *	@irq: Interrupt line to free
  *	@act: irqaction for the interrupt
  *
  * Used to remove interrupts statically setup by the early boot process.
  */
 void remove_irq(unsigned int irq, struct irqaction *act)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (desc && !WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 	    __free_irq(irq, act->dev_id);
 }
 EXPORT_SYMBOL_GPL(remove_irq);
 /**
  *	free_irq - free an interrupt allocated with request_irq
  *	@irq: Interrupt line to free
  *	@dev_id: Device identity to free
  *
  *	Remove an interrupt handler. The handler is removed and if the
  *	interrupt line is no longer in use by any driver it is disabled.
  *	On a shared IRQ the caller must ensure the interrupt is disabled
  *	on the card it drives before calling this function. The function
  *	does not return until any executing interrupts for this IRQ
  *	have completed.
  *
  *	This function must not be called from interrupt context.
  */
 void free_irq(unsigned int irq, void *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (!desc || WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 		return;
 #ifdef CONFIG_SMP
 	if (WARN_ON(desc->affinity_notify))
 		desc->affinity_notify = NULL;
 #endif
 	chip_bus_lock(desc);
 	kfree(__free_irq(irq, dev_id));
 	chip_bus_sync_unlock(desc);
 }
 EXPORT_SYMBOL(free_irq);
 /**
  *	request_threaded_irq - allocate an interrupt line
  *	@irq: Interrupt line to allocate
  *	@handler: Function to be called when the IRQ occurs.
  *		  Primary handler for threaded interrupts
  *		  If NULL and thread_fn != NULL the default
  *		  primary handler is installed
  *	@thread_fn: Function called from the irq handler thread
  *		    If NULL, no irq thread is created
  *	@irqflags: Interrupt type flags
  *	@devname: An ascii name for the claiming device
  *	@dev_id: A cookie passed back to the handler function
  *
  *	This call allocates interrupt resources and enables the
  *	interrupt line and IRQ handling. From the point this
  *	call is made your handler function may be invoked. Since
  *	your handler function must clear any interrupt the board
  *	raises, you must take care both to initialise your hardware
  *	and to set up the interrupt handler in the right order.
  *
  *	If you want to set up a threaded irq handler for your device
  *	then you need to supply @handler and @thread_fn. @handler is
  *	still called in hard interrupt context and has to check
  *	whether the interrupt originates from the device. If yes it
  *	needs to disable the interrupt on the device and return
  *	IRQ_WAKE_THREAD which will wake up the handler thread and run
  *	@thread_fn. This split handler design is necessary to support
  *	shared interrupts.
  *
  *	Dev_id must be globally unique. Normally the address of the
  *	device data structure is used as the cookie. Since the handler
  *	receives this value it makes sense to use it.
  *
  *	If your interrupt is shared you must pass a non NULL dev_id
  *	as this is required when freeing the interrupt.
  *
  *	Flags:
  *
  *	IRQF_SHARED		Interrupt is shared
  *	IRQF_TRIGGER_*		Specify active edge(s) or level
  *
  */
 int request_threaded_irq(unsigned int irq, irq_handler_t handler,
 			 irq_handler_t thread_fn, unsigned long irqflags,
 			 const char *devname, void *dev_id)
 {
 	struct irqaction *action;
 	struct irq_desc *desc;
 	int retval;
 	/*
 	 * Sanity-check: shared interrupts must pass in a real dev-ID,
 	 * otherwise we'll have trouble later trying to figure out
 	 * which interrupt is which (messes up the interrupt freeing
 	 * logic etc).
 	 */
 	if ((irqflags & IRQF_SHARED) && !dev_id)
 		return -EINVAL;
 	desc = irq_to_desc(irq);
 	if (!desc)
 		return -EINVAL;
 	if (!irq_settings_can_request(desc) ||
 	    WARN_ON(irq_settings_is_per_cpu_devid(desc)))
 		return -EINVAL;
 	if (!handler) {
 		if (!thread_fn)
 			return -EINVAL;
 		handler = irq_default_primary_handler;
 	}
 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
 	if (!action)
 		return -ENOMEM;
 	action->handler = handler;
 	action->thread_fn = thread_fn;
 	action->flags = irqflags;
 	action->name = devname;
 	action->dev_id = dev_id;
 	chip_bus_lock(desc);
 	retval = __setup_irq(irq, desc, action);
 	chip_bus_sync_unlock(desc);
 	if (retval)
 		kfree(action);
 #ifdef CONFIG_DEBUG_SHIRQ_FIXME
 	if (!retval && (irqflags & IRQF_SHARED)) {
 		/*
 		 * It's a shared IRQ -- the driver ought to be prepared for it
 		 * to happen immediately, so let's make sure....
 		 * We disable the irq to make sure that a 'real' IRQ doesn't
 		 * run in parallel with our fake.
 		 */
 		unsigned long flags;
 		disable_irq(irq);
 		local_irq_save(flags);
 		handler(irq, dev_id);
 		local_irq_restore(flags);
 		enable_irq(irq);
 	}
 #endif
 	return retval;
 }
 EXPORT_SYMBOL(request_threaded_irq);
 /**
  *	request_any_context_irq - allocate an interrupt line
  *	@irq: Interrupt line to allocate
  *	@handler: Function to be called when the IRQ occurs.
  *		  Threaded handler for threaded interrupts.
  *	@flags: Interrupt type flags
  *	@name: An ascii name for the claiming device
  *	@dev_id: A cookie passed back to the handler function
  *
  *	This call allocates interrupt resources and enables the
  *	interrupt line and IRQ handling. It selects either a
  *	hardirq or threaded handling method depending on the
  *	context.
  *
  *	On failure, it returns a negative value. On success,
  *	it returns either IRQC_IS_HARDIRQ or IRQC_IS_NESTED.
  */
 int request_any_context_irq(unsigned int irq, irq_handler_t handler,
 			    unsigned long flags, const char *name, void *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	int ret;
 	if (!desc)
 		return -EINVAL;
 	if (irq_settings_is_nested_thread(desc)) {
 		ret = request_threaded_irq(irq, NULL, handler,
 					   flags, name, dev_id);
 		return !ret ? IRQC_IS_NESTED : ret;
 	}
 	ret = request_irq(irq, handler, flags, name, dev_id);
 	return !ret ? IRQC_IS_HARDIRQ : ret;
 }
 EXPORT_SYMBOL_GPL(request_any_context_irq);
 void enable_percpu_irq(unsigned int irq, unsigned int type)
 {
 	unsigned int cpu = smp_processor_id();
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
 	if (!desc)
 		return;
 	type &= IRQ_TYPE_SENSE_MASK;
 	if (type != IRQ_TYPE_NONE) {
 		int ret;
 		ret = __irq_set_trigger(desc, irq, type);
 		if (ret) {
 			WARN(1, "failed to set type for IRQ%d\n", irq);
 			goto out;
 		}
 	}
 	irq_percpu_enable(desc, cpu);
 out:
 	irq_put_desc_unlock(desc, flags);
 }
 EXPORT_SYMBOL_GPL(enable_percpu_irq);
 void disable_percpu_irq(unsigned int irq)
 {
 	unsigned int cpu = smp_processor_id();
 	unsigned long flags;
 	struct irq_desc *desc = irq_get_desc_lock(irq, &flags, IRQ_GET_DESC_CHECK_PERCPU);
 	if (!desc)
 		return;
 	irq_percpu_disable(desc, cpu);
 	irq_put_desc_unlock(desc, flags);
 }
 EXPORT_SYMBOL_GPL(disable_percpu_irq);
 /*
  * Internal function to unregister a percpu irqaction.
  */
 static struct irqaction *__free_percpu_irq(unsigned int irq, void __percpu *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	struct irqaction *action;
 	unsigned long flags;
 	WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
 	if (!desc)
 		return NULL;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	action = desc->action;
 	if (!action || action->percpu_dev_id != dev_id) {
 		WARN(1, "Trying to free already-free IRQ %d\n", irq);
 		goto bad;
 	}
 	if (!cpumask_empty(desc->percpu_enabled)) {
 		WARN(1, "percpu IRQ %d still enabled on CPU%d!\n",
 		     irq, cpumask_first(desc->percpu_enabled));
 		goto bad;
 	}
 	/* Found it - now remove it from the list of entries: */
 	desc->action = NULL;
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	unregister_handler_proc(irq, action);
 	module_put(desc->owner);
 	return action;
 bad:
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	return NULL;
 }
 /**
  *	remove_percpu_irq - free a per-cpu interrupt
  *	@irq: Interrupt line to free
  *	@act: irqaction for the interrupt
  *
  * Used to remove interrupts statically setup by the early boot process.
  */
 void remove_percpu_irq(unsigned int irq, struct irqaction *act)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (desc && irq_settings_is_per_cpu_devid(desc))
 	    __free_percpu_irq(irq, act->percpu_dev_id);
 }
 /**
  *	free_percpu_irq - free an interrupt allocated with request_percpu_irq
  *	@irq: Interrupt line to free
  *	@dev_id: Device identity to free
  *
  *	Remove a percpu interrupt handler. The handler is removed, but
  *	the interrupt line is not disabled. This must be done on each
  *	CPU before calling this function. The function does not return
  *	until any executing interrupts for this IRQ have completed.
  *
  *	This function must not be called from interrupt context.
  */
 void free_percpu_irq(unsigned int irq, void __percpu *dev_id)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
 		return;
 	chip_bus_lock(desc);
 	kfree(__free_percpu_irq(irq, dev_id));
 	chip_bus_sync_unlock(desc);
 }
 /**
  *	setup_percpu_irq - setup a per-cpu interrupt
  *	@irq: Interrupt line to setup
  *	@act: irqaction for the interrupt
  *
  * Used to statically setup per-cpu interrupts in the early boot process.
  */
 int setup_percpu_irq(unsigned int irq, struct irqaction *act)
 {
 	struct irq_desc *desc = irq_to_desc(irq);
 	int retval;
 	if (!desc || !irq_settings_is_per_cpu_devid(desc))
 		return -EINVAL;
 	chip_bus_lock(desc);
 	retval = __setup_irq(irq, desc, act);
 	chip_bus_sync_unlock(desc);
 	return retval;
 }
 /**
  *	request_percpu_irq - allocate a percpu interrupt line
  *	@irq: Interrupt line to allocate
  *	@handler: Function to be called when the IRQ occurs.
  *	@devname: An ascii name for the claiming device
  *	@dev_id: A percpu cookie passed back to the handler function
  *
  *	This call allocates interrupt resources, but doesn't
  *	automatically enable the interrupt. It has to be done on each
  *	CPU using enable_percpu_irq().
  *
  *	Dev_id must be globally unique. It is a per-cpu variable, and
  *	the handler gets called with the interrupted CPU's instance of
  *	that variable.
  */
 int request_percpu_irq(unsigned int irq, irq_handler_t handler,
 		       const char *devname, void __percpu *dev_id)
 {
 	struct irqaction *action;
 	struct irq_desc *desc;
 	int retval;
 	if (!dev_id)
 		return -EINVAL;
 	desc = irq_to_desc(irq);
 	if (!desc || !irq_settings_can_request(desc) ||
 	    !irq_settings_is_per_cpu_devid(desc))
 		return -EINVAL;
 	action = kzalloc(sizeof(struct irqaction), GFP_KERNEL);
 	if (!action)
 		return -ENOMEM;
 	action->handler = handler;
 	action->flags = IRQF_PERCPU | IRQF_NO_SUSPEND;
 	action->name = devname;
 	action->percpu_dev_id = dev_id;
 	chip_bus_lock(desc);
 	retval = __setup_irq(irq, desc, action);
 	chip_bus_sync_unlock(desc);
 	if (retval)
 		kfree(action);
 	return retval;
 }