Blame view
arch/x86/kvm/irq_comm.c
11 KB
3b20eb237
|
1 |
// SPDX-License-Identifier: GPL-2.0-only |
3de42dc09
|
2 3 4 5 |
/* * irq_comm.c: Common API for in kernel interrupt controller * Copyright (c) 2007, Intel Corporation. * |
3de42dc09
|
6 7 8 |
* Authors: * Yaozu (Eddie) Dong <Eddie.dong@intel.com> * |
9611c1877
|
9 |
* Copyright 2010 Red Hat, Inc. and/or its affiliates. |
3de42dc09
|
10 11 12 |
*/ #include <linux/kvm_host.h> |
5a0e3ad6a
|
13 |
#include <linux/slab.h> |
c7c9c56ca
|
14 |
#include <linux/export.h> |
b2d091031
|
15 |
#include <linux/rculist.h> |
229456fc3
|
16 |
#include <trace/events/kvm.h> |
79950e107
|
17 |
|
79950e107
|
18 |
#include <asm/msidef.h> |
79950e107
|
19 |
|
3de42dc09
|
20 21 22 |
#include "irq.h" #include "ioapic.h" |
d1ebdbf99
|
23 |
#include "lapic.h" |
5c919412f
|
24 |
#include "hyperv.h" |
520040146
|
25 |
#include "x86.h" |
5c919412f
|
26 |
|
4925663a0
|
27 |
static int kvm_set_pic_irq(struct kvm_kernel_irq_routing_entry *e, |
aa2fbe6d4
|
28 29 |
struct kvm *kvm, int irq_source_id, int level, bool line_status) |
399ec807d
|
30 |
{ |
90bca0529
|
31 |
struct kvm_pic *pic = kvm->arch.vpic; |
1a577b724
|
32 |
return kvm_pic_set_irq(pic, e->irqchip.pin, irq_source_id, level); |
399ec807d
|
33 |
} |
4925663a0
|
34 |
static int kvm_set_ioapic_irq(struct kvm_kernel_irq_routing_entry *e, |
aa2fbe6d4
|
35 36 |
struct kvm *kvm, int irq_source_id, int level, bool line_status) |
399ec807d
|
37 |
{ |
1a6e4a8c2
|
38 |
struct kvm_ioapic *ioapic = kvm->arch.vioapic; |
aa2fbe6d4
|
39 40 |
return kvm_ioapic_set_irq(ioapic, e->irqchip.pin, irq_source_id, level, line_status); |
399ec807d
|
41 |
} |
58c2dde17
|
42 |
int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, |
9e4aabe2b
|
43 |
struct kvm_lapic_irq *irq, struct dest_map *dest_map) |
58c2dde17
|
44 45 46 |
{ int i, r = -1; struct kvm_vcpu *vcpu, *lowest = NULL; |
520040146
|
47 48 |
unsigned long dest_vcpu_bitmap[BITS_TO_LONGS(KVM_MAX_VCPUS)]; unsigned int dest_vcpus = 0; |
58c2dde17
|
49 |
|
dfd146fca
|
50 51 |
if (kvm_irq_delivery_to_apic_fast(kvm, src, irq, &r, dest_map)) return r; |
c96001c57
|
52 53 |
if (irq->dest_mode == APIC_DEST_PHYSICAL && irq->dest_id == 0xff && kvm_lowest_prio_delivery(irq)) { |
343f94fe4
|
54 55 |
printk(KERN_INFO "kvm: apic: phys broadcast and lowest prio "); |
1e08ec4a1
|
56 57 |
irq->delivery_mode = APIC_DM_FIXED; } |
520040146
|
58 |
memset(dest_vcpu_bitmap, 0, sizeof(dest_vcpu_bitmap)); |
988a2cae6
|
59 60 |
kvm_for_each_vcpu(i, vcpu, kvm) { if (!kvm_apic_present(vcpu)) |
343f94fe4
|
61 |
continue; |
58c2dde17
|
62 63 |
if (!kvm_apic_match_dest(vcpu, src, irq->shorthand, irq->dest_id, irq->dest_mode)) |
343f94fe4
|
64 |
continue; |
d1ebdbf99
|
65 |
if (!kvm_lowest_prio_delivery(irq)) { |
58c2dde17
|
66 67 |
if (r < 0) r = 0; |
b4f2225c0
|
68 |
r += kvm_apic_set_irq(vcpu, irq, dest_map); |
7d2296bfa
|
69 |
} else if (kvm_apic_sw_enabled(vcpu->arch.apic)) { |
520040146
|
70 71 72 73 74 75 76 77 78 |
if (!kvm_vector_hashing_enabled()) { if (!lowest) lowest = vcpu; else if (kvm_apic_compare_prio(vcpu, lowest) < 0) lowest = vcpu; } else { __set_bit(i, dest_vcpu_bitmap); dest_vcpus++; } |
e1035715e
|
79 |
} |
343f94fe4
|
80 |
} |
520040146
|
81 82 83 84 85 86 |
if (dest_vcpus != 0) { int idx = kvm_vector_to_index(irq->vector, dest_vcpus, dest_vcpu_bitmap, KVM_MAX_VCPUS); lowest = kvm_get_vcpu(kvm, idx); } |
58c2dde17
|
87 |
if (lowest) |
b4f2225c0
|
88 |
r = kvm_apic_set_irq(lowest, irq, dest_map); |
58c2dde17
|
89 90 |
return r; |
116191b69
|
91 |
} |
371313134
|
92 |
void kvm_set_msi_irq(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
d84f1e075
|
93 |
struct kvm_lapic_irq *irq) |
01f218803
|
94 |
{ |
371313134
|
95 96 97 |
trace_kvm_msi_set_irq(e->msi.address_lo | (kvm->arch.x2apic_format ? (u64)e->msi.address_hi << 32 : 0), e->msi.data); |
01f218803
|
98 99 100 |
irq->dest_id = (e->msi.address_lo & MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT; |
371313134
|
101 102 |
if (kvm->arch.x2apic_format) irq->dest_id |= MSI_ADDR_EXT_DEST_ID(e->msi.address_hi); |
01f218803
|
103 104 |
irq->vector = (e->msi.data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT; |
c96001c57
|
105 106 |
irq->dest_mode = kvm_lapic_irq_dest_mode( !!((1 << MSI_ADDR_DEST_MODE_SHIFT) & e->msi.address_lo)); |
01f218803
|
107 108 |
irq->trig_mode = (1 << MSI_DATA_TRIGGER_SHIFT) & e->msi.data; irq->delivery_mode = e->msi.data & 0x700; |
93bbf0b8b
|
109 110 |
irq->msi_redir_hint = ((e->msi.address_lo & MSI_ADDR_REDIRECTION_LOWPRI) > 0); |
01f218803
|
111 |
irq->level = 1; |
150a84fee
|
112 |
irq->shorthand = APIC_DEST_NOSHORT; |
01f218803
|
113 |
} |
d84f1e075
|
114 |
EXPORT_SYMBOL_GPL(kvm_set_msi_irq); |
01f218803
|
115 |
|
371313134
|
116 117 118 119 120 |
static inline bool kvm_msi_route_invalid(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e) { return kvm->arch.x2apic_format && (e->msi.address_hi & 0xff); } |
bd2b53b20
|
121 |
int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, |
aa2fbe6d4
|
122 |
struct kvm *kvm, int irq_source_id, int level, bool line_status) |
79950e107
|
123 |
{ |
58c2dde17
|
124 |
struct kvm_lapic_irq irq; |
79950e107
|
125 |
|
371313134
|
126 127 |
if (kvm_msi_route_invalid(kvm, e)) return -EINVAL; |
1a6e4a8c2
|
128 129 |
if (!level) return -1; |
371313134
|
130 |
kvm_set_msi_irq(kvm, e, &irq); |
116191b69
|
131 |
|
b4f2225c0
|
132 |
return kvm_irq_delivery_to_apic(kvm, NULL, &irq, NULL); |
79950e107
|
133 |
} |
01f218803
|
134 |
|
a2b07739f
|
135 136 137 138 139 140 141 142 143 |
static int kvm_hv_set_sint(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) { if (!level) return -1; return kvm_hv_synic_set_irq(kvm, e->hv_sint.vcpu, e->hv_sint.sint); } |
b97e6de9c
|
144 145 146 |
int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, int irq_source_id, int level, bool line_status) |
01f218803
|
147 148 149 |
{ struct kvm_lapic_irq irq; int r; |
a2b07739f
|
150 151 152 153 |
switch (e->type) { case KVM_IRQ_ROUTING_HV_SINT: return kvm_hv_set_sint(e, kvm, irq_source_id, level, line_status); |
b97e6de9c
|
154 |
|
a2b07739f
|
155 156 157 |
case KVM_IRQ_ROUTING_MSI: if (kvm_msi_route_invalid(kvm, e)) return -EINVAL; |
371313134
|
158 |
|
a2b07739f
|
159 |
kvm_set_msi_irq(kvm, e, &irq); |
01f218803
|
160 |
|
a2b07739f
|
161 162 163 164 165 166 167 168 169 |
if (kvm_irq_delivery_to_apic_fast(kvm, NULL, &irq, &r, NULL)) return r; break; default: break; } return -EWOULDBLOCK; |
01f218803
|
170 |
} |
5550af4df
|
171 172 173 |
int kvm_request_irq_source_id(struct kvm *kvm) { unsigned long *bitmap = &kvm->arch.irq_sources_bitmap; |
fa40a8214
|
174 175 176 |
int irq_source_id; mutex_lock(&kvm->irq_lock); |
cd5a2685d
|
177 |
irq_source_id = find_first_zero_bit(bitmap, BITS_PER_LONG); |
61552367b
|
178 |
|
cd5a2685d
|
179 |
if (irq_source_id >= BITS_PER_LONG) { |
5550af4df
|
180 181 |
printk(KERN_WARNING "kvm: exhaust allocatable IRQ sources! "); |
0c6ddcebd
|
182 183 |
irq_source_id = -EFAULT; goto unlock; |
61552367b
|
184 185 186 |
} ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); |
7a84428af
|
187 |
ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); |
61552367b
|
188 |
set_bit(irq_source_id, bitmap); |
0c6ddcebd
|
189 |
unlock: |
fa40a8214
|
190 |
mutex_unlock(&kvm->irq_lock); |
61552367b
|
191 |
|
5550af4df
|
192 193 194 195 196 |
return irq_source_id; } void kvm_free_irq_source_id(struct kvm *kvm, int irq_source_id) { |
61552367b
|
197 |
ASSERT(irq_source_id != KVM_USERSPACE_IRQ_SOURCE_ID); |
7a84428af
|
198 |
ASSERT(irq_source_id != KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID); |
61552367b
|
199 |
|
fa40a8214
|
200 |
mutex_lock(&kvm->irq_lock); |
61552367b
|
201 |
if (irq_source_id < 0 || |
cd5a2685d
|
202 |
irq_source_id >= BITS_PER_LONG) { |
5550af4df
|
203 204 |
printk(KERN_ERR "kvm: IRQ source ID out of range! "); |
0c6ddcebd
|
205 |
goto unlock; |
5550af4df
|
206 |
} |
e50212bb5
|
207 |
clear_bit(irq_source_id, &kvm->arch.irq_sources_bitmap); |
900ab14ca
|
208 |
if (!irqchip_kernel(kvm)) |
e50212bb5
|
209 |
goto unlock; |
1a577b724
|
210 |
kvm_ioapic_clear_all(kvm->arch.vioapic, irq_source_id); |
90bca0529
|
211 |
kvm_pic_clear_all(kvm->arch.vpic, irq_source_id); |
0c6ddcebd
|
212 |
unlock: |
fa40a8214
|
213 |
mutex_unlock(&kvm->irq_lock); |
5550af4df
|
214 |
} |
75858a84a
|
215 216 217 218 |
void kvm_register_irq_mask_notifier(struct kvm *kvm, int irq, struct kvm_irq_mask_notifier *kimn) { |
fa40a8214
|
219 |
mutex_lock(&kvm->irq_lock); |
75858a84a
|
220 |
kimn->irq = irq; |
6ef768fac
|
221 |
hlist_add_head_rcu(&kimn->link, &kvm->arch.mask_notifier_list); |
fa40a8214
|
222 |
mutex_unlock(&kvm->irq_lock); |
75858a84a
|
223 224 225 226 227 |
} void kvm_unregister_irq_mask_notifier(struct kvm *kvm, int irq, struct kvm_irq_mask_notifier *kimn) { |
fa40a8214
|
228 |
mutex_lock(&kvm->irq_lock); |
280aa177d
|
229 |
hlist_del_rcu(&kimn->link); |
fa40a8214
|
230 |
mutex_unlock(&kvm->irq_lock); |
719d93cd5
|
231 |
synchronize_srcu(&kvm->irq_srcu); |
75858a84a
|
232 |
} |
4a994358b
|
233 234 |
void kvm_fire_mask_notifiers(struct kvm *kvm, unsigned irqchip, unsigned pin, bool mask) |
75858a84a
|
235 236 |
{ struct kvm_irq_mask_notifier *kimn; |
719d93cd5
|
237 |
int idx, gsi; |
75858a84a
|
238 |
|
719d93cd5
|
239 |
idx = srcu_read_lock(&kvm->irq_srcu); |
9957c86d6
|
240 |
gsi = kvm_irq_map_chip_pin(kvm, irqchip, pin); |
4a994358b
|
241 |
if (gsi != -1) |
6ef768fac
|
242 |
hlist_for_each_entry_rcu(kimn, &kvm->arch.mask_notifier_list, link) |
4a994358b
|
243 244 |
if (kimn->irq == gsi) kimn->func(kimn, mask); |
719d93cd5
|
245 |
srcu_read_unlock(&kvm->irq_srcu, idx); |
75858a84a
|
246 |
} |
5c0aea0e8
|
247 248 249 250 |
bool kvm_arch_can_set_irq_routing(struct kvm *kvm) { return irqchip_in_kernel(kvm); } |
c63cf538e
|
251 252 |
int kvm_set_routing_entry(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e, |
e8cde0939
|
253 |
const struct kvm_irq_routing_entry *ue) |
399ec807d
|
254 |
{ |
5c0aea0e8
|
255 256 257 258 |
/* We can't check irqchip_in_kernel() here as some callers are * currently inititalizing the irqchip. Other callers should therefore * check kvm_arch_can_set_irq_routing() before calling this function. */ |
399ec807d
|
259 260 |
switch (ue->type) { case KVM_IRQ_ROUTING_IRQCHIP: |
8bf463f3b
|
261 |
if (irqchip_split(kvm)) |
43ae312ca
|
262 |
return -EINVAL; |
445ee82d7
|
263 |
e->irqchip.pin = ue->u.irqchip.pin; |
399ec807d
|
264 |
switch (ue->u.irqchip.irqchip) { |
399ec807d
|
265 |
case KVM_IRQCHIP_PIC_SLAVE: |
445ee82d7
|
266 |
e->irqchip.pin += PIC_NUM_PINS / 2; |
df561f668
|
267 |
fallthrough; |
e5dc48777
|
268 |
case KVM_IRQCHIP_PIC_MASTER: |
445ee82d7
|
269 |
if (ue->u.irqchip.pin >= PIC_NUM_PINS / 2) |
43ae312ca
|
270 |
return -EINVAL; |
4925663a0
|
271 |
e->set = kvm_set_pic_irq; |
399ec807d
|
272 273 |
break; case KVM_IRQCHIP_IOAPIC: |
445ee82d7
|
274 |
if (ue->u.irqchip.pin >= KVM_IOAPIC_NUM_PINS) |
43ae312ca
|
275 |
return -EINVAL; |
efbc100c2
|
276 |
e->set = kvm_set_ioapic_irq; |
399ec807d
|
277 278 |
break; default: |
43ae312ca
|
279 |
return -EINVAL; |
399ec807d
|
280 281 |
} e->irqchip.irqchip = ue->u.irqchip.irqchip; |
399ec807d
|
282 |
break; |
79950e107
|
283 284 285 286 287 |
case KVM_IRQ_ROUTING_MSI: e->set = kvm_set_msi; e->msi.address_lo = ue->u.msi.address_lo; e->msi.address_hi = ue->u.msi.address_hi; e->msi.data = ue->u.msi.data; |
371313134
|
288 289 |
if (kvm_msi_route_invalid(kvm, e)) |
43ae312ca
|
290 |
return -EINVAL; |
79950e107
|
291 |
break; |
5c919412f
|
292 293 294 295 296 |
case KVM_IRQ_ROUTING_HV_SINT: e->set = kvm_hv_set_sint; e->hv_sint.vcpu = ue->u.hv_sint.vcpu; e->hv_sint.sint = ue->u.hv_sint.sint; break; |
399ec807d
|
297 |
default: |
43ae312ca
|
298 |
return -EINVAL; |
399ec807d
|
299 |
} |
46e624b95
|
300 |
|
43ae312ca
|
301 |
return 0; |
399ec807d
|
302 |
} |
8feb4a04d
|
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 |
bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, struct kvm_vcpu **dest_vcpu) { int i, r = 0; struct kvm_vcpu *vcpu; if (kvm_intr_is_single_vcpu_fast(kvm, irq, dest_vcpu)) return true; kvm_for_each_vcpu(i, vcpu, kvm) { if (!kvm_apic_present(vcpu)) continue; if (!kvm_apic_match_dest(vcpu, NULL, irq->shorthand, irq->dest_id, irq->dest_mode)) continue; if (++r == 2) return false; *dest_vcpu = vcpu; } return r == 1; } EXPORT_SYMBOL_GPL(kvm_intr_is_single_vcpu); |
399ec807d
|
329 330 |
#define IOAPIC_ROUTING_ENTRY(irq) \ { .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ |
25f97ff45
|
331 |
.u.irqchip = { .irqchip = KVM_IRQCHIP_IOAPIC, .pin = (irq) } } |
399ec807d
|
332 |
#define ROUTING_ENTRY1(irq) IOAPIC_ROUTING_ENTRY(irq) |
3bf58e9ae
|
333 |
#define PIC_ROUTING_ENTRY(irq) \ |
399ec807d
|
334 |
{ .gsi = irq, .type = KVM_IRQ_ROUTING_IRQCHIP, \ |
25f97ff45
|
335 |
.u.irqchip = { .irqchip = SELECT_PIC(irq), .pin = (irq) % 8 } } |
3bf58e9ae
|
336 |
#define ROUTING_ENTRY2(irq) \ |
399ec807d
|
337 |
IOAPIC_ROUTING_ENTRY(irq), PIC_ROUTING_ENTRY(irq) |
399ec807d
|
338 339 340 341 342 343 344 345 346 347 348 349 350 351 |
static const struct kvm_irq_routing_entry default_routing[] = { ROUTING_ENTRY2(0), ROUTING_ENTRY2(1), ROUTING_ENTRY2(2), ROUTING_ENTRY2(3), ROUTING_ENTRY2(4), ROUTING_ENTRY2(5), ROUTING_ENTRY2(6), ROUTING_ENTRY2(7), ROUTING_ENTRY2(8), ROUTING_ENTRY2(9), ROUTING_ENTRY2(10), ROUTING_ENTRY2(11), ROUTING_ENTRY2(12), ROUTING_ENTRY2(13), ROUTING_ENTRY2(14), ROUTING_ENTRY2(15), ROUTING_ENTRY1(16), ROUTING_ENTRY1(17), ROUTING_ENTRY1(18), ROUTING_ENTRY1(19), ROUTING_ENTRY1(20), ROUTING_ENTRY1(21), ROUTING_ENTRY1(22), ROUTING_ENTRY1(23), |
399ec807d
|
352 353 354 355 356 357 358 |
}; int kvm_setup_default_irq_routing(struct kvm *kvm) { return kvm_set_irq_routing(kvm, default_routing, ARRAY_SIZE(default_routing), 0); } |
49df6397e
|
359 360 361 362 363 364 365 |
static const struct kvm_irq_routing_entry empty_routing[] = {}; int kvm_setup_empty_irq_routing(struct kvm *kvm) { return kvm_set_irq_routing(kvm, empty_routing, 0, 0); } |
b053b2aef
|
366 |
|
abdb080f7
|
367 |
void kvm_arch_post_irq_routing_update(struct kvm *kvm) |
b053b2aef
|
368 |
{ |
826da3214
|
369 |
if (!irqchip_split(kvm)) |
b053b2aef
|
370 371 372 |
return; kvm_make_scan_ioapic_request(kvm); } |
6308630bd
|
373 374 |
void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu, ulong *ioapic_handled_vectors) |
b053b2aef
|
375 376 377 378 379 380 |
{ struct kvm *kvm = vcpu->kvm; struct kvm_kernel_irq_routing_entry *entry; struct kvm_irq_routing_table *table; u32 i, nr_ioapic_pins; int idx; |
b053b2aef
|
381 382 383 384 385 386 |
idx = srcu_read_lock(&kvm->irq_srcu); table = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu); nr_ioapic_pins = min_t(u32, table->nr_rt_entries, kvm->arch.nr_reserved_ioapic_pins); for (i = 0; i < nr_ioapic_pins; ++i) { hlist_for_each_entry(entry, &table->map[i], link) { |
3159d36ad
|
387 |
struct kvm_lapic_irq irq; |
b053b2aef
|
388 389 390 |
if (entry->type != KVM_IRQ_ROUTING_MSI) continue; |
3159d36ad
|
391 |
|
371313134
|
392 |
kvm_set_msi_irq(vcpu->kvm, entry, &irq); |
3159d36ad
|
393 |
|
7455a8327
|
394 |
if (irq.trig_mode && |
5c69d5c11
|
395 |
kvm_apic_match_dest(vcpu, NULL, APIC_DEST_NOSHORT, |
3159d36ad
|
396 397 |
irq.dest_id, irq.dest_mode)) __set_bit(irq.vector, ioapic_handled_vectors); |
b053b2aef
|
398 399 400 401 |
} } srcu_read_unlock(&kvm->irq_srcu, idx); } |
5c919412f
|
402 |
|
5c919412f
|
403 404 405 406 |
void kvm_arch_irq_routing_update(struct kvm *kvm) { kvm_hv_irq_routing_update(kvm); } |