Blame view

virt/kvm/arm/arch_timer.c 20.8 KB
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
  /*
   * Copyright (C) 2012 ARM Ltd.
   * Author: Marc Zyngier <marc.zyngier@arm.com>
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU General Public License for more details.
   *
   * You should have received a copy of the GNU General Public License
   * along with this program; if not, write to the Free Software
   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
   */
  
  #include <linux/cpu.h>
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
20
21
22
  #include <linux/kvm.h>
  #include <linux/kvm_host.h>
  #include <linux/interrupt.h>
b452cb520   Christoffer Dall   KVM: arm/arm64: R...
23
  #include <linux/irq.h>
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
24
  #include <linux/uaccess.h>
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
25

372b7c1bc   Mark Rutland   ARM: KVM: arch_ti...
26
  #include <clocksource/arm_arch_timer.h>
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
27
  #include <asm/arch_timer.h>
488f94d72   Jintack Lim   KVM: arm64: Acces...
28
  #include <asm/kvm_hyp.h>
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
29

7275acdfe   Marc Zyngier   ARM: KVM: move GI...
30
31
  #include <kvm/arm_vgic.h>
  #include <kvm/arm_arch_timer.h>
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
32

e21f09108   Christoffer Dall   arm/arm64: KVM: A...
33
  #include "trace.h"
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
34
  static struct timecounter *timecounter;
5ae7f87a5   Anup Patel   ARM: KVM: Allow h...
35
  static unsigned int host_vtimer_irq;
cabdc5c59   Marc Zyngier   KVM: arm/arm64: t...
36
  static u32 host_vtimer_irq_flags;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
37

85e69ad7f   Christoffer Dall   KVM: arm/arm64: M...
38
39
40
41
42
43
44
45
46
  static const struct kvm_irq_level default_ptimer_irq = {
  	.irq	= 30,
  	.level	= 1,
  };
  
  static const struct kvm_irq_level default_vtimer_irq = {
  	.irq	= 27,
  	.level	= 1,
  };
9b4a30044   Marc Zyngier   KVM: arm/arm64: t...
47
48
  void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
  {
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
49
  	vcpu_vtimer(vcpu)->active_cleared_last = false;
9b4a30044   Marc Zyngier   KVM: arm/arm64: t...
50
  }
7b6b46311   Jintack Lim   KVM: arm/arm64: E...
51
  u64 kvm_phys_timer_read(void)
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
  {
  	return timecounter->cc->read(timecounter->cc);
  }
  
  static bool timer_is_armed(struct arch_timer_cpu *timer)
  {
  	return timer->armed;
  }
  
  /* timer_arm: as in "arm the timer", not as in ARM the company */
  static void timer_arm(struct arch_timer_cpu *timer, u64 ns)
  {
  	timer->armed = true;
  	hrtimer_start(&timer->timer, ktime_add_ns(ktime_get(), ns),
  		      HRTIMER_MODE_ABS);
  }
  
  static void timer_disarm(struct arch_timer_cpu *timer)
  {
  	if (timer_is_armed(timer)) {
  		hrtimer_cancel(&timer->timer);
  		cancel_work_sync(&timer->expired);
  		timer->armed = false;
  	}
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  {
  	struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  
  	/*
  	 * We disable the timer in the world switch and let it be
  	 * handled by kvm_timer_sync_hwstate(). Getting a timer
  	 * interrupt at this point is a sure sign of some major
  	 * breakage.
  	 */
  	pr_warn("Unexpected interrupt %d on vcpu %p
  ", irq, vcpu);
  	return IRQ_HANDLED;
  }
1a7484788   Christoffer Dall   arm/arm64: KVM: F...
91
92
93
94
  /*
   * Work function for handling the backup timer that we schedule when a vcpu is
   * no longer running, but had a timer programmed to fire in the future.
   */
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
95
96
97
98
99
  static void kvm_timer_inject_irq_work(struct work_struct *work)
  {
  	struct kvm_vcpu *vcpu;
  
  	vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
100

1a7484788   Christoffer Dall   arm/arm64: KVM: F...
101
102
103
104
  	/*
  	 * If the vcpu is blocked we want to wake it up so that it will see
  	 * the timer has expired when entering the guest.
  	 */
1b6502e5b   Andrew Jones   KVM: arm/arm64: t...
105
  	kvm_vcpu_wake_up(vcpu);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
106
  }
9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
107
  static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
108
  {
a5a1d1c29   Thomas Gleixner   clocksource: Use ...
109
  	u64 cval, now;
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
110

9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
111
112
  	cval = timer_ctx->cnt_cval;
  	now = kvm_phys_timer_read() - timer_ctx->cntvoff;
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
113
114
115
116
117
118
119
120
121
122
123
124
125
  
  	if (now < cval) {
  		u64 ns;
  
  		ns = cyclecounter_cyc2ns(timecounter->cc,
  					 cval - now,
  					 timecounter->mask,
  					 &timecounter->frac);
  		return ns;
  	}
  
  	return 0;
  }
fb280e975   Jintack Lim   KVM: arm/arm64: S...
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
  static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
  {
  	return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
  		(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
  }
  
  /*
   * Returns the earliest expiration time in ns among guest timers.
   * Note that it will return 0 if none of timers can fire.
   */
  static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
  {
  	u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX;
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
  
  	if (kvm_timer_irq_can_fire(vtimer))
  		min_virt = kvm_timer_compute_delta(vtimer);
  
  	if (kvm_timer_irq_can_fire(ptimer))
  		min_phys = kvm_timer_compute_delta(ptimer);
  
  	/* If none of timers can fire, then return 0 */
  	if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX))
  		return 0;
  
  	return min(min_virt, min_phys);
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
154
155
156
  static enum hrtimer_restart kvm_timer_expire(struct hrtimer *hrt)
  {
  	struct arch_timer_cpu *timer;
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
157
158
  	struct kvm_vcpu *vcpu;
  	u64 ns;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
159
  	timer = container_of(hrt, struct arch_timer_cpu, timer);
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
160
161
162
163
164
165
166
  	vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
  
  	/*
  	 * Check that the timer has really expired from the guest's
  	 * PoV (NTP on the host may have forced it to expire
  	 * early). If we should have slept longer, restart it.
  	 */
fb280e975   Jintack Lim   KVM: arm/arm64: S...
167
  	ns = kvm_timer_earliest_exp(vcpu);
1c5631c73   Marc Zyngier   KVM: arm/arm64: H...
168
169
170
171
  	if (unlikely(ns)) {
  		hrtimer_forward_now(hrt, ns_to_ktime(ns));
  		return HRTIMER_RESTART;
  	}
3706feacd   Bhaktipriya Shridhar   KVM: Remove depre...
172
  	schedule_work(&timer->expired);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
173
174
  	return HRTIMER_NORESTART;
  }
9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
175
  bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
1a7484788   Christoffer Dall   arm/arm64: KVM: F...
176
  {
a5a1d1c29   Thomas Gleixner   clocksource: Use ...
177
  	u64 cval, now;
1a7484788   Christoffer Dall   arm/arm64: KVM: F...
178

9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
179
  	if (!kvm_timer_irq_can_fire(timer_ctx))
1a7484788   Christoffer Dall   arm/arm64: KVM: F...
180
  		return false;
9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
181
182
  	cval = timer_ctx->cnt_cval;
  	now = kvm_phys_timer_read() - timer_ctx->cntvoff;
1a7484788   Christoffer Dall   arm/arm64: KVM: F...
183
184
185
  
  	return cval <= now;
  }
d9e139778   Alexander Graf   KVM: arm/arm64: S...
186
187
188
189
190
191
192
193
  /*
   * Reflect the timer output level into the kvm_run structure
   */
  void kvm_timer_update_run(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
  	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
d9e139778   Alexander Graf   KVM: arm/arm64: S...
194
195
196
197
198
199
200
201
  	/* Populate the device bitmap with the timer states */
  	regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
  				    KVM_ARM_DEV_EL1_PTIMER);
  	if (vtimer->irq.level)
  		regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
  	if (ptimer->irq.level)
  		regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
  }
9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
202
203
  static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  				 struct arch_timer_context *timer_ctx)
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
204
205
  {
  	int ret;
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
206

9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
207
208
209
210
  	timer_ctx->active_cleared_last = false;
  	timer_ctx->irq.level = new_level;
  	trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
  				   timer_ctx->irq.level);
11710dec8   Christoffer Dall   KVM: arm/arm64: R...
211

d9e139778   Alexander Graf   KVM: arm/arm64: S...
212
213
214
  	if (likely(irqchip_in_kernel(vcpu->kvm))) {
  		ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
  					  timer_ctx->irq.irq,
cb3f0ad88   Christoffer Dall   KVM: arm/arm64: D...
215
216
  					  timer_ctx->irq.level,
  					  timer_ctx);
d9e139778   Alexander Graf   KVM: arm/arm64: S...
217
218
  		WARN_ON(ret);
  	}
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
219
220
221
222
223
224
  }
  
  /*
   * Check if there was a change in the timer state (should we raise or lower
   * the line level to the GIC).
   */
b22e7df2d   Christoffer Dall   KVM: arm/arm64: C...
225
  static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
226
227
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
228
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
58e0c9732   Jintack Lim   KVM: arm/arm64: U...
229
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
230
231
232
  
  	/*
  	 * If userspace modified the timer registers via SET_ONE_REG before
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
233
  	 * the vgic was initialized, we mustn't set the vtimer->irq.level value
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
234
235
236
  	 * because the guest would never see the interrupt.  Instead wait
  	 * until we call this function from kvm_timer_flush_hwstate.
  	 */
d9e139778   Alexander Graf   KVM: arm/arm64: S...
237
  	if (unlikely(!timer->enabled))
b22e7df2d   Christoffer Dall   KVM: arm/arm64: C...
238
  		return;
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
239

9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
240
241
  	if (kvm_timer_should_fire(vtimer) != vtimer->irq.level)
  		kvm_timer_update_irq(vcpu, !vtimer->irq.level, vtimer);
b3aff6ccb   Andre Przywara   KVM: arm/arm64: F...
242

58e0c9732   Jintack Lim   KVM: arm/arm64: U...
243
244
  	if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
  		kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
245
  }
f242adaf0   Jintack Lim   KVM: arm/arm64: S...
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
  /* Schedule the background timer for the emulated timer. */
  static void kvm_timer_emulate(struct kvm_vcpu *vcpu,
  			      struct arch_timer_context *timer_ctx)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
  
  	if (kvm_timer_should_fire(timer_ctx))
  		return;
  
  	if (!kvm_timer_irq_can_fire(timer_ctx))
  		return;
  
  	/*  The timer has not yet expired, schedule a background timer */
  	timer_arm(timer, kvm_timer_compute_delta(timer_ctx));
  }
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
261
262
263
264
265
266
267
268
  /*
   * Schedule the background timer before calling kvm_vcpu_block, so that this
   * thread is removed from its waitqueue and made runnable when there's a timer
   * interrupt to handle.
   */
  void kvm_timer_schedule(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
9171fa2e0   Jintack Lim   KVM: arm/arm64: D...
269
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
fb280e975   Jintack Lim   KVM: arm/arm64: S...
270
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
271
272
273
274
  
  	BUG_ON(timer_is_armed(timer));
  
  	/*
fb280e975   Jintack Lim   KVM: arm/arm64: S...
275
  	 * No need to schedule a background timer if any guest timer has
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
276
277
278
  	 * already expired, because kvm_vcpu_block will return before putting
  	 * the thread to sleep.
  	 */
fb280e975   Jintack Lim   KVM: arm/arm64: S...
279
  	if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer))
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
280
281
282
  		return;
  
  	/*
fb280e975   Jintack Lim   KVM: arm/arm64: S...
283
  	 * If both timers are not capable of raising interrupts (disabled or
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
284
285
  	 * masked), then there's no more work for us to do.
  	 */
fb280e975   Jintack Lim   KVM: arm/arm64: S...
286
  	if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer))
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
287
  		return;
fb280e975   Jintack Lim   KVM: arm/arm64: S...
288
289
290
291
292
  	/*
  	 * The guest timers have not yet expired, schedule a background timer.
  	 * Set the earliest expiration time among the guest timers.
  	 */
  	timer_arm(timer, kvm_timer_earliest_exp(vcpu));
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
293
294
295
296
297
298
299
  }
  
  void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
  	timer_disarm(timer);
  }
d9e139778   Alexander Graf   KVM: arm/arm64: S...
300
  static void kvm_timer_flush_hwstate_vgic(struct kvm_vcpu *vcpu)
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
301
  {
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
302
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
cff9211eb   Christoffer Dall   arm/arm64: KVM: F...
303
304
  	bool phys_active;
  	int ret;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
305

cff9211eb   Christoffer Dall   arm/arm64: KVM: F...
306
  	/*
0e3dfda91   Christoffer Dall   KVM: arm/arm64: a...
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
  	* If we enter the guest with the virtual input level to the VGIC
  	* asserted, then we have already told the VGIC what we need to, and
  	* we don't need to exit from the guest until the guest deactivates
  	* the already injected interrupt, so therefore we should set the
  	* hardware active state to prevent unnecessary exits from the guest.
  	*
  	* Also, if we enter the guest with the virtual timer interrupt active,
  	* then it must be active on the physical distributor, because we set
  	* the HW bit and the guest must be able to deactivate the virtual and
  	* physical interrupt at the same time.
  	*
  	* Conversely, if the virtual input level is deasserted and the virtual
  	* interrupt is not active, then always clear the hardware active state
  	* to ensure that hardware interrupts from the timer triggers a guest
  	* exit.
  	*/
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
323
324
  	phys_active = vtimer->irq.level ||
  			kvm_vgic_map_is_active(vcpu, vtimer->irq.irq);
cff9211eb   Christoffer Dall   arm/arm64: KVM: F...
325

9b4a30044   Marc Zyngier   KVM: arm/arm64: t...
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
  	/*
  	 * We want to avoid hitting the (re)distributor as much as
  	 * possible, as this is a potentially expensive MMIO access
  	 * (not to mention locks in the irq layer), and a solution for
  	 * this is to cache the "active" state in memory.
  	 *
  	 * Things to consider: we cannot cache an "active set" state,
  	 * because the HW can change this behind our back (it becomes
  	 * "clear" in the HW). We must then restrict the caching to
  	 * the "clear" state.
  	 *
  	 * The cache is invalidated on:
  	 * - vcpu put, indicating that the HW cannot be trusted to be
  	 *   in a sane state on the next vcpu load,
  	 * - any change in the interrupt state
  	 *
  	 * Usage conditions:
  	 * - cached value is "active clear"
  	 * - value to be programmed is "active clear"
  	 */
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
346
  	if (vtimer->active_cleared_last && !phys_active)
9b4a30044   Marc Zyngier   KVM: arm/arm64: t...
347
  		return;
b452cb520   Christoffer Dall   KVM: arm/arm64: R...
348
  	ret = irq_set_irqchip_state(host_vtimer_irq,
cff9211eb   Christoffer Dall   arm/arm64: KVM: F...
349
350
351
  				    IRQCHIP_STATE_ACTIVE,
  				    phys_active);
  	WARN_ON(ret);
9b4a30044   Marc Zyngier   KVM: arm/arm64: t...
352

fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
353
  	vtimer->active_cleared_last = !phys_active;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
354
  }
d9e139778   Alexander Graf   KVM: arm/arm64: S...
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
  bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
  	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
  	bool vlevel, plevel;
  
  	if (likely(irqchip_in_kernel(vcpu->kvm)))
  		return false;
  
  	vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
  	plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
  
  	return vtimer->irq.level != vlevel ||
  	       ptimer->irq.level != plevel;
  }
  
  static void kvm_timer_flush_hwstate_user(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  
  	/*
  	 * To prevent continuously exiting from the guest, we mask the
  	 * physical interrupt such that the guest can make forward progress.
  	 * Once we detect the output level being deasserted, we unmask the
  	 * interrupt again so that we exit from the guest when the timer
  	 * fires.
  	*/
  	if (vtimer->irq.level)
  		disable_percpu_irq(host_vtimer_irq);
  	else
  		enable_percpu_irq(host_vtimer_irq, 0);
  }
  
  /**
   * kvm_timer_flush_hwstate - prepare timers before running the vcpu
   * @vcpu: The vcpu pointer
   *
   * Check if the virtual timer has expired while we were running in the host,
   * and inject an interrupt if that was the case, making sure the timer is
   * masked or disabled on the host so that we keep executing.  Also schedule a
   * software timer for the physical timer if it is enabled.
   */
  void kvm_timer_flush_hwstate(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
  
  	if (unlikely(!timer->enabled))
  		return;
  
  	kvm_timer_update_state(vcpu);
  
  	/* Set the background timer for the physical timer emulation. */
  	kvm_timer_emulate(vcpu, vcpu_ptimer(vcpu));
  
  	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
  		kvm_timer_flush_hwstate_user(vcpu);
  	else
  		kvm_timer_flush_hwstate_vgic(vcpu);
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
415
416
417
418
  /**
   * kvm_timer_sync_hwstate - sync timer state from cpu
   * @vcpu: The vcpu pointer
   *
d9e139778   Alexander Graf   KVM: arm/arm64: S...
419
   * Check if any of the timers have expired while we were running in the guest,
d35268da6   Christoffer Dall   arm/arm64: KVM: a...
420
   * and inject an interrupt if that was the case.
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
421
422
423
424
   */
  void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
425

f242adaf0   Jintack Lim   KVM: arm/arm64: S...
426
427
428
429
430
  	/*
  	 * This is to cancel the background timer for the physical timer
  	 * emulation if it is set.
  	 */
  	timer_disarm(timer);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
431

4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
432
433
434
435
436
  	/*
  	 * The guest could have modified the timer registers or the timer
  	 * could have expired, update the timer state.
  	 */
  	kvm_timer_update_state(vcpu);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
437
  }
85e69ad7f   Christoffer Dall   KVM: arm/arm64: M...
438
  int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
5ae7f87a5   Anup Patel   ARM: KVM: Allow h...
439
  {
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
440
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
a91d18551   Jintack Lim   KVM: arm/arm64: I...
441
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
5ae7f87a5   Anup Patel   ARM: KVM: Allow h...
442
443
  
  	/*
4ad9e16af   Christoffer Dall   arm/arm64: KVM: a...
444
445
446
447
448
  	 * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
  	 * and to 0 for ARMv7.  We provide an implementation that always
  	 * resets the timer to be disabled and unmasked and is compliant with
  	 * the ARMv7 architecture.
  	 */
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
449
  	vtimer->cnt_ctl = 0;
a91d18551   Jintack Lim   KVM: arm/arm64: I...
450
  	ptimer->cnt_ctl = 0;
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
451
  	kvm_timer_update_state(vcpu);
4ad9e16af   Christoffer Dall   arm/arm64: KVM: a...
452

41a54482c   Christoffer Dall   KVM: arm/arm64: M...
453
  	return 0;
5ae7f87a5   Anup Patel   ARM: KVM: Allow h...
454
  }
90de943a4   Jintack Lim   KVM: arm/arm64: M...
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
  /* Make the updates of cntvoff for all vtimer contexts atomic */
  static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
  {
  	int i;
  	struct kvm *kvm = vcpu->kvm;
  	struct kvm_vcpu *tmp;
  
  	mutex_lock(&kvm->lock);
  	kvm_for_each_vcpu(i, tmp, kvm)
  		vcpu_vtimer(tmp)->cntvoff = cntvoff;
  
  	/*
  	 * When called from the vcpu create path, the CPU being created is not
  	 * included in the loop above, so we just set it here as well.
  	 */
  	vcpu_vtimer(vcpu)->cntvoff = cntvoff;
  	mutex_unlock(&kvm->lock);
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
473
474
475
  void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
85e69ad7f   Christoffer Dall   KVM: arm/arm64: M...
476
477
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
478

90de943a4   Jintack Lim   KVM: arm/arm64: M...
479
480
  	/* Synchronize cntvoff across all vtimers of a VM. */
  	update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
a91d18551   Jintack Lim   KVM: arm/arm64: I...
481
  	vcpu_ptimer(vcpu)->cntvoff = 0;
90de943a4   Jintack Lim   KVM: arm/arm64: M...
482

53e724067   Marc Zyngier   ARM: KVM: arch_ti...
483
484
485
  	INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
  	hrtimer_init(&timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
  	timer->timer.function = kvm_timer_expire;
85e69ad7f   Christoffer Dall   KVM: arm/arm64: M...
486
487
488
  
  	vtimer->irq.irq = default_vtimer_irq.irq;
  	ptimer->irq.irq = default_ptimer_irq.irq;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
489
490
491
492
  }
  
  static void kvm_timer_init_interrupt(void *info)
  {
cabdc5c59   Marc Zyngier   KVM: arm/arm64: t...
493
  	enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
494
  }
39735a3a3   Andre Przywara   ARM/KVM: save and...
495
496
  int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
  {
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
497
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
39735a3a3   Andre Przywara   ARM/KVM: save and...
498
499
500
  
  	switch (regid) {
  	case KVM_REG_ARM_TIMER_CTL:
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
501
  		vtimer->cnt_ctl = value;
39735a3a3   Andre Przywara   ARM/KVM: save and...
502
503
  		break;
  	case KVM_REG_ARM_TIMER_CNT:
90de943a4   Jintack Lim   KVM: arm/arm64: M...
504
  		update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
39735a3a3   Andre Przywara   ARM/KVM: save and...
505
506
  		break;
  	case KVM_REG_ARM_TIMER_CVAL:
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
507
  		vtimer->cnt_cval = value;
39735a3a3   Andre Przywara   ARM/KVM: save and...
508
509
510
511
  		break;
  	default:
  		return -1;
  	}
4b4b4512d   Christoffer Dall   arm/arm64: KVM: R...
512
513
  
  	kvm_timer_update_state(vcpu);
39735a3a3   Andre Przywara   ARM/KVM: save and...
514
515
516
517
518
  	return 0;
  }
  
  u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
  {
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
519
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
39735a3a3   Andre Przywara   ARM/KVM: save and...
520
521
522
  
  	switch (regid) {
  	case KVM_REG_ARM_TIMER_CTL:
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
523
  		return vtimer->cnt_ctl;
39735a3a3   Andre Przywara   ARM/KVM: save and...
524
  	case KVM_REG_ARM_TIMER_CNT:
90de943a4   Jintack Lim   KVM: arm/arm64: M...
525
  		return kvm_phys_timer_read() - vtimer->cntvoff;
39735a3a3   Andre Przywara   ARM/KVM: save and...
526
  	case KVM_REG_ARM_TIMER_CVAL:
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
527
  		return vtimer->cnt_cval;
39735a3a3   Andre Przywara   ARM/KVM: save and...
528
529
530
  	}
  	return (u64)-1;
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
531

b3c9950a5   Richard Cochran   arm/kvm/arch_time...
532
  static int kvm_timer_starting_cpu(unsigned int cpu)
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
533
  {
b3c9950a5   Richard Cochran   arm/kvm/arch_time...
534
535
  	kvm_timer_init_interrupt(NULL);
  	return 0;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
536
  }
b3c9950a5   Richard Cochran   arm/kvm/arch_time...
537
538
539
540
541
  static int kvm_timer_dying_cpu(unsigned int cpu)
  {
  	disable_percpu_irq(host_vtimer_irq);
  	return 0;
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
542

53e724067   Marc Zyngier   ARM: KVM: arch_ti...
543
544
  int kvm_timer_hyp_init(void)
  {
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
545
  	struct arch_timer_kvm_info *info;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
546
  	int err;
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
547
548
  	info = arch_timer_get_kvm_info();
  	timecounter = &info->timecounter;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
549

8e1a0476f   Christoffer Dall   KVM: arm/arm64: t...
550
551
552
553
554
  	if (!timecounter->cc) {
  		kvm_err("kvm_arch_timer: uninitialized timecounter
  ");
  		return -ENODEV;
  	}
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
555
556
557
558
  	if (info->virtual_irq <= 0) {
  		kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d
  ",
  			info->virtual_irq);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
559
560
  		return -ENODEV;
  	}
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
561
  	host_vtimer_irq = info->virtual_irq;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
562

cabdc5c59   Marc Zyngier   KVM: arm/arm64: t...
563
564
565
566
567
568
569
570
  	host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
  	if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
  	    host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
  		kvm_err("Invalid trigger for IRQ%d, assuming level low
  ",
  			host_vtimer_irq);
  		host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
  	}
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
571
  	err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
572
573
574
575
  				 "kvm guest timer", kvm_get_running_vcpus());
  	if (err) {
  		kvm_err("kvm_arch_timer: can't request interrupt %d (%d)
  ",
29c2d6ff4   Julien Grall   KVM: arm/arm64: a...
576
  			host_vtimer_irq, err);
5d947a144   Paolo Bonzini   KVM: ARM: cleanup...
577
  		return err;
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
578
  	}
2ffe95e3a   Ard Biesheuvel   KVM: arm/arm64: R...
579
580
  	kvm_debug("virtual timer IRQ%d
  ", host_vtimer_irq);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
581

b3c9950a5   Richard Cochran   arm/kvm/arch_time...
582
  	cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
73c1b41e6   Thomas Gleixner   cpu/hotplug: Clea...
583
  			  "kvm/arm/timer:starting", kvm_timer_starting_cpu,
b3c9950a5   Richard Cochran   arm/kvm/arch_time...
584
  			  kvm_timer_dying_cpu);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
585
586
587
588
589
590
  	return err;
  }
  
  void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
  {
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
591
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
592
593
  
  	timer_disarm(timer);
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
594
  	kvm_vgic_unmap_phys_irq(vcpu, vtimer->irq.irq);
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
595
  }
abcb851da   Christoffer Dall   KVM: arm/arm64: C...
596
  static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
597
  {
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
598
  	int vtimer_irq, ptimer_irq;
abcb851da   Christoffer Dall   KVM: arm/arm64: C...
599
  	int i, ret;
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
600

99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
601
  	vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
abcb851da   Christoffer Dall   KVM: arm/arm64: C...
602
603
604
  	ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
  	if (ret)
  		return false;
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
605

abcb851da   Christoffer Dall   KVM: arm/arm64: C...
606
607
608
  	ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
  	ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
  	if (ret)
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
609
  		return false;
abcb851da   Christoffer Dall   KVM: arm/arm64: C...
610
  	kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
611
612
613
614
615
616
617
  		if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
  		    vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
  			return false;
  	}
  
  	return true;
  }
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
618
  int kvm_timer_enable(struct kvm_vcpu *vcpu)
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
619
  {
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
620
  	struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
621
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
622
623
624
625
626
627
628
  	struct irq_desc *desc;
  	struct irq_data *data;
  	int phys_irq;
  	int ret;
  
  	if (timer->enabled)
  		return 0;
d9e139778   Alexander Graf   KVM: arm/arm64: S...
629
630
631
632
633
634
  	/* Without a VGIC we do not map virtual IRQs to physical IRQs */
  	if (!irqchip_in_kernel(vcpu->kvm))
  		goto no_vgic;
  
  	if (!vgic_initialized(vcpu->kvm))
  		return -ENODEV;
abcb851da   Christoffer Dall   KVM: arm/arm64: C...
635
  	if (!timer_irqs_are_valid(vcpu)) {
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
636
637
638
639
  		kvm_debug("incorrectly configured timer irqs
  ");
  		return -EINVAL;
  	}
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
  	/*
  	 * Find the physical IRQ number corresponding to the host_vtimer_irq
  	 */
  	desc = irq_to_desc(host_vtimer_irq);
  	if (!desc) {
  		kvm_err("%s: no interrupt descriptor
  ", __func__);
  		return -EINVAL;
  	}
  
  	data = irq_desc_get_irq_data(desc);
  	while (data->parent_data)
  		data = data->parent_data;
  
  	phys_irq = data->hwirq;
  
  	/*
  	 * Tell the VGIC that the virtual interrupt is tied to a
  	 * physical interrupt. We do that once per VCPU.
  	 */
fbb4aeec5   Jintack Lim   KVM: arm/arm64: A...
660
  	ret = kvm_vgic_map_phys_irq(vcpu, vtimer->irq.irq, phys_irq);
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
661
662
  	if (ret)
  		return ret;
d9e139778   Alexander Graf   KVM: arm/arm64: S...
663
  no_vgic:
fd5ebf99f   Longpeng(Mike)   arm/arm64: KVM: C...
664
  	timer->enabled = 1;
41a54482c   Christoffer Dall   KVM: arm/arm64: M...
665
  	return 0;
05971120f   Christoffer Dall   arm/arm64: KVM: R...
666
  }
53e724067   Marc Zyngier   ARM: KVM: arch_ti...
667

488f94d72   Jintack Lim   KVM: arm64: Acces...
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
  /*
   * On VHE system, we only need to configure trap on physical timer and counter
   * accesses in EL0 and EL1 once, not for every world switch.
   * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
   * and this makes those bits have no effect for the host kernel execution.
   */
  void kvm_timer_init_vhe(void)
  {
  	/* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
  	u32 cnthctl_shift = 10;
  	u64 val;
  
  	/*
  	 * Disallow physical timer access for the guest.
  	 * Physical counter access is allowed.
  	 */
  	val = read_sysreg(cnthctl_el2);
  	val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift);
  	val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
  	write_sysreg(val, cnthctl_el2);
  }
99a1db7a2   Christoffer Dall   KVM: arm/arm64: A...
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
  
  static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
  {
  	struct kvm_vcpu *vcpu;
  	int i;
  
  	kvm_for_each_vcpu(i, vcpu, kvm) {
  		vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
  		vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
  	}
  }
  
  int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
  {
  	int __user *uaddr = (int __user *)(long)attr->addr;
  	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
  	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
  	int irq;
  
  	if (!irqchip_in_kernel(vcpu->kvm))
  		return -EINVAL;
  
  	if (get_user(irq, uaddr))
  		return -EFAULT;
  
  	if (!(irq_is_ppi(irq)))
  		return -EINVAL;
  
  	if (vcpu->arch.timer_cpu.enabled)
  		return -EBUSY;
  
  	switch (attr->attr) {
  	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
  		set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
  		break;
  	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
  		set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
  		break;
  	default:
  		return -ENXIO;
  	}
  
  	return 0;
  }
  
  int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
  {
  	int __user *uaddr = (int __user *)(long)attr->addr;
  	struct arch_timer_context *timer;
  	int irq;
  
  	switch (attr->attr) {
  	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
  		timer = vcpu_vtimer(vcpu);
  		break;
  	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
  		timer = vcpu_ptimer(vcpu);
  		break;
  	default:
  		return -ENXIO;
  	}
  
  	irq = timer->irq.irq;
  	return put_user(irq, uaddr);
  }
  
  int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
  {
  	switch (attr->attr) {
  	case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
  	case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
  		return 0;
  	}
  
  	return -ENXIO;
  }