time: Add timekeeper lock

Now that all the timekeeping variables are stored in the timekeeper structure, add a new lock to protect the structure. For now, this lock nests under the xtime_lock for writes. For readers, we don't need to take xtime_lock anymore. CC: Thomas Gleixner <tglx@linutronix.de> CC: Eric Dumazet <eric.dumazet@gmail.com> CC: Richard Cochran <richardcochran@gmail.com> Signed-off-by: John Stultz <john.stultz@linaro.org>

time: Add timekeeper lock
Now that all the timekeeping variables are stored in the timekeeper structure, add a new lock to protect the structure. For now, this lock nests under the xtime_lock for writes. For readers, we don't need to take xtime_lock anymore. CC: Thomas Gleixner <tglx@linutronix.de> CC: Eric Dumazet <eric.dumazet@gmail.com> CC: Richard Cochran <richardcochran@gmail.com> Signed-off-by: John Stultz <john.stultz@linaro.org>
John Stultz
1 parent 8fcce546be
Showing 1 changed file with 82 additions and 44 deletions Side-by-side Diff
kernel/time/timekeeping.c
@@ -69,6 +69,9 @@
 	struct timespec total_sleep_time;
 	/* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
 	struct timespec raw_time;
+
+	/* Seqlock for all timekeeper values */
+	seqlock_t lock;
 };
  
 static struct timekeeper timekeeper;
  
@@ -172,10 +175,17 @@
 /* must hold xtime_lock */
 void timekeeping_leap_insert(int leapsecond)
 {
+	unsigned long flags;
+
+	write_seqlock_irqsave(&timekeeper.lock, flags);
+
 	timekeeper.xtime.tv_sec += leapsecond;
 	timekeeper.wall_to_monotonic.tv_sec -= leapsecond;
 	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
 			 timekeeper.clock, timekeeper.mult);
+
+	write_sequnlock_irqrestore(&timekeeper.lock, flags);
+
 }
  
 /**
@@ -222,7 +232,7 @@
 	WARN_ON(timekeeping_suspended);
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
  
 		*ts = timekeeper.xtime;
 		nsecs = timekeeping_get_ns();
@@ -230,7 +240,7 @@
 		/* If arch requires, add in gettimeoffset() */
 		nsecs += arch_gettimeoffset();
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	timespec_add_ns(ts, nsecs);
 }
@@ -245,7 +255,7 @@
 	WARN_ON(timekeeping_suspended);
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		secs = timekeeper.xtime.tv_sec +
 				timekeeper.wall_to_monotonic.tv_sec;
 		nsecs = timekeeper.xtime.tv_nsec +
@@ -254,7 +264,7 @@
 		/* If arch requires, add in gettimeoffset() */
 		nsecs += arch_gettimeoffset();
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
 	/*
 	 * Use ktime_set/ktime_add_ns to create a proper ktime on
 	 * 32-bit architectures without CONFIG_KTIME_SCALAR.
  
@@ -280,14 +290,14 @@
 	WARN_ON(timekeeping_suspended);
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		*ts = timekeeper.xtime;
 		tomono = timekeeper.wall_to_monotonic;
 		nsecs = timekeeping_get_ns();
 		/* If arch requires, add in gettimeoffset() */
 		nsecs += arch_gettimeoffset();
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
 				ts->tv_nsec + tomono.tv_nsec + nsecs);
@@ -315,7 +325,7 @@
 	do {
 		u32 arch_offset;
  
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
  
 		*ts_raw = timekeeper.raw_time;
 		*ts_real = timekeeper.xtime;
@@ -328,7 +338,7 @@
 		nsecs_raw += arch_offset;
 		nsecs_real += arch_offset;
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	timespec_add_ns(ts_raw, nsecs_raw);
 	timespec_add_ns(ts_real, nsecs_real);
  
@@ -362,12 +372,13 @@
 int do_settimeofday(const struct timespec *tv)
 {
 	struct timespec ts_delta;
-	unsigned long flags;
+	unsigned long flags1,flags2;
  
 	if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
 		return -EINVAL;
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	write_seqlock_irqsave(&xtime_lock, flags1);
+	write_seqlock_irqsave(&timekeeper.lock, flags2);
  
 	timekeeping_forward_now();
  
@@ -384,7 +395,8 @@
 	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
 			timekeeper.clock, timekeeper.mult);
  
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags2);
+	write_sequnlock_irqrestore(&xtime_lock, flags1);
  
 	/* signal hrtimers about time change */
 	clock_was_set();
  
@@ -403,12 +415,13 @@
  */
 int timekeeping_inject_offset(struct timespec *ts)
 {
-	unsigned long flags;
+	unsigned long flags1,flags2;
  
 	if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
 		return -EINVAL;
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	write_seqlock_irqsave(&xtime_lock, flags1);
+	write_seqlock_irqsave(&timekeeper.lock, flags2);
  
 	timekeeping_forward_now();
  
@@ -422,7 +435,8 @@
 	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
 			timekeeper.clock, timekeeper.mult);
  
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags2);
+	write_sequnlock_irqrestore(&xtime_lock, flags1);
  
 	/* signal hrtimers about time change */
 	clock_was_set();
  
@@ -494,11 +508,11 @@
 	s64 nsecs;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		nsecs = timekeeping_get_ns_raw();
 		*ts = timekeeper.raw_time;
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	timespec_add_ns(ts, nsecs);
 }
  
  
  
@@ -514,24 +528,30 @@
 	int ret;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
  
 		ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	return ret;
 }
  
 /**
  * timekeeping_max_deferment - Returns max time the clocksource can be deferred
- *
- * Caller must observe xtime_lock via read_seqbegin/read_seqretry to
- * ensure that the clocksource does not change!
  */
 u64 timekeeping_max_deferment(void)
 {
-	return timekeeper.clock->max_idle_ns;
+	unsigned long seq;
+	u64 ret;
+	do {
+		seq = read_seqbegin(&timekeeper.lock);
+
+		ret = timekeeper.clock->max_idle_ns;
+
+	} while (read_seqretry(&timekeeper.lock, seq));
+
+	return ret;
 }
  
 /**
  
  
  
@@ -576,10 +596,13 @@
 	read_persistent_clock(&now);
 	read_boot_clock(&boot);
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	seqlock_init(&timekeeper.lock);
  
+	write_seqlock_irqsave(&xtime_lock, flags);
 	ntp_init();
+	write_sequnlock_irqrestore(&xtime_lock, flags);
  
+	write_seqlock_irqsave(&timekeeper.lock, flags);
 	clock = clocksource_default_clock();
 	if (clock->enable)
 		clock->enable(clock);
@@ -597,7 +620,7 @@
 				-boot.tv_sec, -boot.tv_nsec);
 	timekeeper.total_sleep_time.tv_sec = 0;
 	timekeeper.total_sleep_time.tv_nsec = 0;
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags);
 }
  
 /* time in seconds when suspend began */
@@ -638,7 +661,7 @@
  */
 void timekeeping_inject_sleeptime(struct timespec *delta)
 {
-	unsigned long flags;
+	unsigned long flags1,flags2;
 	struct timespec ts;
  
 	/* Make sure we don't set the clock twice */
@@ -646,7 +669,9 @@
 	if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
 		return;
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	write_seqlock_irqsave(&xtime_lock, flags1);
+	write_seqlock_irqsave(&timekeeper.lock, flags2);
+
 	timekeeping_forward_now();
  
 	__timekeeping_inject_sleeptime(delta);
@@ -656,7 +681,8 @@
 	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
 			timekeeper.clock, timekeeper.mult);
  
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags2);
+	write_sequnlock_irqrestore(&xtime_lock, flags1);
  
 	/* signal hrtimers about time change */
 	clock_was_set();
  
@@ -672,14 +698,15 @@
  */
 static void timekeeping_resume(void)
 {
-	unsigned long flags;
+	unsigned long flags1,flags2;
 	struct timespec ts;
  
 	read_persistent_clock(&ts);
  
 	clocksource_resume();
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	write_seqlock_irqsave(&xtime_lock, flags1);
+	write_seqlock_irqsave(&timekeeper.lock, flags2);
  
 	if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
 		ts = timespec_sub(ts, timekeeping_suspend_time);
@@ -689,7 +716,8 @@
 	timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
 	timekeeper.ntp_error = 0;
 	timekeeping_suspended = 0;
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags2);
+	write_sequnlock_irqrestore(&xtime_lock, flags1);
  
 	touch_softlockup_watchdog();
  
  
@@ -701,13 +729,14 @@
  
 static int timekeeping_suspend(void)
 {
-	unsigned long flags;
+	unsigned long flags1,flags2;
 	struct timespec		delta, delta_delta;
 	static struct timespec	old_delta;
  
 	read_persistent_clock(&timekeeping_suspend_time);
  
-	write_seqlock_irqsave(&xtime_lock, flags);
+	write_seqlock_irqsave(&xtime_lock, flags1);
+	write_seqlock_irqsave(&timekeeper.lock, flags2);
 	timekeeping_forward_now();
 	timekeeping_suspended = 1;
  
@@ -730,7 +759,8 @@
 		timekeeping_suspend_time =
 			timespec_add(timekeeping_suspend_time, delta_delta);
 	}
-	write_sequnlock_irqrestore(&xtime_lock, flags);
+	write_sequnlock_irqrestore(&timekeeper.lock, flags2);
+	write_sequnlock_irqrestore(&xtime_lock, flags1);
  
 	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
 	clocksource_suspend();
  
  
@@ -983,10 +1013,13 @@
 	struct clocksource *clock;
 	cycle_t offset;
 	int shift = 0, maxshift;
+	unsigned long flags;
  
+	write_seqlock_irqsave(&timekeeper.lock, flags);
+
 	/* Make sure we're fully resumed: */
 	if (unlikely(timekeeping_suspended))
-		return;
+		goto out;
  
 	clock = timekeeper.clock;
  
@@ -1067,6 +1100,10 @@
 	/* check to see if there is a new clocksource to use */
 	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
 			timekeeper.clock, timekeeper.mult);
+
+out:
+	write_sequnlock_irqrestore(&timekeeper.lock, flags);
+
 }
  
 /**
  
@@ -1112,13 +1149,13 @@
 	WARN_ON(timekeeping_suspended);
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		*ts = timekeeper.xtime;
 		tomono = timekeeper.wall_to_monotonic;
 		sleep = timekeeper.total_sleep_time;
 		nsecs = timekeeping_get_ns();
  
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
 			ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
  
@@ -1169,10 +1206,10 @@
 	unsigned long seq;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
  
 		now = timekeeper.xtime;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	return now;
 }
  
@@ -1184,11 +1221,11 @@
 	unsigned long seq;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
  
 		now = timekeeper.xtime;
 		mono = timekeeper.wall_to_monotonic;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
  
 	set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
 				now.tv_nsec + mono.tv_nsec);
  
@@ -1220,11 +1257,11 @@
 	unsigned long seq;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		*xtim = timekeeper.xtime;
 		*wtom = timekeeper.wall_to_monotonic;
 		*sleep = timekeeper.total_sleep_time;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
 }
  
 /**
  
@@ -1236,9 +1273,10 @@
 	struct timespec wtom;
  
 	do {
-		seq = read_seqbegin(&xtime_lock);
+		seq = read_seqbegin(&timekeeper.lock);
 		wtom = timekeeper.wall_to_monotonic;
-	} while (read_seqretry(&xtime_lock, seq));
+	} while (read_seqretry(&timekeeper.lock, seq));
+
 	return timespec_to_ktime(wtom);
 }
...	...	@@ -69,6 +69,9 @@
69	69	struct timespec total_sleep_time;
70	70	/* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
71	71	struct timespec raw_time;
	72	+
	73	+ /* Seqlock for all timekeeper values */
	74	+ seqlock_t lock;
72	75	};
73	76
74	77	static struct timekeeper timekeeper;
75	78
...	...	@@ -172,10 +175,17 @@
172	175	/* must hold xtime_lock */
173	176	void timekeeping_leap_insert(int leapsecond)
174	177	{
	178	+ unsigned long flags;
	179	+
	180	+ write_seqlock_irqsave(&timekeeper.lock, flags);
	181	+
175	182	timekeeper.xtime.tv_sec += leapsecond;
176	183	timekeeper.wall_to_monotonic.tv_sec -= leapsecond;
177	184	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
178	185	timekeeper.clock, timekeeper.mult);
	186	+
	187	+ write_sequnlock_irqrestore(&timekeeper.lock, flags);
	188	+
179	189	}
180	190
181	191	/**
...	...	@@ -222,7 +232,7 @@
222	232	WARN_ON(timekeeping_suspended);
223	233
224	234	do {
225		- seq = read_seqbegin(&xtime_lock);
	235	+ seq = read_seqbegin(&timekeeper.lock);
226	236
227	237	*ts = timekeeper.xtime;
228	238	nsecs = timekeeping_get_ns();
...	...	@@ -230,7 +240,7 @@
230	240	/* If arch requires, add in gettimeoffset() */
231	241	nsecs += arch_gettimeoffset();
232	242
233		- } while (read_seqretry(&xtime_lock, seq));
	243	+ } while (read_seqretry(&timekeeper.lock, seq));
234	244
235	245	timespec_add_ns(ts, nsecs);
236	246	}
...	...	@@ -245,7 +255,7 @@
245	255	WARN_ON(timekeeping_suspended);
246	256
247	257	do {
248		- seq = read_seqbegin(&xtime_lock);
	258	+ seq = read_seqbegin(&timekeeper.lock);
249	259	secs = timekeeper.xtime.tv_sec +
250	260	timekeeper.wall_to_monotonic.tv_sec;
251	261	nsecs = timekeeper.xtime.tv_nsec +
...	...	@@ -254,7 +264,7 @@
254	264	/* If arch requires, add in gettimeoffset() */
255	265	nsecs += arch_gettimeoffset();
256	266
257		- } while (read_seqretry(&xtime_lock, seq));
	267	+ } while (read_seqretry(&timekeeper.lock, seq));
258	268	/*
259	269	* Use ktime_set/ktime_add_ns to create a proper ktime on
260	270	* 32-bit architectures without CONFIG_KTIME_SCALAR.
261	271
...	...	@@ -280,14 +290,14 @@
280	290	WARN_ON(timekeeping_suspended);
281	291
282	292	do {
283		- seq = read_seqbegin(&xtime_lock);
	293	+ seq = read_seqbegin(&timekeeper.lock);
284	294	*ts = timekeeper.xtime;
285	295	tomono = timekeeper.wall_to_monotonic;
286	296	nsecs = timekeeping_get_ns();
287	297	/* If arch requires, add in gettimeoffset() */
288	298	nsecs += arch_gettimeoffset();
289	299
290		- } while (read_seqretry(&xtime_lock, seq));
	300	+ } while (read_seqretry(&timekeeper.lock, seq));
291	301
292	302	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
293	303	ts->tv_nsec + tomono.tv_nsec + nsecs);
...	...	@@ -315,7 +325,7 @@
315	325	do {
316	326	u32 arch_offset;
317	327
318		- seq = read_seqbegin(&xtime_lock);
	328	+ seq = read_seqbegin(&timekeeper.lock);
319	329
320	330	*ts_raw = timekeeper.raw_time;
321	331	*ts_real = timekeeper.xtime;
...	...	@@ -328,7 +338,7 @@
328	338	nsecs_raw += arch_offset;
329	339	nsecs_real += arch_offset;
330	340
331		- } while (read_seqretry(&xtime_lock, seq));
	341	+ } while (read_seqretry(&timekeeper.lock, seq));
332	342
333	343	timespec_add_ns(ts_raw, nsecs_raw);
334	344	timespec_add_ns(ts_real, nsecs_real);
335	345
...	...	@@ -362,12 +372,13 @@
362	372	int do_settimeofday(const struct timespec *tv)
363	373	{
364	374	struct timespec ts_delta;
365		- unsigned long flags;
	375	+ unsigned long flags1,flags2;
366	376
367	377	if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
368	378	return -EINVAL;
369	379
370		- write_seqlock_irqsave(&xtime_lock, flags);
	380	+ write_seqlock_irqsave(&xtime_lock, flags1);
	381	+ write_seqlock_irqsave(&timekeeper.lock, flags2);
371	382
372	383	timekeeping_forward_now();
373	384
...	...	@@ -384,7 +395,8 @@
384	395	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
385	396	timekeeper.clock, timekeeper.mult);
386	397
387		- write_sequnlock_irqrestore(&xtime_lock, flags);
	398	+ write_sequnlock_irqrestore(&timekeeper.lock, flags2);
	399	+ write_sequnlock_irqrestore(&xtime_lock, flags1);
388	400
389	401	/* signal hrtimers about time change */
390	402	clock_was_set();
391	403
...	...	@@ -403,12 +415,13 @@
403	415	*/
404	416	int timekeeping_inject_offset(struct timespec *ts)
405	417	{
406		- unsigned long flags;
	418	+ unsigned long flags1,flags2;
407	419
408	420	if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
409	421	return -EINVAL;
410	422
411		- write_seqlock_irqsave(&xtime_lock, flags);
	423	+ write_seqlock_irqsave(&xtime_lock, flags1);
	424	+ write_seqlock_irqsave(&timekeeper.lock, flags2);
412	425
413	426	timekeeping_forward_now();
414	427
...	...	@@ -422,7 +435,8 @@
422	435	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
423	436	timekeeper.clock, timekeeper.mult);
424	437
425		- write_sequnlock_irqrestore(&xtime_lock, flags);
	438	+ write_sequnlock_irqrestore(&timekeeper.lock, flags2);
	439	+ write_sequnlock_irqrestore(&xtime_lock, flags1);
426	440
427	441	/* signal hrtimers about time change */
428	442	clock_was_set();
429	443
...	...	@@ -494,11 +508,11 @@
494	508	s64 nsecs;
495	509
496	510	do {
497		- seq = read_seqbegin(&xtime_lock);
	511	+ seq = read_seqbegin(&timekeeper.lock);
498	512	nsecs = timekeeping_get_ns_raw();
499	513	*ts = timekeeper.raw_time;
500	514
501		- } while (read_seqretry(&xtime_lock, seq));
	515	+ } while (read_seqretry(&timekeeper.lock, seq));
502	516
503	517	timespec_add_ns(ts, nsecs);
504	518	}
505	519
506	520
507	521
...	...	@@ -514,24 +528,30 @@
514	528	int ret;
515	529
516	530	do {
517		- seq = read_seqbegin(&xtime_lock);
	531	+ seq = read_seqbegin(&timekeeper.lock);
518	532
519	533	ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
520	534
521		- } while (read_seqretry(&xtime_lock, seq));
	535	+ } while (read_seqretry(&timekeeper.lock, seq));
522	536
523	537	return ret;
524	538	}
525	539
526	540	/**
527	541	* timekeeping_max_deferment - Returns max time the clocksource can be deferred
528		- *
529		- * Caller must observe xtime_lock via read_seqbegin/read_seqretry to
530		- * ensure that the clocksource does not change!
531	542	*/
532	543	u64 timekeeping_max_deferment(void)
533	544	{
534		- return timekeeper.clock->max_idle_ns;
	545	+ unsigned long seq;
	546	+ u64 ret;
	547	+ do {
	548	+ seq = read_seqbegin(&timekeeper.lock);
	549	+
	550	+ ret = timekeeper.clock->max_idle_ns;
	551	+
	552	+ } while (read_seqretry(&timekeeper.lock, seq));
	553	+
	554	+ return ret;
535	555	}
536	556
537	557	/**
538	558
539	559
540	560
...	...	@@ -576,10 +596,13 @@
576	596	read_persistent_clock(&now);
577	597	read_boot_clock(&boot);
578	598
579		- write_seqlock_irqsave(&xtime_lock, flags);
	599	+ seqlock_init(&timekeeper.lock);
580	600
	601	+ write_seqlock_irqsave(&xtime_lock, flags);
581	602	ntp_init();
	603	+ write_sequnlock_irqrestore(&xtime_lock, flags);
582	604
	605	+ write_seqlock_irqsave(&timekeeper.lock, flags);
583	606	clock = clocksource_default_clock();
584	607	if (clock->enable)
585	608	clock->enable(clock);
...	...	@@ -597,7 +620,7 @@
597	620	-boot.tv_sec, -boot.tv_nsec);
598	621	timekeeper.total_sleep_time.tv_sec = 0;
599	622	timekeeper.total_sleep_time.tv_nsec = 0;
600		- write_sequnlock_irqrestore(&xtime_lock, flags);
	623	+ write_sequnlock_irqrestore(&timekeeper.lock, flags);
601	624	}
602	625
603	626	/* time in seconds when suspend began */
...	...	@@ -638,7 +661,7 @@
638	661	*/
639	662	void timekeeping_inject_sleeptime(struct timespec *delta)
640	663	{
641		- unsigned long flags;
	664	+ unsigned long flags1,flags2;
642	665	struct timespec ts;
643	666
644	667	/* Make sure we don't set the clock twice */
...	...	@@ -646,7 +669,9 @@
646	669	if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
647	670	return;
648	671
649		- write_seqlock_irqsave(&xtime_lock, flags);
	672	+ write_seqlock_irqsave(&xtime_lock, flags1);
	673	+ write_seqlock_irqsave(&timekeeper.lock, flags2);
	674	+
650	675	timekeeping_forward_now();
651	676
652	677	__timekeeping_inject_sleeptime(delta);
...	...	@@ -656,7 +681,8 @@
656	681	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
657	682	timekeeper.clock, timekeeper.mult);
658	683
659		- write_sequnlock_irqrestore(&xtime_lock, flags);
	684	+ write_sequnlock_irqrestore(&timekeeper.lock, flags2);
	685	+ write_sequnlock_irqrestore(&xtime_lock, flags1);
660	686
661	687	/* signal hrtimers about time change */
662	688	clock_was_set();
663	689
...	...	@@ -672,14 +698,15 @@
672	698	*/
673	699	static void timekeeping_resume(void)
674	700	{
675		- unsigned long flags;
	701	+ unsigned long flags1,flags2;
676	702	struct timespec ts;
677	703
678	704	read_persistent_clock(&ts);
679	705
680	706	clocksource_resume();
681	707
682		- write_seqlock_irqsave(&xtime_lock, flags);
	708	+ write_seqlock_irqsave(&xtime_lock, flags1);
	709	+ write_seqlock_irqsave(&timekeeper.lock, flags2);
683	710
684	711	if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
685	712	ts = timespec_sub(ts, timekeeping_suspend_time);
...	...	@@ -689,7 +716,8 @@
689	716	timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
690	717	timekeeper.ntp_error = 0;
691	718	timekeeping_suspended = 0;
692		- write_sequnlock_irqrestore(&xtime_lock, flags);
	719	+ write_sequnlock_irqrestore(&timekeeper.lock, flags2);
	720	+ write_sequnlock_irqrestore(&xtime_lock, flags1);
693	721
694	722	touch_softlockup_watchdog();
695	723
696	724
...	...	@@ -701,13 +729,14 @@
701	729
702	730	static int timekeeping_suspend(void)
703	731	{
704		- unsigned long flags;
	732	+ unsigned long flags1,flags2;
705	733	struct timespec delta, delta_delta;
706	734	static struct timespec old_delta;
707	735
708	736	read_persistent_clock(&timekeeping_suspend_time);
709	737
710		- write_seqlock_irqsave(&xtime_lock, flags);
	738	+ write_seqlock_irqsave(&xtime_lock, flags1);
	739	+ write_seqlock_irqsave(&timekeeper.lock, flags2);
711	740	timekeeping_forward_now();
712	741	timekeeping_suspended = 1;
713	742
...	...	@@ -730,7 +759,8 @@
730	759	timekeeping_suspend_time =
731	760	timespec_add(timekeeping_suspend_time, delta_delta);
732	761	}
733		- write_sequnlock_irqrestore(&xtime_lock, flags);
	762	+ write_sequnlock_irqrestore(&timekeeper.lock, flags2);
	763	+ write_sequnlock_irqrestore(&xtime_lock, flags1);
734	764
735	765	clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
736	766	clocksource_suspend();
737	767
738	768
...	...	@@ -983,10 +1013,13 @@
983	1013	struct clocksource *clock;
984	1014	cycle_t offset;
985	1015	int shift = 0, maxshift;
	1016	+ unsigned long flags;
986	1017
	1018	+ write_seqlock_irqsave(&timekeeper.lock, flags);
	1019	+
987	1020	/* Make sure we're fully resumed: */
988	1021	if (unlikely(timekeeping_suspended))
989		- return;
	1022	+ goto out;
990	1023
991	1024	clock = timekeeper.clock;
992	1025
...	...	@@ -1067,6 +1100,10 @@
1067	1100	/* check to see if there is a new clocksource to use */
1068	1101	update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
1069	1102	timekeeper.clock, timekeeper.mult);
	1103	+
	1104	+out:
	1105	+ write_sequnlock_irqrestore(&timekeeper.lock, flags);
	1106	+
1070	1107	}
1071	1108
1072	1109	/**
1073	1110
...	...	@@ -1112,13 +1149,13 @@
1112	1149	WARN_ON(timekeeping_suspended);
1113	1150
1114	1151	do {
1115		- seq = read_seqbegin(&xtime_lock);
	1152	+ seq = read_seqbegin(&timekeeper.lock);
1116	1153	*ts = timekeeper.xtime;
1117	1154	tomono = timekeeper.wall_to_monotonic;
1118	1155	sleep = timekeeper.total_sleep_time;
1119	1156	nsecs = timekeeping_get_ns();
1120	1157
1121		- } while (read_seqretry(&xtime_lock, seq));
	1158	+ } while (read_seqretry(&timekeeper.lock, seq));
1122	1159
1123	1160	set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
1124	1161	ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
1125	1162
...	...	@@ -1169,10 +1206,10 @@
1169	1206	unsigned long seq;
1170	1207
1171	1208	do {
1172		- seq = read_seqbegin(&xtime_lock);
	1209	+ seq = read_seqbegin(&timekeeper.lock);
1173	1210
1174	1211	now = timekeeper.xtime;
1175		- } while (read_seqretry(&xtime_lock, seq));
	1212	+ } while (read_seqretry(&timekeeper.lock, seq));
1176	1213
1177	1214	return now;
1178	1215	}
1179	1216
...	...	@@ -1184,11 +1221,11 @@
1184	1221	unsigned long seq;
1185	1222
1186	1223	do {
1187		- seq = read_seqbegin(&xtime_lock);
	1224	+ seq = read_seqbegin(&timekeeper.lock);
1188	1225
1189	1226	now = timekeeper.xtime;
1190	1227	mono = timekeeper.wall_to_monotonic;
1191		- } while (read_seqretry(&xtime_lock, seq));
	1228	+ } while (read_seqretry(&timekeeper.lock, seq));
1192	1229
1193	1230	set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1194	1231	now.tv_nsec + mono.tv_nsec);
1195	1232
...	...	@@ -1220,11 +1257,11 @@
1220	1257	unsigned long seq;
1221	1258
1222	1259	do {
1223		- seq = read_seqbegin(&xtime_lock);
	1260	+ seq = read_seqbegin(&timekeeper.lock);
1224	1261	*xtim = timekeeper.xtime;
1225	1262	*wtom = timekeeper.wall_to_monotonic;
1226	1263	*sleep = timekeeper.total_sleep_time;
1227		- } while (read_seqretry(&xtime_lock, seq));
	1264	+ } while (read_seqretry(&timekeeper.lock, seq));
1228	1265	}
1229	1266
1230	1267	/**
1231	1268
...	...	@@ -1236,9 +1273,10 @@
1236	1273	struct timespec wtom;
1237	1274
1238	1275	do {
1239		- seq = read_seqbegin(&xtime_lock);
	1276	+ seq = read_seqbegin(&timekeeper.lock);
1240	1277	wtom = timekeeper.wall_to_monotonic;
1241		- } while (read_seqretry(&xtime_lock, seq));
	1278	+ } while (read_seqretry(&timekeeper.lock, seq));
	1279	+
1242	1280	return timespec_to_ktime(wtom);
1243	1281	}
1244	1282