Blame view

kernel/sched/auto_group.c 6.47 KB
029632fbb   Peter Zijlstra   sched: Make separ...
1
  #include "sched.h"
5091faa44   Mike Galbraith   sched: Add 'autog...
2
3
4
5
  #include <linux/proc_fs.h>
  #include <linux/seq_file.h>
  #include <linux/kallsyms.h>
  #include <linux/utsname.h>
029632fbb   Peter Zijlstra   sched: Make separ...
6
7
  #include <linux/security.h>
  #include <linux/export.h>
5091faa44   Mike Galbraith   sched: Add 'autog...
8
9
10
11
  
  unsigned int __read_mostly sysctl_sched_autogroup_enabled = 1;
  static struct autogroup autogroup_default;
  static atomic_t autogroup_seq_nr;
029632fbb   Peter Zijlstra   sched: Make separ...
12
  void __init autogroup_init(struct task_struct *init_task)
5091faa44   Mike Galbraith   sched: Add 'autog...
13
  {
07e06b011   Yong Zhang   sched: Consolidat...
14
  	autogroup_default.tg = &root_task_group;
5091faa44   Mike Galbraith   sched: Add 'autog...
15
16
17
18
  	kref_init(&autogroup_default.kref);
  	init_rwsem(&autogroup_default.lock);
  	init_task->signal->autogroup = &autogroup_default;
  }
029632fbb   Peter Zijlstra   sched: Make separ...
19
  void autogroup_free(struct task_group *tg)
5091faa44   Mike Galbraith   sched: Add 'autog...
20
21
22
23
24
25
26
  {
  	kfree(tg->autogroup);
  }
  
  static inline void autogroup_destroy(struct kref *kref)
  {
  	struct autogroup *ag = container_of(kref, struct autogroup, kref);
f44937718   Mike Galbraith   sched, autogroup:...
27
28
29
30
31
  #ifdef CONFIG_RT_GROUP_SCHED
  	/* We've redirected RT tasks to the root task group... */
  	ag->tg->rt_se = NULL;
  	ag->tg->rt_rq = NULL;
  #endif
ace783b9b   Li Zefan   sched: split out ...
32
  	sched_offline_group(ag->tg);
5091faa44   Mike Galbraith   sched: Add 'autog...
33
34
35
36
37
38
39
40
41
42
43
44
45
  	sched_destroy_group(ag->tg);
  }
  
  static inline void autogroup_kref_put(struct autogroup *ag)
  {
  	kref_put(&ag->kref, autogroup_destroy);
  }
  
  static inline struct autogroup *autogroup_kref_get(struct autogroup *ag)
  {
  	kref_get(&ag->kref);
  	return ag;
  }
4f8219875   Mike Galbraith   sched, autogroup:...
46
47
48
49
50
51
52
53
54
55
56
57
58
  static inline struct autogroup *autogroup_task_get(struct task_struct *p)
  {
  	struct autogroup *ag;
  	unsigned long flags;
  
  	if (!lock_task_sighand(p, &flags))
  		return autogroup_kref_get(&autogroup_default);
  
  	ag = autogroup_kref_get(p->signal->autogroup);
  	unlock_task_sighand(p, &flags);
  
  	return ag;
  }
5091faa44   Mike Galbraith   sched: Add 'autog...
59
60
61
62
63
64
65
  static inline struct autogroup *autogroup_create(void)
  {
  	struct autogroup *ag = kzalloc(sizeof(*ag), GFP_KERNEL);
  	struct task_group *tg;
  
  	if (!ag)
  		goto out_fail;
07e06b011   Yong Zhang   sched: Consolidat...
66
  	tg = sched_create_group(&root_task_group);
5091faa44   Mike Galbraith   sched: Add 'autog...
67
68
69
70
71
72
73
74
  
  	if (IS_ERR(tg))
  		goto out_free;
  
  	kref_init(&ag->kref);
  	init_rwsem(&ag->lock);
  	ag->id = atomic_inc_return(&autogroup_seq_nr);
  	ag->tg = tg;
f44937718   Mike Galbraith   sched, autogroup:...
75
76
77
78
79
80
  #ifdef CONFIG_RT_GROUP_SCHED
  	/*
  	 * Autogroup RT tasks are redirected to the root task group
  	 * so we don't have to move tasks around upon policy change,
  	 * or flail around trying to allocate bandwidth on the fly.
  	 * A bandwidth exception in __sched_setscheduler() allows
1fe89e1b6   Peter Zijlstra   sched/autogroup: ...
81
  	 * the policy change to proceed.
f44937718   Mike Galbraith   sched, autogroup:...
82
83
84
85
86
  	 */
  	free_rt_sched_group(tg);
  	tg->rt_se = root_task_group.rt_se;
  	tg->rt_rq = root_task_group.rt_rq;
  #endif
5091faa44   Mike Galbraith   sched: Add 'autog...
87
  	tg->autogroup = ag;
41261b6a8   Gerald Schaefer   sched/autogroup: ...
88
  	sched_online_group(tg, &root_task_group);
5091faa44   Mike Galbraith   sched: Add 'autog...
89
90
91
92
93
94
95
96
97
98
99
100
101
  	return ag;
  
  out_free:
  	kfree(ag);
  out_fail:
  	if (printk_ratelimit()) {
  		printk(KERN_WARNING "autogroup_create: %s failure.
  ",
  			ag ? "sched_create_group()" : "kmalloc()");
  	}
  
  	return autogroup_kref_get(&autogroup_default);
  }
029632fbb   Peter Zijlstra   sched: Make separ...
102
  bool task_wants_autogroup(struct task_struct *p, struct task_group *tg)
5091faa44   Mike Galbraith   sched: Add 'autog...
103
104
105
  {
  	if (tg != &root_task_group)
  		return false;
5091faa44   Mike Galbraith   sched: Add 'autog...
106
  	/*
18f649ef3   Oleg Nesterov   sched/autogroup: ...
107
108
109
  	 * If we race with autogroup_move_group() the caller can use the old
  	 * value of signal->autogroup but in this case sched_move_task() will
  	 * be called again before autogroup_kref_put().
8e5bfa8c1   Oleg Nesterov   sched/autogroup: ...
110
111
112
  	 *
  	 * However, there is no way sched_autogroup_exit_task() could tell us
  	 * to avoid autogroup->tg, so we abuse PF_EXITING flag for this case.
5091faa44   Mike Galbraith   sched: Add 'autog...
113
  	 */
8e5bfa8c1   Oleg Nesterov   sched/autogroup: ...
114
115
  	if (p->flags & PF_EXITING)
  		return false;
5091faa44   Mike Galbraith   sched: Add 'autog...
116
117
  	return true;
  }
8e5bfa8c1   Oleg Nesterov   sched/autogroup: ...
118
119
120
121
122
123
124
125
126
  void sched_autogroup_exit_task(struct task_struct *p)
  {
  	/*
  	 * We are going to call exit_notify() and autogroup_move_group() can't
  	 * see this thread after that: we can no longer use signal->autogroup.
  	 * See the PF_EXITING check in task_wants_autogroup().
  	 */
  	sched_move_task(p);
  }
5091faa44   Mike Galbraith   sched: Add 'autog...
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
  static void
  autogroup_move_group(struct task_struct *p, struct autogroup *ag)
  {
  	struct autogroup *prev;
  	struct task_struct *t;
  	unsigned long flags;
  
  	BUG_ON(!lock_task_sighand(p, &flags));
  
  	prev = p->signal->autogroup;
  	if (prev == ag) {
  		unlock_task_sighand(p, &flags);
  		return;
  	}
  
  	p->signal->autogroup = autogroup_kref_get(ag);
18f649ef3   Oleg Nesterov   sched/autogroup: ...
143
144
145
146
147
148
149
  	/*
  	 * We can't avoid sched_move_task() after we changed signal->autogroup,
  	 * this process can already run with task_group() == prev->tg or we can
  	 * race with cgroup code which can read autogroup = prev under rq->lock.
  	 * In the latter case for_each_thread() can not miss a migrating thread,
  	 * cpu_cgroup_attach() must not be possible after cgroup_exit() and it
  	 * can't be removed from thread list, we hold ->siglock.
8e5bfa8c1   Oleg Nesterov   sched/autogroup: ...
150
151
152
  	 *
  	 * If an exiting thread was already removed from thread list we rely on
  	 * sched_autogroup_exit_task().
18f649ef3   Oleg Nesterov   sched/autogroup: ...
153
  	 */
5aface53d   Oleg Nesterov   sched: Change aut...
154
  	for_each_thread(p, t)
5091faa44   Mike Galbraith   sched: Add 'autog...
155
  		sched_move_task(t);
18f649ef3   Oleg Nesterov   sched/autogroup: ...
156

5091faa44   Mike Galbraith   sched: Add 'autog...
157
158
159
160
161
162
163
  	unlock_task_sighand(p, &flags);
  	autogroup_kref_put(prev);
  }
  
  /* Allocates GFP_KERNEL, cannot be called under any spinlock */
  void sched_autogroup_create_attach(struct task_struct *p)
  {
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
164
  	struct autogroup *ag = autogroup_create();
5091faa44   Mike Galbraith   sched: Add 'autog...
165
166
  
  	autogroup_move_group(p, ag);
25985edce   Lucas De Marchi   Fix common misspe...
167
  	/* drop extra reference added by autogroup_create() */
5091faa44   Mike Galbraith   sched: Add 'autog...
168
169
170
171
172
173
174
175
176
177
178
179
180
  	autogroup_kref_put(ag);
  }
  EXPORT_SYMBOL(sched_autogroup_create_attach);
  
  /* Cannot be called under siglock.  Currently has no users */
  void sched_autogroup_detach(struct task_struct *p)
  {
  	autogroup_move_group(p, &autogroup_default);
  }
  EXPORT_SYMBOL(sched_autogroup_detach);
  
  void sched_autogroup_fork(struct signal_struct *sig)
  {
4f8219875   Mike Galbraith   sched, autogroup:...
181
  	sig->autogroup = autogroup_task_get(current);
5091faa44   Mike Galbraith   sched: Add 'autog...
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
  }
  
  void sched_autogroup_exit(struct signal_struct *sig)
  {
  	autogroup_kref_put(sig->autogroup);
  }
  
  static int __init setup_autogroup(char *str)
  {
  	sysctl_sched_autogroup_enabled = 0;
  
  	return 1;
  }
  
  __setup("noautogroup", setup_autogroup);
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
197
198
199
200
201
202
  #ifdef CONFIG_PROC_FS
  
  int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
  {
  	static unsigned long next = INITIAL_JIFFIES;
  	struct autogroup *ag;
83929cce9   Mike Galbraith   sched/autogroup: ...
203
  	unsigned long shares;
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
204
  	int err;
75e45d512   Dongsheng Yang   sched: Replace ha...
205
  	if (nice < MIN_NICE || nice > MAX_NICE)
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
  		return -EINVAL;
  
  	err = security_task_setnice(current, nice);
  	if (err)
  		return err;
  
  	if (nice < 0 && !can_nice(current, nice))
  		return -EPERM;
  
  	/* this is a heavy operation taking global locks.. */
  	if (!capable(CAP_SYS_ADMIN) && time_before(jiffies, next))
  		return -EAGAIN;
  
  	next = HZ / 10 + jiffies;
  	ag = autogroup_task_get(p);
83929cce9   Mike Galbraith   sched/autogroup: ...
221
  	shares = scale_load(sched_prio_to_weight[nice + 20]);
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
222
223
  
  	down_write(&ag->lock);
83929cce9   Mike Galbraith   sched/autogroup: ...
224
  	err = sched_group_set_shares(ag->tg, shares);
c1ad41f1f   Ingo Molnar   Revert "sched/aut...
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
  	if (!err)
  		ag->nice = nice;
  	up_write(&ag->lock);
  
  	autogroup_kref_put(ag);
  
  	return err;
  }
  
  void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m)
  {
  	struct autogroup *ag = autogroup_task_get(p);
  
  	if (!task_group_is_autogroup(ag->tg))
  		goto out;
  
  	down_read(&ag->lock);
  	seq_printf(m, "/autogroup-%ld nice %d
  ", ag->id, ag->nice);
  	up_read(&ag->lock);
  
  out:
  	autogroup_kref_put(ag);
  }
  #endif /* CONFIG_PROC_FS */
5091faa44   Mike Galbraith   sched: Add 'autog...
250
  #ifdef CONFIG_SCHED_DEBUG
029632fbb   Peter Zijlstra   sched: Make separ...
251
  int autogroup_path(struct task_group *tg, char *buf, int buflen)
5091faa44   Mike Galbraith   sched: Add 'autog...
252
  {
511f67a59   Mike Galbraith   sched, autogroup:...
253
  	if (!task_group_is_autogroup(tg))
8ecedd7a0   Bharata B Rao   sched: Display au...
254
  		return 0;
5091faa44   Mike Galbraith   sched: Add 'autog...
255
256
257
  	return snprintf(buf, buflen, "%s-%ld", "/autogroup", tg->autogroup->id);
  }
  #endif /* CONFIG_SCHED_DEBUG */