diff -rup linux-2.6.7-rc2-mm2/arch/i386/Kconfig linux-2.6.7-rc2-mm2-ss/arch/i386/Kconfig --- linux-2.6.7-rc2-mm2/arch/i386/Kconfig Mon Jun 14 15:17:51 2004 +++ linux-2.6.7-rc2-mm2-ss/arch/i386/Kconfig Mon Jun 14 15:44:37 2004 @@ -1527,6 +1527,19 @@ config SCHEDSTATS application, you can say N to avoid the very slight overhead this adds. +config SCHEDSTATS + bool "Collect scheduler statistics" + depends on PROC_FS + default y + help + If you say Y here, additional code will be inserted into the + scheduler and related routines to collect statistics about + scheduler behavior and provide them in /proc/schedstat. These + stats may be useful for both tuning and debugging the scheduler + If you aren't debugging the scheduler or trying to tune a specific + application, you can say N to avoid the very slight overhead + this adds. + config X86_FIND_SMP_CONFIG bool depends on X86_LOCAL_APIC || X86_VOYAGER diff -rup linux-2.6.7-rc2-mm2/arch/ppc/Kconfig linux-2.6.7-rc2-mm2-ss/arch/ppc/Kconfig --- linux-2.6.7-rc2-mm2/arch/ppc/Kconfig Mon Jun 14 15:17:51 2004 +++ linux-2.6.7-rc2-mm2-ss/arch/ppc/Kconfig Mon Jun 14 15:44:37 2004 @@ -1266,6 +1266,19 @@ config SCHEDSTATS application, you can say N to avoid the very slight overhead this adds. +config SCHEDSTATS + bool "Collect scheduler statistics" + depends on PROC_FS + default y + help + If you say Y here, additional code will be inserted into the + scheduler and related routines to collect statistics about + scheduler behavior and provide them in /proc/schedstat. These + stats may be useful for both tuning and debugging the scheduler + If you aren't debugging the scheduler or trying to tune a specific + application, you can say N to avoid the very slight overhead + this adds. + config BOOTX_TEXT bool "Support for early boot text console (BootX or OpenFirmware only)" depends PPC_OF diff -rup linux-2.6.7-rc2-mm2/arch/ppc64/Kconfig linux-2.6.7-rc2-mm2-ss/arch/ppc64/Kconfig --- linux-2.6.7-rc2-mm2/arch/ppc64/Kconfig Mon Jun 14 15:17:51 2004 +++ linux-2.6.7-rc2-mm2-ss/arch/ppc64/Kconfig Mon Jun 14 15:44:37 2004 @@ -425,6 +425,19 @@ config IRQSTACKS for handling hard and soft interrupts. This can help avoid overflowing the process kernel stacks. +config SCHEDSTATS + bool "Collect scheduler statistics" + depends on PROC_FS + default y + help + If you say Y here, additional code will be inserted into the + scheduler and related routines to collect statistics about + scheduler behavior and provide them in /proc/schedstat. These + stats may be useful for both tuning and debugging the scheduler + If you aren't debugging the scheduler or trying to tune a specific + application, you can say N to avoid the very slight overhead + this adds. + endmenu config SCHEDSTATS diff -rup linux-2.6.7-rc2-mm2/fs/proc/array.c linux-2.6.7-rc2-mm2-ss/fs/proc/array.c --- linux-2.6.7-rc2-mm2/fs/proc/array.c Sat May 29 23:26:18 2004 +++ linux-2.6.7-rc2-mm2-ss/fs/proc/array.c Mon Jun 14 15:44:37 2004 @@ -357,9 +357,15 @@ int proc_pid_stat(struct task_struct *ta /* Temporary variable needed for gcc-2.96 */ start_time = jiffies_64_to_clock_t(task->start_time - INITIAL_JIFFIES); +#ifdef CONFIG_SCHEDSTATS + res = sprintf(buffer,"%d (%s) %c %d %d %d %d %d %lu %lu \ +%lu %lu %lu %lu %lu %ld %ld %ld %ld %d %ld %llu %lu %ld %lu %lu %lu %lu %lu \ +%lu %lu %lu %lu %lu %lu %lu %lu %d %d %lu %lu %lu %lu %lu\n", +#else res = sprintf(buffer,"%d (%s) %c %d %d %d %d %d %lu %lu \ %lu %lu %lu %lu %lu %ld %ld %ld %ld %d %ld %llu %lu %ld %lu %lu %lu %lu %lu \ %lu %lu %lu %lu %lu %lu %lu %lu %d %d %lu %lu\n", +#endif task->pid, task->comm, state, @@ -404,7 +410,14 @@ int proc_pid_stat(struct task_struct *ta task->exit_signal, task_cpu(task), task->rt_priority, +#ifdef CONFIG_SCHEDSTATS + task->policy, + task->sched_info.cpu_time, + task->sched_info.run_delay, + task->sched_info.pcnt); +#else task->policy); +#endif /* CONFIG_SCHEDSTATS */ if(mm) mmput(mm); return res; diff -rup linux-2.6.7-rc2-mm2/fs/proc/proc_misc.c linux-2.6.7-rc2-mm2-ss/fs/proc/proc_misc.c --- linux-2.6.7-rc2-mm2/fs/proc/proc_misc.c Mon Jun 14 15:17:53 2004 +++ linux-2.6.7-rc2-mm2-ss/fs/proc/proc_misc.c Mon Jun 14 15:44:37 2004 @@ -289,6 +289,10 @@ static struct file_operations proc_vmsta extern struct file_operations proc_schedstat_operations; #endif +#ifdef CONFIG_SCHEDSTATS +extern struct file_operations proc_schedstat_operations; +#endif + #ifdef CONFIG_PROC_HARDWARE static int hardware_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data) @@ -729,6 +733,9 @@ void __init proc_misc_init(void) create_seq_entry("diskstats", 0, &proc_diskstats_operations); #ifdef CONFIG_MODULES create_seq_entry("modules", 0, &proc_modules_operations); +#endif +#ifdef CONFIG_SCHEDSTATS + create_seq_entry("schedstat", 0, &proc_schedstat_operations); #endif #ifdef CONFIG_SCHEDSTATS create_seq_entry("schedstat", 0, &proc_schedstat_operations); diff -rup linux-2.6.7-rc2-mm2/include/linux/sched.h linux-2.6.7-rc2-mm2-ss/include/linux/sched.h --- linux-2.6.7-rc2-mm2/include/linux/sched.h Mon Jun 14 15:17:53 2004 +++ linux-2.6.7-rc2-mm2-ss/include/linux/sched.h Mon Jun 14 15:46:16 2004 @@ -97,11 +97,13 @@ extern unsigned long nr_uninterruptible( extern unsigned long nr_iowait(void); #ifdef CONFIG_SCHEDSTATS -#define schedstat_inc(s, field) ((s)->field++) -#define schedstat_add(s, field, amt) ((s)->field += amt) +struct sched_info; +extern void cpu_sched_info(struct sched_info *, int); +#define schedstat_inc(rq, field) rq->field++; +#define schedstat_add(rq, field, amt) rq->field += amt; #else -#define schedstat_inc(s, field) do { } while (0) -#define schedstat_add(d, field, amt) do { } while (0) +#define schedstat_inc(rq, field) do { } while (0); +#define schedstat_add(rq, field, amt) do { } while (0); #endif #include @@ -393,6 +395,18 @@ int set_current_groups(struct group_info #define GROUP_AT(gi, i) \ ((gi)->blocks[(i)/NGROUPS_PER_BLOCK][(i)%NGROUPS_PER_BLOCK]) +#ifdef CONFIG_SCHEDSTATS +struct sched_info { + /* cumulative counters */ + unsigned long cpu_time, /* time spent on the cpu */ + run_delay, /* time spent waiting on a runqueue */ + pcnt; /* # of timeslices run on this cpu */ + + /* timestamps */ + unsigned long last_arrival, /* when we last ran on a cpu */ + last_queued; /* when we were last queued to run */ +}; +#endif /* CONFIG_SCHEDSTATS */ struct audit_context; /* See audit.c */ struct mempolicy; @@ -419,6 +433,10 @@ struct task_struct { cpumask_t cpus_allowed; unsigned int time_slice, first_time_slice; +#ifdef CONFIG_SCHEDSTATS + struct sched_info sched_info; +#endif /* CONFIG_SCHEDSTATS */ + struct list_head tasks; struct list_head ptrace_children; struct list_head ptrace_list; @@ -577,6 +595,14 @@ do { if (atomic_dec_and_test(&(tsk)->usa #define SD_WAKE_BALANCE 32 /* Perform balancing at task wakeup */ #define SD_SHARE_CPUPOWER 64 /* Domain members share cpu power */ +enum idle_type +{ + IDLE, + NOT_IDLE, + NEWLY_IDLE, + MAX_IDLE_TYPES +}; + struct sched_group { struct sched_group *next; /* Must be a circular list */ cpumask_t cpumask; @@ -609,33 +635,25 @@ struct sched_domain { unsigned int balance_interval; /* initialise to 1. units in ms. */ unsigned int nr_balance_failed; /* initialise to 0 */ -#ifdef CONFIG_SCHEDSTATS - unsigned long lb_cnt[3]; - unsigned long lb_balanced[3]; - unsigned long lb_failed[3]; - unsigned long lb_pulled[3]; - unsigned long lb_hot_pulled[3]; - unsigned long lb_imbalance[3]; - - /* Active load balancing */ - unsigned long alb_cnt; - unsigned long alb_failed; - unsigned long alb_pushed; - - /* Wakeups */ - unsigned long sched_wake_remote; - /* Passive load balancing */ - unsigned long plb_pulled; - - /* Affine wakeups */ - unsigned long afw_pulled; +#ifdef CONFIG_SCHEDSTATS + /* load_balance() stats */ + unsigned long lb_cnt[MAX_IDLE_TYPES]; + unsigned long lb_balanced[MAX_IDLE_TYPES]; + unsigned long lb_failed[MAX_IDLE_TYPES]; + unsigned long lb_pulled[MAX_IDLE_TYPES]; + unsigned long lb_hot_pulled[MAX_IDLE_TYPES]; + unsigned long lb_imbalance[MAX_IDLE_TYPES]; + unsigned long lb_nobusyg[MAX_IDLE_TYPES]; + unsigned long lb_nobusyq[MAX_IDLE_TYPES]; - /* SD_BALANCE_EXEC balances */ + /* sched_balance_exec() stats */ + unsigned long sbe_attempts; unsigned long sbe_pushed; - /* SD_BALANCE_CLONE balances */ - unsigned long sbc_pushed; + /* try_to_wake_up() stats */ + unsigned long ttwu_try_affine; + unsigned long ttwu_try_balance; #endif }; diff -rup linux-2.6.7-rc2-mm2/kernel/fork.c linux-2.6.7-rc2-mm2-ss/kernel/fork.c --- linux-2.6.7-rc2-mm2/kernel/fork.c Sat May 29 23:25:45 2004 +++ linux-2.6.7-rc2-mm2-ss/kernel/fork.c Mon Jun 14 15:44:37 2004 @@ -964,6 +964,11 @@ struct task_struct *copy_process(unsigne p->security = NULL; p->io_context = NULL; p->audit_context = NULL; + +#ifdef CONFIG_SCHEDSTATS + memset(&p->sched_info, 0, sizeof(p->sched_info)); +#endif /* CONFIG_SCHEDSTATS */ + #ifdef CONFIG_NUMA p->mempolicy = mpol_copy(p->mempolicy); if (IS_ERR(p->mempolicy)) { diff -rup linux-2.6.7-rc2-mm2/kernel/sched.c linux-2.6.7-rc2-mm2-ss/kernel/sched.c --- linux-2.6.7-rc2-mm2/kernel/sched.c Mon Jun 14 15:17:53 2004 +++ linux-2.6.7-rc2-mm2-ss/kernel/sched.c Mon Jun 14 16:01:54 2004 @@ -237,7 +237,11 @@ struct runqueue { task_t *migration_thread; struct list_head migration_queue; #endif + #ifdef CONFIG_SCHEDSTATS + /* latency stats */ + struct sched_info rq_sched_info; + /* sys_sched_yield stats */ unsigned long yld_exp_empty; unsigned long yld_act_empty; @@ -245,13 +249,29 @@ struct runqueue { unsigned long yld_cnt; /* schedule stats */ - unsigned long sched_cnt; + unsigned long sched_noswitch; unsigned long sched_switch; - unsigned long sched_idle; + unsigned long sched_cnt; - /* wake stats */ - unsigned long sched_wake; - unsigned long sched_wake_local; + /* pull_task() stats */ + unsigned long pt_gained[MAX_IDLE_TYPES]; + unsigned long pt_lost[MAX_IDLE_TYPES]; + + /* active_load_balance() stats */ + unsigned long alb_cnt; + unsigned long alb_lost; + unsigned long alb_gained; + unsigned long alb_failed; + + /* try_to_wake_up stats */ + unsigned long ttwu_cnt; + unsigned long ttwu_moved; + + /* sched_migrate_task() */ + unsigned long smt_cnt; + + /* sched_balance_exec() */ + unsigned long sbe_cnt; #endif }; @@ -301,65 +321,71 @@ static inline void task_rq_unlock(runque #ifdef CONFIG_SCHEDSTATS - /* * bump this up when changing the output format or the meaning of an existing * format, so that tools can adapt (or abort) */ -#define SCHEDSTAT_VERSION 7 +#define SCHEDSTAT_VERSION 8 static int show_schedstat(struct seq_file *seq, void *v) { - int i; + int cpu; + enum idle_type itype; seq_printf(seq, "version %d\n", SCHEDSTAT_VERSION); seq_printf(seq, "timestamp %lu\n", jiffies); - for_each_cpu(i) { - /* Include offline CPUs */ - runqueue_t *rq = cpu_rq(i); -#ifdef CONFIG_SMP + for_each_online_cpu (cpu) { + + int dcnt = 0; + + runqueue_t *rq = cpu_rq(cpu); struct sched_domain *sd; - int j = 0; -#endif - seq_printf(seq, - "cpu%d %lu %lu %lu %lu %lu %lu %lu %lu %lu", - i, - rq->yld_both_empty, rq->yld_act_empty, - rq->yld_exp_empty, rq->yld_cnt, + /* runqueue-specific stats */ + seq_printf(seq, + "cpu%d %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu " + "%lu %lu %lu %lu %lu %lu", + cpu, rq->yld_both_empty, + rq->yld_act_empty, rq->yld_exp_empty, + rq->yld_cnt, rq->sched_noswitch, rq->sched_switch, rq->sched_cnt, - rq->sched_idle, rq->sched_wake, rq->sched_wake_local); -#ifdef CONFIG_SMP - for_each_domain(i, sd) { - char str[NR_CPUS]; - int k; - cpumask_scnprintf(str, NR_CPUS, sd->span); - seq_printf(seq, " domain%d %s", j++, str); - - for (k = 0; k < 3; k++) { - seq_printf(seq, " %lu %lu %lu %lu %lu %lu", - sd->lb_cnt[k], sd->lb_balanced[k], - sd->lb_failed[k], sd->lb_pulled[k], - sd->lb_hot_pulled[k], sd->lb_imbalance[k]); - } - - seq_printf(seq, " %lu %lu %lu %lu %lu %lu %lu %lu", - sd->alb_cnt, sd->alb_failed, - sd->alb_pushed, sd->sched_wake_remote, - sd->plb_pulled, sd->afw_pulled, - sd->sbe_pushed, sd->sbc_pushed); - } -#endif + rq->alb_cnt, rq->alb_gained, rq->alb_lost, + rq->alb_failed, + rq->ttwu_cnt, rq->ttwu_moved, + rq->smt_cnt, rq->sbe_cnt, rq->rq_sched_info.cpu_time, + rq->rq_sched_info.run_delay, rq->rq_sched_info.pcnt); + + for (itype = IDLE; itype < MAX_IDLE_TYPES; itype++) + seq_printf(seq, " %lu %lu", rq->pt_gained[itype], + rq->pt_lost[itype]); seq_printf(seq, "\n"); - } + /* domain-specific stats */ + for_each_domain(cpu, sd) { + char mask_str[NR_CPUS]; + + cpumask_scnprintf(mask_str, NR_CPUS, sd->span); + seq_printf(seq, "domain%d %s", dcnt++, mask_str); + for (itype = IDLE; itype < MAX_IDLE_TYPES; itype++) { + seq_printf(seq, " %lu %lu %lu %lu %lu", + sd->lb_cnt[itype], + sd->lb_failed[itype], + sd->lb_imbalance[itype], + sd->lb_nobusyg[itype], + sd->lb_nobusyq[itype]); + } + seq_printf(seq, " %lu %lu %lu %lu\n", + sd->sbe_pushed, sd->sbe_attempts, + sd->ttwu_try_affine, sd->ttwu_try_balance); + } + } return 0; } static int schedstat_open(struct inode *inode, struct file *file) { - unsigned size = 4096 * (1 + num_online_cpus() / 32); + unsigned size = PAGE_SIZE * (1 + num_online_cpus() / 32); char *buf = kmalloc(size, GFP_KERNEL); struct seq_file *m; int res; @@ -384,6 +410,7 @@ struct file_operations proc_schedstat_op }; #endif + /* * rq_lock - lock a given runqueue and disable interrupts. */ @@ -403,6 +430,113 @@ static inline void rq_unlock(runqueue_t spin_unlock_irq(&rq->lock); } +#ifdef CONFIG_SCHEDSTATS +/* + * Called when a process is dequeued from the active array and given + * the cpu. We should note that with the exception of interactive + * tasks, the expired queue will become the active queue after the active + * queue is empty, without explicitly dequeuing and requeuing tasks in the + * expired queue. (Interactive tasks may be requeued directly to the + * active queue, thus delaying tasks in the expired queue from running; + * see scheduler_tick()). + * + * This function is only called from sched_info_arrive(), rather than + * dequeue_task(). Even though a task may be queued and dequeued multiple + * times as it is shuffled about, we're really interested in knowing how + * long it was from the *first* time it was queued to the time that it + * finally hit a cpu. + */ +static inline void sched_info_dequeued(task_t *t) +{ + t->sched_info.last_queued = 0; +} + +/* + * Called when a task finally hits the cpu. We can now calculate how + * long it was waiting to run. We also note when it began so that we + * can keep stats on how long its timeslice is. + */ +static inline void sched_info_arrive(task_t *t) +{ + unsigned long now = jiffies; + unsigned long diff = 0; + struct runqueue *rq = task_rq(t); + + if (t->sched_info.last_queued) + diff = now - t->sched_info.last_queued; + sched_info_dequeued(t); + t->sched_info.run_delay += diff; + t->sched_info.last_arrival = now; + t->sched_info.pcnt++; + + if (!rq) + return; + + rq->rq_sched_info.run_delay += diff; + rq->rq_sched_info.pcnt++; +} + +/* + * Called when a process is queued into either the active or expired + * array. The time is noted and later used to determine how long we + * had to wait for us to reach the cpu. Since the expired queue will + * become the active queue after active queue is empty, without dequeuing + * and requeuing any tasks, we are interested in queuing to either. It + * is unusual but not impossible for tasks to be dequeued and immediately + * requeued in the same or another array: this can happen in sched_yield(), + * set_user_nice(), and even load_balance() as it moves tasks from runqueue + * to runqueue. + * + * This function is only called from enqueue_task(), but also only updates + * the timestamp if it is already not set. It's assumed that + * sched_info_dequeued() will clear that stamp when appropriate. + */ +static inline void sched_info_queued(task_t *t) +{ + if (!t->sched_info.last_queued) + t->sched_info.last_queued = jiffies; +} + +/* + * Called when a process ceases being the active-running process, either + * voluntarily or involuntarily. Now we can calculate how long we ran. + */ +static inline void sched_info_depart(task_t *t) +{ + struct runqueue *rq = task_rq(t); + unsigned long diff = jiffies - t->sched_info.last_arrival; + + t->sched_info.cpu_time += diff; + + if (rq) + rq->rq_sched_info.cpu_time += diff; +} + +/* + * Called when tasks are switched involuntarily due, typically, to expiring + * their time slice. (This may also be called when switching to or from + * the idle task.) We are only called when prev != next. + */ +static inline void sched_info_switch(task_t *prev, task_t *next) +{ + struct runqueue *rq = task_rq(prev); + + /* + * prev now departs the cpu. It's not interesting to record + * stats about how efficient we were at scheduling the idle + * process, however. + */ + if (prev != rq->idle) + sched_info_depart(prev); + + if (next != rq->idle) + sched_info_arrive(next); +} +#else +#define sched_info_queued(t) {} +#define sched_info_switch(t, next) {} +#endif /* CONFIG_SCHEDSTATS */ + /* * Adding/removing a task to/from a priority array: */ @@ -416,6 +550,7 @@ static void dequeue_task(struct task_str static void enqueue_task(struct task_struct *p, prio_array_t *array) { + sched_info_queued(p); list_add_tail(&p->run_list, array->queue + p->prio); __set_bit(p->prio, array->bitmap); array->nr_active++; @@ -846,6 +981,7 @@ static int try_to_wake_up(task_t * p, un #endif rq = task_rq_lock(p, &flags); + schedstat_inc(rq, ttwu_cnt); old_state = p->state; if (!(old_state & state)) goto out; @@ -856,24 +992,7 @@ static int try_to_wake_up(task_t * p, un cpu = task_cpu(p); this_cpu = smp_processor_id(); - schedstat_inc(rq, sched_wake); -#ifndef CONFIG_SMP - schedstat_inc(rq, sched_wake_local); -#endif - #ifdef CONFIG_SMP -#ifdef CONFIG_SCHEDSTATS - if (cpu == this_cpu) - schedstat_inc(rq, sched_wake_local); - else { - for_each_domain(this_cpu, sd) - if (cpu_isset(cpu, sd->span)) - break; - if (sd) - schedstat_inc(sd, sched_wake_remote); - } -#endif - if (unlikely(task_running(rq, p))) goto out_activate; @@ -903,33 +1022,32 @@ static int try_to_wake_up(task_t * p, un */ imbalance = sd->imbalance_pct + (sd->imbalance_pct - 100) / 2; - if ( ((sd->flags & SD_WAKE_AFFINE) && - !task_hot(p, rq->timestamp_last_tick, sd)) - || ((sd->flags & SD_WAKE_BALANCE) && - imbalance*this_load <= 100*load) ) { - + if ((sd->flags & SD_WAKE_AFFINE) && + !task_hot(p, rq->timestamp_last_tick, sd)) { + /* + * This domain has SD_WAKE_AFFINE and p is cache cold + * in this domain. + */ + schedstat_inc(sd, ttwu_try_affine); + if (cpu_isset(cpu, sd->span)) + goto out_set_cpu; + } else if ((sd->flags & SD_WAKE_BALANCE) && + imbalance*this_load <= 100*load) { /* - * Now sd has SD_WAKE_AFFINE and p is cache cold in sd - * or sd has SD_WAKE_BALANCE and there is an imbalance + * This domain has SD_WAKE_BALANCE and there is + * an imbalance. */ - if (cpu_isset(cpu, sd->span)) { -#ifdef CONFIG_SCHEDSTATS - if ((sd->flags & SD_WAKE_AFFINE) && - !task_hot(p, rq->timestamp_last_tick, sd)) - schedstat_inc(sd, afw_pulled); - else if ((sd->flags & SD_WAKE_BALANCE) && - imbalance*this_load <= 100*load) - schedstat_inc(sd, plb_pulled); -#endif - goto out_set_cpu; - } - } + schedstat_inc(sd, ttwu_try_balance); + if (cpu_isset(cpu, sd->span)) + goto out_set_cpu; + } } new_cpu = cpu; /* Could not wake to this_cpu. Wake to cpu instead */ out_set_cpu: new_cpu = wake_idle(new_cpu, p); if (new_cpu != cpu && cpu_isset(new_cpu, p->cpus_allowed)) { + schedstat_inc(rq, ttwu_moved); set_task_cpu(p, new_cpu); task_rq_unlock(rq, &flags); /* might preempt at this point */ @@ -981,7 +1099,7 @@ out: int fastcall wake_up_process(task_t * p) { return try_to_wake_up(p, TASK_STOPPED | - TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); + TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE, 0); } EXPORT_SYMBOL(wake_up_process); @@ -1281,13 +1399,6 @@ static void double_rq_unlock(runqueue_t spin_unlock(&rq2->lock); } -enum idle_type -{ - IDLE, - NOT_IDLE, - NEWLY_IDLE, -}; - #ifdef CONFIG_SMP /* @@ -1406,7 +1517,6 @@ lock_again: rq->nr_running++; } } else { - schedstat_inc(sd, sbc_pushed); /* Not the local CPU - must adjust timestamp */ p->timestamp = (p->timestamp - this_rq->timestamp_last_tick) + rq->timestamp_last_tick; @@ -1437,6 +1547,7 @@ static void sched_migrate_task(task_t *p || unlikely(cpu_is_offline(dest_cpu))) goto out; + schedstat_inc(rq, smt_cnt); /* force the process onto the specified CPU */ if (migrate_task(p, dest_cpu, &req)) { /* Need to wait for migration thread (might exit: take ref). */ @@ -1465,6 +1576,7 @@ void sched_balance_exec(void) struct sched_domain *tmp, *sd = NULL; int new_cpu, this_cpu = get_cpu(); + schedstat_inc(this_rq(), sbe_cnt); /* Prefer the current CPU if there's only this task running */ if (this_rq()->nr_running <= 1) goto out; @@ -1473,6 +1585,7 @@ void sched_balance_exec(void) if (tmp->flags & SD_BALANCE_EXEC) sd = tmp; + schedstat_inc(sd, sbe_attempts); if (sd) { new_cpu = find_idlest_cpu(current, this_cpu, sd); if (new_cpu != this_cpu) { @@ -1549,13 +1662,6 @@ int can_migrate_task(task_t *p, runqueue return 0; } -#ifdef CONFIG_SCHEDSTATS - if (!task_hot(p, rq->timestamp_last_tick, sd)) - schedstat_inc(sd, lb_pulled[idle]); - else - schedstat_inc(sd, lb_hot_pulled[idle]); -#endif - return 1; } @@ -1622,6 +1728,15 @@ skip_queue: idx++; goto skip_bitmap; } + + /* + * Right now, this is the only place pull_task() is called, + * so we can safely collect pull_task() stats here rather than + * inside pull_task(). + */ + schedstat_inc(this_rq, pt_gained[idle]); + schedstat_inc(busiest, pt_lost[idle]); + pull_task(busiest, array, tmp, this_rq, dst_array, this_cpu); pulled++; @@ -1817,17 +1932,17 @@ static int load_balance(int this_cpu, ru spin_lock(&this_rq->lock); schedstat_inc(sd, lb_cnt[idle]); - - group = find_busiest_group(sd, this_cpu, &imbalance, idle); + + group = find_busiest_group(sd, this_cpu, &imbalance, idle); if (!group) { - schedstat_inc(sd, lb_balanced[idle]); - goto out_balanced; + schedstat_inc(sd, lb_nobusyg[idle]); + goto out_balanced; } - - busiest = find_busiest_queue(group); + + busiest = find_busiest_queue(group); if (!busiest) { - schedstat_inc(sd, lb_balanced[idle]); - goto out_balanced; + schedstat_inc(sd, lb_nobusyq[idle]); + goto out_balanced; } /* @@ -1912,19 +2029,19 @@ static int load_balance_newidle(int this unsigned long imbalance; int nr_moved = 0; - schedstat_inc(sd, lb_cnt[NEWLY_IDLE]); - group = find_busiest_group(sd, this_cpu, &imbalance, NEWLY_IDLE); + schedstat_inc(sd, lb_cnt[NEWLY_IDLE]); + group = find_busiest_group(sd, this_cpu, &imbalance, NEWLY_IDLE); if (!group) { - schedstat_inc(sd, lb_balanced[NEWLY_IDLE]); - goto out; + schedstat_inc(sd, lb_nobusyg[NEWLY_IDLE]); + goto out; } - - busiest = find_busiest_queue(group); + + busiest = find_busiest_queue(group); if (!busiest || busiest == this_rq) { - schedstat_inc(sd, lb_balanced[NEWLY_IDLE]); - goto out; + schedstat_inc(sd, lb_nobusyq[NEWLY_IDLE]); + goto out; } - + schedstat_add(sd, lb_imbalance[NEWLY_IDLE], imbalance); /* Attempt to move tasks */ @@ -1970,8 +2087,8 @@ static void active_load_balance(runqueue struct sched_domain *sd; struct sched_group *group, *busy_group; int i; - int moved = 0; + schedstat_inc(busiest, alb_cnt); if (busiest->nr_running <= 1) return; @@ -1982,7 +2099,6 @@ static void active_load_balance(runqueue WARN_ON(1); return; } - schedstat_inc(sd, alb_cnt); group = sd->groups; while (!cpu_isset(busiest_cpu, group->cpumask)) @@ -2019,16 +2135,16 @@ static void active_load_balance(runqueue if (unlikely(busiest == rq)) goto next_group; double_lock_balance(busiest, rq); - moved += move_tasks(rq, push_cpu, busiest, 1, sd, IDLE); + if (move_tasks(rq, push_cpu, busiest, 1, sd, IDLE)) { + schedstat_inc(busiest, alb_lost); + schedstat_inc(rq, alb_gained); + } else { + schedstat_inc(busiest, alb_failed); + } spin_unlock(&rq->lock); next_group: group = group->next; } while (group != sd->groups); - - if (moved) - schedstat_add(sd, alb_pushed, moved); - else - schedstat_inc(sd, alb_failed); } /* @@ -2406,7 +2522,6 @@ need_resched: next = rq->idle; rq->expired_timestamp = 0; wake_sleeping_dependent(cpu, rq); - schedstat_inc(rq, sched_idle); goto switch_tasks; } } @@ -2422,7 +2537,8 @@ need_resched: array = rq->active; rq->expired_timestamp = 0; rq->best_expired_prio = MAX_PRIO; - } + } else + schedstat_inc(rq, sched_noswitch); idx = sched_find_first_bit(array->bitmap); queue = array->queue + idx; @@ -2458,6 +2574,7 @@ switch_tasks: } prev->timestamp = now; + sched_info_switch(prev, next); if (likely(prev != next)) { next->timestamp = now; rq->nr_switches++; @@ -3172,6 +3289,15 @@ asmlinkage long sys_sched_yield(void) if (unlikely(rt_task(current))) target = rq->active; + if (current->array->nr_active == 1) { + schedstat_inc(rq, yld_act_empty); + if (!rq->expired->nr_active) { + schedstat_inc(rq, yld_both_empty); + } + } else if (!rq->expired->nr_active) { + schedstat_inc(rq, yld_exp_empty); + } + dequeue_task(current, array); enqueue_task(current, target); @@ -3759,7 +3885,7 @@ static int migration_call(struct notifie rq->idle->static_prio = MAX_PRIO; __setscheduler(rq->idle, SCHED_NORMAL, 0); task_rq_unlock(rq, &flags); - BUG_ON(rq->nr_running != 0); + BUG_ON(rq->nr_running != 0); /* No need to migrate the tasks: it was best-effort if * they didn't do lock_cpu_hotplug(). Just wake up @@ -3774,7 +3900,7 @@ static int migration_call(struct notifie complete(&req->done); } spin_unlock_irq(&rq->lock); - break; + break; #endif } return NOTIFY_OK;