diff -rup linux-2.6.0-test1/fs/proc/proc_misc.c linux-2.6.0-ss/fs/proc/proc_misc.c --- linux-2.6.0-test1/fs/proc/proc_misc.c Sun Jul 13 20:30:43 2003 +++ linux-2.6.0-ss/fs/proc/proc_misc.c Fri Jul 18 16:16:21 2003 @@ -286,6 +286,11 @@ static struct file_operations proc_vmsta .release = seq_release, }; +#ifdef CONFIG_SCHEDSTATS +extern int schedstats_read_proc(char *page, char **start, off_t off, + int count, int *eof, void *data); +#endif + #ifdef CONFIG_PROC_HARDWARE static int hardware_read_proc(char *page, char **start, off_t off, int count, int *eof, void *data) @@ -632,6 +637,9 @@ void __init proc_misc_init(void) #endif {"locks", locks_read_proc}, {"execdomains", execdomains_read_proc}, +#ifdef CONFIG_SCHEDSTATS + {"schedstat", schedstats_read_proc}, +#endif {NULL,} }; for (p = simple_ones; p->name; p++) diff -rup linux-2.6.0-test1/kernel/sched.c linux-2.6.0-ss/kernel/sched.c --- linux-2.6.0-test1/kernel/sched.c Sun Jul 13 20:37:14 2003 +++ linux-2.6.0-ss/kernel/sched.c Fri Jul 18 16:39:48 2003 @@ -235,6 +235,113 @@ __init void node_nr_running_init(void) #endif /* CONFIG_NUMA */ + +#ifdef CONFIG_SCHEDSTATS +struct schedstat { + /* sys_sched_yield stats */ + unsigned long yld_exp_empty; + unsigned long yld_act_empty; + unsigned long yld_both_empty; + unsigned long yld_cnt; + + /* schedule stats */ + unsigned long sched_noswitch; + unsigned long sched_switch; + unsigned long sched_cnt; + + /* load_balance stats */ + unsigned long lb_imbalance; + unsigned long lb_idle; + unsigned long lb_busy; + unsigned long lb_resched; + unsigned long lb_cnt; + unsigned long lb_nobusy; + unsigned long lb_bnode; + + /* pull_task stats */ + unsigned long pt_gained; + unsigned long pt_lost; + unsigned long pt_node_gained; + unsigned long pt_node_lost; + + /* balance_node stats */ + unsigned long bn_cnt; + unsigned long bn_idle; +} ____cacheline_aligned; + +/* + * bump this up when changing the output format or the meaning of an existing + * format, so that tools can adapt (or abort) + */ +#define SCHEDSTAT_VERSION 2 + +struct schedstat schedstats[NR_CPUS]; + +/* + * This could conceivably exceed a page's worth of output on machines with + * large number of cpus, where large == about 4096/100 or 40ish. Start + * worrying when we pass 32, probably. Then this has to stop being a + * "simple" entry in proc/proc_misc.c and needs to be an actual seq_file. + */ +int schedstats_read_proc(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + struct schedstat sums; + int i, len; + + memset(&sums, 0, sizeof(sums)); + len = sprintf(page, "version %d\n", SCHEDSTAT_VERSION); + for (i = 0; i < NR_CPUS; i++) { + if (!cpu_online(i)) continue; + sums.yld_exp_empty += schedstats[i].yld_exp_empty; + sums.yld_act_empty += schedstats[i].yld_act_empty; + sums.yld_both_empty += schedstats[i].yld_both_empty; + sums.yld_cnt += schedstats[i].yld_cnt; + sums.sched_noswitch += schedstats[i].sched_noswitch; + sums.sched_switch += schedstats[i].sched_switch; + sums.sched_cnt += schedstats[i].sched_cnt; + sums.lb_idle += schedstats[i].lb_idle; + sums.lb_busy += schedstats[i].lb_busy; + sums.lb_resched += schedstats[i].lb_resched; + sums.lb_cnt += schedstats[i].lb_cnt; + sums.lb_imbalance += schedstats[i].lb_imbalance; + sums.lb_nobusy += schedstats[i].lb_nobusy; + sums.lb_bnode += schedstats[i].lb_bnode; + sums.pt_node_gained += schedstats[i].pt_node_gained; + sums.pt_node_lost += schedstats[i].pt_node_lost; + sums.pt_gained += schedstats[i].pt_gained; + sums.pt_lost += schedstats[i].pt_lost; + sums.bn_cnt += schedstats[i].bn_cnt; + sums.bn_idle += schedstats[i].bn_idle; + len += sprintf(page + len, + "cpu%d %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu " + "%lu %lu %lu %lu %lu %lu %lu %lu\n", + i, schedstats[i].yld_both_empty, + schedstats[i].yld_act_empty, schedstats[i].yld_exp_empty, + schedstats[i].yld_cnt, schedstats[i].sched_noswitch, + schedstats[i].sched_switch, schedstats[i].sched_cnt, + schedstats[i].lb_idle, schedstats[i].lb_busy, + schedstats[i].lb_resched, + schedstats[i].lb_cnt, schedstats[i].lb_imbalance, + schedstats[i].lb_nobusy, schedstats[i].lb_bnode, + schedstats[i].pt_gained, schedstats[i].pt_lost, + schedstats[i].pt_node_gained, schedstats[i].pt_node_lost, + schedstats[i].bn_cnt, schedstats[i].bn_idle); + } + len += sprintf(page + len, + "totals %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu %lu " + "%lu %lu %lu %lu %lu %lu %lu\n", + sums.yld_both_empty, sums.yld_act_empty, sums.yld_exp_empty, + sums.yld_cnt, sums.sched_noswitch, sums.sched_switch, + sums.sched_cnt, sums.lb_idle, sums.lb_busy, sums.lb_resched, + sums.lb_cnt, sums.lb_imbalance, sums.lb_nobusy, sums.lb_bnode, + sums.pt_gained, sums.pt_lost, sums.pt_node_gained, + sums.pt_node_lost, sums.bn_cnt, sums.bn_idle); + + return len; +} +#endif + /* * task_rq_lock - lock the runqueue a given task resides on and disable * interrupts. Note the ordering: we can safely lookup the task_rq without @@ -888,7 +995,7 @@ static inline unsigned int double_lock_b /* * find_busiest_queue - find the busiest runqueue among the cpus in cpumask. */ -static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance, unsigned long cpumask) +static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance, unsigned long cpumask, int *remote_cpu) { int nr_running, load, max_load, i; runqueue_t *busiest, *rq_src; @@ -935,6 +1042,7 @@ static inline runqueue_t *find_busiest_q if ((load > max_load) && (rq_src != this_rq)) { busiest = rq_src; + *remote_cpu = i; max_load = load; } } @@ -967,8 +1075,16 @@ out: * pull_task - move a task from a remote runqueue to the local runqueue. * Both runqueues must be locked. */ -static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu) +static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu, int remote_cpu) { +#ifdef CONFIG_SCHEDSTATS + if (cpu_to_node(this_cpu) != cpu_to_node(remote_cpu)) { + schedstats[this_cpu].pt_node_gained++; + schedstats[remote_cpu].pt_node_lost++; + } + schedstats[this_cpu].pt_gained++; + schedstats[remote_cpu].pt_lost++; +#endif dequeue_task(p, src_array); nr_running_dec(src_rq); set_task_cpu(p, this_cpu); @@ -997,15 +1113,26 @@ static inline void pull_task(runqueue_t */ static void load_balance(runqueue_t *this_rq, int idle, unsigned long cpumask) { - int imbalance, idx, this_cpu = smp_processor_id(); + int imbalance, idx, this_cpu = smp_processor_id(), remote_cpu; runqueue_t *busiest; prio_array_t *array; struct list_head *head, *curr; task_t *tmp; - busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance, cpumask); - if (!busiest) - goto out; +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_cnt++; +#endif + busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance, cpumask, &remote_cpu); + if (!busiest) { +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_nobusy++; +#endif + goto out; + } + +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_imbalance += imbalance; +#endif /* * We first consider expired tasks. Those will likely not be @@ -1059,7 +1186,7 @@ skip_queue: idx++; goto skip_bitmap; } - pull_task(busiest, array, tmp, this_rq, this_cpu); + pull_task(busiest, array, tmp, this_rq, this_cpu, remote_cpu); if (!idle && --imbalance) { if (curr != head) goto skip_queue; @@ -1094,9 +1221,15 @@ static void balance_node(runqueue_t *thi int node = find_busiest_node(cpu_to_node(this_cpu)); unsigned long cpumask, this_cpumask = 1UL << this_cpu; +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].bn_cnt++; +#endif if (node >= 0) { cpumask = node_to_cpumask(node) | this_cpumask; spin_lock(&this_rq->lock); +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_bnode++; +#endif load_balance(this_rq, idle, cpumask); spin_unlock(&this_rq->lock); } @@ -1105,9 +1238,7 @@ static void balance_node(runqueue_t *thi static void rebalance_tick(runqueue_t *this_rq, int idle) { -#ifdef CONFIG_NUMA int this_cpu = smp_processor_id(); -#endif unsigned long j = jiffies; /* @@ -1120,11 +1251,18 @@ static void rebalance_tick(runqueue_t *t */ if (idle) { #ifdef CONFIG_NUMA - if (!(j % IDLE_NODE_REBALANCE_TICK)) + if (!(j % IDLE_NODE_REBALANCE_TICK)) { +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].bn_idle++; +#endif balance_node(this_rq, idle, this_cpu); + } #endif if (!(j % IDLE_REBALANCE_TICK)) { spin_lock(&this_rq->lock); +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_idle++; +#endif load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); spin_unlock(&this_rq->lock); } @@ -1136,6 +1274,9 @@ static void rebalance_tick(runqueue_t *t #endif if (!(j % BUSY_REBALANCE_TICK)) { spin_lock(&this_rq->lock); +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_busy++; +#endif load_balance(this_rq, idle, cpu_to_node_mask(this_cpu)); spin_unlock(&this_rq->lock); } @@ -1262,13 +1403,17 @@ asmlinkage void schedule(void) runqueue_t *rq; prio_array_t *array; struct list_head *queue; - int idx; + int idx, this_cpu = smp_processor_id(); + /* * Test if we are atomic. Since do_exit() needs to call into * schedule() atomically, we ignore that path for now. * Otherwise, whine if we are scheduling when we should not be. */ +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].sched_cnt++; +#endif if (likely(!(current->state & (TASK_DEAD | TASK_ZOMBIE)))) { if (unlikely(in_atomic())) { printk(KERN_ERR "bad: scheduling while atomic!\n"); @@ -1306,6 +1451,9 @@ need_resched: pick_next_task: if (unlikely(!rq->nr_running)) { #ifdef CONFIG_SMP +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].lb_resched++; +#endif load_balance(rq, 1, cpu_to_node_mask(smp_processor_id())); if (rq->nr_running) goto pick_next_task; @@ -1320,11 +1468,17 @@ pick_next_task: /* * Switch the active and expired arrays. */ +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].sched_switch++; +#endif rq->active = rq->expired; rq->expired = array; array = rq->active; rq->expired_timestamp = 0; } +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].sched_noswitch++; +#endif idx = sched_find_first_bit(array->bitmap); queue = array->queue + idx; @@ -1984,6 +2138,7 @@ asmlinkage long sys_sched_yield(void) { runqueue_t *rq = this_rq_lock(); prio_array_t *array = current->array; + int this_cpu = smp_processor_id(); /* * We implement yielding by moving the task into the expired @@ -1992,7 +2147,19 @@ asmlinkage long sys_sched_yield(void) * (special rule: RT tasks will just roundrobin in the active * array.) */ +#ifdef CONFIG_SCHEDSTATS + schedstats[this_cpu].yld_cnt++; +#endif if (likely(!rt_task(current))) { +#ifdef CONFIG_SCHEDSTATS + if (current->array->nr_active == 1) { + schedstats[this_cpu].yld_act_empty++; + if (!rq->expired->nr_active) + schedstats[this_cpu].yld_both_empty++; + } else if (!rq->expired->nr_active) { + schedstats[this_cpu].yld_exp_empty++; + } +#endif dequeue_task(current, array); enqueue_task(current, rq->expired); } else {