Mailing List Archive

# HG changeset patch
# User Keir Fraser <keir.fraser@citrix.com>
# Date 1273572624 -3600
# Node ID e984d995f360d101b188fc4973243e2e4048e0c4
# Parent 4ede18e45935dcf1a6a4d0824873dbd05a63d148
tasklet: Improve scheduler interaction.

Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
---
xen/common/sched_credit.c | 5 ++--
xen/common/sched_credit2.c | 5 ++--
xen/common/sched_sedf.c | 15 ++++--------
xen/common/schedule.c | 21 +++++++++++++++++-
xen/common/tasklet.c | 52 ++++++++++++++++++++++++---------------------
xen/include/xen/sched-if.h | 3 +-
xen/include/xen/sched.h | 6 +++--
xen/include/xen/tasklet.h | 8 ++++++
8 files changed, 72 insertions(+), 43 deletions(-)

diff -r 4ede18e45935 -r e984d995f360 xen/common/sched_credit.c
--- a/xen/common/sched_credit.c Tue May 11 08:39:01 2010 +0100
+++ b/xen/common/sched_credit.c Tue May 11 11:10:24 2010 +0100
@@ -1243,7 +1243,8 @@ csched_load_balance(struct csched_privat
* fast for the common case.
*/
static struct task_slice
-csched_schedule(const struct scheduler *ops, s_time_t now)
+csched_schedule(
+ const struct scheduler *ops, s_time_t now, bool_t tasklet_work_scheduled)
{
const int cpu = smp_processor_id();
struct list_head * const runq = RUNQ(cpu);
@@ -1278,7 +1279,7 @@ csched_schedule(const struct scheduler *
snext = __runq_elem(runq->next);

/* Tasklet work (which runs in idle VCPU context) overrides all else. */
- if ( !tasklet_queue_empty(cpu) )
+ if ( tasklet_work_scheduled )
{
snext = CSCHED_VCPU(idle_vcpu[cpu]);
snext->pri = CSCHED_PRI_TS_BOOST;
diff -r 4ede18e45935 -r e984d995f360 xen/common/sched_credit2.c
--- a/xen/common/sched_credit2.c Tue May 11 08:39:01 2010 +0100
+++ b/xen/common/sched_credit2.c Tue May 11 11:10:24 2010 +0100
@@ -897,7 +897,8 @@ void __dump_execstate(void *unused);
* fast for the common case.
*/
static struct task_slice
-csched_schedule(const struct scheduler *ops, s_time_t now)
+csched_schedule(
+ const struct scheduler *ops, s_time_t now, bool_t tasklet_work_scheduled)
{
const int cpu = smp_processor_id();
struct csched_runqueue_data *rqd = RQD(ops, cpu);
@@ -921,7 +922,7 @@ csched_schedule(const struct scheduler *
burn_credits(rqd, scurr, now);

/* Tasklet work (which runs in idle VCPU context) overrides all else. */
- if ( !tasklet_queue_empty(cpu) )
+ if ( tasklet_work_scheduled )
{
snext = CSCHED_VCPU(idle_vcpu[cpu]);
goto out;
diff -r 4ede18e45935 -r e984d995f360 xen/common/sched_sedf.c
--- a/xen/common/sched_sedf.c Tue May 11 08:39:01 2010 +0100
+++ b/xen/common/sched_sedf.c Tue May 11 11:10:24 2010 +0100
@@ -790,7 +790,8 @@ static struct task_slice sedf_do_extra_s
-timeslice for the current period used up
-domain on waitqueue has started it's period
-and various others ;) in general: determine which domain to run next*/
-static struct task_slice sedf_do_schedule(const struct scheduler *ops, s_time_t now)
+static struct task_slice sedf_do_schedule(
+ const struct scheduler *ops, s_time_t now, bool_t tasklet_work_scheduled)
{
int cpu = smp_processor_id();
struct list_head *runq = RUNQ(cpu);
@@ -826,18 +827,13 @@ static struct task_slice sedf_do_schedul
check_waitq:
update_queues(now, runq, waitq);

- if ( unlikely(!cpu_isset(cpu, *SEDF_CPUONLINE(per_cpu(cpupool, cpu)))) )
- {
- ret.task = IDLETASK(cpu);
- ret.time = SECONDS(1);
- goto sched_done;
- }
-
/*now simply pick the first domain from the runqueue, which has the
earliest deadline, because the list is sorted*/

/* Tasklet work (which runs in idle VCPU context) overrides all else. */
- if ( !tasklet_queue_empty(cpu) || (list_empty(runq) && list_empty(waitq)) )
+ if ( tasklet_work_scheduled ||
+ (list_empty(runq) && list_empty(waitq)) ||
+ unlikely(!cpu_isset(cpu, *SEDF_CPUONLINE(per_cpu(cpupool, cpu)))) )
{
ret.task = IDLETASK(cpu);
ret.time = SECONDS(1);
@@ -870,7 +866,6 @@ static struct task_slice sedf_do_schedul
extraq, cpu);
}

- sched_done:
/*TODO: Do something USEFUL when this happens and find out, why it
still can happen!!!*/
if ( ret.time < 0)
diff -r 4ede18e45935 -r e984d995f360 xen/common/schedule.c
--- a/xen/common/schedule.c Tue May 11 08:39:01 2010 +0100
+++ b/xen/common/schedule.c Tue May 11 11:10:24 2010 +0100
@@ -930,6 +930,8 @@ static void schedule(void)
struct vcpu *prev = current, *next = NULL;
s_time_t now = NOW();
struct scheduler *sched = this_cpu(scheduler);
+ unsigned long *tasklet_work = &this_cpu(tasklet_work_to_do);
+ bool_t tasklet_work_scheduled = 0;
struct schedule_data *sd;
struct task_slice next_slice;

@@ -939,13 +941,30 @@ static void schedule(void)
perfc_incr(sched_run);

sd = &this_cpu(schedule_data);
+
+ /* Update tasklet scheduling status. */
+ switch ( *tasklet_work )
+ {
+ case TASKLET_enqueued:
+ set_bit(_TASKLET_scheduled, tasklet_work);
+ case TASKLET_enqueued|TASKLET_scheduled:
+ tasklet_work_scheduled = 1;
+ break;
+ case TASKLET_scheduled:
+ clear_bit(_TASKLET_scheduled, tasklet_work);
+ case 0:
+ /*tasklet_work_scheduled = 0;*/
+ break;
+ default:
+ BUG();
+ }

spin_lock_irq(sd->schedule_lock);

stop_timer(&sd->s_timer);

/* get policy-specific decision on scheduling... */
- next_slice = sched->do_schedule(sched, now);
+ next_slice = sched->do_schedule(sched, now, tasklet_work_scheduled);

next = next_slice.task;

diff -r 4ede18e45935 -r e984d995f360 xen/common/tasklet.c
--- a/xen/common/tasklet.c Tue May 11 08:39:01 2010 +0100
+++ b/xen/common/tasklet.c Tue May 11 11:10:24 2010 +0100
@@ -20,14 +20,22 @@
/* Some subsystems call into us before we are initialised. We ignore them. */
static bool_t tasklets_initialised;

-/*
- * NB. Any modification to a tasklet_list requires the scheduler to run
- * on the related CPU so that its idle VCPU's priority is set correctly.
- */
+DEFINE_PER_CPU(unsigned long, tasklet_work_to_do);
+
static DEFINE_PER_CPU(struct list_head, tasklet_list);

/* Protects all lists and tasklet structures. */
static DEFINE_SPINLOCK(tasklet_lock);
+
+static void tasklet_enqueue(struct tasklet *t)
+{
+ unsigned int cpu = t->scheduled_on;
+ unsigned long *work_to_do = &per_cpu(tasklet_work_to_do, cpu);
+
+ list_add_tail(&t->list, &per_cpu(tasklet_list, cpu));
+ if ( !test_and_set_bit(_TASKLET_enqueued, work_to_do) )
+ cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
+}

void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu)
{
@@ -41,8 +49,7 @@ void tasklet_schedule_on_cpu(struct task
if ( !t->is_running )
{
list_del(&t->list);
- list_add_tail(&t->list, &per_cpu(tasklet_list, cpu));
- cpu_raise_softirq(cpu, SCHEDULE_SOFTIRQ);
+ tasklet_enqueue(t);
}
}

@@ -57,19 +64,21 @@ void do_tasklet(void)
void do_tasklet(void)
{
unsigned int cpu = smp_processor_id();
+ unsigned long *work_to_do = &per_cpu(tasklet_work_to_do, cpu);
struct list_head *list = &per_cpu(tasklet_list, cpu);
struct tasklet *t;

- if ( likely(list_empty(list)) )
+ /*
+ * Work must be enqueued *and* scheduled. Otherwise there is no work to
+ * do, and/or scheduler needs to run to update idle vcpu priority.
+ */
+ if ( likely(*work_to_do != (TASKLET_enqueued|TASKLET_scheduled)) )
return;

spin_lock_irq(&tasklet_lock);

if ( unlikely(list_empty(list)) )
- {
- spin_unlock_irq(&tasklet_lock);
- return;
- }
+ goto out;

t = list_entry(list->next, struct tasklet, list);
list_del_init(&t->list);
@@ -88,19 +97,17 @@ void do_tasklet(void)
if ( t->scheduled_on >= 0 )
{
BUG_ON(t->is_dead || !list_empty(&t->list));
- list_add_tail(&t->list, &per_cpu(tasklet_list, t->scheduled_on));
- if ( t->scheduled_on != cpu )
- cpu_raise_softirq(t->scheduled_on, SCHEDULE_SOFTIRQ);
+ tasklet_enqueue(t);
}

- raise_softirq(SCHEDULE_SOFTIRQ);
+ out:
+ if ( list_empty(list) )
+ {
+ clear_bit(_TASKLET_enqueued, work_to_do);
+ raise_softirq(SCHEDULE_SOFTIRQ);
+ }

spin_unlock_irq(&tasklet_lock);
-}
-
-bool_t tasklet_queue_empty(unsigned int cpu)
-{
- return list_empty(&per_cpu(tasklet_list, cpu));
}

void tasklet_kill(struct tasklet *t)
@@ -113,7 +120,6 @@ void tasklet_kill(struct tasklet *t)
{
BUG_ON(t->is_dead || t->is_running || (t->scheduled_on < 0));
list_del_init(&t->list);
- cpu_raise_softirq(t->scheduled_on, SCHEDULE_SOFTIRQ);
}

t->scheduled_on = -1;
@@ -143,10 +149,8 @@ void migrate_tasklets_from_cpu(unsigned
BUG_ON(t->scheduled_on != cpu);
t->scheduled_on = smp_processor_id();
list_del(&t->list);
- list_add_tail(&t->list, &this_cpu(tasklet_list));
+ tasklet_enqueue(t);
}
-
- raise_softirq(SCHEDULE_SOFTIRQ);

spin_unlock_irqrestore(&tasklet_lock, flags);
}
diff -r 4ede18e45935 -r e984d995f360 xen/include/xen/sched-if.h
--- a/xen/include/xen/sched-if.h Tue May 11 08:39:01 2010 +0100
+++ b/xen/include/xen/sched-if.h Tue May 11 11:10:24 2010 +0100
@@ -110,7 +110,8 @@ struct scheduler {
void (*wake) (const struct scheduler *, struct vcpu *);
void (*context_saved) (const struct scheduler *, struct vcpu *);

- struct task_slice (*do_schedule) (const struct scheduler *, s_time_t);
+ struct task_slice (*do_schedule) (const struct scheduler *, s_time_t,
+ bool_t tasklet_work_scheduled);

int (*pick_cpu) (const struct scheduler *, struct vcpu *);
int (*adjust) (const struct scheduler *, struct domain *,
diff -r 4ede18e45935 -r e984d995f360 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h Tue May 11 08:39:01 2010 +0100
+++ b/xen/include/xen/sched.h Tue May 11 11:10:24 2010 +0100
@@ -595,8 +595,10 @@ uint64_t get_cpu_idle_time(unsigned int
* Used by idle loop to decide whether there is work to do:
* (1) Run softirqs; or (2) Play dead; or (3) Run tasklets.
*/
-#define cpu_is_haltable(cpu) \
- (!softirq_pending(cpu) && cpu_online(cpu) && tasklet_queue_empty(cpu))
+#define cpu_is_haltable(cpu) \
+ (!softirq_pending(cpu) && \
+ cpu_online(cpu) && \
+ !per_cpu(tasklet_work_to_do, cpu))

#define IS_PRIV(_d) ((_d)->is_privileged)
#define IS_PRIV_FOR(_d, _t) (IS_PRIV(_d) || ((_d)->target && (_d)->target == (_t)))
diff -r 4ede18e45935 -r e984d995f360 xen/include/xen/tasklet.h
--- a/xen/include/xen/tasklet.h Tue May 11 08:39:01 2010 +0100
+++ b/xen/include/xen/tasklet.h Tue May 11 11:10:24 2010 +0100
@@ -24,10 +24,16 @@ struct tasklet
#define DECLARE_TASKLET(name, func, data) \
struct tasklet name = { LIST_HEAD_INIT(name.list), -1, 0, 0, func, data }

+/* Indicates status of tasklet work on each CPU. */
+DECLARE_PER_CPU(unsigned long, tasklet_work_to_do);
+#define _TASKLET_enqueued 0 /* Tasklet work is enqueued for this CPU. */
+#define _TASKLET_scheduled 1 /* Scheduler has scheduled do_tasklet(). */
+#define TASKLET_enqueued (1ul << _TASKLET_enqueued)
+#define TASKLET_scheduled (1ul << _TASKLET_scheduled)
+
void tasklet_schedule_on_cpu(struct tasklet *t, unsigned int cpu);
void tasklet_schedule(struct tasklet *t);
void do_tasklet(void);
-bool_t tasklet_queue_empty(unsigned int cpu);
void tasklet_kill(struct tasklet *t);
void migrate_tasklets_from_cpu(unsigned int cpu);
void tasklet_init(

_______________________________________________
Xen-changelog mailing list
Xen-changelog@lists.xensource.com
http://lists.xensource.com/xen-changelog