78 lines
2.3 KiB
Diff
78 lines
2.3 KiB
Diff
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
|
|
--- a/kernel/sched/fair.c
|
|
+++ b/kernel/sched/fair.c
|
|
@@ -13934,29 +13934,62 @@ static inline int on_null_domain(struct rq *rq)
|
|
*/
|
|
static inline int find_new_ilb(void)
|
|
{
|
|
- int this_cpu = smp_processor_id();
|
|
- const struct cpumask *hk_mask;
|
|
- int ilb_cpu;
|
|
+ struct cpumask *ilb_cpus;
|
|
+ int ilb_cpu, fallback = -1;
|
|
|
|
- hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE);
|
|
+ lockdep_assert_irqs_disabled();
|
|
|
|
- for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) {
|
|
- if (ilb_cpu == this_cpu)
|
|
+ /*
|
|
+ * Reuse the per-CPU select_rq_mask, which is protected from concurrent
|
|
+ * use on this CPU by having interrupts disabled.
|
|
+ */
|
|
+ ilb_cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
|
|
+ cpumask_and(ilb_cpus, nohz.idle_cpus_mask,
|
|
+ housekeeping_cpumask(HK_TYPE_KERNEL_NOISE));
|
|
+
|
|
+ for_each_cpu(ilb_cpu, ilb_cpus) {
|
|
+ if (!idle_cpu(ilb_cpu)) {
|
|
+ /*
|
|
+ * Once an idle fallback exists, a busy CPU proves that
|
|
+ * this core cannot be fully idle. Skip its siblings.
|
|
+ */
|
|
+ if (sched_smt_active() && fallback >= 0)
|
|
+ cpumask_andnot(ilb_cpus, ilb_cpus, cpu_smt_mask(ilb_cpu));
|
|
continue;
|
|
+ }
|
|
|
|
- if (idle_cpu(ilb_cpu))
|
|
- return ilb_cpu;
|
|
+ /*
|
|
+ * Running the idle load balancer on an idle sibling of a busy
|
|
+ * SMT core can reduce the capacity available to its sibling. Prefer
|
|
+ * a CPU whose entire core is idle, but retain the first idle CPU as
|
|
+ * a fallback so idle balancing can still make progress when no fully
|
|
+ * idle core exists.
|
|
+ */
|
|
+ if (sched_smt_active() && !is_core_idle(ilb_cpu)) {
|
|
+ if (fallback < 0)
|
|
+ fallback = ilb_cpu;
|
|
+
|
|
+ /*
|
|
+ * The core is not idle, so there is no need to check
|
|
+ * any of its other SMT siblings.
|
|
+ */
|
|
+ cpumask_andnot(ilb_cpus, ilb_cpus,
|
|
+ cpu_smt_mask(ilb_cpu));
|
|
+ continue;
|
|
+ }
|
|
+
|
|
+ return ilb_cpu;
|
|
}
|
|
|
|
- return -1;
|
|
+ return fallback;
|
|
}
|
|
|
|
/*
|
|
* Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU
|
|
* SMP function call (IPI).
|
|
*
|
|
- * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set
|
|
- * (if there is one).
|
|
+ * Prefer a CPU on a fully idle core in the HK_TYPE_KERNEL_NOISE housekeeping
|
|
+ * set. Fall back to the first idle CPU when no fully idle core exists.
|
|
*/
|
|
static void kick_ilb(unsigned int flags)
|
|
{
|