omarchy-pkgs/pkgbuilds/linux-omarchy-muqss/0110-muqss-0.31.patch

diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c
--- a/arch/m68k/coldfire/pci.c
+++ b/arch/m68k/coldfire/pci.c
@@ -226,7 +226,7 @@ static int __init mcf_pci_init(void)
 	/* Turn of PCI reset, and wait for devices to settle */
 	mcf_write32(0, PCIGSCR);
 	set_current_state(TASK_UNINTERRUPTIBLE);
-	schedule_timeout(msecs_to_jiffies(200));
+	schedule_msec_hrtimeout(200);
 
 
 	pci_add_resource(&bridge->windows, &ioport_resource);
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -1185,7 +1185,7 @@ static void poll_window_credits(struct pnv_vas_window *window)
 	if (creds < window->vas_win.wcreds_max) {
 		val = 0;
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout(10);
 		count++;
 		/*
 		 * Process can not close send window until all credits are
@@ -1218,7 +1218,7 @@ static void poll_window_busy_state(struct pnv_vas_window *window)
 	if (busy) {
 		val = 0;
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout(10);
 		count++;
 		/*
 		 * Takes around few milliseconds to process all pending
diff --git a/block/Makefile b/block/Makefile
--- a/block/Makefile
+++ b/block/Makefile
@@ -14,6 +14,7 @@ obj-y		:= bdev.o fops.o bio.o elevator.o blk-core.o blk-sysfs.o \
 			disk-events.o blk-ia-ranges.o early-lookup.o
 
 obj-$(CONFIG_BLK_ERROR_INJECTION) += error-injection.o
+obj-$(CONFIG_MUQSS_IOTIME)	+= blk-iotime.o
 obj-$(CONFIG_BLK_DEV_BSG_COMMON) += bsg.o
 obj-$(CONFIG_BLK_DEV_BSGLIB)	+= bsg-lib.o
 obj-$(CONFIG_BLK_CGROUP)	+= blk-cgroup.o
diff --git a/block/bio.c b/block/bio.c
--- a/block/bio.c
+++ b/block/bio.c
@@ -19,6 +19,7 @@
 #include <linux/blk-crypto.h>
 #include <linux/xarray.h>
 #include <linux/kmemleak.h>
+#include <linux/muqss_iotime.h>
 
 #include <trace/events/block.h>
 #include "blk.h"
@@ -185,6 +186,8 @@ void bio_uninit(struct bio *bio)
 		bio->bi_blkg = NULL;
 	}
 #endif
+	muqss_iotime_put_owner(bio);
+
 	if (bio_integrity(bio))
 		bio_integrity_free(bio);
 
@@ -234,13 +237,18 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
 	bio->bi_private = NULL;
 #ifdef CONFIG_BLK_CGROUP
 	bio->bi_blkg = NULL;
-	bio->issue_time_ns = 0;
 	if (bdev)
 		bio_associate_blkg(bio);
 #ifdef CONFIG_BLK_CGROUP_IOCOST
 	bio->bi_iocost_cost = 0;
 #endif
 #endif
+#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME)
+	bio->issue_time_ns = 0;
+#endif
+#ifdef CONFIG_MUQSS_IOTIME
+	bio->bi_muqss_owner = NULL;
+#endif
 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
 	bio->bi_crypt_context = NULL;
 #endif
@@ -868,6 +876,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp)
 			bio_set_flag(bio, BIO_REMAPPED);
 		bio_clone_blkg_association(bio, bio_src);
 	}
+	muqss_iotime_clone_owner(bio, bio_src);
 
 	if (bio_crypt_clone(bio, bio_src, gfp) < 0)
 		return -ENOMEM;
@@ -1865,6 +1874,7 @@ void bio_endio(struct bio *bio)
 		bio->bi_blkg = NULL;
 	}
 #endif
+	muqss_iotime_put_owner(bio);
 
 	if (bio->bi_end_io)
 		bio->bi_end_io(bio);
diff --git a/block/blk-iotime.c b/block/blk-iotime.c
new file mode 100644
--- /dev/null
+++ b/block/blk-iotime.c
@@ -0,0 +1,753 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * I/O aware scheduling support for MuQSS.
+ *
+ * The CPU scheduler can see how much CPU a task uses but nothing of what it
+ * costs the storage device. This attaches an rq_qos policy that attributes
+ * block device time back to the task that caused the I/O, so that cost is
+ * visible to the scheduler.
+ *
+ * Two different quantities are recorded and deliberately kept apart:
+ *
+ * Latency is what a waiting task experiences, measured per bio from
+ * bio->issue_time_ns, which is stamped in blk_mq_submit_bio() before merging
+ * and request allocation. It therefore includes plug, merge and queue wait as
+ * well as service.
+ *
+ * Occupancy is what the task cost everybody else, measured per request from
+ * rq->io_start_time_ns at dispatch to rq->muqss_done_ns at completion. This is
+ * the quantity the scheduler charges for.
+ *
+ * Attribution has two halves. Reads and synchronous writes are submitted by
+ * the originating task, so the owner is simply current. Writeback is not: the
+ * dirtier is recorded on the folio when it is dirtied and recovered when the
+ * flusher thread submits the I/O. See the owner table below.
+ *
+ * The owner table earns its keep a second time over. A kworker running a work
+ * item declares itself to be acting for whoever queued that work, which both
+ * attributes the I/O the work item issues and lets the CPU time it costs be
+ * charged back as kern_debt_ns. See muqss_kerntime_begin().
+ *
+ * The occupancy is charged one for one against the deadline, with nothing to
+ * tune it by: the accounting below and the scheduling it feeds are one
+ * feature, and only CONFIG_MUQSS_IOTIME=n removes either. See
+ * consume_iotime_penalty() in kernel/sched/MuQSS.c and
+ * MuQSS-iotime-design.md.
+ */
+#include <linux/kernel.h>
+#include <linux/blkdev.h>
+#include <linux/blk-mq.h>
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/slab.h>
+#include <linux/workqueue_types.h>
+#include <linux/muqss_iotime.h>
+
+#include "blk.h"
+#include "blk-mq.h"
+#include "blk-mq-debugfs.h"
+#include "blk-rq-qos.h"
+#include "blk-stat.h"
+
+struct blk_iotime {
+	struct rq_qos		rqos;
+
+	/* Unattributed device busy time, see comment above. */
+	atomic64_t		occupancy_ns;
+	atomic64_t		requests;
+
+	/*
+	 * Bios that completed with no owner recorded. Counted so the size of
+	 * the attribution gap stays visible rather than silently skewing the
+	 * per task numbers.
+	 */
+	atomic64_t		unattributed;
+
+	/* Requests disowned because a merge brought in another task's bio. */
+	atomic64_t		mixed;
+
+	/*
+	 * Requests that reached completion with no captured timestamp, and so
+	 * could not be measured at all. See muqss_iotime_done().
+	 */
+	atomic64_t		unstamped;
+
+	/* Bios that completed with an owner but no submit timestamp. */
+	atomic64_t		nostamp_bio;
+};
+
+static inline struct blk_iotime *BLK_IOTIME(struct rq_qos *rqos)
+{
+	return container_of(rqos, struct blk_iotime, rqos);
+}
+
+/*
+ * Owner table.
+ *
+ * Reads are attributable at submit time because the submitting task is the
+ * originating one. Writeback is not: the folio is dirtied by one task and
+ * written out much later by a flusher thread. The dirtier is therefore
+ * recorded on the folio itself, in spare page->flags bits, as an index into
+ * this table.
+ *
+ * Only MUQSS_IOWNER_WIDTH bits are available, so the table is small and the
+ * index carries no generation. Slot 0 is reserved to mean "no owner". A slot
+ * is held from the task's first dirty until it exits, and the table holds a
+ * reference for that whole period, so a resolved pointer is always a live
+ * task_struct.
+ *
+ * Slots are released on exit and never stolen from a live task, so a resolved
+ * index is never charged to the wrong task. The cost of that choice is that
+ * beyond IOWNER_SLOTS concurrent dirtiers, further tasks get no slot and their
+ * writeback goes unattributed; iowner_exhausted counts it.
+ *
+ * Writeback still in flight for a task that has exited resolves to nothing,
+ * which is correct: there is no longer anybody to charge.
+ */
+#define IOWNER_SLOTS		(1U << MUQSS_IOWNER_SHIFT)
+
+static struct task_struct *iowner_table[IOWNER_SLOTS];
+/* Work can be queued with a raw runqueue lock held (for example by PSI). */
+static DEFINE_RAW_SPINLOCK(iowner_lock);
+
+static atomic64_t iowner_exhausted;
+
+static atomic64_t iowner_proxied;
+
+/* Work items charged back to a task, and the CPU time they cost. */
+static atomic64_t kernwork_charged;
+static atomic64_t kernwork_ns;
+
+/*
+ * Slot for @tsk, allocating one on first use.
+ *
+ * @tsk is not necessarily current: an io-wq worker allocates on behalf of the
+ * task that queued the request it is running, so that pages it dirties are
+ * tagged with the real owner. The caller holds a reference to @tsk.
+ */
+static unsigned int iotime_task_slot(struct task_struct *tsk)
+{
+	unsigned int slot, scan;
+	unsigned long flags;
+
+	if (!MUQSS_IOWNER_WIDTH)
+		return 0;
+
+	slot = READ_ONCE(tsk->io_owner_slot);
+	if (slot)
+		return slot;
+
+	if (tsk->flags & (PF_KTHREAD | PF_EXITING))
+		return 0;
+
+	raw_spin_lock_irqsave(&iowner_lock, flags);
+
+	/*
+	 * Recheck PF_EXITING under the lock. It is set well before
+	 * muqss_iotime_release_slot() runs, and that also takes this lock, so
+	 * a task seen here as not exiting either has not reached exit or will
+	 * release the slot we are about to hand it. Without the recheck a slot
+	 * claimed for a remote task racing through exit would never be freed.
+	 */
+	if (tsk->flags & PF_EXITING)
+		goto out;
+
+	/* Raced with another dirty on the same task. */
+	if (tsk->io_owner_slot) {
+		slot = tsk->io_owner_slot;
+		goto out;
+	}
+
+	/* Prefer a free slot. */
+	for (scan = 1; scan < IOWNER_SLOTS; scan++) {
+		if (!iowner_table[scan]) {
+			slot = scan;
+			goto claim;
+		}
+	}
+
+	/*
+	 * Table full of live dirtiers. Leave this task unattributed rather
+	 * than evicting somebody still using their slot: a stolen slot would
+	 * silently charge one task's writeback to another, which is worse
+	 * than not charging it at all.
+	 */
+	atomic64_inc(&iowner_exhausted);
+	slot = 0;
+	goto out;
+
+claim:
+	iowner_table[slot] = get_task_struct(tsk);
+	WRITE_ONCE(tsk->io_owner_slot, slot);
+out:
+	raw_spin_unlock_irqrestore(&iowner_lock, flags);
+	return slot;
+}
+
+unsigned int muqss_iotime_owner_slot(void)
+{
+	struct task_struct *tsk = current;
+	struct task_struct *owner;
+
+	/*
+	 * A thread running inside a proxy window dirties pages on behalf of
+	 * the task that window belongs to, so tag them with that task. Doing
+	 * otherwise would both misattribute the writeback and, for io-wq
+	 * workers, let a churn of short lived workers consume the whole slot
+	 * table.
+	 *
+	 * An io-wq worker outside a window has nobody to speak for and must
+	 * not claim a slot of its own. A kworker outside a window needs no
+	 * such test: iotime_task_slot() refuses PF_KTHREAD outright.
+	 */
+	owner = READ_ONCE(tsk->io_owner_override);
+	if (owner)
+		tsk = owner;
+	else if (tsk->flags & PF_IO_WORKER)
+		return 0;
+
+	return iotime_task_slot(tsk);
+}
+
+struct task_struct *muqss_iotime_proxy_begin(struct task_struct *owner)
+{
+	struct task_struct *prev = current->io_owner_override;
+
+	WRITE_ONCE(current->io_owner_override, owner);
+	return prev;
+}
+
+void muqss_iotime_proxy_end(struct task_struct *prev)
+{
+	WRITE_ONCE(current->io_owner_override, prev);
+}
+
+void muqss_work_set_owner(struct work_struct *work)
+{
+	/* Interrupt work does not belong to the task it interrupted. */
+	work->muqss_owner_slot = in_task() ? muqss_iotime_owner_slot() : 0;
+}
+
+void muqss_kerntime_begin(struct muqss_kern_window *w, unsigned int slot)
+{
+	w->prev = NULL;
+	w->start = 0;
+
+	/*
+	 * Resolving the slot is what costs here, so it gates everything else.
+	 * Work queued by kernel threads on their own account carries slot 0
+	 * and leaves this function having done one branch, which matters:
+	 * timers, readahead, RCU and any amount of driver housekeeping run
+	 * through work items that belong to nobody.
+	 */
+	w->owner = muqss_iotime_owner_task(slot);
+	if (!w->owner)
+		return;
+
+	w->prev = muqss_iotime_proxy_begin(w->owner);
+	w->start = muqss_task_runtime_live();
+}
+
+void muqss_kerntime_end(struct muqss_kern_window *w)
+{
+	s64 ns;
+
+	if (!w->owner)
+		return;
+
+	ns = muqss_task_runtime_live() - w->start;
+	muqss_iotime_proxy_end(w->prev);
+
+	/*
+	 * Runtime rather than elapsed time, so a worker preempted for a whole
+	 * scheduling round does not hand the bill for it to the task it is
+	 * working for. Negative or absurd deltas are possible if the two ends
+	 * of the window landed either side of a clock warp, and are dropped
+	 * rather than clamped: there is no sensible value to substitute.
+	 */
+	if (ns > 0) {
+		atomic64_add(ns, &w->owner->kern_time_ns);
+		atomic64_add(ns, &w->owner->kern_debt_ns);
+		atomic64_inc(&kernwork_charged);
+		atomic64_add(ns, &kernwork_ns);
+	}
+
+	put_task_struct(w->owner);
+	w->owner = NULL;
+}
+
+struct task_struct *muqss_iotime_owner_task(unsigned int slot)
+{
+	struct task_struct *tsk = NULL;
+	unsigned long flags;
+
+	if (!slot || slot >= IOWNER_SLOTS)
+		return NULL;
+
+	raw_spin_lock_irqsave(&iowner_lock, flags);
+	if (iowner_table[slot])
+		tsk = get_task_struct(iowner_table[slot]);
+	raw_spin_unlock_irqrestore(&iowner_lock, flags);
+
+	return tsk;
+}
+
+void muqss_iotime_release_slot(struct task_struct *p)
+{
+	unsigned int slot = READ_ONCE(p->io_owner_slot);
+	unsigned long flags;
+
+	if (!slot)
+		return;
+
+	raw_spin_lock_irqsave(&iowner_lock, flags);
+	if (iowner_table[slot] == p) {
+		iowner_table[slot] = NULL;
+		WRITE_ONCE(p->io_owner_slot, 0);
+		raw_spin_unlock_irqrestore(&iowner_lock, flags);
+		put_task_struct(p);
+		return;
+	}
+	WRITE_ONCE(p->io_owner_slot, 0);
+	raw_spin_unlock_irqrestore(&iowner_lock, flags);
+}
+
+void muqss_iotime_dirty_folio(struct folio *folio)
+{
+	unsigned int slot = muqss_iotime_owner_slot();
+
+	folio_set_io_owner(folio, slot);
+}
+
+void muqss_iotime_task_init(struct task_struct *p)
+{
+	atomic64_set(&p->io_latency_ns, 0);
+	atomic64_set(&p->io_count, 0);
+	atomic64_set(&p->io_occupancy_ns, 0);
+	atomic64_set(&p->io_debt_ns, 0);
+	atomic64_set(&p->kern_time_ns, 0);
+	atomic64_set(&p->kern_debt_ns, 0);
+	p->io_owner_slot = 0;
+	p->io_owner_override = NULL;
+}
+
+void muqss_iotime_set_owner(struct bio *bio)
+{
+	struct task_struct *tsk = current;
+
+	if (bio->bi_muqss_owner)
+		return;
+
+	/*
+	 * Not in task context: current is whatever this interrupted, which
+	 * has nothing to do with the I/O. Bios submitted from softirq or hard
+	 * IRQ, such as a filesystem completing one write by starting another,
+	 * would otherwise be charged to a bystander.
+	 */
+	if (!in_task())
+		return;
+
+	/*
+	 * Passthrough requests are not issued on any task's behalf. Neither
+	 * is the flush machinery's own empty bio, but REQ_PREFLUSH is also set
+	 * on an ordinary write that wants a cache flush ahead of it, and that
+	 * write is somebody's. Test for the payload rather than the flag.
+	 */
+	if (blk_op_is_passthrough(bio->bi_opf))
+		return;
+	if ((bio->bi_opf & REQ_PREFLUSH) && !bio_has_data(bio))
+		return;
+
+	if (tsk->flags & (PF_KTHREAD | PF_IO_WORKER)) {
+		struct task_struct *owner;
+		struct folio *folio;
+
+		/*
+		 * io_uring punted this submission to an io-wq worker, which
+		 * published the task that queued the request. Charge that
+		 * task: the worker is an anonymous thread that exists only to
+		 * run somebody else's I/O.
+		 *
+		 * Writeback that happens to be issued from inside the same
+		 * window is charged here too, which is a small misattribution
+		 * within one application rather than a loss.
+		 */
+		owner = READ_ONCE(tsk->io_owner_override);
+		if (owner) {
+			atomic64_inc(&iowner_proxied);
+			bio->bi_muqss_owner = get_task_struct(owner);
+			return;
+		}
+
+		/*
+		 * Writeback. current is a flusher thread rather than the task
+		 * that dirtied the data, so recover the owner from the folio,
+		 * where it was recorded at dirty time.
+		 *
+		 * The first page decides for the whole bio. A writeback bio
+		 * normally covers one file's pages dirtied by one task, and
+		 * reading every page to take a vote would put a loop on the
+		 * submit path for a heuristic that does not need it.
+		 *
+		 * Cloned bios are skipped: their bi_io_vec belongs to the
+		 * parent, and muqss_iotime_clone_owner() has already carried
+		 * the owner across.
+		 */
+		if (op_is_write(bio->bi_opf) && bio->bi_vcnt &&
+		    !bio_flagged(bio, BIO_CLONED)) {
+			folio = page_folio(bio->bi_io_vec[0].bv_page);
+			bio->bi_muqss_owner =
+				muqss_iotime_owner_task(folio_io_owner(folio));
+			return;
+		}
+
+		/*
+		 * An io_uring SQPOLL thread submits for whoever shares its
+		 * ring, and the request does not record which task that was;
+		 * with IORING_SETUP_ATTACH_WQ it need not be a single one.
+		 * The thread is a long lived schedulable task in its own
+		 * right though, so charge it directly rather than lose the
+		 * time: the device cost is real, and the sq thread is the
+		 * entity the scheduler can actually act on.
+		 *
+		 * Kernel threads proper get nothing.
+		 */
+		if (tsk->flags & PF_IO_WORKER)
+			bio->bi_muqss_owner = get_task_struct(tsk);
+		return;
+	}
+
+	bio->bi_muqss_owner = get_task_struct(tsk);
+}
+
+void muqss_iotime_put_owner(struct bio *bio)
+{
+	if (bio->bi_muqss_owner) {
+		put_task_struct(bio->bi_muqss_owner);
+		bio->bi_muqss_owner = NULL;
+	}
+}
+
+void muqss_iotime_clone_owner(struct bio *bio, struct bio *bio_src)
+{
+	muqss_iotime_put_owner(bio);
+
+	if (bio_src->bi_muqss_owner)
+		bio->bi_muqss_owner = get_task_struct(bio_src->bi_muqss_owner);
+}
+
+/*
+ * Per bio completion. The owner reference is not dropped here: bio_endio()
+ * and bio_uninit() release it alongside the blkg reference, which keeps the
+ * lifetime rules identical to the ones the block layer already follows.
+ */
+static void muqss_iotime_done_bio(struct rq_qos *rqos, struct bio *bio)
+{
+	struct blk_iotime *bit = BLK_IOTIME(rqos);
+	struct task_struct *tsk = bio->bi_muqss_owner;
+	u64 now;
+
+	if (!tsk) {
+		atomic64_inc(&bit->unattributed);
+		return;
+	}
+	if (!bio->issue_time_ns) {
+		atomic64_inc(&bit->nostamp_bio);
+		return;
+	}
+
+	now = blk_time_get_ns();
+	if (now <= bio->issue_time_ns)
+		return;
+
+	atomic64_add(now - bio->issue_time_ns, &tsk->io_latency_ns);
+	atomic64_inc(&tsk->io_count);
+}
+
+/*
+ * A request is being built from its first bio. Take our own reference for the
+ * request: the bio's reference is released by bio_endio() during
+ * blk_update_request(), which runs before rq_qos_done(), so borrowing it would
+ * leave a dangling pointer at completion.
+ */
+static void muqss_iotime_track(struct rq_qos *rqos, struct request *rq,
+			       struct bio *bio)
+{
+	if (bio->bi_muqss_owner)
+		rq->muqss_owner = get_task_struct(bio->bi_muqss_owner);
+}
+
+/*
+ * A further bio is merging into an existing request. Occupancy is measured
+ * per request and cannot be split between owners, so a request that ends up
+ * serving more than one task is disowned rather than charged to whichever
+ * task happened to be first.
+ */
+static void muqss_iotime_merge(struct rq_qos *rqos, struct request *rq,
+			       struct bio *bio)
+{
+	if (!rq->muqss_owner || rq->muqss_owner == bio->bi_muqss_owner)
+		return;
+
+	put_task_struct(rq->muqss_owner);
+	rq->muqss_owner = NULL;
+	atomic64_inc(&BLK_IOTIME(rqos)->mixed);
+}
+
+/*
+ * Two whole requests are being merged: next's bios are about to be appended to
+ * rq and next freed. The rq_qos ->merge hook does not cover this, it only sees
+ * a bio being merged into a request, so without this the survivor keeps its own
+ * owner and is charged for both requests' device time.
+ *
+ * Called from attempt_merge() rather than being an rq_qos op because the
+ * framework has no request-to-request callback to hang it on.
+ */
+void muqss_iotime_merge_requests(struct request *rq, struct request *next)
+{
+	struct rq_qos *rqos;
+
+	if (!rq->muqss_owner || rq->muqss_owner == next->muqss_owner)
+		return;
+
+	put_task_struct(rq->muqss_owner);
+	rq->muqss_owner = NULL;
+
+	rqos = rq_qos_id(rq->q, RQ_QOS_MUQSS_IOTIME);
+	if (rqos)
+		atomic64_inc(&BLK_IOTIME(rqos)->mixed);
+}
+
+/* Per request completion. This is where device occupancy is attributed. */
+static void muqss_iotime_done(struct rq_qos *rqos, struct request *rq)
+{
+	struct blk_iotime *bit = BLK_IOTIME(rqos);
+	struct task_struct *tsk = rq->muqss_owner;
+	u64 now, occupancy;
+
+	if (!(rq->rq_flags & RQF_STATS) || !rq->io_start_time_ns)
+		goto out;
+
+	/*
+	 * Use the timestamp captured when the request completed, not one
+	 * taken here: rq_qos_done() runs after blk_update_request() has
+	 * completed every bio, so measuring now would fold the completion
+	 * path into what is supposed to be device service time. That bias is
+	 * roughly constant per request, which would quietly turn part of the
+	 * charge into a per-I/O-count charge and penalise small random I/O.
+	 *
+	 * Consuming the stamp also makes this idempotent. rq_qos_done() runs
+	 * twice for a request with an end_io handler that frees it: once from
+	 * __blk_mq_end_request() and again from blk_mq_free_request(). Every
+	 * policy has to tolerate that. Clearing muqss_owner alone stopped the
+	 * task being charged twice but left the device totals below counting
+	 * the same request on both passes.
+	 *
+	 * A request that never went through blk_mq_end_request() has no stamp
+	 * and is not measured. Timing it here instead would report the
+	 * completion path as device time, which is the bias this exists to
+	 * avoid; unstamped counts them so the loss stays visible.
+	 */
+	now = rq->muqss_done_ns;
+	if (!now) {
+		atomic64_inc(&bit->unstamped);
+		goto out;
+	}
+	rq->muqss_done_ns = 0;
+
+	if (now <= rq->io_start_time_ns)
+		goto out;
+
+	occupancy = now - rq->io_start_time_ns;
+	atomic64_add(occupancy, &bit->occupancy_ns);
+	atomic64_inc(&bit->requests);
+
+	if (tsk) {
+		atomic64_add(occupancy, &tsk->io_occupancy_ns);
+		atomic64_add(occupancy, &tsk->io_debt_ns);
+	}
+out:
+	if (tsk) {
+		put_task_struct(tsk);
+		rq->muqss_owner = NULL;
+	}
+}
+
+static void muqss_iotime_exit(struct rq_qos *rqos)
+{
+	struct blk_iotime *bit = BLK_IOTIME(rqos);
+
+	/*
+	 * QUEUE_FLAG_BIO_ISSUE_TIME is deliberately left set. blk-iolatency
+	 * uses the same flag, and the queue is being torn down here anyway.
+	 */
+	blk_stat_disable_accounting(rqos->disk->queue);
+	kfree(bit);
+}
+
+#ifdef CONFIG_BLK_DEBUG_FS
+static int muqss_iotime_occupancy_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n",
+		   (u64)atomic64_read(&BLK_IOTIME(rqos)->occupancy_ns));
+	return 0;
+}
+
+static int muqss_iotime_requests_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n",
+		   (u64)atomic64_read(&BLK_IOTIME(rqos)->requests));
+	return 0;
+}
+
+static int muqss_iotime_unattributed_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n",
+		   (u64)atomic64_read(&BLK_IOTIME(rqos)->unattributed));
+	return 0;
+}
+
+static int muqss_iotime_mixed_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n", (u64)atomic64_read(&BLK_IOTIME(rqos)->mixed));
+	return 0;
+}
+
+static int muqss_iotime_unstamped_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n",
+		   (u64)atomic64_read(&BLK_IOTIME(rqos)->unstamped));
+	return 0;
+}
+
+static int muqss_iotime_nostamp_bio_show(void *data, struct seq_file *m)
+{
+	struct rq_qos *rqos = data;
+
+	seq_printf(m, "%llu\n",
+		   (u64)atomic64_read(&BLK_IOTIME(rqos)->nostamp_bio));
+	return 0;
+}
+
+static int muqss_iotime_exhausted_show(void *data, struct seq_file *m)
+{
+	seq_printf(m, "%llu\n", (u64)atomic64_read(&iowner_exhausted));
+	return 0;
+}
+
+static int muqss_iotime_proxied_show(void *data, struct seq_file *m)
+{
+	seq_printf(m, "%llu\n", (u64)atomic64_read(&iowner_proxied));
+	return 0;
+}
+
+static int muqss_iotime_owner_width_show(void *data, struct seq_file *m)
+{
+	seq_printf(m, "%u\n", MUQSS_IOWNER_WIDTH);
+	return 0;
+}
+
+static int muqss_iotime_kernwork_show(void *data, struct seq_file *m)
+{
+	seq_printf(m, "%llu\n", (u64)atomic64_read(&kernwork_charged));
+	return 0;
+}
+
+static int muqss_iotime_kernwork_ns_show(void *data, struct seq_file *m)
+{
+	seq_printf(m, "%llu\n", (u64)atomic64_read(&kernwork_ns));
+	return 0;
+}
+
+static const struct blk_mq_debugfs_attr muqss_iotime_debugfs_attrs[] = {
+	{"occupancy_ns", 0400, muqss_iotime_occupancy_show},
+	{"requests", 0400, muqss_iotime_requests_show},
+	{"unattributed", 0400, muqss_iotime_unattributed_show},
+	{"mixed", 0400, muqss_iotime_mixed_show},
+	{"unstamped", 0400, muqss_iotime_unstamped_show},
+	{"nostamp_bio", 0400, muqss_iotime_nostamp_bio_show},
+	{"owner_slots_exhausted", 0400, muqss_iotime_exhausted_show},
+	{"proxied", 0400, muqss_iotime_proxied_show},
+	{"owner_tag_bits", 0400, muqss_iotime_owner_width_show},
+	{"kernwork", 0400, muqss_iotime_kernwork_show},
+	{"kernwork_ns", 0400, muqss_iotime_kernwork_ns_show},
+	{},
+};
+#endif
+
+static const struct rq_qos_ops muqss_iotime_ops = {
+	.track		= muqss_iotime_track,
+	.merge		= muqss_iotime_merge,
+	.done		= muqss_iotime_done,
+	.done_bio	= muqss_iotime_done_bio,
+	.exit		= muqss_iotime_exit,
+#ifdef CONFIG_BLK_DEBUG_FS
+	.debugfs_attrs	= muqss_iotime_debugfs_attrs,
+#endif
+};
+
+void muqss_iotime_enable(struct gendisk *disk)
+{
+	struct request_queue *q = disk->queue;
+	unsigned int memflags;
+	struct blk_iotime *bit;
+	int ret;
+
+	if (!queue_is_mq(q))
+		return;
+
+	/*
+	 * Only attach to queues backed by real hardware.
+	 *
+	 * A stacking driver that uses blk-mq -- loop is the common one --
+	 * serves its requests by doing I/O to another block device, which runs
+	 * this policy too. Accounting both charges the task twice for one
+	 * trip to the disk, and with the deadline charge live that is a real
+	 * distortion rather than a cosmetic one.
+	 *
+	 * Drivers for physical devices pass their parent through
+	 * device_add_disk(): nvme passes ctrl->device, sd its scsi_device,
+	 * virtio_blk its virtio device. Virtual ones call add_disk(), which
+	 * is device_add_disk(NULL, ...) -- loop, brd, zram, nbd. So the
+	 * absence of a parent is a serviceable test for "not a device whose
+	 * busy time means anything".
+	 *
+	 * Bio based stacking drivers (dm, md) never reach here at all, having
+	 * failed queue_is_mq() above.
+	 */
+	if (!disk_to_dev(disk)->parent)
+		return;
+
+	bit = kzalloc_obj(*bit);
+	if (!bit)
+		return;
+
+	mutex_lock(&q->rq_qos_mutex);
+	ret = rq_qos_add(&bit->rqos, disk, RQ_QOS_MUQSS_IOTIME,
+			 &muqss_iotime_ops);
+	mutex_unlock(&q->rq_qos_mutex);
+	if (ret) {
+		kfree(bit);
+		return;
+	}
+
+	/* Needed for rq->io_start_time_ns to be stamped at dispatch. */
+	blk_stat_enable_accounting(q);
+	/* Needed for bio->issue_time_ns to be stamped at submit. */
+	blk_queue_flag_set(QUEUE_FLAG_BIO_ISSUE_TIME, q);
+
+	memflags = blk_debugfs_lock(q);
+	blk_mq_debugfs_register_rq_qos(q);
+	blk_debugfs_unlock(q, memflags);
+}
diff --git a/block/blk-merge.c b/block/blk-merge.c
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -12,6 +12,8 @@
 
 #include <trace/events/block.h>
 
+#include <linux/muqss_iotime.h>
+
 #include "blk.h"
 #include "blk-mq-sched.h"
 #include "blk-rq-qos.h"
@@ -848,6 +850,7 @@ static struct request *attempt_merge(struct request_queue *q,
 	 * 'next' is going away, so update stats accordingly
 	 */
 	blk_account_io_merge_request(next);
+	muqss_iotime_merge_requests(req, next);
 
 	trace_block_rq_merge(next);
 
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -757,6 +757,8 @@ static const char *rq_qos_id_to_name(enum rq_qos_id id)
 		return "latency";
 	case RQ_QOS_COST:
 		return "cost";
+	case RQ_QOS_MUQSS_IOTIME:
+		return "iotime";
 	}
 	return "unknown";
 }
diff --git a/block/blk-mq.c b/block/blk-mq.c
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -41,6 +41,7 @@
 #include "blk-stat.h"
 #include "blk-mq-sched.h"
 #include "blk-rq-qos.h"
+#include <linux/muqss_iotime.h>
 
 static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
 static DEFINE_PER_CPU(call_single_data_t, blk_cpu_csd);
@@ -398,12 +399,48 @@ static inline void blk_mq_rq_time_init(struct request *rq, u64 alloc_time_ns)
 #endif
 }
 
+/*
+ * Record the completion timestamp for MuQSS I/O accounting.
+ *
+ * rq_qos_done() is reached only after blk_update_request() has run bio_endio()
+ * on every bio of the request, so a timestamp taken inside the policy is later
+ * than the request actually finished, and lands after the per bio latency
+ * stamp. Occupancy then measures longer than the end to end latency it is
+ * supposed to be a subset of.
+ *
+ * Does not overwrite: the first caller to record a stamp is the earliest and
+ * therefore the most accurate one.
+ */
+static inline void blk_mq_set_iotime_done(struct request *rq, u64 now)
+{
+#ifdef CONFIG_MUQSS_IOTIME
+	if (!rq->muqss_done_ns)
+		rq->muqss_done_ns = now;
+#endif
+}
+
+/*
+ * Take the completion stamp before bio completion. Costs an extra timestamp
+ * on the completion path, which cannot be avoided by reusing the one taken
+ * for blk-stat: that one is deliberately read after blk_update_request() and
+ * moving it would change accounting shared with everybody else.
+ */
+static inline void blk_mq_capture_iotime_done(struct request *rq)
+{
+#ifdef CONFIG_MUQSS_IOTIME
+	if (blk_mq_need_time_stamp(rq))
+		rq->muqss_done_ns = blk_time_get_ns();
+#endif
+}
+
 static inline void blk_mq_bio_issue_init(struct request_queue *q,
 					 struct bio *bio)
 {
-#ifdef CONFIG_BLK_CGROUP
-	if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags))
+#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME)
+	if (test_bit(QUEUE_FLAG_BIO_ISSUE_TIME, &q->queue_flags)) {
 		bio->issue_time_ns = blk_time_get_ns();
+		muqss_iotime_set_owner(bio);
+	}
 #endif
 }
 
@@ -435,6 +472,10 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
 
 	rq->part = NULL;
 	rq->io_start_time_ns = 0;
+#ifdef CONFIG_MUQSS_IOTIME
+	rq->muqss_owner = NULL;
+	rq->muqss_done_ns = 0;
+#endif
 	rq->stats_sectors = 0;
 	rq->nr_phys_segments = 0;
 	rq->nr_integrity_segments = 0;
@@ -1127,8 +1168,12 @@ static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
 
 inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
 {
-	if (blk_mq_need_time_stamp(rq))
-		__blk_mq_end_request_acct(rq, blk_time_get_ns());
+	if (blk_mq_need_time_stamp(rq)) {
+		u64 now = blk_time_get_ns();
+
+		blk_mq_set_iotime_done(rq, now);
+		__blk_mq_end_request_acct(rq, now);
+	}
 
 	blk_mq_finish_request(rq);
 
@@ -1144,6 +1189,7 @@ EXPORT_SYMBOL(__blk_mq_end_request);
 
 void blk_mq_end_request(struct request *rq, blk_status_t error)
 {
+	blk_mq_capture_iotime_done(rq);
 	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
 		BUG();
 	__blk_mq_end_request(rq, error);
@@ -1178,8 +1224,10 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob)
 		prefetch(rq->rq_next);
 
 		blk_complete_request(rq);
-		if (iob->need_ts)
+		if (iob->need_ts) {
+			blk_mq_set_iotime_done(rq, now);
 			__blk_mq_end_request_acct(rq, now);
+		}
 
 		blk_mq_finish_request(rq);
 
diff --git a/block/blk-rq-qos.h b/block/blk-rq-qos.h
--- a/block/blk-rq-qos.h
+++ b/block/blk-rq-qos.h
@@ -17,6 +17,7 @@ enum rq_qos_id {
 	RQ_QOS_WBT,
 	RQ_QOS_LATENCY,
 	RQ_QOS_COST,
+	RQ_QOS_MUQSS_IOTIME,
 };
 
 struct rq_wait {
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -20,6 +20,7 @@
 #include "blk-cgroup.h"
 #include "blk-throttle.h"
 #include "error-injection.h"
+#include <linux/muqss_iotime.h>
 
 struct queue_sysfs_entry {
 	struct attribute attr;
@@ -993,6 +994,7 @@ int blk_register_queue(struct gendisk *disk)
 
 	blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 	wbt_init_enable_default(disk);
+	muqss_iotime_enable(disk);
 
 	/* Now everything is ready and send out KOBJ_ADD uevent */
 	kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
diff --git a/drivers/accessibility/speakup/speakup_acntpc.c b/drivers/accessibility/speakup/speakup_acntpc.c
--- a/drivers/accessibility/speakup/speakup_acntpc.c
+++ b/drivers/accessibility/speakup/speakup_acntpc.c
@@ -207,7 +207,7 @@ static void do_catch_up(struct spk_synth *synth)
 		full_time_val = full_time->u.n.value;
 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 		if (synth_full()) {
-			schedule_timeout(msecs_to_jiffies(full_time_val));
+			schedule_msec_hrtimeout(full_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -235,7 +235,7 @@ static void do_catch_up(struct spk_synth *synth)
 			jiffy_delta_val = jiffy_delta->u.n.value;
 			delay_time_val = delay_time->u.n.value;
 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			jiff_max = jiffies + jiffy_delta_val;
 		}
 	}
diff --git a/drivers/accessibility/speakup/speakup_apollo.c b/drivers/accessibility/speakup/speakup_apollo.c
--- a/drivers/accessibility/speakup/speakup_apollo.c
+++ b/drivers/accessibility/speakup/speakup_apollo.c
@@ -178,7 +178,7 @@ static void do_catch_up(struct spk_synth *synth)
 		if (!synth->io_ops->synth_out(synth, ch)) {
 			synth->io_ops->tiocmset(synth, 0, UART_MCR_RTS);
 			synth->io_ops->tiocmset(synth, UART_MCR_RTS, 0);
-			schedule_timeout(msecs_to_jiffies(full_time_val));
+			schedule_msec_hrtimeout(full_time_val);
 			continue;
 		}
 		if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) {
@@ -188,11 +188,9 @@ static void do_catch_up(struct spk_synth *synth)
 			delay_time_val = delay_time->u.n.value;
 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 			if (synth->io_ops->synth_out(synth, synth->procspeech))
-				schedule_timeout(msecs_to_jiffies
-						 (delay_time_val));
+				schedule_msec_hrtimeout(delay_time_val);
 			else
-				schedule_timeout(msecs_to_jiffies
-						 (full_time_val));
+				schedule_msec_hrtimeout(full_time_val);
 			jiff_max = jiffies + jiffy_delta_val;
 		}
 		set_current_state(TASK_RUNNING);
diff --git a/drivers/accessibility/speakup/speakup_decext.c b/drivers/accessibility/speakup/speakup_decext.c
--- a/drivers/accessibility/speakup/speakup_decext.c
+++ b/drivers/accessibility/speakup/speakup_decext.c
@@ -189,7 +189,7 @@ static void do_catch_up(struct spk_synth *synth)
 		if (ch == '\n')
 			ch = 0x0D;
 		if (synth_full() || !synth->io_ops->synth_out(synth, ch)) {
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -213,8 +213,7 @@ static void do_catch_up(struct spk_synth *synth)
 				delay_time_val = delay_time->u.n.value;
 				spin_unlock_irqrestore(&speakup_info.spinlock,
 						       flags);
-				schedule_timeout(msecs_to_jiffies
-						 (delay_time_val));
+				schedule_msec_hrtimeout(delay_time_val);
 				jiff_max = jiffies + jiffy_delta_val;
 			}
 		}
diff --git a/drivers/accessibility/speakup/speakup_decpc.c b/drivers/accessibility/speakup/speakup_decpc.c
--- a/drivers/accessibility/speakup/speakup_decpc.c
+++ b/drivers/accessibility/speakup/speakup_decpc.c
@@ -409,7 +409,7 @@ static void do_catch_up(struct spk_synth *synth)
 		if (ch == '\n')
 			ch = 0x0D;
 		if (dt_sendchar(ch)) {
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -432,8 +432,7 @@ static void do_catch_up(struct spk_synth *synth)
 				delay_time_val = delay_time->u.n.value;
 				spin_unlock_irqrestore(&speakup_info.spinlock,
 						       flags);
-				schedule_timeout(msecs_to_jiffies
-						 (delay_time_val));
+				schedule_msec_hrtimeout(delay_time_val);
 				jiff_max = jiffies + jiffy_delta_val;
 			}
 		}
diff --git a/drivers/accessibility/speakup/speakup_dectlk.c b/drivers/accessibility/speakup/speakup_dectlk.c
--- a/drivers/accessibility/speakup/speakup_dectlk.c
+++ b/drivers/accessibility/speakup/speakup_dectlk.c
@@ -265,7 +265,7 @@ static void do_catch_up(struct spk_synth *synth)
 		if (ch == '\n')
 			ch = 0x0D;
 		if (synth_full_val || !synth->io_ops->synth_out(synth, ch)) {
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -289,8 +289,7 @@ static void do_catch_up(struct spk_synth *synth)
 				delay_time_val = delay_time->u.n.value;
 				spin_unlock_irqrestore(&speakup_info.spinlock,
 						       flags);
-				schedule_timeout(msecs_to_jiffies
-						 (delay_time_val));
+				schedule_msec_hrtimeout(delay_time_val);
 				jiff_max = jiffies + jiffy_delta_val;
 			}
 		}
diff --git a/drivers/accessibility/speakup/speakup_dtlk.c b/drivers/accessibility/speakup/speakup_dtlk.c
--- a/drivers/accessibility/speakup/speakup_dtlk.c
+++ b/drivers/accessibility/speakup/speakup_dtlk.c
@@ -221,7 +221,7 @@ static void do_catch_up(struct spk_synth *synth)
 		delay_time_val = delay_time->u.n.value;
 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 		if (synth_full()) {
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -237,7 +237,7 @@ static void do_catch_up(struct spk_synth *synth)
 			delay_time_val = delay_time->u.n.value;
 			jiffy_delta_val = jiffy_delta->u.n.value;
 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			jiff_max = jiffies + jiffy_delta_val;
 		}
 	}
diff --git a/drivers/accessibility/speakup/speakup_keypc.c b/drivers/accessibility/speakup/speakup_keypc.c
--- a/drivers/accessibility/speakup/speakup_keypc.c
+++ b/drivers/accessibility/speakup/speakup_keypc.c
@@ -208,7 +208,7 @@ static void do_catch_up(struct spk_synth *synth)
 		full_time_val = full_time->u.n.value;
 		spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 		if (synth_full()) {
-			schedule_timeout(msecs_to_jiffies(full_time_val));
+			schedule_msec_hrtimeout(full_time_val);
 			continue;
 		}
 		set_current_state(TASK_RUNNING);
@@ -241,7 +241,7 @@ static void do_catch_up(struct spk_synth *synth)
 			jiffy_delta_val = jiffy_delta->u.n.value;
 			delay_time_val = delay_time->u.n.value;
 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
-			schedule_timeout(msecs_to_jiffies(delay_time_val));
+			schedule_msec_hrtimeout(delay_time_val);
 			jiff_max = jiffies + jiffy_delta_val;
 		}
 	}
diff --git a/drivers/accessibility/speakup/synth.c b/drivers/accessibility/speakup/synth.c
--- a/drivers/accessibility/speakup/synth.c
+++ b/drivers/accessibility/speakup/synth.c
@@ -98,7 +98,7 @@ static void _spk_do_catch_up(struct spk_synth *synth, int unicode)
 		else
 			ret = synth->io_ops->synth_out(synth, ch);
 		if (!ret) {
-			schedule_timeout(msecs_to_jiffies(full_time_val));
+			schedule_msec_hrtimeout(full_time_val);
 			continue;
 		}
 		if (time_after_eq(jiffies, jiff_max) && (ch == SPACE)) {
@@ -108,11 +108,9 @@ static void _spk_do_catch_up(struct spk_synth *synth, int unicode)
 			full_time_val = full_time->u.n.value;
 			spin_unlock_irqrestore(&speakup_info.spinlock, flags);
 			if (synth->io_ops->synth_out(synth, synth->procspeech))
-				schedule_timeout(
-					msecs_to_jiffies(delay_time_val));
+				schedule_msec_hrtimeout(delay_time_val);
 			else
-				schedule_timeout(
-					msecs_to_jiffies(full_time_val));
+				schedule_msec_hrtimeout(full_time_val);
 			jiff_max = jiffies + jiffy_delta_val;
 		}
 		set_current_state(TASK_RUNNING);
diff --git a/drivers/auxdisplay/hd44780_common.c b/drivers/auxdisplay/hd44780_common.c
--- a/drivers/auxdisplay/hd44780_common.c
+++ b/drivers/auxdisplay/hd44780_common.c
@@ -34,7 +34,7 @@
 /* sleeps that many milliseconds with a reschedule */
 static void long_sleep(int ms)
 {
-	schedule_timeout_interruptible(msecs_to_jiffies(ms));
+	schedule_msec_hrtimeout_interruptible(ms);
 }
 
 int hd44780_common_print(struct charlcd *lcd, int c)
diff --git a/drivers/block/swim.c b/drivers/block/swim.c
--- a/drivers/block/swim.c
+++ b/drivers/block/swim.c
@@ -330,7 +330,7 @@ static inline void swim_motor(struct swim __iomem *base,
 			if (swim_readbit(base, MOTOR_ON))
 				break;
 			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(1);
+			schedule_min_hrtimeout();
 		}
 	} else if (action == OFF) {
 		swim_action(base, MOTOR_OFF);
@@ -349,7 +349,7 @@ static inline void swim_eject(struct swim __iomem *base)
 		if (!swim_readbit(base, DISK_IN))
 			break;
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 	}
 	swim_select(base, RELAX);
 }
@@ -373,7 +373,7 @@ static inline int swim_step(struct swim __iomem *base)
 	for (wait = 0; wait < HZ; wait++) {
 
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 
 		swim_select(base, RELAX);
 		if (!swim_readbit(base, STEP))
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -3804,7 +3804,7 @@ static void cleanup_smi_msgs(struct ipmi_smi *intf)
 	/* Current message first, to preserve order */
 	while (intf->curr_msg && !list_empty(&intf->waiting_rcv_msgs)) {
 		/* Wait for the message to clear out. */
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 	}
 
 	/* No need for locks, the interface is down. */
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -1282,7 +1282,7 @@ static void shutdown_ssif(void *send_info)
 
 	/* make sure the driver is not looking for flags any more. */
 	while (ssif_info->ssif_state != SSIF_IDLE)
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 
 	ssif_info->stopping = true;
 	timer_delete_sync(&ssif_info->watch_timer);
diff --git a/drivers/comedi/drivers/ni_mio_common.c b/drivers/comedi/drivers/ni_mio_common.c
--- a/drivers/comedi/drivers/ni_mio_common.c
+++ b/drivers/comedi/drivers/ni_mio_common.c
@@ -4762,7 +4762,7 @@ static int cs5529_wait_for_idle(struct comedi_device *dev)
 		if ((status & NI67XX_CAL_STATUS_BUSY) == 0)
 			break;
 		set_current_state(TASK_INTERRUPTIBLE);
-		if (schedule_timeout(1))
+		if (schedule_min_hrtimeout())
 			return -EIO;
 	}
 	if (i == timeout) {
diff --git a/drivers/crypto/ccp/tee-dev.c b/drivers/crypto/ccp/tee-dev.c
--- a/drivers/crypto/ccp/tee-dev.c
+++ b/drivers/crypto/ccp/tee-dev.c
@@ -270,7 +270,7 @@ static int tee_submit_cmd(struct psp_tee_device *tee, enum tee_cmd_id cmd_id,
 
 		/* Wait if ring buffer is full or TEE is processing data */
 		mutex_unlock(&tee->rb_mgr.mutex);
-		schedule_timeout_interruptible(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout_interruptible(10);
 		mutex_lock(&tee->rb_mgr.mutex);
 
 	} while (--nloop);
diff --git a/drivers/gpib/nec7210/nec7210.c b/drivers/gpib/nec7210/nec7210.c
--- a/drivers/gpib/nec7210/nec7210.c
+++ b/drivers/gpib/nec7210/nec7210.c
@@ -317,7 +317,7 @@ int nec7210_go_to_standby(struct gpib_board *board, struct nec7210_priv *priv)
 	if (i == timeout) {
 		for (i = 0; i < HZ; i++) {
 			set_current_state(TASK_INTERRUPTIBLE);
-			if (schedule_timeout(1))
+			if (schedule_min_hrtimeout())
 				return -ERESTARTSYS;
 			adsr_bits = read_byte(priv, ADSR);
 			if (adsr_bits & HR_NATN)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmd.c
@@ -215,7 +215,7 @@ static int vmw_fifo_wait_noirq(struct vmw_private *dev_priv,
 			DRM_ERROR("SVGA device lockup.\n");
 			break;
 		}
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 		if (interruptible && signal_pending(current)) {
 			ret = -ERESTARTSYS;
 			break;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_irq.c
@@ -200,7 +200,7 @@ int vmw_fallback_wait(struct vmw_private *dev_priv,
 			break;
 		}
 		if (lazy)
-			schedule_timeout(1);
+			schedule_min_hrtimeout();
 		else if ((++count & 0x0F) == 0) {
 			/**
 			 * FIXME: Use schedule_hr_timeout here for
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -332,7 +332,7 @@ static int adt7470_update_thread(void *p)
 		if (kthread_should_stop())
 			break;
 
-		schedule_timeout_interruptible(msecs_to_jiffies(data->auto_update_interval));
+		schedule_msec_hrtimeout_interruptible(data->auto_update_interval);
 	}
 
 	return 0;
diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
--- a/drivers/hwmon/fam15h_power.c
+++ b/drivers/hwmon/fam15h_power.c
@@ -221,7 +221,7 @@ static ssize_t power1_average_show(struct device *dev,
 		prev_ptsc[cu] = data->cpu_sw_pwr_ptsc[cu];
 	}
 
-	leftover = schedule_timeout_interruptible(msecs_to_jiffies(data->power_period));
+	leftover = schedule_msec_hrtimeout_interruptible(data->power_period);
 	if (leftover)
 		return 0;
 
diff --git a/drivers/hwmon/ibmaem.c b/drivers/hwmon/ibmaem.c
--- a/drivers/hwmon/ibmaem.c
+++ b/drivers/hwmon/ibmaem.c
@@ -827,9 +827,8 @@ static ssize_t aem_show_power(struct device *dev,
 	time = ktime_get_ns();
 	before = data->energy[attr->index];
 
-	leftover = schedule_timeout_interruptible(
-			msecs_to_jiffies(data->power_period[attr->index])
-		   );
+	leftover = schedule_msec_hrtimeout_interruptible(
+			data->power_period[attr->index]);
 	if (leftover) {
 		mutex_unlock(&data->lock);
 		return 0;
diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c
--- a/drivers/iio/light/tsl2563.c
+++ b/drivers/iio/light/tsl2563.c
@@ -274,11 +274,7 @@ static void tsl2563_wait_adc(struct tsl2563_chip *chip)
 	default:
 		delay = 402;
 	}
-	/*
-	 * TODO: Make sure that we wait at least required delay but why we
-	 * have to extend it one tick more?
-	 */
-	schedule_timeout_interruptible(msecs_to_jiffies(delay) + 2);
+	schedule_msec_hrtimeout_interruptible(delay + 1);
 }
 
 static int tsl2563_adjust_gainlevel(struct tsl2563_chip *chip, u16 adc)
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -1846,8 +1846,7 @@ static void bch_btree_gc(struct cache_set *c)
 		cond_resched();
 
 		if (ret == -EAGAIN)
-			schedule_timeout_interruptible(
-				msecs_to_jiffies(btree_gc_sleep_ms(c)));
+			schedule_msec_hrtimeout_interruptible(btree_gc_sleep_ms(c));
 		else if (ret)
 			pr_warn("gc failed!\n");
 	} while (ret && !test_bit(CACHE_SET_IO_DISABLE, &c->flags));
diff --git a/drivers/media/i2c/bt866.c b/drivers/media/i2c/bt866.c
--- a/drivers/media/i2c/bt866.c
+++ b/drivers/media/i2c/bt866.c
@@ -65,7 +65,7 @@ static int bt866_write(struct bt866 *encoder, u8 subaddr, u8 data)
 		err++;
 		v4l_warn(client, "error #%d writing to 0x%02x\n",
 				err, subaddr);
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 	}
 	if (err == 3) {
 		v4l_warn(client, "giving up\n");
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -698,7 +698,7 @@ static int zilog_tx(struct rc_dev *rcdev, unsigned int *txbuf,
 	 */
 	for (i = 0; i < 20; ++i) {
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(50));
+		schedule_msec_hrtimeout(50);
 		ret = i2c_master_send(ir->tx_c, buf, 1);
 		if (ret == 1)
 			break;
diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c
--- a/drivers/media/i2c/msp3400-driver.c
+++ b/drivers/media/i2c/msp3400-driver.c
@@ -170,7 +170,7 @@ static int msp_read(struct i2c_client *client, int dev, int addr)
 			break;
 		dev_warn(&client->dev, "I/O error #%d (read 0x%02x/0x%02x)\n", err,
 		       dev, addr);
-		schedule_timeout_interruptible(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout_interruptible(10);
 	}
 	if (err == 3) {
 		dev_warn(&client->dev, "resetting chip, sound will go off.\n");
@@ -211,7 +211,7 @@ static int msp_write(struct i2c_client *client, int dev, int addr, int val)
 			break;
 		dev_warn(&client->dev, "I/O error #%d (write 0x%02x/0x%02x)\n", err,
 		       dev, addr);
-		schedule_timeout_interruptible(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout_interruptible(10);
 	}
 	if (err == 3) {
 		dev_warn(&client->dev, "resetting chip, sound will go off.\n");
diff --git a/drivers/media/i2c/saa7110.c b/drivers/media/i2c/saa7110.c
--- a/drivers/media/i2c/saa7110.c
+++ b/drivers/media/i2c/saa7110.c
@@ -184,7 +184,7 @@ static v4l2_std_id determine_norm(struct v4l2_subdev *sd)
 	saa7110_write_block(sd, initseq, sizeof(initseq));
 	saa7110_selmux(sd, decoder->input);
 	prepare_to_wait(&decoder->wq, &wait, TASK_UNINTERRUPTIBLE);
-	schedule_timeout(msecs_to_jiffies(250));
+	schedule_msec_hrtimeout(250);
 	finish_wait(&decoder->wq, &wait);
 	status = saa7110_read(sd);
 	if (status & 0x40) {
@@ -219,7 +219,7 @@ static v4l2_std_id determine_norm(struct v4l2_subdev *sd)
 	/*saa7110_write(sd,0x2E,0x9A);*/
 
 	prepare_to_wait(&decoder->wq, &wait, TASK_UNINTERRUPTIBLE);
-	schedule_timeout(msecs_to_jiffies(250));
+	schedule_msec_hrtimeout(250);
 	finish_wait(&decoder->wq, &wait);
 
 	status = saa7110_read(sd);
diff --git a/drivers/media/pci/cx18/cx18-gpio.c b/drivers/media/pci/cx18/cx18-gpio.c
--- a/drivers/media/pci/cx18/cx18-gpio.c
+++ b/drivers/media/pci/cx18/cx18-gpio.c
@@ -81,11 +81,11 @@ static void gpio_reset_seq(struct cx18 *cx, u32 active_lo, u32 active_hi,
 
 	/* Assert */
 	gpio_update(cx, mask, ~active_lo);
-	schedule_timeout_uninterruptible(msecs_to_jiffies(assert_msecs));
+	schedule_msec_hrtimeout_uninterruptible(assert_msecs);
 
 	/* Deassert */
 	gpio_update(cx, mask, ~active_hi);
-	schedule_timeout_uninterruptible(msecs_to_jiffies(recovery_msecs));
+	schedule_msec_hrtimeout_uninterruptible(recovery_msecs);
 }
 
 /*
diff --git a/drivers/media/pci/hws/hws_pci.c b/drivers/media/pci/hws/hws_pci.c
--- a/drivers/media/pci/hws/hws_pci.c
+++ b/drivers/media/pci/hws/hws_pci.c
@@ -215,7 +215,7 @@ static int main_ks_thread_handle(void *data)
 		/* If we're suspending, don't touch hardware; just sleep/freeze. */
 		if (READ_ONCE(pdx->suspended)) {
 			try_to_freeze();
-			schedule_timeout_interruptible(msecs_to_jiffies(1000));
+			schedule_msec_hrtimeout_interruptible(1000);
 			continue;
 		}
 
@@ -225,7 +225,7 @@ static int main_ks_thread_handle(void *data)
 		try_to_freeze(); /* cooperate with freezer each loop */
 
 		/* Sleep 1s or until signaled to wake/stop */
-		schedule_timeout_interruptible(msecs_to_jiffies(1000));
+		schedule_msec_hrtimeout_interruptible(1000);
 	}
 
 	dev_dbg(&pdx->pdev->dev, "%s: exiting\n", __func__);
diff --git a/drivers/media/pci/ivtv/ivtv-gpio.c b/drivers/media/pci/ivtv/ivtv-gpio.c
--- a/drivers/media/pci/ivtv/ivtv-gpio.c
+++ b/drivers/media/pci/ivtv/ivtv-gpio.c
@@ -105,7 +105,7 @@ void ivtv_reset_ir_gpio(struct ivtv *itv)
 	curout = (curout & ~0xF) | 1;
 	write_reg(curout, IVTV_REG_GPIO_OUT);
 	/* We could use something else for smaller time */
-	schedule_timeout_interruptible(msecs_to_jiffies(1));
+	schedule_msec_hrtimeout_interruptible(1);
 	curout |= 2;
 	write_reg(curout, IVTV_REG_GPIO_OUT);
 	curdir &= ~0x80;
@@ -125,11 +125,11 @@ int ivtv_reset_tuner_gpio(void *dev, int component, int cmd, int value)
 	curout = read_reg(IVTV_REG_GPIO_OUT);
 	curout &= ~(1 << itv->card->xceive_pin);
 	write_reg(curout, IVTV_REG_GPIO_OUT);
-	schedule_timeout_interruptible(msecs_to_jiffies(1));
+	schedule_msec_hrtimeout_interruptible(1);
 
 	curout |= 1 << itv->card->xceive_pin;
 	write_reg(curout, IVTV_REG_GPIO_OUT);
-	schedule_timeout_interruptible(msecs_to_jiffies(1));
+	schedule_msec_hrtimeout_interruptible(1);
 	return 0;
 }
 
diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
+++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
@@ -1141,7 +1141,7 @@ void ivtv_s_std_dec(struct ivtv *itv, v4l2_std_id std)
 				TASK_UNINTERRUPTIBLE);
 		if ((read_reg(IVTV_REG_DEC_LINE_FIELD) >> 16) < 100)
 			break;
-		schedule_timeout(msecs_to_jiffies(25));
+		schedule_msec_hrtimeout(25);
 	}
 	finish_wait(&itv->vsync_waitq, &wait);
 	mutex_lock(&itv->serialize_lock);
diff --git a/drivers/media/pci/ivtv/ivtv-streams.c b/drivers/media/pci/ivtv/ivtv-streams.c
--- a/drivers/media/pci/ivtv/ivtv-streams.c
+++ b/drivers/media/pci/ivtv/ivtv-streams.c
@@ -835,7 +835,7 @@ int ivtv_stop_v4l2_encode_stream(struct ivtv_stream *s, int gop_end)
 			while (!test_bit(IVTV_F_I_EOS, &itv->i_flags) &&
 				time_before(jiffies,
 					    then + msecs_to_jiffies(2000))) {
-				schedule_timeout(msecs_to_jiffies(10));
+				schedule_msec_hrtimeout(10);
 			}
 
 			/* To convert jiffies to ms, we must multiply by 1000
diff --git a/drivers/media/pci/ivtv/ivtvfb.c b/drivers/media/pci/ivtv/ivtvfb.c
--- a/drivers/media/pci/ivtv/ivtvfb.c
+++ b/drivers/media/pci/ivtv/ivtvfb.c
@@ -474,7 +474,7 @@ static int ivtvfb_ioctl(struct fb_info *info, unsigned int cmd, unsigned long ar
 
 		case FBIO_WAITFORVSYNC:
 			prepare_to_wait(&itv->vsync_waitq, &wait, TASK_INTERRUPTIBLE);
-			if (!schedule_timeout(msecs_to_jiffies(50)))
+			if (!schedule_msec_hrtimeout(50))
 				rc = -ETIMEDOUT;
 			finish_wait(&itv->vsync_waitq, &wait);
 			return rc;
diff --git a/drivers/media/pci/saa7134/saa7134-tvaudio.c b/drivers/media/pci/saa7134/saa7134-tvaudio.c
--- a/drivers/media/pci/saa7134/saa7134-tvaudio.c
+++ b/drivers/media/pci/saa7134/saa7134-tvaudio.c
@@ -307,8 +307,7 @@ static int tvaudio_sleep(struct saa7134_dev *dev, int timeout)
 			set_current_state(TASK_INTERRUPTIBLE);
 			schedule();
 		} else {
-			schedule_timeout_interruptible
-						(msecs_to_jiffies(timeout));
+			schedule_msec_hrtimeout_interruptible(timeout);
 		}
 	}
 	return dev->thread.scan1 != dev->thread.scan2;
diff --git a/drivers/media/radio/radio-mr800.c b/drivers/media/radio/radio-mr800.c
--- a/drivers/media/radio/radio-mr800.c
+++ b/drivers/media/radio/radio-mr800.c
@@ -366,7 +366,7 @@ static int vidioc_s_hw_freq_seek(struct file *file, void *priv,
 			retval = -ENODATA;
 			break;
 		}
-		if (schedule_timeout_interruptible(msecs_to_jiffies(10))) {
+		if (schedule_msec_hrtimeout_interruptible(10)) {
 			retval = -ERESTARTSYS;
 			break;
 		}
diff --git a/drivers/media/radio/radio-tea5777.c b/drivers/media/radio/radio-tea5777.c
--- a/drivers/media/radio/radio-tea5777.c
+++ b/drivers/media/radio/radio-tea5777.c
@@ -235,7 +235,7 @@ static int radio_tea5777_update_read_reg(struct radio_tea5777 *tea, int wait)
 	}
 
 	if (wait) {
-		if (schedule_timeout_interruptible(msecs_to_jiffies(wait)))
+		if (schedule_msec_hrtimeout_interruptible(wait))
 			return -ERESTARTSYS;
 	}
 
diff --git a/drivers/media/radio/tea575x.c b/drivers/media/radio/tea575x.c
--- a/drivers/media/radio/tea575x.c
+++ b/drivers/media/radio/tea575x.c
@@ -401,7 +401,7 @@ int snd_tea575x_s_hw_freq_seek(struct file *file, struct snd_tea575x *tea,
 	for (;;) {
 		if (time_after(jiffies, timeout))
 			break;
-		if (schedule_timeout_interruptible(msecs_to_jiffies(10))) {
+		if (schedule_msec_hrtimeout_interruptible(10)) {
 			/* some signal arrived, stop search */
 			tea->val &= ~TEA575X_BIT_SEARCH;
 			snd_tea575x_set_freq(tea);
diff --git a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
--- a/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
+++ b/drivers/media/usb/gspca/m5602/m5602_s5k83a.c
@@ -297,7 +297,7 @@ static int rotation_thread_function(void *data)
 	__s32 vflip, hflip;
 
 	set_current_state(TASK_INTERRUPTIBLE);
-	while (!schedule_timeout(msecs_to_jiffies(100))) {
+	while (!schedule_msec_hrtimeout(100)) {
 		if (mutex_lock_interruptible(&sd->gspca_dev.usb_lock))
 			break;
 
diff --git a/drivers/mfd/ucb1x00-core.c b/drivers/mfd/ucb1x00-core.c
--- a/drivers/mfd/ucb1x00-core.c
+++ b/drivers/mfd/ucb1x00-core.c
@@ -253,7 +253,7 @@ unsigned int ucb1x00_adc_read(struct ucb1x00 *ucb, int adc_channel, int sync)
 			break;
 		/* yield to other processes */
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 	}
 
 	return UCB_ADC_DAT(val);
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1464,7 +1464,7 @@ int mei_me_polling_thread(void *_dev)
 						    MEI_POLLING_TIMEOUT_IDLE);
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(polling_timeout));
+		schedule_msec_hrtimeout_interruptible(polling_timeout);
 	}
 
 	return 0;
diff --git a/drivers/misc/sgi-xp/xpc_channel.c b/drivers/misc/sgi-xp/xpc_channel.c
--- a/drivers/misc/sgi-xp/xpc_channel.c
+++ b/drivers/misc/sgi-xp/xpc_channel.c
@@ -834,7 +834,7 @@ xpc_allocate_msg_wait(struct xpc_channel *ch)
 
 	atomic_inc(&ch->n_on_msg_allocate_wq);
 	prepare_to_wait(&ch->msg_allocate_wq, &wait, TASK_INTERRUPTIBLE);
-	ret = schedule_timeout(1);
+	ret = schedule_min_hrtimeout();
 	finish_wait(&ch->msg_allocate_wq, &wait);
 	atomic_dec(&ch->n_on_msg_allocate_wq);
 
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb.c b/drivers/net/can/usb/peak_usb/pcan_usb.c
--- a/drivers/net/can/usb/peak_usb/pcan_usb.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb.c
@@ -308,7 +308,7 @@ static int pcan_usb_write_mode(struct peak_usb_device *dev, u8 onoff)
 	} else {
 		/* the PCAN-USB needs time to init */
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(PCAN_USB_STARTUP_TIMEOUT));
+		schedule_msec_hrtimeout(PCAN_USB_STARTUP_TIMEOUT);
 	}
 
 	return err;
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
--- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c
@@ -1090,7 +1090,7 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs)
 	struct napi_struct *napi, *n;
 	int ret;
 
-	schedule_timeout_uninterruptible(msecs_to_jiffies(100));
+	schedule_msec_hrtimeout_uninterruptible(100);
 
 	if (wait_for_pending_requests(oct))
 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -758,7 +758,7 @@ static int liquidio_watchdog(void *param)
 	while (!kthread_should_stop()) {
 		/* sleep for a couple of seconds so that we don't hog the CPU */
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(2000));
+		schedule_msec_hrtimeout(2000);
 
 		mask_of_crashed_or_stuck_cores =
 		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -3514,7 +3514,7 @@ static void lan78xx_terminate_urbs(struct lan78xx_net *dev)
 	/* maybe wait for deletions to finish. */
 	while (!skb_queue_empty(&dev->rxq) ||
 	       !skb_queue_empty(&dev->txq)) {
-		schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS));
+		schedule_msec_hrtimeout(UNLINK_TIMEOUT_MS);
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		netif_dbg(dev, ifdown, dev->net,
 			  "waited for %d urb completions", temp);
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -791,7 +791,7 @@ static void wait_skb_queue_empty(struct sk_buff_head *q)
 	spin_lock_irqsave(&q->lock, flags);
 	while (!skb_queue_empty(q)) {
 		spin_unlock_irqrestore(&q->lock, flags);
-		schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS));
+		schedule_msec_hrtimeout(UNLINK_TIMEOUT_MS);
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		spin_lock_irqsave(&q->lock, flags);
 	}
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
@@ -799,7 +799,7 @@ static int ipw2100_hw_send_command(struct ipw2100_priv *priv,
 	 * doesn't seem to have as many firmware restart cycles...
 	 *
 	 * As a test, we're sticking in a 1/100s delay here */
-	schedule_timeout_uninterruptible(msecs_to_jiffies(10));
+	schedule_msec_hrtimeout_uninterruptible(10);
 
 	return 0;
 
@@ -1250,7 +1250,7 @@ static int ipw2100_start_adapter(struct ipw2100_priv *priv)
 	IPW_DEBUG_FW("Waiting for f/w initialization to complete...\n");
 	i = 5000;
 	do {
-		schedule_timeout_uninterruptible(msecs_to_jiffies(40));
+		schedule_msec_hrtimeout_uninterruptible(40);
 		/* Todo... wait for sync command ... */
 
 		read_register(priv->net_dev, IPW_REG_INTA, &inta);
diff --git a/drivers/parport/ieee1284.c b/drivers/parport/ieee1284.c
--- a/drivers/parport/ieee1284.c
+++ b/drivers/parport/ieee1284.c
@@ -202,7 +202,7 @@ int parport_wait_peripheral(struct parport *port,
 			/* parport_wait_event didn't time out, but the
 			 * peripheral wasn't actually ready either.
 			 * Wait for another 10ms. */
-			schedule_timeout_interruptible(msecs_to_jiffies(10));
+			schedule_msec_hrtimeout_interruptible(10);
 		}
 	}
 
diff --git a/drivers/parport/ieee1284_ops.c b/drivers/parport/ieee1284_ops.c
--- a/drivers/parport/ieee1284_ops.c
+++ b/drivers/parport/ieee1284_ops.c
@@ -520,7 +520,7 @@ size_t parport_ieee1284_ecp_read_data (struct parport *port,
 			/* Yield the port for a while. */
 			if (dev->port->irq != PARPORT_IRQ_NONE) {
 				parport_release (dev);
-				schedule_timeout_interruptible(msecs_to_jiffies(40));
+				schedule_msec_hrtimeout_interruptible(40);
 				parport_claim_or_block (dev);
 			}
 			else
diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c
--- a/drivers/parport/parport_pc.c
+++ b/drivers/parport/parport_pc.c
@@ -177,8 +177,7 @@ static int change_mode(struct parport *p, int m)
 				if (time_after_eq(jiffies, expire))
 					/* The FIFO is stuck. */
 					return -EBUSY;
-				schedule_timeout_interruptible(
-							msecs_to_jiffies(10));
+				schedule_msec_hrtimeout_interruptible(10);
 				if (signal_pending(current))
 					break;
 			}
diff --git a/drivers/platform/x86/intel_ips.c b/drivers/platform/x86/intel_ips.c
--- a/drivers/platform/x86/intel_ips.c
+++ b/drivers/platform/x86/intel_ips.c
@@ -806,7 +806,7 @@ static int ips_adjust(void *data)
 			ips_gpu_lower(ips);
 
 sleep:
-		schedule_timeout_interruptible(msecs_to_jiffies(IPS_ADJUST_PERIOD));
+		schedule_msec_hrtimeout_interruptible(IPS_ADJUST_PERIOD);
 	} while (!kthread_should_stop());
 
 	dev_dbg(ips->dev, "ips-adjust thread stopped\n");
@@ -984,7 +984,7 @@ static int ips_monitor(void *data)
 	seqno_timestamp = get_jiffies_64();
 
 	old_cpu_power = thm_readl(THM_CEC);
-	schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
+	schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD);
 
 	/* Collect an initial average */
 	for (i = 0; i < IPS_SAMPLE_COUNT; i++) {
@@ -1011,7 +1011,7 @@ static int ips_monitor(void *data)
 			mchp_samples[i] = mchp;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
+		schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD);
 		if (kthread_should_stop())
 			break;
 	}
@@ -1038,7 +1038,7 @@ static int ips_monitor(void *data)
 	 * us to reduce the sample frequency if the CPU and GPU are idle.
 	 */
 	old_cpu_power = thm_readl(THM_CEC);
-	schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
+	schedule_msec_hrtimeout_interruptible(IPS_SAMPLE_PERIOD);
 	last_sample_period = IPS_SAMPLE_PERIOD;
 
 	timer_setup(&ips->timer, monitor_timeout, TIMER_DEFERRABLE);
diff --git a/drivers/rtc/rtc-wm8350.c b/drivers/rtc/rtc-wm8350.c
--- a/drivers/rtc/rtc-wm8350.c
+++ b/drivers/rtc/rtc-wm8350.c
@@ -114,7 +114,7 @@ static int wm8350_rtc_settime(struct device *dev, struct rtc_time *tm)
 	/* Wait until confirmation of stopping */
 	do {
 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout_uninterruptible(1);
 	} while (--retries && !(rtc_ctrl & WM8350_RTC_STS));
 
 	if (!retries) {
@@ -197,7 +197,7 @@ static int wm8350_rtc_stop_alarm(struct wm8350 *wm8350)
 	/* Wait until confirmation of stopping */
 	do {
 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout_uninterruptible(1);
 	} while (retries-- && !(rtc_ctrl & WM8350_RTC_ALMSTS));
 
 	if (!(rtc_ctrl & WM8350_RTC_ALMSTS))
@@ -220,7 +220,7 @@ static int wm8350_rtc_start_alarm(struct wm8350 *wm8350)
 	/* Wait until confirmation */
 	do {
 		rtc_ctrl = wm8350_reg_read(wm8350, WM8350_RTC_TIME_CONTROL);
-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout_uninterruptible(1);
 	} while (retries-- && rtc_ctrl & WM8350_RTC_ALMSTS);
 
 	if (rtc_ctrl & WM8350_RTC_ALMSTS)
diff --git a/drivers/s390/char/sclp_sdias.c b/drivers/s390/char/sclp_sdias.c
--- a/drivers/s390/char/sclp_sdias.c
+++ b/drivers/s390/char/sclp_sdias.c
@@ -68,7 +68,7 @@ static int sdias_sclp_send(struct sclp_req *req)
 			/* not initiated, wait some time and retry */
 			set_current_state(TASK_INTERRUPTIBLE);
 			TRACE("add request failed: rc = %i\n",rc);
-			schedule_timeout(msecs_to_jiffies(500));
+			schedule_msec_hrtimeout(500);
 			continue;
 		}
 		/* initiated, wait for completion of service call */
diff --git a/drivers/scsi/be2iscsi/be_cmds.c b/drivers/scsi/be2iscsi/be_cmds.c
--- a/drivers/scsi/be2iscsi/be_cmds.c
+++ b/drivers/scsi/be2iscsi/be_cmds.c
@@ -611,7 +611,7 @@ static int be_mbox_db_ready_poll(struct be_ctrl_info *ctrl)
 		if (time_after(jiffies, timeout))
 			break;
 		/* 1ms sleep is enough in most cases */
-		schedule_timeout_uninterruptible(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout_uninterruptible(1);
 	} while (!ready);
 
 	beiscsi_log(phba, KERN_ERR,
diff --git a/drivers/scsi/esas2r/esas2r_flash.c b/drivers/scsi/esas2r/esas2r_flash.c
--- a/drivers/scsi/esas2r/esas2r_flash.c
+++ b/drivers/scsi/esas2r/esas2r_flash.c
@@ -965,7 +965,7 @@ static bool esas2r_flash_access(struct esas2r_adapter *a, u32 function)
 				break;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > timeout) {
 			/*
diff --git a/drivers/scsi/esas2r/esas2r_init.c b/drivers/scsi/esas2r/esas2r_init.c
--- a/drivers/scsi/esas2r/esas2r_init.c
+++ b/drivers/scsi/esas2r/esas2r_init.c
@@ -994,7 +994,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a)
 			break;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > 180000) {
 			esas2r_hdebug("FW ready TMO");
@@ -1017,7 +1017,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a)
 			break;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(50));
+		schedule_msec_hrtimeout_interruptible(50);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > 3000) {
 			esas2r_hdebug("timeout waiting for interface down");
@@ -1106,7 +1106,7 @@ bool esas2r_check_adapter(struct esas2r_adapter *a)
 			break;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > 3000) {
 			esas2r_hdebug(
@@ -1356,7 +1356,7 @@ bool esas2r_init_adapter_hw(struct esas2r_adapter *a, bool init_poll)
 			atomic_dec(&a->disable_cnt);
 
 		while (test_bit(AF_DISC_PENDING, &a->flags)) {
-			schedule_timeout_interruptible(msecs_to_jiffies(100));
+			schedule_msec_hrtimeout_interruptible(100);
 
 			/*
 			 * Determine the need for a timer tick based on the
@@ -1512,7 +1512,7 @@ static void esas2r_power_down_notify_firmware(struct esas2r_adapter *a)
 			break;
 		}
 
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > 30000) {
 			esas2r_hdebug("Timeout waiting for power down");
@@ -1557,7 +1557,7 @@ void esas2r_power_down(struct esas2r_adapter *a)
 				break;
 			}
 
-			schedule_timeout_interruptible(msecs_to_jiffies(100));
+			schedule_msec_hrtimeout_interruptible(100);
 
 			if ((jiffies_to_msecs(jiffies) - starttime) > 3000) {
 				esas2r_hdebug(
diff --git a/drivers/scsi/esas2r/esas2r_main.c b/drivers/scsi/esas2r/esas2r_main.c
--- a/drivers/scsi/esas2r/esas2r_main.c
+++ b/drivers/scsi/esas2r/esas2r_main.c
@@ -1247,7 +1247,7 @@ void esas2r_wait_request(struct esas2r_adapter *a, struct esas2r_request *rq)
 		if (rq->req_stat != RS_STARTED)
 			break;
 
-		schedule_timeout_interruptible(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout_interruptible(100);
 
 		if ((jiffies_to_msecs(jiffies) - starttime) > timeout) {
 			esas2r_hdebug("request TMO");
diff --git a/drivers/scsi/fnic/fnic_scsi.c b/drivers/scsi/fnic/fnic_scsi.c
--- a/drivers/scsi/fnic/fnic_scsi.c
+++ b/drivers/scsi/fnic/fnic_scsi.c
@@ -258,7 +258,7 @@ int fnic_fw_reset_handler(struct fnic *fnic)
 
 	/* wait for io cmpl */
 	while (atomic_read(&fnic->in_flight))
-		schedule_timeout(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout(1);
 
 	spin_lock_irqsave(&fnic->wq_copy_lock[0], flags);
 
@@ -1634,7 +1634,7 @@ static void fnic_cleanup_io(struct fnic *fnic, int exclude_id)
 		  io_count,
 		  atomic64_read(&fnic->fnic_stats.io_stats.active_ios));
 
-		schedule_timeout(msecs_to_jiffies(100));
+		schedule_msec_hrtimeout(100);
 	}
 }
 
@@ -1912,7 +1912,7 @@ void fnic_rport_exch_reset(struct fnic *fnic, u32 port_id)
 	atomic_dec(&fnic->in_flight);
 
 	while ((io_count = fnic_count_ioreqs(fnic, port_id)))
-		schedule_timeout(msecs_to_jiffies(1000));
+		schedule_msec_hrtimeout(1000);
 
 	FNIC_SCSI_DBG(KERN_DEBUG, fnic,
 				  "rport: 0x%x remaining portid-io-count: %d ",
@@ -2510,7 +2510,7 @@ static int fnic_clean_pending_aborts(struct fnic *fnic,
 		ret = iter_data.ret;
 		goto clean_pending_aborts_end;
 	}
-	schedule_timeout(msecs_to_jiffies(2 * fnic->config.ed_tov));
+	schedule_msec_hrtimeout(2 * fnic->config.ed_tov);
 
 	/* walk again to check, if IOs are still pending in fw */
 	if (fnic_is_abts_pending(fnic, lr_sc))
@@ -2811,7 +2811,7 @@ int fnic_device_reset(struct scsi_cmnd *sc)
 		}
 		FNIC_SCSI_DBG(KERN_ERR, fnic,
 					  "Cannot clean up all IOs for the LUN\n");
-		schedule_timeout(msecs_to_jiffies(1000));
+		schedule_msec_hrtimeout(1000);
 		count++;
 	}
 
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -5920,7 +5920,7 @@ lpfc_chk_tgt_mapped(struct lpfc_vport *vport, struct fc_rport *rport)
 			return FAILED;
 		if (pnode->nlp_state == NLP_STE_MAPPED_NODE)
 			return SUCCESS;
-		schedule_timeout_uninterruptible(msecs_to_jiffies(500));
+		schedule_msec_hrtimeout_uninterruptible(500);
 		rdata = rport->dd_data;
 		if (!rdata)
 			return FAILED;
@@ -5962,7 +5962,7 @@ lpfc_reset_flush_io_context(struct lpfc_vport *vport, uint16_t tgt_id,
 					tgt_id, lun_id, context);
 	later = secs_to_jiffies(2 * vport->cfg_devloss_tmo) + jiffies;
 	while (time_after(later, jiffies) && cnt) {
-		schedule_timeout_uninterruptible(msecs_to_jiffies(20));
+		schedule_msec_hrtimeout_uninterruptible(20);
 		cnt = lpfc_sli_sum_iocb(vport, tgt_id, lun_id, context);
 	}
 	if (cnt) {
diff --git a/drivers/scsi/qla4xxx/ql4_nx.c b/drivers/scsi/qla4xxx/ql4_nx.c
--- a/drivers/scsi/qla4xxx/ql4_nx.c
+++ b/drivers/scsi/qla4xxx/ql4_nx.c
@@ -1552,7 +1552,7 @@ static int qla4_82xx_cmdpeg_ready(struct scsi_qla_host *ha, int pegtune_val)
 			    (val == PHAN_INITIALIZE_ACK))
 				return 0;
 			set_current_state(TASK_UNINTERRUPTIBLE);
-			schedule_timeout(msecs_to_jiffies(500));
+			schedule_msec_hrtimeout(500);
 
 		} while (--retries);
 
diff --git a/drivers/scsi/snic/snic_scsi.c b/drivers/scsi/snic/snic_scsi.c
--- a/drivers/scsi/snic/snic_scsi.c
+++ b/drivers/scsi/snic/snic_scsi.c
@@ -1863,7 +1863,7 @@ snic_dr_clean_pending_req(struct snic *snic, struct scsi_cmnd *lr_sc)
 		}
 	}
 
-	schedule_timeout(msecs_to_jiffies(100));
+	schedule_msec_hrtimeout(100);
 
 	/* Walk through all the cmds and check abts status. */
 	if (snic_is_abts_pending(snic, lr_sc))
@@ -2321,7 +2321,7 @@ snic_reset(struct Scsi_Host *shost, struct scsi_cmnd *sc)
 
 	/* Wait for all the IOs that are entered in Qcmd */
 	while (atomic_read(&snic->ios_inflight))
-		schedule_timeout(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout(1);
 
 	ret = snic_issue_hba_reset(snic, sc);
 	if (ret) {
diff --git a/drivers/usb/class/usblp.c b/drivers/usb/class/usblp.c
--- a/drivers/usb/class/usblp.c
+++ b/drivers/usb/class/usblp.c
@@ -923,7 +923,7 @@ static int usblp_wwait(struct usblp *usblp, int nonblock)
 		if (rc <= 0)
 			break;
 
-		if (schedule_timeout(msecs_to_jiffies(1500)) == 0) {
+		if (schedule_msec_hrtimeout(1500) == 0) {
 			if (usblp->flags & LP_ABORT) {
 				err = usblp_check_status(usblp, err);
 				if (err == 1) {	/* Paper out */
diff --git a/drivers/video/fbdev/omap/hwa742.c b/drivers/video/fbdev/omap/hwa742.c
--- a/drivers/video/fbdev/omap/hwa742.c
+++ b/drivers/video/fbdev/omap/hwa742.c
@@ -926,7 +926,7 @@ static void hwa742_resume(void)
 		if (hwa742_read_reg(HWA742_PLL_DIV_REG) & (1 << 7))
 			break;
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(5));
+		schedule_msec_hrtimeout(5);
 	}
 	hwa742_set_update_mode(hwa742.update_mode_before_suspend);
 }
diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c
--- a/drivers/video/fbdev/pxafb.c
+++ b/drivers/video/fbdev/pxafb.c
@@ -1272,7 +1272,7 @@ static int pxafb_smart_thread(void *arg)
 		mutex_unlock(&fbi->ctrlr_lock);
 
 		set_current_state(TASK_INTERRUPTIBLE);
-		schedule_timeout(msecs_to_jiffies(30));
+		schedule_msec_hrtimeout(30);
 	}
 
 	pr_debug("%s(): task ending\n", __func__);
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -28,7 +28,7 @@ static inline void get_lock_elapsed_time(struct f2fs_time_stat *ts)
 {
 	ts->total_time = ktime_get();
 #ifdef CONFIG_64BIT
-	ts->running_time = current->se.sum_exec_runtime;
+	ts->running_time = tsk_seruntime(current);
 #endif
 #if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS)
 	ts->runnable_time = current->sched_info.run_delay;
diff --git a/fs/jffs2/background.c b/fs/jffs2/background.c
--- a/fs/jffs2/background.c
+++ b/fs/jffs2/background.c
@@ -110,7 +110,7 @@ static int jffs2_garbage_collect_thread(void *_c)
 		 * This forces the GCD to slow the hell down.   Pulling an
 		 * inode in with read_inode() is much preferable to having
 		 * the GC thread get there first. */
-		schedule_timeout_interruptible(msecs_to_jiffies(50));
+		schedule_msec_hrtimeout_interruptible(50);
 
 		if (kthread_should_stop()) {
 			jffs2_dbg(1, "%s(): kthread_stop() called\n", __func__);
diff --git a/fs/proc/base.c b/fs/proc/base.c
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -513,7 +513,7 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
 		seq_puts(m, "0 0 0\n");
 	else
 		seq_printf(m, "%llu %llu %lu\n",
-		   (unsigned long long)task->se.sum_exec_runtime,
+		   (unsigned long long)tsk_seruntime(task),
 		   (unsigned long long)task->sched_info.run_delay,
 		   task->sched_info.pcount);
 
@@ -521,6 +521,108 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
 }
 #endif
 
+#ifdef CONFIG_MUQSS_IOTIME
+/*
+ * Provides /proc/PID/iotime
+ *
+ * Block device time consumed on this task's behalf, in nanoseconds.
+ *
+ * io_latency_ns is end to end per bio and so includes queue wait: it is what
+ * this task waited for. io_occupancy_ns is device service time per request:
+ * it is what this task cost everybody else, and is what scheduling charges
+ * are based on. io_debt_ns is occupancy not yet charged to the deadline.
+ * See block/blk-iotime.c.
+ *
+ * kern_time_ns is CPU time rather than device time: kernel work done for this
+ * task in another thread's context, which today means kworkers running work
+ * items it queued. kern_debt_ns is the part of it not yet charged. Work the
+ * task asks for in its own context is not counted here, because that already
+ * comes out of its own timeslice.
+ *
+ * Access is restricted exactly as /proc/PID/io is, and for the same reason:
+ * these counters say when a task used the device and for how long, which is
+ * enough to infer a good deal about what it is doing. The exec_update_lock
+ * keeps the permission check and the read on the same side of an exec, so
+ * credentials cannot change between the two.
+ *
+ * Also as /proc/PID/io: the tgid file sums the thread group and the tid file
+ * reports one thread. Charging is per thread, since each has its own deadline,
+ * but a multithreaded writer's total is only meaningful summed.
+ */
+struct iotime_acct {
+	u64 latency_ns;
+	u64 count;
+	u64 occupancy_ns;
+	u64 debt_ns;
+	u64 kern_time_ns;
+	u64 kern_debt_ns;
+};
+
+static void iotime_add(struct iotime_acct *acct, struct task_struct *task)
+{
+	acct->latency_ns += atomic64_read(&task->io_latency_ns);
+	acct->count += atomic64_read(&task->io_count);
+	acct->occupancy_ns += atomic64_read(&task->io_occupancy_ns);
+	acct->debt_ns += atomic64_read(&task->io_debt_ns);
+	acct->kern_time_ns += atomic64_read(&task->kern_time_ns);
+	acct->kern_debt_ns += atomic64_read(&task->kern_debt_ns);
+}
+
+static int do_iotime_accounting(struct task_struct *task, struct seq_file *m,
+				int whole)
+{
+	struct iotime_acct acct = { };
+	int result;
+
+	result = down_read_killable(&task->signal->exec_update_lock);
+	if (result)
+		return result;
+
+	if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) {
+		result = -EACCES;
+		goto out_unlock;
+	}
+
+	if (whole) {
+		struct task_struct *t;
+
+		rcu_read_lock();
+		for_each_thread(task, t)
+			iotime_add(&acct, t);
+		rcu_read_unlock();
+	} else {
+		iotime_add(&acct, task);
+	}
+
+	seq_printf(m,
+		   "io_latency_ns %llu\n"
+		   "io_count %llu\n"
+		   "io_occupancy_ns %llu\n"
+		   "io_debt_ns %llu\n"
+		   "kern_time_ns %llu\n"
+		   "kern_debt_ns %llu\n",
+		   acct.latency_ns, acct.count, acct.occupancy_ns,
+		   acct.debt_ns, acct.kern_time_ns, acct.kern_debt_ns);
+
+out_unlock:
+	up_read(&task->signal->exec_update_lock);
+	return result;
+}
+
+static int proc_tgid_iotime(struct seq_file *m, struct pid_namespace *ns,
+			    struct pid *pid, struct task_struct *task)
+{
+	return do_iotime_accounting(task, m, 1);
+}
+
+static int proc_tid_iotime(struct seq_file *m, struct pid_namespace *ns,
+			   struct pid *pid, struct task_struct *task)
+{
+	return do_iotime_accounting(task, m, 0);
+}
+
+#endif
+
 #ifdef CONFIG_LATENCYTOP
 static int lstats_show_proc(struct seq_file *m, void *v)
 {
@@ -3336,6 +3438,9 @@ static const struct pid_entry tgid_base_stuff[] = {
 #ifdef CONFIG_SCHED_INFO
 	ONE("schedstat",  S_IRUGO, proc_pid_schedstat),
 #endif
+#ifdef CONFIG_MUQSS_IOTIME
+	ONE("iotime",     S_IRUSR, proc_tgid_iotime),
+#endif
 #ifdef CONFIG_LATENCYTOP
 	REG("latency",  S_IRUGO, proc_lstats_operations),
 #endif
@@ -3691,6 +3796,9 @@ static const struct pid_entry tid_base_stuff[] = {
 #ifdef CONFIG_SCHED_INFO
 	ONE("schedstat", S_IRUGO, proc_pid_schedstat),
 #endif
+#ifdef CONFIG_MUQSS_IOTIME
+	ONE("iotime",	S_IRUSR, proc_tid_iotime),
+#endif
 #ifdef CONFIG_LATENCYTOP
 	REG("latency",  S_IRUGO, proc_lstats_operations),
 #endif
diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c
--- a/fs/xfs/xfs_trans_ail.c
+++ b/fs/xfs/xfs_trans_ail.c
@@ -714,7 +714,7 @@ xfsaild(
 		spin_unlock(&ailp->ail_lock);
 
 		if (tout)
-			schedule_timeout(msecs_to_jiffies(tout));
+			schedule_msec_hrtimeout(tout);
 
 		__set_current_state(TASK_RUNNING);
 
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -137,6 +137,23 @@ struct request {
 	/* Time that I/O was submitted to the device. */
 	u64 io_start_time_ns;
 
+#ifdef CONFIG_MUQSS_IOTIME
+	/*
+	 * Task this request is being served for, holding a reference. Set
+	 * once from the first bio and cleared if a merge brings in a bio
+	 * belonging to somebody else, since occupancy cannot be split
+	 * between owners at completion time.
+	 */
+	struct task_struct *muqss_owner;
+	/*
+	 * Completion time, captured where the block layer already takes it.
+	 * rq_qos_done() runs after blk_update_request() has completed every
+	 * bio, so a timestamp taken there would include the completion path
+	 * and overstate device service time.
+	 */
+	u64 muqss_done_ns;
+#endif
+
 #ifdef CONFIG_BLK_WBT
 	unsigned short wbt_flags;
 #endif
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -252,13 +252,25 @@ struct bio {
 	 * on release of the bio.
 	 */
 	struct blkcg_gq		*bi_blkg;
-	/* Time that this bio was issued. */
-	u64			issue_time_ns;
 #ifdef CONFIG_BLK_CGROUP_IOCOST
 	u64			bi_iocost_cost;
 #endif
 #endif
 
+	/*
+	 * Time that this bio was issued. Wanted by blkcg accounting and by
+	 * MuQSS I/O aware scheduling, which must work on kernels built
+	 * without any block cgroup support.
+	 */
+#if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_MUQSS_IOTIME)
+	u64			issue_time_ns;
+#endif
+
+#ifdef CONFIG_MUQSS_IOTIME
+	/* Task this I/O is being done for. Holds a reference. */
+	struct task_struct	*bi_muqss_owner;
+#endif
+
 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
 	struct bio_crypt_ctx	*bi_crypt_context;
 #endif
diff --git a/include/linux/freezer.h b/include/linux/freezer.h
--- a/include/linux/freezer.h
+++ b/include/linux/freezer.h
@@ -91,6 +91,7 @@ static inline bool try_to_freeze(void) { return false; }
 
 static inline void set_freezable(void) {}
 
+#define pm_freezing (false)
 #endif /* !CONFIG_FREEZER */
 
 #endif	/* FREEZER_H_INCLUDED */
diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h
--- a/include/linux/ioprio.h
+++ b/include/linux/ioprio.h
@@ -29,6 +29,8 @@ static inline bool ioprio_valid(unsigned short ioprio)
  */
 static inline int task_nice_ioprio(struct task_struct *task)
 {
+	if (iso_task(task))
+		return 0;
 	return (task_nice(task) + 20) / 5;
 }
 
diff --git a/include/linux/mm.h b/include/linux/mm.h
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2460,6 +2460,61 @@ static inline bool folio_use_access_time(struct folio *folio)
 }
 #endif /* CONFIG_NUMA_BALANCING */
 
+#if MUQSS_IOWNER_WIDTH
+/*
+ * MuQSS I/O owner tag: an index into the owner table in block/blk-iotime.c,
+ * recorded when a folio is dirtied so that the writeback issued for it later
+ * by a flusher thread can still be charged to the task responsible. Zero
+ * means no owner.
+ *
+ * The tag itself is lossy by design: which of two racing dirtiers wins the
+ * field does not matter, and a wrong answer costs one misattributed
+ * writeback. The *word* is not ours to be lossy with, though. Other fields in
+ * folio->flags are updated by atomic bit operations from contexts that do not
+ * hold the folio lock -- PG_waiters is set by lock waiters, and losing it
+ * skips the wakeup on unlock -- so a plain read-modify-write store here would
+ * discard concurrent updates to unrelated bits. Update the field the way
+ * folio_xchg_last_cpupid() does, with a cmpxchg loop that touches only our
+ * own bits.
+ */
+static inline unsigned int folio_io_owner(const struct folio *folio)
+{
+	return (READ_ONCE(folio->flags.f) >> MUQSS_IOWNER_PGSHIFT) &
+	       MUQSS_IOWNER_MASK;
+}
+
+static inline void folio_set_io_owner(struct folio *folio, unsigned int slot)
+{
+	unsigned long old_flags, flags;
+
+	old_flags = READ_ONCE(folio->flags.f);
+	do {
+		flags = old_flags;
+		flags &= ~(MUQSS_IOWNER_MASK << MUQSS_IOWNER_PGSHIFT);
+		flags |= ((unsigned long)slot & MUQSS_IOWNER_MASK) <<
+			 MUQSS_IOWNER_PGSHIFT;
+	} while (unlikely(!try_cmpxchg(&folio->flags.f, &old_flags, flags)));
+}
+
+static inline void page_io_owner_reset(struct page *page)
+{
+	page->flags.f &= ~(MUQSS_IOWNER_MASK << MUQSS_IOWNER_PGSHIFT);
+}
+#else
+static inline unsigned int folio_io_owner(const struct folio *folio)
+{
+	return 0;
+}
+
+static inline void folio_set_io_owner(struct folio *folio, unsigned int slot)
+{
+}
+
+static inline void page_io_owner_reset(struct page *page)
+{
+}
+#endif /* MUQSS_IOWNER_WIDTH */
+
 #if defined(CONFIG_KASAN_SW_TAGS) || defined(CONFIG_KASAN_HW_TAGS)
 
 /*
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1240,6 +1240,7 @@ static inline bool zone_is_empty(const struct zone *zone)
 #define KASAN_TAG_PGOFF		(LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
 #define LRU_GEN_PGOFF		(KASAN_TAG_PGOFF - LRU_GEN_WIDTH)
 #define LRU_REFS_PGOFF		(LRU_GEN_PGOFF - LRU_REFS_WIDTH)
+#define MUQSS_IOWNER_PGOFF	(LRU_REFS_PGOFF - MUQSS_IOWNER_WIDTH)
 
 /*
  * Define the bit shifts to access each section.  For non-existent
@@ -1251,6 +1252,7 @@ static inline bool zone_is_empty(const struct zone *zone)
 #define ZONES_PGSHIFT		(ZONES_PGOFF * (ZONES_WIDTH != 0))
 #define LAST_CPUPID_PGSHIFT	(LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
 #define KASAN_TAG_PGSHIFT	(KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0))
+#define MUQSS_IOWNER_PGSHIFT	(MUQSS_IOWNER_PGOFF * (MUQSS_IOWNER_WIDTH != 0))
 
 /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
 #ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -1269,6 +1271,7 @@ static inline bool zone_is_empty(const struct zone *zone)
 #define NODES_MASK		((1UL << NODES_WIDTH) - 1)
 #define SECTIONS_MASK		((1UL << SECTIONS_WIDTH) - 1)
 #define LAST_CPUPID_MASK	((1UL << LAST_CPUPID_SHIFT) - 1)
+#define MUQSS_IOWNER_MASK	((1UL << MUQSS_IOWNER_WIDTH) - 1)
 #define KASAN_TAG_MASK		((1UL << KASAN_TAG_WIDTH) - 1)
 #define ZONEID_MASK		((1UL << ZONEID_SHIFT) - 1)
 
diff --git a/include/linux/muqss.h b/include/linux/muqss.h
new file mode 100644
--- /dev/null
+++ b/include/linux/muqss.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Scheduler-neutral accessors for task fields that MuQSS keeps somewhere
+ * other than the mainline scheduling entities.
+ *
+ * Kept out of <linux/sched.h> so that enabling MuQSS costs that header only
+ * the task_struct members themselves. Include this after task_struct is
+ * defined; <linux/sched.h> already does so on your behalf.
+ */
+#ifndef _LINUX_MUQSS_H
+#define _LINUX_MUQSS_H
+
+#ifdef CONFIG_SCHED_MUQSS
+
+/* MuQSS accounts runtime and RT timeouts directly on the task. */
+#define tsk_seruntime(t)	((t)->sched_time)
+#define tsk_rttimeout(t)	((t)->rt_timeout)
+
+/*
+ * The rtmutex PI chain sorts equal-priority waiters by SCHED_DEADLINE
+ * deadline. MuQSS implements no deadline class, and that tie-break is only
+ * consulted when dl_prio(prio) holds - i.e. prio < MAX_DL_PRIO, which is 0 -
+ * so it is unreachable here. Report 0 rather than carry a dl entity.
+ */
+#define tsk_dl_deadline(t)	(0)
+
+/* No deadline class: no budget to overrun and no bandwidth to migrate. */
+#define tsk_dl_overrun(t)	(0)
+#define tsk_dl_bw(t)		(0)
+
+static inline bool iso_task(struct task_struct *p)
+{
+	return (p->policy == SCHED_ISO);
+}
+
+#else /* CONFIG_SCHED_MUQSS */
+
+#define tsk_seruntime(t)	((t)->se.sum_exec_runtime)
+#define tsk_rttimeout(t)	((t)->rt.timeout)
+#define tsk_dl_deadline(t)	((t)->dl.deadline)
+#define tsk_dl_overrun(t)	((t)->dl.dl_overrun)
+#define tsk_dl_bw(t)		((t)->dl.dl_bw)
+
+static inline bool iso_task(struct task_struct *p)
+{
+	return false;
+}
+
+#endif /* CONFIG_SCHED_MUQSS */
+
+#endif /* _LINUX_MUQSS_H */
diff --git a/include/linux/muqss_iotime.h b/include/linux/muqss_iotime.h
new file mode 100644
--- /dev/null
+++ b/include/linux/muqss_iotime.h
@@ -0,0 +1,168 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * I/O aware scheduling support for MuQSS.
+ *
+ * Attributes block device time back to the task that caused the I/O, so the
+ * CPU scheduler can see what a task costs the storage device rather than only
+ * what it costs the CPU.
+ *
+ * The attributed time is charged against the task's virtual deadline one for
+ * one, a nanosecond of deadline for each nanosecond of device time, so that
+ * I/O and CPU cost a task the same.
+ *
+ * The same attribution carries CPU time as well as device time. A kworker
+ * running a work item declares itself to be acting for whoever queued that
+ * work, so the time it spends is charged back to them rather than lost to a
+ * kernel thread. That half is charged one for one: a nanosecond of CPU time
+ * costs the same wherever the kernel chose to spend it, and work a task does
+ * in its own context already costs it exactly that.
+ *
+ * See MuQSS-iotime-design.md.
+ */
+#ifndef _LINUX_MUQSS_IOTIME_H
+#define _LINUX_MUQSS_IOTIME_H
+
+#include <linux/types.h>
+
+struct bio;
+struct folio;
+struct request;
+struct gendisk;
+struct task_struct;
+struct work_struct;
+
+#ifdef CONFIG_MUQSS_IOTIME
+
+/*
+ * A window during which current is running on behalf of another task, opened
+ * by a kworker around one work item. Lives on the stack of the thread doing
+ * the work; see muqss_kerntime_begin().
+ */
+struct muqss_kern_window {
+	struct task_struct *owner;	/* who to charge, NULL for nobody */
+	struct task_struct *prev;	/* override to restore on close */
+	u64 start;			/* current's runtime when opened */
+};
+
+/*
+ * Tag @bio with the task that caused it. Called from the block layer submit
+ * path, where current is the originating task for reads, readahead and
+ * synchronous writes alike. Takes a reference that is dropped by
+ * muqss_iotime_put_owner().
+ */
+void muqss_iotime_set_owner(struct bio *bio);
+
+/*
+ * Drop the owner reference taken above. Idempotent, so it can be called from
+ * every site that releases bio state without tracking which ran first.
+ */
+void muqss_iotime_put_owner(struct bio *bio);
+
+/*
+ * Propagate ownership from @bio_src to @bio. Stacked drivers (md, dm, loop)
+ * resubmit clones from their own threads, where current is a kernel thread
+ * rather than the originating task, so without this attribution is lost on
+ * every RAID and LUKS setup.
+ */
+void muqss_iotime_clone_owner(struct bio *bio, struct bio *bio_src);
+
+/* Attach the accounting policy to a newly registered queue. */
+void muqss_iotime_enable(struct gendisk *disk);
+
+/* Zero a new task's counters so nothing is inherited across fork. */
+void muqss_iotime_task_init(struct task_struct *p);
+
+/*
+ * Owner table, used to attribute writeback. A dirtying task is given a slot,
+ * whose index is small enough to store in spare page->flags bits; writeback
+ * issued later by a flusher thread reads the index back off the folio and
+ * resolves it to the task that dirtied it.
+ */
+
+/* Slot for current, allocating one on first use. 0 if none is available. */
+unsigned int muqss_iotime_owner_slot(void);
+
+/* Resolve a slot to its task, taking a reference. NULL if the slot is stale. */
+struct task_struct *muqss_iotime_owner_task(unsigned int slot);
+
+/* Give up a task's slot. Called from exit. */
+void muqss_iotime_release_slot(struct task_struct *p);
+
+/* Record current as the dirtier of @folio, if it can be attributed. */
+void muqss_iotime_dirty_folio(struct folio *folio);
+
+/*
+ * Declare that current is about to do I/O on behalf of @owner, and undo it.
+ * Used by io_uring's io-wq workers, which issue requests that another task
+ * queued. Returns the previous value, to be handed back to _proxy_end().
+ *
+ * The caller must keep a reference to @owner across the window.
+ */
+struct task_struct *muqss_iotime_proxy_begin(struct task_struct *owner);
+void muqss_iotime_proxy_end(struct task_struct *prev);
+
+/*
+ * Two requests are being merged into one. Disowns @rq if @next belongs to a
+ * different task, so the survivor is not charged for both.
+ */
+void muqss_iotime_merge_requests(struct request *rq, struct request *next);
+
+/*
+ * Record who queued @work, so the kworker that eventually runs it can be
+ * charged to them. Called from the queueing task's own context, which rules
+ * out the deferred re-entries into __queue_work() made by the delayed work
+ * timer and the rcu_work callback: those run in interrupt and softirq
+ * context, where current is whoever was unlucky enough to be interrupted.
+ */
+void muqss_work_set_owner(struct work_struct *work);
+
+/*
+ * Open and close a window in which current is running work on behalf of the
+ * task in @slot. The CPU time current consumes across the window is charged
+ * to that task, and any I/O the work issues is attributed to it too.
+ *
+ * Cheap when there is nobody to charge, which is the common case: an unowned
+ * slot costs one comparison and reads no clocks.
+ */
+void muqss_kerntime_begin(struct muqss_kern_window *w, unsigned int slot);
+void muqss_kerntime_end(struct muqss_kern_window *w);
+
+/*
+ * The CPU time current has consumed so far, including the part not yet banked
+ * into ->sched_time. Implemented by the scheduler; see MuQSS.c.
+ */
+u64 muqss_task_runtime_live(void);
+
+#else  /* !CONFIG_MUQSS_IOTIME */
+
+struct muqss_kern_window { };
+
+static inline void muqss_iotime_set_owner(struct bio *bio) { }
+static inline void muqss_iotime_put_owner(struct bio *bio) { }
+static inline void muqss_iotime_clone_owner(struct bio *bio,
+					    struct bio *bio_src) { }
+static inline void muqss_iotime_enable(struct gendisk *disk) { }
+static inline void muqss_iotime_task_init(struct task_struct *p) { }
+static inline unsigned int muqss_iotime_owner_slot(void) { return 0; }
+static inline struct task_struct *muqss_iotime_owner_task(unsigned int slot)
+{
+	return NULL;
+}
+static inline void muqss_iotime_release_slot(struct task_struct *p) { }
+static inline void muqss_iotime_dirty_folio(struct folio *folio) { }
+static inline struct task_struct *
+muqss_iotime_proxy_begin(struct task_struct *owner)
+{
+	return NULL;
+}
+static inline void muqss_iotime_proxy_end(struct task_struct *prev) { }
+static inline void muqss_iotime_merge_requests(struct request *rq,
+					       struct request *next) { }
+static inline void muqss_work_set_owner(struct work_struct *work) { }
+static inline void muqss_kerntime_begin(struct muqss_kern_window *w,
+					unsigned int slot) { }
+static inline void muqss_kerntime_end(struct muqss_kern_window *w) { }
+
+#endif /* CONFIG_MUQSS_IOTIME */
+
+#endif /* _LINUX_MUQSS_IOTIME_H */
diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h
--- a/include/linux/page-flags-layout.h
+++ b/include/linux/page-flags-layout.h
@@ -113,9 +113,39 @@
 			    ZONES_WIDTH - LRU_GEN_WIDTH - SECTIONS_WIDTH - \
 			    NODES_WIDTH - KASAN_TAG_WIDTH - LAST_CPUPID_WIDTH)
 
+/*
+ * MuQSS I/O aware scheduling stores an index into its owner table here, so
+ * that a page dirtied now can be charged to the task that dirtied it when it
+ * is written back later by a flusher thread.
+ *
+ * These bits are affordable because MuQSS excludes NUMA balancing, which is
+ * the only user of LAST_CPUPID: with CONFIG_NUMA_BALANCING off,
+ * LAST_CPUPID_SHIFT is 0 and its share of page->flags is unclaimed.
+ *
+ * Taken last so nothing else has to shrink. __LRU_REFS_WIDTH rather than
+ * LRU_REFS_WIDTH is used below because the latter is a min() expression and
+ * so cannot be evaluated by the preprocessor; the two are equal whenever
+ * there is room for MGLRU's full allocation, which is the only case in which
+ * anything is left over for us anyway.
+ */
+#ifdef CONFIG_MUQSS_IOTIME
+#define MUQSS_IOWNER_SHIFT	8
+#else
+#define MUQSS_IOWNER_SHIFT	0
+#endif
+
+#if ZONES_WIDTH + LRU_GEN_WIDTH + SECTIONS_WIDTH + NODES_WIDTH + \
+	KASAN_TAG_WIDTH + LAST_CPUPID_WIDTH + __LRU_REFS_WIDTH + \
+	MUQSS_IOWNER_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
+#define MUQSS_IOWNER_WIDTH	MUQSS_IOWNER_SHIFT
+#else
+#define MUQSS_IOWNER_WIDTH	0
+#endif
+
 #define NR_NON_PAGEFLAG_BITS	(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH + \
 				LAST_CPUPID_SHIFT + KASAN_TAG_WIDTH + \
-				LRU_GEN_WIDTH + LRU_REFS_WIDTH)
+				LRU_GEN_WIDTH + LRU_REFS_WIDTH + \
+				MUQSS_IOWNER_WIDTH)
 
 #define NR_UNUSED_PAGEFLAG_BITS	(BITS_PER_LONG - \
 				(NR_NON_PAGEFLAG_BITS + NR_PAGEFLAGS))
diff --git a/include/linux/sched.h b/include/linux/sched.h
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -44,6 +44,9 @@
 #include <linux/restart_block.h>
 #include <linux/rseq_types.h>
 #include <linux/seqlock_types.h>
+#ifdef CONFIG_SCHED_MUQSS
+#include <linux/skip_list.h>
+#endif
 #include <linux/kcsan.h>
 #include <linux/rv.h>
 #include <linux/uidgid_types.h>
@@ -334,6 +337,12 @@ extern long schedule_timeout_interruptible(long timeout);
 extern long schedule_timeout_killable(long timeout);
 extern long schedule_timeout_uninterruptible(long timeout);
 extern long schedule_timeout_idle(long timeout);
+
+extern long schedule_msec_hrtimeout(long timeout);
+extern long schedule_min_hrtimeout(void);
+extern long schedule_msec_hrtimeout_interruptible(long timeout);
+extern long schedule_msec_hrtimeout_uninterruptible(long timeout);
+
 asmlinkage void schedule(void);
 extern void schedule_preempt_disabled(void);
 asmlinkage void preempt_schedule_irq(void);
@@ -877,6 +886,79 @@ struct task_struct {
 	int				normal_prio;
 	unsigned int			rt_priority;
 
+#ifdef CONFIG_SCHED_MUQSS
+	int				time_slice;
+	u64				deadline;
+	skiplist_node			node; /* Skip list node */
+	u64				last_ran;
+	u64				sched_time; /* sched_clock time spent running */
+#ifdef CONFIG_SMT_NICE
+	int				smt_bias; /* Policy/nice level bias across smt siblings */
+#endif
+#ifdef CONFIG_HOTPLUG_CPU
+	bool				zerobound; /* Bound to CPU0 for hotplug */
+#endif
+	unsigned long			rt_timeout;
+	/* Unbanked cpu time */
+	unsigned long			utime_ns, stime_ns;
+#ifdef CONFIG_MUQSS_IOTIME
+	/*
+	 * Block device time consumed on this task's behalf. Updated from
+	 * I/O completion, which may be any CPU, so these are atomic. Read
+	 * via /proc/<pid>/iotime. Not inherited across fork. Bytes are not
+	 * counted here, task_io_accounting already has them.
+	 *
+	 * io_latency_ns is end to end per bio, so it includes queue wait and
+	 * is what a waiting task experiences. io_occupancy_ns is device
+	 * service time per request, which is what this task cost everybody
+	 * else, and is the only one of the two fit to charge for.
+	 *
+	 * io_debt_ns is occupancy not yet charged to the deadline. The
+	 * scheduler consumes and zeroes it; see consume_iotime_penalty().
+	 */
+	atomic64_t			io_latency_ns;
+	atomic64_t			io_count;
+	atomic64_t			io_occupancy_ns;
+	atomic64_t			io_debt_ns;
+
+	/*
+	 * CPU time burnt in some other thread's context on this task's
+	 * behalf, currently kworkers running work items it queued. Unlike the
+	 * I/O counters above this is CPU time, which the task would have been
+	 * charged against its own time_slice had the kernel done the work
+	 * synchronously instead of handing it to a worker.
+	 *
+	 * kern_time_ns is cumulative and only for reporting. kern_debt_ns is
+	 * the part not yet charged to the deadline; the scheduler consumes
+	 * and zeroes it, see consume_kerntime_penalty().
+	 */
+	atomic64_t			kern_time_ns;
+	atomic64_t			kern_debt_ns;
+
+	/*
+	 * Index of this task's slot in the I/O owner table, stamped into
+	 * page->flags when it dirties a folio so writeback can be charged
+	 * back to it. 0 means no slot; allocated lazily on first dirty and
+	 * released on exit.
+	 */
+	unsigned int			io_owner_slot;
+
+	/*
+	 * The task this one is currently working on behalf of, or NULL.
+	 * io_uring's io-wq workers publish the task that queued the request
+	 * here for the duration of the issue, so a submission punted to a
+	 * worker is still charged to the task that asked for it. kworkers do
+	 * the same around each work item, which additionally attributes any
+	 * I/O the work item submits, and any pages it dirties, to the task
+	 * that queued the work rather than losing them to a kernel thread.
+	 *
+	 * Only ever written by the task itself and only read while it is
+	 * running, so it needs no locking. The publisher holds a reference
+	 * to the pointee for the whole window.
+	 */
+	struct task_struct		*io_owner_override;
+#endif
+#else /* CONFIG_SCHED_MUQSS */
 	struct sched_entity		se;
 	struct sched_rt_entity		rt;
 	struct sched_dl_entity		dl;
@@ -885,6 +967,7 @@ struct task_struct {
 	struct sched_ext_entity		scx;
 #endif
 	const struct sched_class	*sched_class;
+#endif /* CONFIG_SCHED_MUQSS */
 
 #ifdef CONFIG_SCHED_CORE
 	struct rb_node			core_node;
@@ -1677,6 +1760,8 @@ struct task_struct {
 	randomized_struct_fields_end
 } __attribute__ ((aligned (64)));
 
+#include <linux/muqss.h>
+
 #ifdef CONFIG_SCHED_PROXY_EXEC
 DECLARE_STATIC_KEY_TRUE(__sched_proxy_exec);
 static inline bool sched_proxy_exec(void)
@@ -2293,7 +2378,12 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
 
 static inline bool task_is_runnable(struct task_struct *p)
 {
+#ifdef CONFIG_SCHED_MUQSS
+	/* MuQSS has no delayed dequeue: queued means runnable. */
+	return p->on_rq;
+#else
 	return p->on_rq && !p->se.sched_delayed;
+#endif
 }
 
 extern bool sched_task_on_rq(struct task_struct *p);
diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h
--- a/include/linux/sched/deadline.h
+++ b/include/linux/sched/deadline.h
@@ -30,6 +30,24 @@ static inline bool dl_time_before(u64 a, u64 b)
 }
 
 struct root_domain;
+#ifdef CONFIG_SCHED_MUQSS
+/*
+ * MuQSS implements no deadline scheduling class, so there is never any DL
+ * bandwidth to account for or migrate between root domains.
+ */
+static inline void dl_add_task_root_domain(struct task_struct *p) { }
+static inline void dl_clear_root_domain(struct root_domain *rd) { }
+static inline void dl_clear_root_domain_cpu(int cpu) { }
+static inline bool dl_task_needs_bw_move(struct task_struct *p,
+					 const struct cpumask *new_mask)
+{
+	return false;
+}
+static inline bool dl_bw_visited(int cpu, u64 cookie)
+{
+	return false;
+}
+#else /* CONFIG_SCHED_MUQSS */
 extern void dl_add_task_root_domain(struct task_struct *p);
 extern void dl_clear_root_domain(struct root_domain *rd);
 extern void dl_clear_root_domain_cpu(int cpu);
@@ -43,19 +61,23 @@ extern void dl_clear_root_domain_cpu(int cpu);
 extern bool dl_task_needs_bw_move(struct task_struct *p,
 				  const struct cpumask *new_mask);
 
-extern u64 dl_cookie;
 extern bool dl_bw_visited(int cpu, u64 cookie);
+#endif /* CONFIG_SCHED_MUQSS */
+
+extern u64 dl_cookie;
 
 static inline bool dl_server(struct sched_dl_entity *dl_se)
 {
 	return dl_se->dl_server;
 }
 
+#ifndef CONFIG_SCHED_MUQSS
 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
 {
 	BUG_ON(dl_server(dl_se));
 	return container_of(dl_se, struct task_struct, dl);
 }
+#endif
 
 /*
  * Regarding the deadline, a task with implicit deadline has a relative
diff --git a/include/linux/sched/nohz.h b/include/linux/sched/nohz.h
--- a/include/linux/sched/nohz.h
+++ b/include/linux/sched/nohz.h
@@ -13,7 +13,7 @@ extern int get_nohz_timer_target(void);
 static inline void nohz_balance_enter_idle(int cpu) { }
 #endif
 
-#ifdef CONFIG_NO_HZ_COMMON
+#if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_SCHED_MUQSS)
 void calc_load_nohz_start(void);
 void calc_load_nohz_remote(struct rq *rq);
 void calc_load_nohz_stop(void);
diff --git a/include/linux/sched/prio.h b/include/linux/sched/prio.h
--- a/include/linux/sched/prio.h
+++ b/include/linux/sched/prio.h
@@ -16,6 +16,21 @@
 #define MAX_RT_PRIO		100
 #define MAX_DL_PRIO		0
 
+#ifdef CONFIG_SCHED_MUQSS
+/*
+ * MuQSS ranks its non-RT policies immediately below the RT range, and has no
+ * deadline class. SCHED_IDLEPRIO is an alias for the SCHED_IDLE policy value.
+ */
+#define ISO_PRIO		(MAX_RT_PRIO)
+#define NORMAL_PRIO		(MAX_RT_PRIO + 1)
+#define IDLE_PRIO		(MAX_RT_PRIO + 2)
+#define PRIO_LIMIT		((IDLE_PRIO) + 1)
+
+#define SCHED_IDLEPRIO		SCHED_IDLE
+#define SCHED_MAX		(SCHED_IDLEPRIO)
+#define SCHED_RANGE(policy)	((policy) <= SCHED_MAX)
+#endif /* CONFIG_SCHED_MUQSS */
+
 #define MAX_PRIO		(MAX_RT_PRIO + NICE_WIDTH)
 #define DEFAULT_PRIO		(MAX_RT_PRIO + NICE_WIDTH / 2)
 
diff --git a/include/linux/skip_list.h b/include/linux/skip_list.h
new file mode 100644
--- /dev/null
+++ b/include/linux/skip_list.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_SKIP_LISTS_H
+#define _LINUX_SKIP_LISTS_H
+
+/*
+ * Four levels is enough: randomLevel() returns 0..3, and insert only
+ * grows the list one level at a time, so next/prev[4..] were unused.
+ */
+#define SKIPLIST_MAXLEVEL 4
+
+typedef u64 keyType;
+
+typedef struct nodeStructure skiplist_node;
+
+struct nodeStructure {
+	keyType key;
+	skiplist_node *next[SKIPLIST_MAXLEVEL];
+	skiplist_node *prev[SKIPLIST_MAXLEVEL];
+};
+
+typedef struct listStructure {
+	int entries;
+	int level;	/* Maximum level of the list
+			(1 more than the number of levels in the list) */
+	u64 best_key;	/* Earliest queued key; lockless hint for EDT */
+	skiplist_node *header; /* pointer to header */
+} skiplist;
+
+void skiplist_init(skiplist_node *slnode);
+void skiplist_cache_init(void);
+skiplist *new_skiplist(skiplist_node *slnode);
+void skiplist_free(skiplist *l);
+void free_skiplist(skiplist *l);
+void skiplist_node_init(skiplist_node *node);
+void skiplist_insert(skiplist *l, skiplist_node *node, keyType key, unsigned int randseed);
+bool skiplist_delete(skiplist *l, skiplist_node *node);
+
+static inline bool skiplist_node_empty(skiplist_node *node) {
+	return (!node->next[0]);
+}
+#endif /* _LINUX_SKIP_LISTS_H */
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -288,6 +288,7 @@ static inline unsigned int work_static(struct work_struct *work) { return 0; }
 		lockdep_init_map(&(_work)->lockdep_map, "(work_completion)"#_work, (_key), 0); \
 		INIT_LIST_HEAD(&(_work)->entry);			\
 		(_work)->func = (_func);				\
+		muqss_work_init_owner(_work);				\
 	} while (0)
 #else
 #define __INIT_WORK_KEY(_work, _func, _onstack, _key)			\
@@ -296,6 +297,7 @@ static inline unsigned int work_static(struct work_struct *work) { return 0; }
 		(_work)->data = (atomic_long_t) WORK_DATA_INIT();	\
 		INIT_LIST_HEAD(&(_work)->entry);			\
 		(_work)->func = (_func);				\
+		muqss_work_init_owner(_work);				\
 	} while (0)
 #endif
 
diff --git a/include/linux/workqueue_types.h b/include/linux/workqueue_types.h
--- a/include/linux/workqueue_types.h
+++ b/include/linux/workqueue_types.h
@@ -17,9 +17,51 @@ struct work_struct {
 	atomic_long_t data;
 	struct list_head entry;
 	work_func_t func;
+#ifdef CONFIG_MUQSS_IOTIME
+	/*
+	 * Index in the I/O owner table of the task that queued this work, so
+	 * the CPU time the kworker spends running it can be charged back to
+	 * whoever asked for it. 0 means nobody to charge, which is the case
+	 * for the great deal of work queued by kernel threads on their own
+	 * account.
+	 *
+	 * A slot index rather than a task pointer deliberately: work items
+	 * are cancelled and disabled through half a dozen paths that clear
+	 * the pending bit without ever running the work, and none of them
+	 * would drop a reference taken here. A stale slot simply resolves to
+	 * NULL and the work is charged to nobody.
+	 */
+	unsigned int muqss_owner_slot;
+#endif
 #ifdef CONFIG_LOCKDEP
 	struct lockdep_map lockdep_map;
 #endif
 };
 
+#ifdef CONFIG_MUQSS_IOTIME
+static inline unsigned int muqss_work_owner(struct work_struct *work)
+{
+	return work->muqss_owner_slot;
+}
+
+/*
+ * Every queueing path stamps the owner, so this only matters for a work item
+ * initialised on memory that was never zeroed and then reaching execution by
+ * some path that does not. It is here so that such a path, if one exists,
+ * charges nobody rather than charging whichever unlucky task holds the slot
+ * the garbage happens to name.
+ */
+static inline void muqss_work_init_owner(struct work_struct *work)
+{
+	work->muqss_owner_slot = 0;
+}
+#else
+static inline unsigned int muqss_work_owner(struct work_struct *work)
+{
+	return 0;
+}
+
+static inline void muqss_work_init_owner(struct work_struct *work) { }
+#endif
+
 #endif /* _LINUX_WORKQUEUE_TYPES_H */
diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
--- a/include/uapi/linux/sched.h
+++ b/include/uapi/linux/sched.h
@@ -125,7 +125,7 @@ struct clone_args {
 #define SCHED_FIFO		1
 #define SCHED_RR		2
 #define SCHED_BATCH		3
-/* SCHED_ISO: reserved but not implemented yet */
+#define SCHED_ISO		4	/* Implemented on MuQSS only */
 #define SCHED_IDLE		5
 #define SCHED_DEADLINE		6
 #define SCHED_EXT		7
diff --git a/init/Kconfig b/init/Kconfig
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -218,6 +218,8 @@ config THREAD_INFO_IN_TASK
 
 menu "General setup"
 
+source "kernel/Kconfig.MuQSS"
+
 config BROKEN
 	bool
 	help
@@ -887,6 +889,7 @@ menu "Scheduler features"
 config UCLAMP_TASK
 	bool "Enable utilization clamping for RT/FAIR tasks"
 	depends on CPU_FREQ_GOV_SCHEDUTIL
+	depends on !SCHED_MUQSS
 	help
 	  This feature enables the scheduler to track the clamped utilization
 	  of each CPU based on RUNNABLE tasks scheduled on that CPU.
@@ -935,6 +938,8 @@ config UCLAMP_BUCKETS_COUNT
 
 config SCHED_PROXY_EXEC
 	bool "Proxy Execution"
+	# MuQSS has no sched_class donor/proxy plumbing
+	depends on !SCHED_MUQSS
 	# Avoid some build failures w/ PREEMPT_RT until it can be fixed
 	depends on !PREEMPT_RT
 	# Need to investigate how to inform sched_ext of split contexts
@@ -1012,6 +1017,7 @@ config ARCH_WANT_NUMA_VARIABLE_LOCALITY
 
 config NUMA_BALANCING
 	bool "Memory placement aware NUMA scheduler"
+	depends on !SCHED_MUQSS
 	depends on ARCH_SUPPORTS_NUMA_BALANCING
 	depends on !ARCH_WANT_NUMA_VARIABLE_LOCALITY
 	depends on SMP && NUMA_MIGRATION && !PREEMPT_RT
@@ -1026,6 +1032,8 @@ config SCHED_CACHE
 	bool "Cache aware load balance"
 	default y
 	depends on SMP
+	# Cache aware load balancing lives in CFS, which MuQSS replaces
+	depends on !SCHED_MUQSS
 	help
 	  When enabled, the scheduler will attempt to aggregate tasks from
 	  the same process onto a single Last Level Cache (LLC) domain when
@@ -1148,7 +1156,7 @@ menuconfig CGROUP_SCHED
 	  bandwidth allocation to such task groups. It uses cgroups to group
 	  tasks.
 
-if CGROUP_SCHED
+if CGROUP_SCHED && !SCHED_MUQSS
 config GROUP_SCHED_WEIGHT
 	def_bool n
 
@@ -1210,7 +1218,7 @@ config EXT_SUB_SCHED
 
 config SCHED_MM_CID
 	def_bool y
-	depends on SMP && RSEQ
+	depends on SMP && RSEQ && !SCHED_MUQSS
 
 config UCLAMP_TASK_GROUP
 	bool "Utilization clamping per group of tasks"
@@ -1337,6 +1345,7 @@ config CGROUP_DEVICE
 
 config CGROUP_CPUACCT
 	bool "Simple CPU accounting controller"
+	depends on !SCHED_MUQSS
 	help
 	  Provides a simple controller for monitoring the
 	  total CPU consumed by the tasks in a cgroup.
@@ -1482,6 +1491,7 @@ config CHECKPOINT_RESTORE
 
 config SCHED_AUTOGROUP
 	bool "Automatic process group scheduling"
+	depends on !SCHED_MUQSS
 	select CGROUPS
 	select CGROUP_SCHED
 	select FAIR_GROUP_SCHED
@@ -2005,6 +2015,7 @@ config RSEQ
 config RSEQ_SLICE_EXTENSION
 	bool "Enable rseq-based time slice extension mechanism"
 	depends on RSEQ && HIGH_RES_TIMERS && GENERIC_ENTRY && HAVE_GENERIC_TIF_BITS
+	depends on !SCHED_MUQSS
 	help
 	  Allows userspace to request a limited time slice extension when
 	  returning from an interrupt to user space via the RSEQ shared
diff --git a/init/init_task.c b/init/init_task.c
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -111,9 +111,17 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.stack		= init_stack,
 	.usage		= REFCOUNT_INIT(2),
 	.flags		= PF_KTHREAD,
+#ifdef CONFIG_SCHED_MUQSS
+	.prio		= NORMAL_PRIO,
+	.static_prio	= MAX_PRIO - 20,
+	.normal_prio	= NORMAL_PRIO,
+	.deadline	= 0,
+	.time_slice	= 1000000,
+#else
 	.prio		= MAX_PRIO - 20,
 	.static_prio	= MAX_PRIO - 20,
 	.normal_prio	= MAX_PRIO - 20,
+#endif
 	.policy		= SCHED_NORMAL,
 	.cpus_ptr	= &init_task.cpus_mask,
 	.user_cpus_ptr	= NULL,
@@ -126,6 +134,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.restart_block	= {
 		.fn = do_no_restart_syscall,
 	},
+#ifndef CONFIG_SCHED_MUQSS
 	.se		= {
 		.group_node 	= LIST_HEAD_INIT(init_task.se.group_node),
 	},
@@ -133,6 +142,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 		.run_list	= LIST_HEAD_INIT(init_task.rt.run_list),
 		.time_slice	= RR_TIMESLICE,
 	},
+#endif
 	.tasks		= LIST_HEAD_INIT(init_task.tasks),
 #ifdef CONFIG_SMP
 	.pushable_tasks	= PLIST_NODE_INIT(init_task.pushable_tasks, MAX_PRIO),
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -56,6 +56,7 @@
 #include <linux/task_work.h>
 #include <linux/io_uring.h>
 #include <linux/io_uring/cmd.h>
+#include <linux/muqss_iotime.h>
 #include <linux/audit.h>
 #include <linux/security.h>
 #include <linux/jump_label.h>
@@ -1531,7 +1532,19 @@ void io_wq_submit_work(struct io_wq_work *work)
 	}
 
 	do {
+		struct task_struct *prev;
+
+		/*
+		 * Anything this issue submits to the block layer is being done
+		 * on behalf of the task that queued the request, not this
+		 * worker. Publish it so the I/O is charged to the right task;
+		 * req->tctx pins it for the duration.
+		 */
+		prev = muqss_iotime_proxy_begin(req->tctx ? req->tctx->task :
+						NULL);
 		ret = io_issue_sqe(req, issue_flags);
+		muqss_iotime_proxy_end(prev);
+
 		if (ret != -EAGAIN)
 			break;
 
diff --git a/kernel/Kconfig.MuQSS b/kernel/Kconfig.MuQSS
new file mode 100644
--- /dev/null
+++ b/kernel/Kconfig.MuQSS
@@ -0,0 +1,220 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# MuQSS - The Multiple Queue Skiplist Scheduler by Con Kolivas.
+#
+# Everything MuQSS needs from Kconfig lives in this file, including the
+# negative dependencies that disable mainline scheduler features MuQSS
+# replaces or does not implement. Kconfig merges properties from repeated
+# "config" blocks, so those features are gated here rather than by editing
+# each mainline Kconfig in place. The only change required outside this file
+# is a single source line in init/Kconfig.
+
+config SCHED_MUQSS
+	bool "MuQSS cpu scheduler"
+	# Architectures the port is built and tested on. Widening this needs
+	# arch review: <linux/sched/prio.h> gains a NORMAL_PRIO definition,
+	# which collides with the private one in
+	# arch/powerpc/platforms/cell/spufs/sched.c.
+	depends on X86 || ARM64
+	select HIGH_RES_TIMERS
+	default y
+	help
+	  The Multiple Queue Skiplist Scheduler for excellent interactivity and
+	  responsiveness on the desktop and highly scalable deterministic
+	  low latency on any hardware.
+
+	  Say Y here.
+
+config SMT_NICE
+	bool "SMT (Hyperthreading) aware nice priority and policy support"
+	depends on SCHED_MUQSS && SCHED_SMT
+	default y
+	help
+	  Enabling Hyperthreading on Intel CPUs decreases the effectiveness
+	  of the use of 'nice' levels and different scheduling policies
+	  (e.g. realtime) due to sharing of CPU power between hyperthreads.
+	  SMT nice support makes each logical CPU aware of what is running on
+	  its hyperthread siblings, maintaining appropriate distribution of
+	  CPU according to nice levels and scheduling policies at the expense
+	  of slightly increased overhead.
+
+	  If unsure say Y here.
+
+choice
+	depends on SMP
+	prompt "CPU scheduler runqueue sharing"
+	default RQ_MC if SCHED_MUQSS
+	default RQ_NONE
+
+config RQ_NONE
+	bool "No sharing"
+	help
+	  This is the default behaviour where the CPU scheduler has one runqueue
+	  per CPU, whether it is a physical or logical CPU (hyperthread).
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=none
+
+	  If unsure, say N.
+
+config RQ_SMT
+	bool "SMT (hyperthread) siblings"
+	depends on SCHED_SMT && SCHED_MUQSS
+
+	help
+	  With this option enabled, the CPU scheduler will have one runqueue
+	  shared by SMT (hyperthread) siblings. As these logical cores share
+	  one physical core, sharing the runqueue resource can lead to decreased
+	  overhead, lower latency and higher throughput.
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=smt
+
+	  If unsure, say N.
+
+config RQ_MC
+	bool "Multicore siblings"
+	depends on SCHED_MC && SCHED_MUQSS
+	help
+	  With this option enabled, the CPU scheduler will have one runqueue
+	  shared by multicore siblings in addition to any SMT siblings.
+	  As these physical cores share caches, sharing the runqueue resource
+	  will lead to lower latency, but its effects on overhead and throughput
+	  are less predictable. As a general rule, 6 or fewer cores will likely
+	  benefit from this, while larger CPUs will only derive a latency
+	  benefit. If your workloads are primarily single threaded, this will
+	  possibly worsen throughput. If you are only concerned about latency
+	  then enable this regardless of how many cores you have.
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=mc
+
+	  If unsure, say Y.
+
+config RQ_MC_LLC
+	bool "Multicore siblings (LLC)"
+	depends on SCHED_MC && SCHED_MUQSS
+	help
+	  With this option enabled, the CPU scheduler will behave similarly as
+	  with "Multicore siblings".
+	  This option takes LLC cache into account when scheduling tasks.
+	  Option may benefit CPUs with multiple LLC caches, such as Ryzen
+	  and Xeon CPUs.
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=llc
+
+	  If unsure, say N.
+
+config RQ_SMP
+	bool "Symmetric Multi-Processing"
+	depends on SMP && SCHED_MUQSS
+	help
+	  With this option enabled, the CPU scheduler will have one runqueue
+	  shared by all physical CPUs unless they are on separate NUMA nodes.
+	  As physical CPUs usually do not share resources, sharing the runqueue
+	  will normally worsen throughput but improve latency. If you only
+	  care about latency enable this.
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=smp
+
+	  If unsure, say N.
+
+config RQ_ALL
+	bool "NUMA"
+	depends on SMP && SCHED_MUQSS
+	help
+	  With this option enabled, the CPU scheduler will have one runqueue
+	  regardless of the architecture configuration, including across NUMA
+	  nodes. This can substantially decrease throughput in NUMA
+	  configurations, but light NUMA designs will not be dramatically
+	  affected. This option should only be chosen if latency is the prime
+	  concern.
+
+	  This can still be enabled runtime with the boot parameter
+	  rqshare=all
+
+	  If unsure, say N.
+endchoice
+
+config SHARERQ
+	int
+	default 0 if RQ_NONE
+	default 1 if RQ_SMT
+	default 2 if RQ_MC
+	default 3 if RQ_MC_LLC
+	default 4 if RQ_SMP
+	default 5 if RQ_ALL
+
+config MUQSS_IOTIME
+	bool "I/O aware CPU scheduling support"
+	depends on SCHED_MUQSS && BLOCK
+	default y
+	help
+	  Ordinarily the CPU scheduler can see how much CPU a task uses, but it
+	  is blind to how much block device time that task consumes. A task
+	  that keeps a disk busy on everyone else's behalf looks identical to
+	  one that is merely idle, so it keeps full CPU priority while other
+	  tasks wait behind its I/O.
+
+	  This option attaches the scheduler to the block layer so that the
+	  time a device spends servicing each request is attributed back to the
+	  task that asked for it, and accumulates that as a per task I/O time
+	  figure. It is exported per task through /proc/<pid>/iotime and per
+	  device through debugfs.
+
+	  That figure is charged against the task's virtual deadline, demoting a
+	  task in proportion to the device time it consumed. This stops a task
+	  that keeps a disk busy on everybody else's behalf from retaining full
+	  CPU priority for free, while leaving tasks that merely wait on I/O as
+	  responsive as they are now.
+
+	  The charge is one nanosecond of deadline per nanosecond of device
+	  time, so I/O and CPU cost a task equally. That rate is what the
+	  feature means rather than a setting within it, so there is nothing to
+	  tune it by.
+
+	  The same machinery charges a task for kernel work done in another
+	  thread's context on its behalf, which today means kworkers running
+	  work items it queued. Kernel work a task asks for in its own context
+	  needs no such help, since it already comes out of that task's own
+	  timeslice; what escapes the scheduler is the work handed to somebody
+	  else, and forcing IRQ threading moves more work into that blind spot.
+	  That is charged one for one, since a nanosecond of CPU time costs the
+	  same wherever the kernel chose to spend it. It is reported alongside
+	  the I/O figures.
+
+	  Doing this costs a timestamp and an owner reference on every block
+	  request, and an accounting update on every completion. On very fast
+	  devices, or under heavy small random I/O, that overhead is measurable
+	  and may reduce peak I/O throughput. Say N if you are tuning purely
+	  for storage benchmarks.
+
+	  When disabled this is compiled out entirely and costs nothing.
+
+	  If unsure, say Y.
+
+#
+# Mainline features MuQSS replaces or does not implement are gated with
+# "depends on !SCHED_MUQSS" at their point of definition. Kconfig scopes a
+# "depends on" to the properties declared in the same config block, so a
+# dependency cannot be attached to a symbol from here; each gate has to live
+# beside the definition it constrains. They are:
+#
+#   init/Kconfig                              UCLAMP_TASK
+#                                             SCHED_PROXY_EXEC
+#                                             NUMA_BALANCING
+#                                             SCHED_CACHE
+#                                             SCHED_MM_CID
+#                                             CGROUP_CPUACCT
+#                                             SCHED_AUTOGROUP
+#                                             RSEQ_SLICE_EXTENSION
+#                                             if CGROUP_SCHED (group scheduling
+#                                               block: FAIR_GROUP_SCHED,
+#                                               CFS_BANDWIDTH, RT_GROUP_SCHED,
+#                                               GROUP_SCHED_*, EXT_GROUP_SCHED)
+#   kernel/Kconfig.preempt                    PREEMPT_RT
+#                                             SCHED_CORE
+#                                             SCHED_CLASS_EXT
+#   kernel/trace/rv/monitors/deadline/Kconfig RV_MON_DEADLINE
diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz
--- a/kernel/Kconfig.hz
+++ b/kernel/Kconfig.hz
@@ -3,9 +3,29 @@
 # Timer Interrupt Frequency Configuration
 #
 
+#
+# Silent symbols keep CONFIG_HZ_* available to the rest of the tree and to
+# inherited .configs, but they are not choice members. The promptable
+# *_NODEF names are what olddefconfig can stick on; renaming every rate
+# (including 100) means a saved "# CONFIG_HZ_100 is not set" cannot pin the
+# choice away from the MuQSS default.
+#
+config HZ_100
+	bool
+
+config HZ_250
+	bool
+
+config HZ_300
+	bool
+
+config HZ_1000
+	bool
+
 choice
 	prompt "Timer frequency"
-	default HZ_250
+	default HZ_100_NODEF if SCHED_MUQSS
+	default HZ_250_NODEF if !SCHED_MUQSS
 	help
 	 Allows the configuration of the timer frequency. It is customary
 	 to have the timer interrupt run at 1000 Hz but 100 Hz may be more
@@ -17,31 +37,45 @@ choice
 	 per second.
 
 
-	config HZ_100
+	config HZ_100_NODEF
 		bool "100 HZ"
+		select HZ_100
 	help
+	  100 Hz is a suitable choice in combination with MuQSS which does
+	  not rely on ticks for rescheduling interrupts, and is not Hz limited
+	  for timeouts and sleeps from both the kernel and userspace.
+	  This allows us to benefit from the lower overhead and higher
+	  throughput of fewer timer ticks.
+
+	  Non-MuQSS kernels:
 	  100 Hz is a typical choice for servers, SMP and NUMA systems
 	  with lots of processors that may show reduced performance if
 	  too many timer interrupts are occurring.
 
-	config HZ_250
+	config HZ_250_NODEF
 		bool "250 HZ"
+		select HZ_250
 	help
 	 250 Hz is a good compromise choice allowing server performance
 	 while also showing good interactive responsiveness even
 	 on SMP and NUMA systems. If you are going to be using NTSC video
 	 or multimedia, select 300Hz instead.
 
-	config HZ_300
+	 250 Hz is the default choice for the mainline scheduler but not
+	 advantageous in combination with MuQSS.
+
+	config HZ_300_NODEF
 		bool "300 HZ"
+		select HZ_300
 	help
 	 300 Hz is a good compromise choice allowing server performance
 	 while also showing good interactive responsiveness even
 	 on SMP and NUMA systems and exactly dividing by both PAL and
 	 NTSC frame rates for video and multimedia work.
 
-	config HZ_1000
+	config HZ_1000_NODEF
 		bool "1000 HZ"
+		select HZ_1000
 	help
 	 1000 Hz is the preferred choice for desktop systems and other
 	 systems requiring fast interactive responses to events.
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -14,15 +14,35 @@ config PREEMPT_BUILD
 config ARCH_HAS_PREEMPT_LAZY
 	bool
 
+#
+# Silent symbols keep CONFIG_PREEMPT{,_NONE,_VOLUNTARY,_LAZY} available to
+# the rest of the tree and to inherited .configs, but they are not choice
+# members. The promptable *_NODEF names are what olddefconfig can stick on;
+# renaming every model (including PREEMPT) means a saved
+# "# CONFIG_PREEMPT is not set" or CONFIG_PREEMPT_{NONE,VOLUNTARY,LAZY}=y
+# cannot pin the choice away from full preempt.
+#
+config PREEMPT_NONE
+	bool
+
+config PREEMPT_VOLUNTARY
+	bool
+
+config PREEMPT
+	bool
+
+config PREEMPT_LAZY
+	bool
+
 choice
 	prompt "Preemption Model"
-	default PREEMPT_LAZY if ARCH_HAS_PREEMPT_LAZY
-	default PREEMPT_NONE
+	default PREEMPT_NODEF
 
-config PREEMPT_NONE
+config PREEMPT_NONE_NODEF
 	bool "No Forced Preemption (Server)"
 	depends on !PREEMPT_RT
 	depends on ARCH_NO_PREEMPT
+	select PREEMPT_NONE
 	select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC
 	help
 	  This is the traditional Linux preemption model, geared towards
@@ -35,11 +55,12 @@ config PREEMPT_NONE
 	  raw processing power of the kernel, irrespective of scheduling
 	  latencies.
 
-config PREEMPT_VOLUNTARY
-	bool "Voluntary Kernel Preemption (Desktop)"
+config PREEMPT_VOLUNTARY_NODEF
+	bool "Voluntary Kernel Preemption (Nothing)"
 	depends on !ARCH_HAS_PREEMPT_LAZY
 	depends on !ARCH_NO_PREEMPT
 	depends on !PREEMPT_RT
+	select PREEMPT_VOLUNTARY
 	select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC
 	help
 	  This option reduces the latency of the kernel by adding more
@@ -54,11 +75,13 @@ config PREEMPT_VOLUNTARY
 	  applications to run more 'smoothly' even when the system is
 	  under load.
 
-	  Select this if you are building a kernel for a desktop system.
+	  Select this for no system in particular (choose Preemptible
+	  instead on a desktop if you know what's good for you).
 
-config PREEMPT
+config PREEMPT_NODEF
 	bool "Preemptible Kernel (Low-Latency Desktop)"
 	depends on !ARCH_NO_PREEMPT
+	select PREEMPT
 	select PREEMPT_BUILD if !PREEMPT_DYNAMIC
 	help
 	  This option reduces the latency of the kernel by making
@@ -75,10 +98,11 @@ config PREEMPT
 	  embedded system with latency requirements in the milliseconds
 	  range.
 
-config PREEMPT_LAZY
+config PREEMPT_LAZY_NODEF
 	bool "Scheduler controlled preemption model"
 	depends on !ARCH_NO_PREEMPT
 	depends on ARCH_HAS_PREEMPT_LAZY
+	select PREEMPT_LAZY
 	select PREEMPT_BUILD if !PREEMPT_DYNAMIC
 	help
 	  This option provides a scheduler driven preemption model that
@@ -87,11 +111,17 @@ config PREEMPT_LAZY
 	  reduce lock holder preemption and recover some of the performance
 	  gains seen from using Voluntary preemption.
 
+	  It trades the latency that makes a desktop feel responsive for
+	  throughput, so choose Preemptible instead unless you are building
+	  for a throughput oriented system.
+
 endchoice
 
 config PREEMPT_RT
 	bool "Fully Preemptible Kernel (Real-Time)"
 	depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST
+	# MuQSS does not implement the PREEMPT_RT scheduler/locking contract
+	depends on !SCHED_MUQSS
 	select PREEMPTION
 	help
 	  This option turns the kernel into a real-time kernel by replacing
@@ -150,7 +180,7 @@ config PREEMPT_DYNAMIC
 
 config SCHED_CORE
 	bool "Core Scheduling for SMT"
-	depends on SCHED_SMT
+	depends on SCHED_SMT && !SCHED_MUQSS
 	help
 	  This option permits Core Scheduling, a means of coordinated task
 	  selection across SMT siblings. When enabled -- see
@@ -168,7 +198,7 @@ config SCHED_CORE
 
 config SCHED_CLASS_EXT
 	bool "Extensible Scheduling Class"
-	depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF
+	depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF && !SCHED_MUQSS
 	select GENERIC_ALLOCATOR
 	select STACKTRACE if STACKTRACE_SUPPORT
 	help
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -3083,7 +3083,7 @@ static int cpuset_can_attach(struct cgroup_taskset *tset)
 			 */
 			cs->nr_migrate_dl_tasks++;
 			if (dl_task_needs_bw_move(task, cs->effective_cpus))
-				cs->sum_migrate_dl_bw += task->dl.dl_bw;
+				cs->sum_migrate_dl_bw += tsk_dl_bw(task);
 		}
 	}
 
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -170,7 +170,7 @@ int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
 	 */
 	t1 = tsk->sched_info.pcount;
 	t2 = tsk->sched_info.run_delay;
-	t3 = tsk->se.sum_exec_runtime;
+	t3 = tsk_seruntime(tsk);
 
 	d->cpu_count += t1;
 
diff --git a/kernel/exit.c b/kernel/exit.c
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -34,6 +34,7 @@
 #include <linux/mount.h>
 #include <linux/proc_fs.h>
 #include <linux/kthread.h>
+#include <linux/muqss_iotime.h>
 #include <linux/mempolicy.h>
 #include <linux/taskstats_kern.h>
 #include <linux/delayacct.h>
@@ -207,7 +208,7 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts
 	sig->inblock += task_io_get_inblock(tsk);
 	sig->oublock += task_io_get_oublock(tsk);
 	task_io_accounting_add(&sig->ioac, &tsk->ioac);
-	sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
+	sig->sum_sched_runtime += tsk_seruntime(tsk);
 	sig->nr_threads--;
 	__unhash_process(post, tsk, group_dead);
 	write_sequnlock(&sig->stats_lock);
@@ -298,8 +299,8 @@ void release_task(struct task_struct *p)
 	proc_flush_pid(thread_pid);
 	put_pid(thread_pid);
 	exit_cred_namespaces(p);
-	add_device_randomness(&p->se.sum_exec_runtime,
-			      sizeof(p->se.sum_exec_runtime));
+	add_device_randomness(&tsk_seruntime(p),
+			      sizeof(tsk_seruntime(p)));
 	free_pids(post.pids);
 	release_thread(p);
 	/*
@@ -1037,6 +1038,9 @@ void __noreturn do_exit(long code)
 	if (tsk->io_context)
 		exit_io_context(tsk);
 
+	/* Give up the I/O owner table slot used to attribute writeback. */
+	muqss_iotime_release_slot(tsk);
+
 	if (tsk->splice_pipe)
 		free_pipe_info(tsk->splice_pipe);
 
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -372,7 +372,7 @@ waiter_update_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)
 	lockdep_assert(RB_EMPTY_NODE(&waiter->tree.entry));
 
 	waiter->tree.prio = __waiter_prio(task);
-	waiter->tree.deadline = task->dl.deadline;
+	waiter->tree.deadline = tsk_dl_deadline(task);
 }
 
 /*
@@ -393,7 +393,7 @@ waiter_clone_prio(struct rt_mutex_waiter *waiter, struct task_struct *task)
  * Only use with rt_waiter_node_{less,equal}()
  */
 #define task_to_waiter_node(p)	\
-	&(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = (p)->dl.deadline }
+	&(struct rt_waiter_node){ .prio = __waiter_prio(p), .deadline = tsk_dl_deadline(p) }
 #define task_to_waiter(p)	\
 	&(struct rt_mutex_waiter){ .tree = *task_to_waiter_node(p) }
 
diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
--- a/kernel/sched/Makefile
+++ b/kernel/sched/Makefile
@@ -23,12 +23,14 @@ ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y)
 # I turn this off for IA-64 only.  Andreas Schwab says it's also needed on m68k
 # to get a correct value for the wait-channel (WCHAN in ps). --davidm
 CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
+CFLAGS_MuQSS.o := $(PROFILING) -fno-omit-frame-pointer
 endif
 
 # Branch profiling isn't noinstr-safe
 ifdef CONFIG_TRACE_BRANCH_PROFILING
 CFLAGS_build_policy.o += -DDISABLE_BRANCH_PROFILING
 CFLAGS_build_utility.o += -DDISABLE_BRANCH_PROFILING
+CFLAGS_build_muqss.o += -DDISABLE_BRANCH_PROFILING
 endif
 #
 # Build efficiency:
@@ -36,7 +38,12 @@ endif
 # These compilation units have roughly the same size and complexity - so their
 # build parallelizes well and finishes roughly at once:
 #
+ifdef CONFIG_SCHED_MUQSS
+obj-y += MuQSS.o
+obj-y += build_muqss.o
+else
 obj-y += core.o
 obj-y += fair.o
 obj-y += build_policy.o
 obj-y += build_utility.o
+endif
diff --git a/kernel/sched/MuQSS.c b/kernel/sched/MuQSS.c
new file mode 100644
--- /dev/null
+++ b/kernel/sched/MuQSS.c
@@ -0,0 +1,10795 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  kernel/sched/MuQSS.c, was kernel/sched.c
+ *
+ *  Kernel scheduler and related syscalls
+ *
+ *  Copyright (C) 1991-2002  Linus Torvalds
+ *
+ *  1996-12-23  Modified by Dave Grothe to fix bugs in semaphores and
+ *		make semaphores SMP safe
+ *  1998-11-19	Implemented schedule_timeout() and related stuff
+ *		by Andrea Arcangeli
+ *  2002-01-04	New ultra-scalable O(1) scheduler by Ingo Molnar:
+ *		hybrid priority-list and round-robin design with
+ *		an array-switch method of distributing timeslices
+ *		and per-CPU runqueues.  Cleanups and useful suggestions
+ *		by Davide Libenzi, preemptible kernel bits by Robert Love.
+ *  2003-09-03	Interactivity tuning by Con Kolivas.
+ *  2004-04-02	Scheduler domains code by Nick Piggin
+ *  2007-04-15  Work begun on replacing all interactivity tuning with a
+ *              fair scheduling design by Con Kolivas.
+ *  2007-05-05  Load balancing (smp-nice) and other improvements
+ *              by Peter Williams
+ *  2007-05-06  Interactivity improvements to CFS by Mike Galbraith
+ *  2007-07-01  Group scheduling enhancements by Srivatsa Vaddagiri
+ *  2007-11-29  RT balancing improvements by Steven Rostedt, Gregory Haskins,
+ *              Thomas Gleixner, Mike Kravetz
+ *  2009-08-13	Brainfuck deadline scheduling policy by Con Kolivas deletes
+ *              a whole lot of those previous things.
+ *  2016-10-01  Multiple Queue Skiplist Scheduler scalable evolution of BFS
+ * 		scheduler by Con Kolivas.
+ *  2019-08-31  LLC bits by Eduards Bezverhijs
+ */
+/* Out-of-line migrate_{disable,enable} for modules; see linux/sched.h. */
+#define INSTANTIATE_EXPORTED_MIGRATE_DISABLE
+#include <linux/sched/isolation.h>
+#include <linux/sched/loadavg.h>
+
+#include <linux/binfmts.h>
+#include <linux/blkdev.h>
+#include <linux/compat.h>
+#include <linux/context_tracking.h>
+#include <linux/cpuset.h>
+#include <linux/delayacct.h>
+#include <linux/init_task.h>
+#include <linux/kcov.h>
+#include <linux/kprobes.h>
+#include <linux/mmu_context.h>
+#include <linux/module.h>
+#include <linux/nmi.h>
+#include <linux/prefetch.h>
+#include <linux/profile.h>
+#include <linux/rcupdate_wait.h>
+#include <linux/sched.h>
+#include <linux/scs.h>
+#include <linux/security.h>
+#include <linux/skip_list.h>
+#include <linux/syscalls.h>
+#include <linux/tick.h>
+#include <linux/wait_bit.h>
+#include <linux/completion.h>
+#include <linux/hrtimer_rearm.h>
+#include <linux/livepatch_sched.h>
+#include <linux/muqss_iotime.h>
+#include <linux/smp.h>
+
+#include <asm/irq_regs.h>
+#include <asm/switch_to.h>
+#include <asm/tlb.h>
+
+#include "../workqueue_internal.h"
+#include "../../io_uring/io-wq.h"
+#include "../smpboot.h"
+
+#define CREATE_TRACE_POINTS
+#include <linux/sched/rseq_api.h>
+#include <trace/events/sched.h>
+#include <trace/events/ipi.h>
+#undef CREATE_TRACE_POINTS
+
+#include "MuQSS.h"
+#include "smp.h"
+
+/* Modules probe these; core.c exports the same set under CFS. */
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu);
+EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask);
+EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
+
+#define rt_prio(prio)		unlikely((prio) < MAX_RT_PRIO)
+#define rt_task(p)		rt_prio((p)->prio)
+#define batch_task(p)		(unlikely((p)->policy == SCHED_BATCH))
+#define is_rt_policy(policy)	((policy) == SCHED_FIFO || \
+					(policy) == SCHED_RR)
+#define has_rt_policy(p)	unlikely(is_rt_policy((p)->policy))
+
+#define is_idle_policy(policy)	((policy) == SCHED_IDLEPRIO)
+#define idleprio_task(p)	unlikely(is_idle_policy((p)->policy))
+#define task_running_idle(p)	unlikely((p)->prio == IDLE_PRIO)
+
+#define is_iso_policy(policy)	((policy) == SCHED_ISO)
+#define iso_task(p)		unlikely(is_iso_policy((p)->policy))
+#define task_running_iso(p)	unlikely((p)->prio == ISO_PRIO)
+
+#define rq_idle(rq)		((rq)->rq_prio == PRIO_LIMIT)
+
+#define ISO_PERIOD		(5 * HZ)
+
+/*
+ * 'User priority' is the nice value converted to something we
+ * can work with better when scaling various scheduler parameters,
+ * it's a [ 0 ... 39 ] range.
+ */
+#define USER_PRIO(p)		((p)-MAX_RT_PRIO)
+#define TASK_USER_PRIO(p)	USER_PRIO((p)->static_prio)
+#define MAX_USER_PRIO		(USER_PRIO(MAX_PRIO))
+#define STOP_PRIO		(MAX_RT_PRIO - 1)
+
+/*
+ * Some helpers for converting to/from various scales. Use shifts to get
+ * approximate multiples of ten for less overhead. These are internal scales
+ * only - niffies themselves are real nanoseconds, so anything converting the
+ * tick into niffies must use TICK_NSEC rather than the approximations here.
+ */
+#define APPROX_NS_PS		(1073741824) /* Approximate ns per second */
+#define JIFFY_NS		(APPROX_NS_PS / HZ)
+#define NS_TO_JIFFIES(TIME)	((TIME) / JIFFY_NS)
+#define HALF_JIFFY_NS		(APPROX_NS_PS / HZ / 2)
+/*
+ * time_slice is banked in NS_TO_US() of real nanoseconds, so half a tick in
+ * those units has to come from TICK_NSEC, not from the approximate scale.
+ */
+#define HALF_JIFFY_US		(NS_TO_US(TICK_NSEC) / 2)
+#define MS_TO_NS(TIME)		((TIME) << 20)
+#define MS_TO_US(TIME)		((TIME) << 10)
+#define NS_TO_MS(TIME)		((TIME) >> 20)
+#define NS_TO_US(TIME)		((TIME) >> 10)
+#define US_TO_NS(TIME)		((TIME) << 10)
+#define TICK_APPROX_NS		((APPROX_NS_PS+HZ/2)/HZ)
+
+#define RESCHED_US	(100) /* Reschedule if less than this many μs left */
+
+static void print_scheduler_version(void)
+{
+	printk(KERN_INFO "MuQSS CPU scheduler v0.31 by Con Kolivas.\n");
+}
+
+/*
+ * This is the time all tasks within the same priority round robin.
+ * Value is in ms and set to a minimum of 6ms.
+ * Tunable via /proc interface.
+ */
+int rr_interval __read_mostly = 6;
+
+/*
+ * Tunable to choose whether to prioritise latency or throughput, simple
+ * binary yes or no
+ */
+int sched_interactive __read_mostly = 1;
+
+/*
+ * sched_iso_cpu - sysctl which determines the cpu percentage SCHED_ISO tasks
+ * are allowed to run five seconds as real time tasks. This is the total over
+ * all online cpus.
+ */
+int sched_iso_cpu __read_mostly = 70;
+
+/*
+ * sched_yield_type - Choose what sort of yield sched_yield will perform.
+ * 0: No yield.
+ * 1: Yield only to better priority/deadline tasks. (default)
+ * 2: Expire timeslice and recalculate deadline.
+ */
+int sched_yield_type __read_mostly = 1;
+
+/*
+ * The relative length of deadline for each priority(nice) level.
+ */
+static int prio_ratios[NICE_WIDTH] __read_mostly;
+
+
+/*
+ * The quota handed out to tasks of all priority levels when refilling their
+ * time_slice.
+ */
+static inline int timeslice(void)
+{
+	return MS_TO_US(rr_interval);
+}
+
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+
+#ifdef CONFIG_SMP
+
+/* Define RQ share levels */
+#define RQSHARE_NONE 0
+#define RQSHARE_SMT 1
+#define RQSHARE_MC 2
+#define RQSHARE_MC_LLC 3
+#define RQSHARE_SMP 4
+#define RQSHARE_ALL 5
+
+/* Define locality levels */
+#define LOCALITY_SAME 0
+#define LOCALITY_SMT 1
+#define LOCALITY_MC_LLC 2
+#define LOCALITY_MC 3
+#define LOCALITY_SMP 4
+#define LOCALITY_DISTANT 5
+
+/*
+ * This determines what level of runqueue sharing will be done and is
+ * configurable at boot time with the bootparam rqshare =
+ */
+static int rqshare __read_mostly = CONFIG_SHARERQ; /* Default RQSHARE_MC */
+
+static int __init set_rqshare(char *str)
+{
+	if (!strncmp(str, "none", 4)) {
+		rqshare = RQSHARE_NONE;
+		return 1;
+	}
+	if (!strncmp(str, "smt", 3)) {
+		rqshare = RQSHARE_SMT;
+		return 1;
+	}
+	if (!strncmp(str, "mc", 2)) {
+		rqshare = RQSHARE_MC;
+		return 1;
+	}
+	if (!strncmp(str, "llc", 3)) {
+		rqshare = RQSHARE_MC_LLC;
+		return 1;
+	}
+	if (!strncmp(str, "smp", 3)) {
+		rqshare = RQSHARE_SMP;
+		return 1;
+	}
+	if (!strncmp(str, "all", 3)) {
+		rqshare = RQSHARE_ALL;
+		return 1;
+	}
+	return 0;
+}
+__setup("rqshare=", set_rqshare);
+
+/*
+ * Total number of runqueues. Equals number of CPUs when there is no runqueue
+ * sharing but is usually less with SMT/MC sharing of runqueues.
+ */
+static int total_runqueues __read_mostly = 1;
+
+static cpumask_t cpu_idle_map ____cacheline_aligned_in_smp;
+
+/*
+ * For asym packing, by default the lower numbered cpu has higher priority.
+ */
+int __weak arch_asym_cpu_priority(int cpu)
+{
+	return -cpu;
+}
+
+#else
+struct rq *uprq;
+#endif /* CONFIG_SMP */
+
+/*
+ * sched_smt_active() is unconditionally available now that cpu_smt_mask() is
+ * cpumask_of(cpu) for !CONFIG_SCHED_SMT, so the key must always be defined.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_smt_present);
+EXPORT_SYMBOL_GPL(sched_smt_present);
+
+#include "stats.h"
+
+/*
+ * All common locking functions performed on rq->lock. rq->clock is local to
+ * the CPU accessing it so it can be modified just with interrupts disabled
+ * when we're not updating niffies.
+ * Looking up task_rq must be done under rq->lock to be safe.
+ */
+
+/*
+ * RQ-clock updating methods:
+ */
+
+#ifdef HAVE_SCHED_AVG_IRQ
+static void update_irq_load_avg(struct rq *rq, long delta);
+#else
+static inline void update_irq_load_avg(struct rq *rq, long delta) {}
+#endif
+
+/* Use CONFIG_PARAVIRT as this will avoid more #ifdef in arch code. */
+#ifdef CONFIG_PARAVIRT
+struct static_key paravirt_steal_rq_enabled;
+#endif
+
+static void update_rq_clock_task(struct rq *rq, s64 delta)
+{
+/*
+ * In theory, the compile should just see 0 here, and optimize out the call
+ * to sched_rt_avg_update. But I don't trust it...
+ */
+	s64 __maybe_unused steal = 0, irq_delta = 0;
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+	irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time;
+
+	/*
+	 * Since irq_time is only updated on {soft,}irq_exit, we might run into
+	 * this case when a previous update_rq_clock() happened inside a
+	 * {soft,}irq region.
+	 *
+	 * When this happens, we stop ->clock_task and only update the
+	 * prev_irq_time stamp to account for the part that fit, so that a next
+	 * update will consume the rest. This ensures ->clock_task is
+	 * monotonic.
+	 *
+	 * It does however cause some slight miss-attribution of {soft,}irq
+	 * time, a more accurate solution would be to update the irq_time using
+	 * the current rq->clock timestamp, except that would require using
+	 * atomic ops.
+	 */
+	if (irq_delta > delta)
+		irq_delta = delta;
+
+	rq->prev_irq_time += irq_delta;
+	delta -= irq_delta;
+#endif
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	if (static_key_false((&paravirt_steal_rq_enabled))) {
+		steal = paravirt_steal_clock(cpu_of(rq));
+		steal -= rq->prev_steal_time_rq;
+
+		if (unlikely(steal > delta))
+			steal = delta;
+
+		rq->prev_steal_time_rq += steal;
+		delta -= steal;
+	}
+#endif
+	rq->clock_task += delta;
+
+#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
+	if (irq_delta + steal)
+		update_irq_load_avg(rq, irq_delta + steal);
+#endif
+}
+
+static inline void update_rq_clock(struct rq *rq)
+{
+	s64 delta = sched_clock_cpu(cpu_of(rq)) - rq->clock;
+
+	if (unlikely(delta < 0))
+		return;
+	rq->clock += delta;
+	update_rq_clock_task(rq, delta);
+}
+
+/*
+ * Niffies are a globally increasing nanosecond counter. They're only used by
+ * update_load_avg and time_slice_expired, however deadlines are based on them
+ * across CPUs. Update them whenever we will call one of those functions, and
+ * synchronise them across CPUs whenever we hold both runqueue locks.
+ *
+ * Niffies are the highest of two absolute values that each advance at real
+ * time: this runqueue's clock, and a tick line advanced by whole jiffies so
+ * that we keep counting if the rq clock stalls. Because both are absolute,
+ * time contributed by the tick line, or imported by synchronise_niffies(), is
+ * never counted again when the rq clock catches up - niffies simply stall
+ * until it does. Accumulating the maximum of the two *deltas* instead keeps
+ * every overshoot for good and drifts upwards without bound.
+ */
+static inline void update_clocks(struct rq *rq)
+{
+	long jdiff;
+
+	update_rq_clock(rq);
+	jdiff = jiffies - rq->last_jiffy;
+	if (jdiff > 0) {
+		rq->last_jiffy += jdiff;
+		rq->jiffy_niffies += (u64)jdiff * TICK_NSEC;
+	}
+	if (rq->niffies < rq->jiffy_niffies)
+		rq->niffies = rq->jiffy_niffies;
+	if (rq->niffies < rq->clock)
+		rq->niffies = rq->clock;
+}
+
+/*
+ * Any time we have two runqueues locked we use that as an opportunity to
+ * synchronise niffies to the highest value as idle ticks may have artificially
+ * kept niffies low on one CPU and the truth can only be later.
+ */
+static inline void synchronise_niffies(struct rq *rq1, struct rq *rq2)
+{
+	if (rq1->niffies > rq2->niffies)
+		rq2->niffies = rq1->niffies;
+	else
+		rq1->niffies = rq2->niffies;
+}
+
+/*
+ * double_rq_lock - safely lock two runqueues
+ *
+ * Note this does not disable interrupts like task_rq_lock,
+ * you need to do so manually before calling.
+ */
+
+/* For when we know rq1 != rq2 */
+static inline void __double_rq_lock(struct rq *rq1, struct rq *rq2)
+	__acquires(rq1->lock)
+	__acquires(rq2->lock)
+{
+	if (rq1 < rq2) {
+		raw_spin_lock(rq1->lock);
+		raw_spin_lock_nested(rq2->lock, SINGLE_DEPTH_NESTING);
+	} else {
+		raw_spin_lock(rq2->lock);
+		raw_spin_lock_nested(rq1->lock, SINGLE_DEPTH_NESTING);
+	}
+}
+
+static inline void double_rq_lock(struct rq *rq1, struct rq *rq2)
+	__acquires(rq1->lock)
+	__acquires(rq2->lock)
+{
+	BUG_ON(!irqs_disabled());
+	if (rq1->lock == rq2->lock) {
+		raw_spin_lock(rq1->lock);
+		__acquire(rq2->lock);	/* Fake it out ;) */
+	} else
+		__double_rq_lock(rq1, rq2);
+	synchronise_niffies(rq1, rq2);
+}
+
+/*
+ * double_rq_unlock - safely unlock two runqueues
+ *
+ * Note this does not restore interrupts like task_rq_unlock,
+ * you need to do so manually after calling.
+ */
+static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2)
+	__releases(rq1->lock)
+	__releases(rq2->lock)
+{
+	raw_spin_unlock(rq1->lock);
+	if (rq1->lock != rq2->lock)
+		raw_spin_unlock(rq2->lock);
+	else
+		__release(rq2->lock);
+}
+
+static inline void lock_all_rqs(void)
+{
+	int cpu;
+
+	preempt_disable();
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		do_raw_spin_lock(rq->lock);
+	}
+}
+
+static inline void unlock_all_rqs(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		do_raw_spin_unlock(rq->lock);
+	}
+	preempt_enable();
+}
+
+/* Specially nest trylock an rq */
+static inline bool trylock_rq(struct rq *this_rq, struct rq *rq)
+{
+	if (unlikely(!do_raw_spin_trylock(rq->lock)))
+		return false;
+	spin_acquire(&rq->lock->dep_map, SINGLE_DEPTH_NESTING, 1, _RET_IP_);
+	synchronise_niffies(this_rq, rq);
+	return true;
+}
+
+/* Unlock a specially nested trylocked rq */
+static inline void unlock_rq(struct rq *rq)
+{
+	spin_release(&rq->lock->dep_map, _RET_IP_);
+	do_raw_spin_unlock(rq->lock);
+}
+
+/*
+ * cmpxchg based fetch_or, macro so it works for different integer types
+ */
+#define fetch_or(ptr, mask)						\
+	({								\
+		typeof(ptr) _ptr = (ptr);				\
+		typeof(mask) _mask = (mask);				\
+		typeof(*_ptr) _old, _val = *_ptr;			\
+									\
+		for (;;) {						\
+			_old = cmpxchg(_ptr, _val, _val | _mask);	\
+			if (_old == _val)				\
+				break;					\
+			_val = _old;					\
+		}							\
+	_old;								\
+})
+
+#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
+/*
+ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
+ * this avoids any races wrt polling state changes and thereby avoids
+ * spurious IPIs.
+ */
+static bool set_nr_and_not_polling(struct task_struct *p)
+{
+	struct thread_info *ti = task_thread_info(p);
+	return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG);
+}
+
+/*
+ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
+ *
+ * If this returns true, then the idle task promises to call
+ * sched_ttwu_pending() and reschedule soon.
+ */
+static bool set_nr_if_polling(struct task_struct *p)
+{
+	struct thread_info *ti = task_thread_info(p);
+	typeof(ti->flags) old, val = READ_ONCE(ti->flags);
+
+	for (;;) {
+		if (!(val & _TIF_POLLING_NRFLAG))
+			return false;
+		if (val & _TIF_NEED_RESCHED)
+			return true;
+		old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED);
+		if (old == val)
+			break;
+		val = old;
+	}
+	return true;
+}
+
+#else
+static bool set_nr_and_not_polling(struct task_struct *p)
+{
+	set_tsk_need_resched(p);
+	return true;
+}
+
+#ifdef CONFIG_SMP
+static bool set_nr_if_polling(struct task_struct *p)
+{
+	return false;
+}
+#endif
+#endif
+
+static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
+{
+	struct wake_q_node *node = &task->wake_q;
+
+	/*
+	 * Atomically grab the task, if ->wake_q is !nil already it means
+	 * it's already queued (either by us or someone else) and will get the
+	 * wakeup due to that.
+	 *
+	 * In order to ensure that a pending wakeup will observe our pending
+	 * state, even in the failed case, an explicit smp_mb() must be used.
+	 */
+	smp_mb__before_atomic();
+	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
+		return false;
+
+	/*
+	 * The head is context local, there can be no concurrency.
+	 */
+	*head->lastp = node;
+	head->lastp = &node->next;
+	return true;
+}
+
+/**
+ * wake_q_add() - queue a wakeup for 'later' waking.
+ * @head: the wake_q_head to add @task to
+ * @task: the task to queue for 'later' wakeup
+ *
+ * Queue a task for later wakeup, most likely by the wake_up_q() call in the
+ * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
+ * instantly.
+ *
+ * This function must be used as-if it were wake_up_process(); IOW the task
+ * must be ready to be woken at this location.
+ */
+void wake_q_add(struct wake_q_head *head, struct task_struct *task)
+{
+	if (__wake_q_add(head, task))
+		get_task_struct(task);
+}
+
+/**
+ * wake_q_add_safe() - safely queue a wakeup for 'later' waking.
+ * @head: the wake_q_head to add @task to
+ * @task: the task to queue for 'later' wakeup
+ *
+ * Queue a task for later wakeup, most likely by the wake_up_q() call in the
+ * same context, _HOWEVER_ this is not guaranteed, the wakeup can come
+ * instantly.
+ *
+ * This function must be used as-if it were wake_up_process(); IOW the task
+ * must be ready to be woken at this location.
+ *
+ * This function is essentially a task-safe equivalent to wake_q_add(). Callers
+ * that already hold reference to @task can call the 'safe' version and trust
+ * wake_q to do the right thing depending whether or not the @task is already
+ * queued for wakeup.
+ */
+void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
+{
+	if (!__wake_q_add(head, task))
+		put_task_struct(task);
+}
+
+void wake_up_q(struct wake_q_head *head)
+{
+	struct wake_q_node *node = head->first;
+
+	while (node != WAKE_Q_TAIL) {
+		struct task_struct *task;
+
+		task = container_of(node, struct task_struct, wake_q);
+		BUG_ON(!task);
+		/* Task can safely be re-inserted now */
+		node = node->next;
+		task->wake_q.next = NULL;
+
+		/*
+		 * wake_up_process() executes a full barrier, which pairs with
+		 * the queueing in wake_q_add() so as not to miss wakeups.
+		 */
+		wake_up_process(task);
+		put_task_struct(task);
+	}
+}
+
+static inline void smp_sched_reschedule(int cpu)
+{
+	if (likely(cpu_online(cpu)))
+		smp_send_reschedule(cpu);
+}
+
+/*
+ * resched_task - mark a task 'to be rescheduled now'.
+ *
+ * On UP this means the setting of the need_resched flag, on SMP it
+ * might also involve a cross-CPU call to trigger the scheduler on
+ * the target CPU.
+ */
+#ifdef CONFIG_PREEMPT_DYNAMIC
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy);
+static __always_inline bool dynamic_preempt_lazy(void)
+{
+	return static_branch_unlikely(&sk_dynamic_preempt_lazy);
+}
+#else
+static __always_inline bool dynamic_preempt_lazy(void)
+{
+	return IS_ENABLED(CONFIG_PREEMPT_LAZY);
+}
+#endif
+
+static void resched_task(struct task_struct *p)
+{
+	int cpu;
+#ifdef CONFIG_LOCKDEP
+	/* Kernel threads call this when creating workqueues while still
+	 * inactive from __kthread_bind_mask, holding only the pi_lock */
+	if (!(p->flags & PF_KTHREAD)) {
+		struct rq *rq = task_rq(p);
+
+		lockdep_assert_held(rq->lock);
+	}
+#endif
+	if (test_tsk_need_resched(p))
+		return;
+
+	cpu = task_cpu(p);
+	if (cpu == smp_processor_id()) {
+		set_tsk_need_resched(p);
+		set_preempt_need_resched();
+		return;
+	}
+
+	if (set_nr_and_not_polling(p))
+		smp_sched_reschedule(cpu);
+	else
+		trace_sched_wake_idle_without_ipi(cpu);
+}
+
+/*
+ * A task that is not running or queued will not have a node set.
+ * A task that is queued but not running will have a node set.
+ * A task that is currently running will have ->on_cpu set but no node set.
+ */
+static inline bool task_queued(struct task_struct *p)
+{
+	return !skiplist_node_empty(&p->node);
+}
+
+static void enqueue_task(struct rq *rq, struct task_struct *p, int flags);
+static inline void resched_if_idle(struct rq *rq);
+
+static inline bool deadline_before(u64 deadline, u64 time)
+{
+	return (deadline < time);
+}
+
+/*
+ * Deadline is "now" in niffies + (offset by priority). Setting the deadline
+ * is the key to everything. It distributes cpu fairly amongst tasks of the
+ * same nice value, it proportions cpu according to nice level, it means the
+ * task that last woke up the longest ago has the earliest deadline, thus
+ * ensuring that interactive tasks get low latency on wake up. The CPU
+ * proportion works out to the square of the virtual deadline difference, so
+ * this equation will give nice 19 3% CPU compared to nice 0.
+ */
+static inline u64 prio_deadline_diff(int user_prio)
+{
+	return (prio_ratios[user_prio] * rr_interval * (MS_TO_NS(1) / 128));
+}
+
+static inline u64 task_deadline_diff(struct task_struct *p)
+{
+	return prio_deadline_diff(TASK_USER_PRIO(p));
+}
+
+static inline u64 static_deadline_diff(int static_prio)
+{
+	return prio_deadline_diff(USER_PRIO(static_prio));
+}
+
+static inline int longest_deadline_diff(void)
+{
+	return prio_deadline_diff(39);
+}
+
+static inline int ms_longest_deadline_diff(void)
+{
+	return NS_TO_MS(longest_deadline_diff());
+}
+
+#ifdef CONFIG_MUQSS_IOTIME
+/*
+ * Scale an I/O time penalty by nice level, exactly as prio_deadline_diff()
+ * scales rr_interval. A deadline is virtual time, already stretched by
+ * prio_ratios[] before it means anything; adding raw nanoseconds of device
+ * time to it mixes two different currencies. Passing the penalty through the
+ * same ratio makes it commensurate with the deadline it is added to, so a
+ * given amount of I/O costs a task the same share of its own allotment
+ * whatever its nice level, and a niced down task is pushed back further in
+ * absolute terms than a niced up one for the same device time.
+ *
+ * The multiply is 64 bit, as it is in prio_deadline_diff(). Wrapping it would
+ * need a penalty over U64_MAX / prio_ratios[39], which is some forty days of
+ * device occupancy charged to one task between two scheduling events, so it
+ * is not a case worth writing code for.
+ */
+static inline u64 prio_penalty_diff(int user_prio, u64 penalty)
+{
+	return penalty * prio_ratios[user_prio] / 128;
+}
+
+/*
+ * Idleprio tasks are charged at the highest nice level whatever nice value
+ * they happen to carry. SCHED_IDLEPRIO is a class beneath the whole nice
+ * range rather than a position within it, and a task there has said it should
+ * run only when nothing else wants the CPU. Scaling by its own nice would let
+ * an idleprio task sitting at nice -20 be charged the lightest rate of all
+ * for keeping the disk busy, which is backwards. Their deadlines already sort
+ * below everything else in enqueue_task(); this keeps what they are charged
+ * for I/O consistent with that.
+ */
+static inline u64 task_penalty_diff(struct task_struct *p, u64 penalty)
+{
+	if (idleprio_task(p))
+		return prio_penalty_diff(39, penalty);
+
+	return prio_penalty_diff(TASK_USER_PRIO(p), penalty);
+}
+#endif
+
+#ifdef CONFIG_MUQSS_IOTIME
+/*
+ * Take the block device time accumulated on this task's behalf since it was
+ * last charged, and convert it to an amount of virtual deadline to push the
+ * task back by. The debt is consumed as it is read, so each nanosecond of
+ * device time demotes the task exactly once no matter which caller gets to it
+ * first. The charge is added to the deadline, so it is a demotion, not a
+ * boost: a task that keeps a device busy for everybody else stops getting
+ * full CPU priority for free.
+ *
+ * Charged one for one, a nanosecond of deadline for each nanosecond of device
+ * time consumed. That rate says device time and CPU time cost a task the
+ * same, which is the premise of the feature rather than a setting within it,
+ * so there is nothing to weight one against the other with.
+ *
+ * The charge is then scaled by nice through task_penalty_diff(), the same way
+ * rr_interval is, so that it is in the same virtual currency as the deadline
+ * it is added to. One for one is therefore one for one in deadline terms
+ * rather than in raw nanoseconds: at nice 0 the ratio is prio_ratios[20] /
+ * 128, about 6.7, exactly as an ordinary nice 0 timeslice is worth 6.7
+ * rr_intervals of deadline.
+ *
+ * There is no ceiling. A task that keeps a device busy can be demoted
+ * arbitrarily far behind, past the deadline of the lowest nice level. See
+ * MuQSS-iotime-design.md.
+ *
+ * This has to be consumed from both time_slice_expired() and enqueue_task().
+ * Expiry alone misses the streaming reader, which blocks before exhausting its
+ * slice and so never refreshes its deadline. Enqueue alone is not enough
+ * either: time_slice_expired() assigns the deadline absolutely, resetting the
+ * task to an uncharged baseline, and two of its callers (sched_yield() and
+ * yield_to()) have no enqueue behind them to reapply the charge.
+ */
+static inline u64 consume_iotime_penalty(struct task_struct *p)
+{
+	u64 debt = atomic64_xchg(&p->io_debt_ns, 0);
+
+	if (!debt)
+		return 0;
+
+	return task_penalty_diff(p, debt);
+}
+
+/*
+ * The same for CPU time spent in another thread's context on this task's
+ * behalf, which today means kworkers running work items it queued.
+ *
+ * Charged one for one, as device time is. A nanosecond of CPU time is a
+ * nanosecond of CPU time wherever it was spent, and the whole premise
+ * here is that the thread it was spent in should not decide whether it counts.
+ * Work a task asks for in its own context is already charged at exactly that
+ * rate: update_cpu_clock_switch() and update_cpu_clock_tick() deduct it from
+ * ->time_slice with no regard for which mode it was spent in, so a task that
+ * burns its whole quantum inside a syscall is demoted as surely as one that
+ * spun in userspace. Charging deferred work at anything other than parity
+ * would say that where the kernel chose to do the work changes what it cost,
+ * which is the bug this exists to fix. Forcing IRQ threading, as -ck does,
+ * only moves more work into that blind spot.
+ *
+ * It is still scaled by nice through task_penalty_diff(), like every other
+ * addition to a deadline, so that it is in the same virtual currency as the
+ * deadline it lands on.
+ *
+ * As with iotime there is no ceiling. The runaway this invites is a different
+ * shape to the I/O case and worth naming, because here CPU consumption is
+ * being punished with CPU demotion: a demoted task need not stop generating
+ * the work, so it can be starved while kworkers keep charging it. It is
+ * bounded in practice because the debt a task can accrue is bounded by the
+ * work it queued, and a task demoted enough to stop running stops queueing
+ * more.
+ *
+ * Kept as a separate counter from io_debt_ns rather than folded into it,
+ * because the two are quantities of very different size: work item runtimes
+ * are microseconds where device occupancy is milliseconds, and sharing an
+ * accumulator would let one vanish into the rounding of the other. Separate
+ * also keeps them separable in /proc/<pid>/iotime when the question is why a
+ * task was demoted.
+ */
+static inline u64 consume_kerntime_penalty(struct task_struct *p)
+{
+	u64 debt = atomic64_xchg(&p->kern_debt_ns, 0);
+
+	if (!debt)
+		return 0;
+
+	return task_penalty_diff(p, debt);
+}
+#else
+static inline u64 consume_iotime_penalty(struct task_struct *p)
+{
+	return 0;
+}
+
+static inline u64 consume_kerntime_penalty(struct task_struct *p)
+{
+	return 0;
+}
+#endif
+
+/*
+ * Everything charged against a task's deadline that did not come out of its
+ * own time_slice.
+ */
+static inline u64 consume_task_penalty(struct task_struct *p)
+{
+	return consume_iotime_penalty(p) + consume_kerntime_penalty(p);
+}
+
+static inline bool rq_local(struct rq *rq);
+
+#ifndef SCHED_CAPACITY_SCALE
+#define SCHED_CAPACITY_SCALE 1024
+#endif
+
+static inline int rq_load(struct rq *rq)
+{
+	return rq->nr_running;
+}
+
+/*
+ * Update the load average for feeding into cpu frequency governors. Use a
+ * rough estimate of a rolling average with ~ time constant of 32ms.
+ * 80/128 ~ 0.63. * 80 / 32768 / 128 == * 5 / 262144
+ * Make sure a call to update_clocks has been made before calling this to get
+ * an updated rq->niffies.
+ */
+static void update_load_avg(struct rq *rq, unsigned int flags)
+{
+	long us_interval, load;
+
+	us_interval = NS_TO_US(rq->niffies - rq->load_update);
+	if (unlikely(us_interval <= 0))
+		return;
+
+	load = rq->load_avg - (rq->load_avg * us_interval * 5 / 262144);
+	if (unlikely(load < 0))
+		load = 0;
+	load += rq_load(rq) * SCHED_CAPACITY_SCALE * us_interval * 5 / 262144;
+	rq->load_avg = load;
+
+	rq->load_update = rq->niffies;
+	update_irq_load_avg(rq, 0);
+	if (likely(rq_local(rq)))
+		cpufreq_trigger(rq, flags);
+}
+
+#ifdef HAVE_SCHED_AVG_IRQ
+/*
+ * IRQ variant of update_load_avg below. delta is actually time in nanoseconds
+ * here so we scale curload to how long it's been since the last update.
+ */
+static void update_irq_load_avg(struct rq *rq, long delta)
+{
+	long us_interval, load;
+
+	us_interval = NS_TO_US(rq->niffies - rq->irq_load_update);
+	if (unlikely(us_interval <= 0))
+		return;
+
+	load = rq->irq_load_avg - (rq->irq_load_avg * us_interval * 5 / 262144);
+	if (unlikely(load < 0))
+		load = 0;
+	load += NS_TO_US(delta) * SCHED_CAPACITY_SCALE * 5 / 262144;
+	rq->irq_load_avg = load;
+
+	rq->irq_load_update = rq->niffies;
+}
+#endif
+
+static inline void update_best_key(struct rq *rq)
+{
+	WRITE_ONCE(rq->sl->best_key, rq->node->next[0]->key);
+}
+
+/*
+ * Removing from the runqueue. Enter with rq locked. Deleting a task
+ * from the skip list is done via the stored node reference in the task struct
+ * and does not require a full look up. Thus it occurs in O(k) time where k
+ * is the "level" of the list the task was stored at - usually 0, max 3.
+ */
+static void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
+{
+	/*
+	 * Everything below is accounting for @p having been on @rq, so none of
+	 * it may happen if it was not: a task whose node is unlinked is on no
+	 * runqueue at all and skiplist_delete() has left the list untouched.
+	 *
+	 * nr_running in particular is unsigned, so a single spurious decrement
+	 * wraps it to ~0 and the runqueue never looks empty again. That wedges
+	 * sched_cpu_wait_empty(), whose only exit is nr_running <= 1, and a CPU
+	 * offline then hangs forever with the dying CPU idle. psi_dequeue()
+	 * would likewise clear state that is counted somewhere else. Leaving
+	 * best_key alone is right for the same reason - the list did not
+	 * change.
+	 *
+	 * skiplist_delete() has already warned; this only keeps a lost race in
+	 * the caller from becoming permanent corruption of the runqueue.
+	 */
+	if (unlikely(!skiplist_delete(rq->sl, &p->node)))
+		return;
+
+	update_best_key(rq);
+	update_clocks(rq);
+
+	if (!(flags & DEQUEUE_SAVE)) {
+		sched_info_dequeue(rq, p);
+		/* Pass full flags: DEQUEUE_SLEEP vs migration clear all state. */
+		psi_dequeue(p, flags);
+	}
+	rq->nr_running--;
+	if (rt_task(p))
+		rq->rt_nr_running--;
+	update_load_avg(rq, flags);
+}
+
+#ifdef CONFIG_PREEMPT_RCU
+static bool rcu_read_critical(struct task_struct *p)
+{
+	return p->rcu_read_unlock_special.b.blocked;
+}
+#else /* CONFIG_PREEMPT_RCU */
+#define rcu_read_critical(p) (false)
+#endif /* CONFIG_PREEMPT_RCU */
+
+/*
+ * To determine if it's safe for a task of SCHED_IDLEPRIO to actually run as
+ * an idle task, we ensure none of the following conditions are met.
+ */
+static bool idleprio_suitable(struct task_struct *p)
+{
+	return (!(p->sched_contributes_to_load) && !(p->flags & (PF_EXITING)) &&
+		!signal_pending(p) && !rcu_read_critical(p) && !freezing(p));
+}
+
+/*
+ * To determine if a task of SCHED_ISO can run in pseudo-realtime, we check
+ * that the iso_refractory flag is not set.
+ */
+static inline bool isoprio_suitable(struct rq *rq)
+{
+	return !rq->iso_refractory;
+}
+
+static inline void inc_nr_running(struct rq *rq)
+{
+	rq->nr_running++;
+	if (trace_sched_update_nr_running_tp_enabled()) {
+		call_trace_sched_update_nr_running(rq, 1);
+	}
+}
+
+static inline void dec_nr_running(struct rq *rq)
+{
+	rq->nr_running--;
+	if (trace_sched_update_nr_running_tp_enabled()) {
+		call_trace_sched_update_nr_running(rq, -1);
+	}
+}
+
+/*
+ * A running task is off the skiplist, so its share of rt_nr_running is not
+ * maintained by the enqueue/dequeue pair; __schedule() hands that slot over on
+ * every switch instead. When a *running* task's priority crosses the rt
+ * boundary neither happens, so fix the count up here.
+ */
+static inline void rt_running_reprio(struct rq *rq, int oldprio, int newprio)
+{
+	if (!rt_prio(newprio) == !rt_prio(oldprio))
+		return;
+
+	if (rt_prio(newprio))
+		rq->rt_nr_running++;
+	else
+		rq->rt_nr_running--;
+}
+
+/*
+ * Adding to the runqueue. Enter with rq locked.
+ */
+static void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
+{
+	unsigned int randseed, cflags = 0;
+	u64 sl_id, penalty;
+
+	if (!rt_task(p)) {
+		/* Check it hasn't gotten rt from PI */
+		if ((idleprio_task(p) && idleprio_suitable(p)) ||
+		   (iso_task(p) && isoprio_suitable(rq)))
+			p->prio = p->normal_prio;
+		else
+			p->prio = NORMAL_PRIO;
+	} else
+		rq->rt_nr_running++;
+	/*
+	 * The sl_id key passed to the skiplist generates a sorted list.
+	 * Realtime and sched iso tasks run FIFO so they only need be sorted
+	 * according to priority. The skiplist will put tasks of the same
+	 * key inserted later in FIFO order. Tasks of sched normal, batch
+	 * and idleprio are sorted according to their deadlines. Idleprio
+	 * tasks are offset by an impossibly large deadline value ensuring
+	 * they get sorted into last positions, but still according to their
+	 * own deadlines. This creates a "landscape" of skiplists running
+	 * from priority 0 realtime in first place to the lowest priority
+	 * idleprio tasks last. Skiplist insertion is an O(log n) process.
+	 */
+	/*
+	 * Charge for device time and kernel work consumed since this task was
+	 * last queued. Consumed unconditionally so debt cannot accumulate
+	 * while a task is realtime, but only applied below, since realtime and
+	 * iso tasks sort by priority rather than by deadline and so are never
+	 * demoted.
+	 */
+	penalty = consume_task_penalty(p);
+
+	if (p->prio <= ISO_PRIO) {
+		sl_id = p->prio;
+	} else {
+		p->deadline += penalty;
+		sl_id = p->deadline;
+		if (idleprio_task(p)) {
+			if (p->prio == IDLE_PRIO)
+				sl_id |= 0xF000000000000000;
+			else
+				sl_id += longest_deadline_diff();
+		}
+	}
+	/*
+	 * Some architectures don't have better than microsecond resolution
+	 * so mask out ~microseconds as the random seed for skiplist insertion.
+	 */
+	update_clocks(rq);
+	if (!(flags & ENQUEUE_RESTORE)) {
+		sched_info_enqueue(rq, p);
+		/* Full flags so ENQUEUE_MIGRATED is visible to psi_enqueue(). */
+		psi_enqueue(p, flags);
+	}
+
+	randseed = (rq->niffies >> 10) & 0xFFFFFFFF;
+	skiplist_insert(rq->sl, &p->node, sl_id, randseed);
+	update_best_key(rq);
+	if (p->in_iowait)
+		cflags |= SCHED_CPUFREQ_IOWAIT;
+	inc_nr_running(rq);
+	update_load_avg(rq, cflags);
+}
+
+/*
+ * Returns the relative length of deadline all compared to the shortest
+ * deadline which is that of nice -20.
+ */
+static inline int task_prio_ratio(struct task_struct *p)
+{
+	return prio_ratios[TASK_USER_PRIO(p)];
+}
+
+/*
+ * task_timeslice - all tasks of all priorities get the exact same timeslice
+ * length. CPU distribution is handled by giving different deadlines to
+ * tasks of different priorities. Use 128 as the base value for fast shifts.
+ */
+static inline int task_timeslice(struct task_struct *p)
+{
+	return (rr_interval * task_prio_ratio(p) / 128);
+}
+
+#ifdef CONFIG_SMP
+/* Entered with rq locked */
+static inline void resched_if_idle(struct rq *rq)
+{
+	if (rq_idle(rq))
+		resched_task(rq->curr);
+}
+
+static inline bool rq_local(struct rq *rq)
+{
+	return (rq->cpu == smp_processor_id());
+}
+#ifdef CONFIG_SMT_NICE
+static const cpumask_t *thread_cpumask(int cpu);
+
+/* Find the best real time priority running on any SMT siblings of cpu and if
+ * none are running, the static priority of the best deadline task running.
+ * The lookups to the other runqueues is done lockless as the occasional wrong
+ * value would be harmless. */
+static int best_smt_bias(struct rq *this_rq)
+{
+	int other_cpu, best_bias = 0;
+
+	for_each_cpu(other_cpu, &this_rq->thread_mask) {
+		struct rq *rq = cpu_rq(other_cpu);
+
+		if (rq_idle(rq))
+			continue;
+		if (unlikely(!rq->online))
+			continue;
+		if (!rq->rq_mm)
+			continue;
+		if (likely(rq->rq_smt_bias > best_bias))
+			best_bias = rq->rq_smt_bias;
+	}
+	return best_bias;
+}
+
+static int task_prio_bias(struct task_struct *p)
+{
+	if (rt_task(p))
+		return 1 << 30;
+	else if (task_running_iso(p))
+		return 1 << 29;
+	else if (task_running_idle(p))
+		return 0;
+	return MAX_PRIO - p->static_prio;
+}
+
+static DEFINE_STATIC_KEY_FALSE(smt_nice_enabled);
+
+/* We've already decided p can run on CPU, now test if it shouldn't for SMT
+ * nice reasons. */
+static bool smt_should_schedule(struct task_struct *p, struct rq *this_rq)
+{
+	int best_bias, task_bias;
+
+	if (!idleprio_suitable(p))
+		return true;
+	best_bias = best_smt_bias(this_rq);
+	/* The smt siblings are all idle or running IDLEPRIO */
+	if (best_bias < 1)
+		return true;
+	task_bias = task_prio_bias(p);
+	if (task_bias < 1)
+		return false;
+	if (task_bias >= best_bias)
+		return true;
+	/* Dither 25% cpu of normal tasks regardless of nice difference */
+	if (best_bias % 4 == 1)
+		return true;
+	/* Sorry, you lose */
+	return false;
+}
+
+static inline bool smt_schedule(struct task_struct *p, struct rq *this_rq)
+{
+	if (!static_branch_unlikely(&smt_nice_enabled))
+		return true;
+	/* Kernel threads and RT tasks always run */
+	if (unlikely(!p->mm) || rt_task(p))
+		return true;
+	return smt_should_schedule(p, this_rq);
+}
+#else /* CONFIG_SMT_NICE */
+#define smt_schedule(p, this_rq) (true)
+#endif /* CONFIG_SMT_NICE */
+
+static inline void atomic_set_cpu(int cpu, cpumask_t *cpumask)
+{
+	set_bit(cpu, (volatile unsigned long *)cpumask);
+}
+
+/*
+ * The cpu_idle_map stores a bitmap of all the CPUs currently idle to
+ * allow easy lookup of whether any suitable idle CPUs are available.
+ * It's cheaper to maintain a binary yes/no if there are any idle CPUs on the
+ * idle_cpus variable than to do a full bitmask check when we are busy. The
+ * bits are set atomically but read locklessly as occasional false positive /
+ * negative is harmless.
+ */
+static inline void set_cpuidle_map(int cpu)
+{
+	/*
+	 * Only an active CPU may advertise itself as a wakeup target.
+	 * select_best_cpu() returns resched_best_idle()'s pick directly,
+	 * and that pick comes straight out of this map without consulting
+	 * is_cpu_allowed(), so a CPU on its way down (online but no longer
+	 * active) would keep re-arming itself here every time it idled and
+	 * collect fresh wakeups that sched_cpu_wait_empty() has to chase.
+	 * set_rq_offline() clears the map; this keeps it clear.
+	 */
+	if (likely(cpu_active(cpu)))
+		atomic_set_cpu(cpu, &cpu_idle_map);
+}
+
+static inline void atomic_clear_cpu(int cpu, cpumask_t *cpumask)
+{
+	clear_bit(cpu, (volatile unsigned long *)cpumask);
+}
+
+static inline void clear_cpuidle_map(int cpu)
+{
+	atomic_clear_cpu(cpu, &cpu_idle_map);
+}
+
+static bool suitable_idle_cpus(struct task_struct *p)
+{
+	return (cpumask_intersects(p->cpus_ptr, &cpu_idle_map));
+}
+
+/*
+ * Resched current on rq. We don't know if rq is local to this CPU nor if it
+ * is locked so we do not use an intermediate variable for the task to avoid
+ * having it dereferenced.
+ */
+static void resched_curr(struct rq *rq)
+{
+	int cpu;
+
+	if (test_tsk_need_resched(rq->curr))
+		return;
+
+	rq->preempt = rq->curr;
+	cpu = rq->cpu;
+
+	/* We're doing this without holding the rq lock if it's not task_rq */
+
+	if (cpu == smp_processor_id()) {
+		set_tsk_need_resched(rq->curr);
+		set_preempt_need_resched();
+		return;
+	}
+
+	if (set_nr_and_not_polling(rq->curr))
+		smp_sched_reschedule(cpu);
+	else
+		trace_sched_wake_idle_without_ipi(cpu);
+}
+
+#define CPUIDLE_NO_SIBLING      (1)
+#define CPUIDLE_DIFF_THREAD     (2)
+#define CPUIDLE_DIFF_CORE_LLC   (4)
+#define CPUIDLE_DIFF_CORE       (8)
+#define CPUIDLE_CACHE_BUSY      (16)
+#define CPUIDLE_DIFF_CPU        (32)
+#define CPUIDLE_THREAD_BUSY     (64)
+#define CPUIDLE_DIFF_NODE       (128)
+
+/*
+ * The best idle CPU is chosen according to the CPUIDLE ranking above where the
+ * lowest value would give the most suitable CPU to schedule p onto next. The
+ * order works out to be the following:
+ *
+ * Same thread, idle or busy cache, idle or busy threads
+ * Other core, same cache, idle or busy cache, idle threads.
+ * Same node, other CPU, idle cache, idle threads.
+ * Same node, other CPU, busy cache, idle threads.
+ * Other core, same cache, busy threads.
+ * Same node, other CPU, busy threads.
+ * Other node, other CPU, idle cache, idle threads.
+ * Other node, other CPU, busy cache, idle threads.
+ * Other node, other CPU, busy threads.
+ *
+ * CPUIDLE_NO_SIBLING is the least significant rank so it only separates cores
+ * that are otherwise equal, preferring a core with an idle SMT sibling over one
+ * with no siblings at all. On hybrid CPUs that indirectly prefers P cores over
+ * E cores, since only the former have siblings. A core whose sibling is busy
+ * still ranks below a core with no siblings as it only offers half a core.
+ * This does not treat CPUs with a single offline sibling as idle for
+ * simplicity.
+ */
+static int best_mask_cpu(int best_cpu, struct rq *rq, cpumask_t *tmpmask)
+{
+	int best_ranking = CPUIDLE_DIFF_NODE | CPUIDLE_THREAD_BUSY |
+		CPUIDLE_DIFF_CPU | CPUIDLE_CACHE_BUSY | CPUIDLE_DIFF_CORE |
+		CPUIDLE_DIFF_CORE_LLC | CPUIDLE_DIFF_THREAD | CPUIDLE_NO_SIBLING;
+	unsigned long best_idle_jiffy = 0;
+	int cpu_tmp;
+
+	if (cpumask_test_cpu(best_cpu, tmpmask))
+		goto out;
+
+	for_each_cpu(cpu_tmp, tmpmask) {
+		int ranking, locality;
+		struct rq *tmp_rq;
+
+		ranking = 0;
+		tmp_rq = cpu_rq(cpu_tmp);
+
+		locality = rq->cpu_locality[cpu_tmp];
+#ifdef CONFIG_NUMA
+		if (locality > LOCALITY_SMP)
+			ranking |= CPUIDLE_DIFF_NODE;
+		else
+#endif
+			if (locality > LOCALITY_MC)
+				ranking |= CPUIDLE_DIFF_CPU;
+#ifdef CONFIG_SCHED_MC
+			else if (locality == LOCALITY_MC_LLC)
+				ranking |= CPUIDLE_DIFF_CORE_LLC;
+			else if (locality == LOCALITY_MC)
+				ranking |= CPUIDLE_DIFF_CORE;
+		if (!(tmp_rq->cache_idle(tmp_rq)))
+			ranking |= CPUIDLE_CACHE_BUSY;
+#endif
+#ifdef CONFIG_SCHED_SMT
+		if (locality == LOCALITY_SMT)
+			ranking |= CPUIDLE_DIFF_THREAD;
+		if (!tmp_rq->has_smt_sibling)
+			ranking |= CPUIDLE_NO_SIBLING;
+		else if (!cpumask_subset(&tmp_rq->thread_mask, &cpu_idle_map))
+			ranking |= CPUIDLE_THREAD_BUSY;
+#endif
+		/* Also look for the most recently idled CPU as it will likely
+		 * be still at a higher CPU frequency */
+		if (ranking < best_ranking ||
+		    (ranking == best_ranking && tmp_rq->idle_jiffy > best_idle_jiffy)) {
+			best_cpu = cpu_tmp;
+			best_ranking = ranking;
+			best_idle_jiffy = tmp_rq->idle_jiffy;
+		}
+	}
+out:
+	return best_cpu;
+}
+
+bool cpus_share_cache(int this_cpu, int that_cpu)
+{
+	struct rq *this_rq = cpu_rq(this_cpu);
+
+	return (this_rq->cpu_locality[that_cpu] < LOCALITY_SMP);
+}
+
+/* As per resched_curr but only will resched idle task */
+static inline void resched_idle(struct rq *rq)
+{
+	if (test_tsk_need_resched(rq->idle))
+		return;
+
+	rq->preempt = rq->idle;
+
+	if (rq_local(rq)) {
+		set_tsk_need_resched(rq->idle);
+		set_preempt_need_resched();
+		return;
+	}
+
+	/*
+	 * Atomically set NEED_RESCHED and only IPI if the idle task is not
+	 * polling — same protocol as resched_curr / wake_up_idle_cpu.
+	 */
+	if (set_nr_and_not_polling(rq->idle))
+		smp_sched_reschedule(rq->cpu);
+	else
+		trace_sched_wake_idle_without_ipi(rq->cpu);
+}
+
+DEFINE_PER_CPU(cpumask_t, idlemask);
+
+static struct rq *resched_best_idle(struct task_struct *p, int cpu)
+{
+	cpumask_t *tmpmask = &(per_cpu(idlemask, cpu));
+	struct rq *rq;
+	int best_cpu;
+
+	cpumask_and(tmpmask, p->cpus_ptr, &cpu_idle_map);
+	best_cpu = best_mask_cpu(cpu, task_rq(p), tmpmask);
+	rq = cpu_rq(best_cpu);
+	if (!smt_schedule(p, rq))
+		return NULL;
+	rq->preempt = p;
+	resched_idle(rq);
+	return rq;
+}
+
+static inline void resched_suitable_idle(struct task_struct *p)
+{
+	if (suitable_idle_cpus(p))
+		resched_best_idle(p, task_cpu(p));
+}
+
+static inline struct rq *rq_order(struct rq *rq, int cpu)
+{
+	return rq->rq_order[cpu];
+}
+#else /* CONFIG_SMP */
+static inline void set_cpuidle_map(int cpu)
+{
+}
+
+static inline void clear_cpuidle_map(int cpu)
+{
+}
+
+static inline bool suitable_idle_cpus(struct task_struct *p)
+{
+	return uprq->curr == uprq->idle;
+}
+
+static inline void resched_suitable_idle(struct task_struct *p)
+{
+}
+
+static inline void resched_curr(struct rq *rq)
+{
+	resched_task(rq->curr);
+}
+
+static inline void resched_if_idle(struct rq *rq)
+{
+}
+
+static inline bool rq_local(struct rq *rq)
+{
+	return true;
+}
+
+static inline struct rq *rq_order(struct rq *rq, int cpu)
+{
+	return rq;
+}
+
+static inline bool smt_schedule(struct task_struct *p, struct rq *rq)
+{
+	return true;
+}
+
+/* One CPU shares its cache with itself. */
+bool cpus_share_cache(int this_cpu, int that_cpu)
+{
+	return true;
+}
+#endif /* CONFIG_SMP */
+
+static inline int normal_prio(struct task_struct *p)
+{
+	if (has_rt_policy(p))
+		return MAX_RT_PRIO - 1 - p->rt_priority;
+	if (idleprio_task(p))
+		return IDLE_PRIO;
+	if (iso_task(p))
+		return ISO_PRIO;
+	return NORMAL_PRIO;
+}
+
+/*
+ * Calculate the current priority, i.e. the priority
+ * taken into account by the scheduler. This value might
+ * be boosted by RT tasks as it will be RT if the task got
+ * RT-boosted. If not then it returns p->normal_prio.
+ */
+static int effective_prio(struct task_struct *p)
+{
+	p->normal_prio = normal_prio(p);
+	/*
+	 * If we are RT tasks or we were boosted to RT priority,
+	 * keep the priority unchanged. Otherwise, update priority
+	 * to the normal priority:
+	 */
+	if (!rt_prio(p->prio))
+		return p->normal_prio;
+	return p->prio;
+}
+
+/*
+ * activate_task - move a task to the runqueue. Enter with rq locked.
+ */
+static void activate_task(struct rq *rq, struct task_struct *p, int flags)
+{
+	resched_if_idle(rq);
+
+	/* Sleep profiling (SLEEP_PROFILING) was removed upstream. */
+
+	p->prio = effective_prio(p);
+	enqueue_task(rq, p, flags);
+	p->on_rq = TASK_ON_RQ_QUEUED;
+}
+
+/*
+ * deactivate_task - If it's running, it's not on the runqueue and we can just
+ * decrement the nr_running. Enter with rq locked.
+ */
+static inline void deactivate_task(struct task_struct *p, struct rq *rq)
+{
+	p->on_rq = 0;
+	sched_info_dequeue(rq, p);
+	/* deactivate_task is always DEQUEUE_SLEEP in muqss */
+	psi_dequeue(p, DEQUEUE_SLEEP);
+}
+
+/*
+ * PSI counts the state a task is in per CPU, indexed by task_cpu(), so
+ * whenever task_cpu() changes for a task that still carries state, that state
+ * has to be moved with it. That is TSK_RUNNING for a task being taken to
+ * another runqueue, and TSK_IOWAIT for one that is still asleep. Whoever
+ * clears them next does so against task_cpu(p), and if that is no longer the
+ * CPU they were counted on the per-CPU counter underflows.
+ *
+ * TSK_ONCPU is deliberately not moved. It belongs to psi_sched_switch(), and
+ * a task whose CPU is being changed here is by definition not on one.
+ */
+#ifdef CONFIG_PSI
+static inline unsigned int psi_migrate_begin(struct task_struct *p)
+{
+	unsigned int migrate = p->psi_flags & ~TSK_ONCPU;
+
+	if (migrate)
+		psi_task_change(p, migrate, 0);
+	return migrate;
+}
+
+static inline void psi_migrate_end(struct task_struct *p, unsigned int migrate)
+{
+	if (migrate)
+		psi_task_change(p, 0, migrate);
+}
+#else /* !CONFIG_PSI */
+static inline unsigned int psi_migrate_begin(struct task_struct *p)
+{
+	return 0;
+}
+
+static inline void psi_migrate_end(struct task_struct *p, unsigned int migrate)
+{
+}
+#endif /* CONFIG_PSI */
+
+#ifdef CONFIG_SMP
+void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
+{
+	unsigned int migrate;
+	struct rq *rq;
+
+	if (task_cpu(p) == new_cpu)
+		return;
+
+	/* Do NOT call set_task_cpu on a currently queued task as we will not
+	 * be reliably holding the rq lock after changing CPU. */
+	BUG_ON(task_queued(p));
+	rq = task_rq(p);
+
+#ifdef CONFIG_LOCKDEP
+	/*
+	 * The caller should hold either p->pi_lock or rq->lock, when changing
+	 * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
+	 *
+	 * Furthermore, all task_rq users should acquire both locks, see
+	 * task_rq_lock().
+	 */
+	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
+				      lockdep_is_held(rq->lock)));
+#endif
+
+	trace_sched_migrate_task(p, new_cpu);
+	rseq_sched_set_ids_changed(p);
+	perf_event_task_migrate(p);
+
+	/*
+	 * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be
+	 * successfully executed on another CPU. We must ensure that updates of
+	 * per-task data have been completed by this moment.
+	 */
+	smp_wmb();
+
+	p->wake_cpu = new_cpu;
+
+	if (task_running(rq, p)) {
+		/*
+		 * We should only be calling this on a running task if we're
+		 * holding rq lock.
+		 */
+		lockdep_assert_held(rq->lock);
+
+		/*
+		 * We can't change the task_thread_info CPU on a running task
+		 * as p will still be protected by the rq lock of the CPU it
+		 * is still running on so we only set the wake_cpu for it to be
+		 * lazily updated once off the CPU.
+		 */
+		return;
+	}
+
+	migrate = psi_migrate_begin(p);
+	WRITE_ONCE(task_thread_info(p)->cpu, new_cpu);
+	psi_migrate_end(p, migrate);
+	/* We're no longer protecting p after this point since we're holding
+	 * the wrong runqueue lock. */
+}
+#endif /* CONFIG_SMP */
+
+/*
+ * Move a task off the runqueue and take it to a cpu for it will
+ * become the running task.
+ */
+static inline void take_task(struct rq *rq, int cpu, struct task_struct *p)
+{
+	struct rq *p_rq = task_rq(p);
+
+	dequeue_task(p_rq, p, DEQUEUE_SAVE);
+	if (p_rq != rq) {
+		sched_info_dequeue(p_rq, p);
+		sched_info_enqueue(rq, p);
+	}
+	/*
+	 * DEQUEUE_SAVE left TSK_RUNNING in place, and set_task_cpu() moves it
+	 * to @cpu along with the task.
+	 */
+	set_task_cpu(p, cpu);
+}
+
+/*
+ * Returns a descheduling task to the runqueue unless it is being
+ * deactivated.
+ */
+static inline void return_task(struct task_struct *p, struct rq *rq,
+			       int cpu, bool deactivate)
+{
+	if (deactivate)
+		deactivate_task(p, rq);
+	else {
+#ifdef CONFIG_SMP
+		/*
+		 * set_task_cpu was called on the running task that doesn't
+		 * want to deactivate so it has to be enqueued to a different
+		 * CPU and we need its lock. Tag it to be moved with as the
+		 * lock is dropped in finish_lock_switch.
+		 */
+		if (unlikely(p->wake_cpu != cpu))
+			WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
+		else
+#endif
+			enqueue_task(rq, p, ENQUEUE_RESTORE);
+	}
+}
+
+/* Enter with rq lock held. We know p is on the local cpu */
+static inline void __set_tsk_resched(struct task_struct *p)
+{
+	set_tsk_need_resched(p);
+	set_preempt_need_resched();
+}
+
+/**
+ * task_curr - is this task currently executing on a CPU?
+ * @p: the task in question.
+ *
+ * Return: 1 if the task is currently executing. 0 otherwise.
+ */
+inline int task_curr(const struct task_struct *p)
+{
+	return cpu_curr(task_cpu(p)) == p;
+}
+
+static __always_inline
+int __task_state_match(struct task_struct *p, unsigned int state)
+{
+	if (READ_ONCE(p->__state) & state)
+		return 1;
+
+	if (READ_ONCE(p->saved_state) & state)
+		return -1;
+
+	return 0;
+}
+
+static __always_inline
+int task_state_match(struct task_struct *p, unsigned int state)
+{
+	int match;
+
+	/*
+	 * Serialize against current_save_and_set_rtlock_wait_state(),
+	 * current_restore_rtlock_saved_state(), and __refrigerator().
+	 */
+	raw_spin_lock_irq(&p->pi_lock);
+	match = __task_state_match(p, state);
+	raw_spin_unlock_irq(&p->pi_lock);
+
+	return match;
+}
+
+/*
+ * wait_task_inactive - wait for a thread to unschedule.
+ *
+ * Wait for the thread to block in any of the states set in @match_state.
+ * If it changes, i.e. @p might have woken up, then return zero.  When we
+ * succeed in waiting for @p to be off its CPU, we return a positive number
+ * (its total switch count).  If a second call a short while later returns
+ * the same number, the caller can be sure that @p has remained unscheduled
+ * the whole time.
+ *
+ * The caller must ensure that the task *will* unschedule sometime soon,
+ * else this function might spin for a *long* time. This function can't
+ * be called with interrupts off, or it may introduce deadlock with
+ * smp_call_function() if an IPI is sent by the same process we are
+ * waiting to become inactive.
+ */
+unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
+{
+	int running, queued, match;
+	struct rq_flags rf;
+	unsigned long ncsw;
+	struct rq *rq;
+
+	for (;;) {
+		rq = task_rq(p);
+
+		/*
+		 * If the task is actively running on another CPU
+		 * still, just relax and busy-wait without holding
+		 * any locks.
+		 *
+		 * NOTE! Since we don't hold any locks, it's not
+		 * even sure that "rq" stays as the right runqueue!
+		 * But we don't care, since this will return false
+		 * if the runqueue has changed and p is actually now
+		 * running somewhere else!
+		 */
+		while (task_running(rq, p)) {
+			if (!task_state_match(p, match_state))
+				return 0;
+			cpu_relax();
+		}
+
+		/*
+		 * Ok, time to look more closely! We need the rq
+		 * lock now, to be *sure*. If we're wrong, we'll
+		 * just go back and repeat.
+		 */
+		rq = task_rq_lock(p, &rf);
+		trace_sched_wait_task(p);
+		running = task_running(rq, p);
+		queued = task_on_rq_queued(p);
+		ncsw = 0;
+		if ((match = __task_state_match(p, match_state))) {
+			/*
+			 * When matching on p->saved_state, consider this task
+			 * still queued so it will wait.
+			 */
+			if (match < 0)
+				queued = 1;
+			ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
+		}
+		task_rq_unlock(rq, p, &rf);
+
+		/*
+		 * If it changed from the expected state, bail out now.
+		 */
+		if (unlikely(!ncsw))
+			break;
+
+		/*
+		 * Was it really running after all now that we
+		 * checked with the proper locks actually held?
+		 *
+		 * Oops. Go back and try again..
+		 */
+		if (unlikely(running)) {
+			cpu_relax();
+			continue;
+		}
+
+		/*
+		 * It's not enough that it's not actively running,
+		 * it must be off the runqueue _entirely_, and not
+		 * preempted!
+		 *
+		 * So if it was still runnable (but just not actively
+		 * running right now), it's preempted, and we should
+		 * yield - it could be a while.
+		 */
+		if (unlikely(queued)) {
+			ktime_t to = NSEC_PER_SEC / HZ;
+
+			set_current_state(TASK_UNINTERRUPTIBLE);
+			schedule_hrtimeout(&to, HRTIMER_MODE_REL);
+			continue;
+		}
+
+		/*
+		 * Ahh, all good. It wasn't running, and it wasn't
+		 * runnable, which means that it will never become
+		 * running in the future either. We're all done!
+		 */
+		break;
+	}
+
+	return ncsw;
+}
+
+/***
+ * kick_process - kick a running thread to enter/exit the kernel
+ * @p: the to-be-kicked thread
+ *
+ * Cause a process which is running on another CPU to enter
+ * kernel-mode, without any delay. (to get signals handled.)
+ *
+ * NOTE: this function doesn't have to take the runqueue lock,
+ * because all it wants to ensure is that the remote task enters
+ * the kernel. If the IPI races and the task has been migrated
+ * to another CPU then no harm is done and the purpose has been
+ * achieved as well.
+ */
+#ifdef CONFIG_SMP
+void kick_process(struct task_struct *p)
+{
+	int cpu;
+
+	preempt_disable();
+	cpu = task_cpu(p);
+	if ((cpu != smp_processor_id()) && task_curr(p))
+		smp_sched_reschedule(cpu);
+	preempt_enable();
+}
+#else /* !CONFIG_SMP */
+/* @p can only be running on the CPU we are already running on. */
+void kick_process(struct task_struct *p)
+{
+}
+#endif /* CONFIG_SMP */
+EXPORT_SYMBOL_GPL(kick_process);
+
+/*
+ * RT tasks preempt purely on priority. SCHED_NORMAL tasks preempt on the
+ * basis of earlier deadlines. SCHED_IDLEPRIO don't preempt anything else or
+ * between themselves, they cooperatively multitask. An idle rq scores as
+ * prio PRIO_LIMIT so it is always preempted.
+ */
+static inline bool
+can_preempt(struct task_struct *p, int prio, u64 deadline)
+{
+	/* Better static priority RT task or better policy preemption */
+	if (p->prio < prio)
+		return true;
+	if (p->prio > prio)
+		return false;
+	if (p->policy == SCHED_BATCH)
+		return false;
+	/* SCHED_NORMAL and ISO will preempt based on deadline */
+	if (!deadline_before(p->deadline, deadline))
+		return false;
+	return true;
+}
+
+#ifdef CONFIG_SMP
+
+/*
+ * Per-CPU kthreads are allowed to run on !active && online CPUs, see
+ * __set_cpus_allowed_ptr().
+ */
+static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
+{
+	if (!cpumask_test_cpu(cpu, p->cpus_ptr))
+		return false;
+
+	/* migrate_disable() must be allowed to finish on an online CPU. */
+	if (is_migration_disabled(p))
+		return cpu_online(cpu);
+
+	if (!(p->flags & PF_KTHREAD))
+		return cpu_active(cpu);
+
+	/* KTHREAD_IS_PER_CPU is always allowed. */
+	if (kthread_is_per_cpu(p))
+		return cpu_online(cpu);
+
+	/* Regular kernel threads don't get to stay during offline. */
+	if (cpu_dying(cpu))
+		return false;
+
+	/* But are allowed during online. */
+	return cpu_online(cpu);
+}
+
+/*
+ * Check to see if p can run on cpu, and if not, whether there are any online
+ * CPUs it can run on instead. This only happens with the hotplug threads that
+ * bring up the CPUs.
+ */
+static inline bool sched_other_cpu(struct task_struct *p, int cpu)
+{
+	if (unlikely(is_migration_disabled(p) && task_cpu(p) != cpu))
+		return true;
+	/*
+	 * Defer to the single placement predicate: userspace is refused a
+	 * deactivated CPU, a regular kthread is refused a dying one, while
+	 * per-CPU kthreads and a migrate_disable() pin on this CPU still
+	 * may run there.
+	 */
+	if (likely(cpumask_test_cpu(cpu, p->cpus_ptr)))
+		return !is_cpu_allowed(p, cpu);
+	if (p->nr_cpus_allowed == 1) {
+		cpumask_t valid_mask;
+
+		/*
+		 * Nowhere left to send it, so it may as well run here. The mask
+		 * has to be the same one valid_task_cpu() uses to pick the
+		 * destination, or the two disagree and the task is placed on a
+		 * CPU that then refuses to run it.
+		 *
+		 * That is what a regular kthread affine to a single dying CPU
+		 * hits: the CPU is still online, so intersecting with
+		 * cpu_online_mask alone leaves it non-empty and this escape
+		 * hatch never fires, while is_cpu_allowed() has already refused
+		 * the dying CPU itself. It ends up runnable with no CPU willing
+		 * to pick it. bind_zero() would free it by overriding the
+		 * affinity, but that only runs from sched_cpu_wait_empty(),
+		 * several hotplug states later - and anything the CPU going
+		 * down waits for in between, such as blk_mq_hctx_notify_offline()
+		 * waiting on a threaded completion interrupt, deadlocks against
+		 * it.
+		 */
+		cpumask_and(&valid_mask, p->cpus_ptr, cpu_online_mask);
+		if (!kthread_is_per_cpu(p))
+			cpumask_andnot(&valid_mask, &valid_mask, cpu_dying_mask);
+		if (unlikely(cpumask_empty(&valid_mask)))
+			return false;
+	}
+	return true;
+}
+
+static inline bool needs_other_cpu(struct task_struct *p, int cpu)
+{
+	if (unlikely(is_migration_disabled(p)))
+		return task_cpu(p) != cpu;
+	return !is_cpu_allowed(p, cpu);
+}
+
+/*
+ * Pin p->cpus_ptr to this CPU so EDT / select_best_cpu cannot steal a
+ * preempted migrate_disable() task. Called under rq->lock on prev.
+ */
+static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
+{
+	if (likely(!p->migration_disabled))
+		return;
+	if (p->cpus_ptr != &p->cpus_mask)
+		return;
+	p->cpus_ptr = cpumask_of(cpu_of(rq));
+}
+
+#define cpu_online_map		(*(cpumask_t *)cpu_online_mask)
+
+static void try_preempt(struct task_struct *p, struct rq *this_rq)
+{
+	int i, this_entries = rq_load(this_rq);
+	cpumask_t tmp;
+
+	/*
+	 * An idle CPU first: waking one costs less than bouncing a task
+	 * that is already running, and it is what keeps work spread out.
+	 * Only if none is available do we consider preemption, and the
+	 * loop below starts at cpu_order[0] - this_rq - so @p still gets
+	 * to preempt its own dest before any remote runqueue.
+	 */
+	if (suitable_idle_cpus(p) && resched_best_idle(p, task_cpu(p)))
+		return;
+
+	/* IDLEPRIO tasks never preempt anything but idle */
+	if (p->policy == SCHED_IDLEPRIO)
+		return;
+
+	cpumask_and(&tmp, &cpu_online_map, p->cpus_ptr);
+
+	for (i = 0; i < num_online_cpus(); i++) {
+		struct rq *rq = this_rq->cpu_order[i];
+
+		if (!cpumask_test_cpu(rq->cpu, &tmp))
+			continue;
+
+		if (!sched_interactive && rq != this_rq && rq_load(rq) <= this_entries)
+			continue;
+		if (smt_schedule(p, rq) && can_preempt(p, rq->rq_prio, rq->rq_deadline)) {
+			/* We set rq->preempting lockless, it's a hint only */
+			rq->preempting = p;
+			resched_curr(rq);
+			return;
+		}
+	}
+}
+
+static int __set_cpus_allowed_ptr(struct task_struct *p,
+				  const struct cpumask *new_mask,
+				 u32 flags);
+#else /* CONFIG_SMP */
+static inline bool needs_other_cpu(struct task_struct *p, int cpu)
+{
+	return false;
+}
+
+static void try_preempt(struct task_struct *p, struct rq *this_rq)
+{
+	if (p->policy == SCHED_IDLEPRIO)
+		return;
+	if (can_preempt(p, uprq->rq_prio, uprq->rq_deadline))
+		resched_curr(uprq);
+}
+
+static inline int __set_cpus_allowed_ptr(struct task_struct *p,
+					 const struct cpumask *new_mask,
+					 u32 __always_unused flags)
+{
+	return set_cpus_allowed_ptr(p, new_mask);
+}
+#endif /* CONFIG_SMP */
+
+static void
+ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
+{
+	struct rq *rq;
+
+	if (!schedstat_enabled())
+		return;
+
+	rq = this_rq();
+
+#ifdef CONFIG_SMP
+	if (cpu == rq->cpu) {
+		__schedstat_inc(rq->ttwu_local);
+	} else {
+		struct sched_domain *sd;
+
+		rcu_read_lock();
+		for_each_domain(rq->cpu, sd) {
+			if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
+				__schedstat_inc(sd->ttwu_wake_remote);
+				break;
+			}
+		}
+		rcu_read_unlock();
+	}
+
+#endif /* CONFIG_SMP */
+
+	__schedstat_inc(rq->ttwu_count);
+}
+
+/*
+ * Mark the task runnable and perform wakeup-preemption.
+ */
+static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags)
+{
+	/*
+	 * WF_SYNC means the waker will leave the CPU shortly. Prefer an
+	 * idle CPU and do not preempt the waker itself — it will pick the
+	 * wakee up on the next schedule(). A remote dest still needs
+	 * kicking, or the wakee sits until dest's next tick (or forever
+	 * on a nohz_full hog).
+	 */
+	if ((wake_flags & WF_SYNC) && rq == this_rq())
+		resched_suitable_idle(p);
+	else
+		try_preempt(p, rq);
+	WRITE_ONCE(p->__state, TASK_RUNNING);
+	trace_sched_wakeup(p);
+}
+
+static void
+ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags)
+{
+	int en_flags = ENQUEUE_WAKEUP;
+
+	lockdep_assert_held(rq->lock);
+
+	if (p->sched_contributes_to_load)
+		rq->nr_uninterruptible--;
+
+#ifdef CONFIG_SMP
+	if (wake_flags & WF_MIGRATED)
+		en_flags |= ENQUEUE_MIGRATED;
+	else
+#endif
+	if (p->in_iowait) {
+		delayacct_blkio_end(p);
+		atomic_dec(&task_rq(p)->nr_iowait);
+	}
+
+	activate_task(rq, p, en_flags);
+	ttwu_do_wakeup(rq, p, wake_flags);
+}
+
+/*
+ * Consider @p being inside a wait loop:
+ *
+ *   for (;;) {
+ *      set_current_state(TASK_UNINTERRUPTIBLE);
+ *
+ *      if (CONDITION)
+ *         break;
+ *
+ *      schedule();
+ *   }
+ *   __set_current_state(TASK_RUNNING);
+ *
+ * between set_current_state() and schedule(). In this case @p is still
+ * runnable, so all that needs doing is change p->__state back to TASK_RUNNING in
+ * an atomic manner.
+ *
+ * By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
+ * then schedule() must still happen and p->__state can be changed to
+ * TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
+ * need to do a full wakeup with enqueue.
+ *
+ * Returns: %true when the wakeup is done,
+ *          %false otherwise.
+ */
+static int ttwu_runnable(struct task_struct *p, int wake_flags)
+{
+	struct rq *rq;
+	int ret = 0;
+
+	rq = __task_rq_lock(p, NULL);
+	if (likely(task_on_rq_queued(p))) {
+		ttwu_do_wakeup(rq, p, wake_flags);
+		ret = 1;
+	}
+	__task_rq_unlock(rq, p, NULL);
+
+	return ret;
+}
+
+#ifdef CONFIG_SMP
+void sched_ttwu_pending(void *arg)
+{
+	struct llist_node *llist = arg;
+	struct rq *rq = this_rq();
+	struct task_struct *p, *t;
+	struct rq_flags rf;
+
+	if (!llist)
+		return;
+
+	/*
+	 * rq::ttwu_pending racy indication of out-standing wakeups.
+	 * Races such that false-negatives are possible, since they
+	 * are shorter lived that false-positives would be.
+	 */
+	WRITE_ONCE(rq->ttwu_pending, 0);
+
+	rq_lock_irqsave(rq, &rf);
+
+	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
+		if (WARN_ON_ONCE(p->on_cpu))
+			smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+		if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
+			set_task_cpu(p, cpu_of(rq));
+
+		ttwu_do_activate(rq, p, p->sched_remote_wakeup ? WF_MIGRATED : 0);
+	}
+
+	rq_unlock_irqrestore(rq, &rf);
+}
+
+/*
+ * Called from smp.c: report whether the caller still needs to send the IPI,
+ * having woken a polling idle task in passing.
+ */
+bool call_function_single_prep_ipi(int cpu)
+{
+	if (set_nr_if_polling(cpu_rq(cpu)->idle)) {
+		trace_sched_wake_idle_without_ipi(cpu);
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * Queue a task on the target CPUs wake_list and wake the CPU via IPI if
+ * necessary. The wakee CPU on receipt of the IPI will queue the task
+ * via sched_ttwu_wakeup() for activation so the wakee incurs the cost
+ * of the wakeup instead of the waker.
+ */
+static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	/*
+	 * Carry WF_MIGRATED across to the CPU that will do the enqueue. Losing
+	 * it there makes ttwu_do_activate() treat an already migrated wakeup
+	 * as a local one, decrementing rq->nr_iowait a second time and telling
+	 * PSI to clear an iowait that psi_ttwu_dequeue() has already dropped.
+	 */
+	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+
+	WRITE_ONCE(rq->ttwu_pending, 1);
+	__smp_call_single_queue(cpu, &p->wake_entry.llist);
+}
+
+void wake_up_if_idle(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	rcu_read_lock();
+
+	if (!is_idle_task(rcu_dereference(rq->curr)))
+		goto out;
+
+	/*
+	 * Match mainline: always go through resched_curr so TIF_NEED_RESCHED
+	 * is set before any IPI.  Sending a reschedule IPI alone does nothing
+	 * useful — scheduler_ipi() only folds an already-set need_resched.
+	 */
+	rq_lock_irqsave(rq, &rf);
+	if (is_idle_task(rq->curr))
+		resched_curr(rq);
+	rq_unlock_irqrestore(rq, &rf);
+
+out:
+	rcu_read_unlock();
+}
+
+static inline bool ttwu_queue_cond(int cpu, int wake_flags)
+{
+	/*
+	 * Do not complicate things with the async wake_list while the CPU is
+	 * in hotplug state.
+	 */
+	if (!cpu_active(cpu))
+		return false;
+
+	/*
+	 * If the CPU does not share cache, then queue the task on the
+	 * remote rqs wakelist to avoid accessing remote data.
+	 */
+	if (!cpus_share_cache(smp_processor_id(), cpu))
+		return true;
+
+	/*
+	 * If the task is descheduling and the only running task on the
+	 * CPU then use the wakelist to offload the task activation to
+	 * the soon-to-be-idle CPU as the current CPU is likely busy.
+	 * nr_running is checked to avoid unnecessary task stacking.
+	 */
+	if ((wake_flags & WF_ON_CPU) && cpu_rq(cpu)->nr_running <= 1)
+		return true;
+
+	return false;
+}
+
+static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+{
+	/* CFS would require sched_feat(TTWU_QUEUE) here but that is
+	 * fixed enabled */
+	if (ttwu_queue_cond(cpu, wake_flags)) {
+		if (WARN_ON_ONCE(cpu == smp_processor_id()))
+			return false;
+
+		sched_clock_cpu(cpu); /* Sync clocks across CPUs */
+		__ttwu_queue_wakelist(p, cpu, wake_flags);
+		return true;
+	}
+
+	return false;
+}
+
+static int valid_task_cpu(struct task_struct *p)
+{
+	cpumask_t valid_mask;
+
+	if (unlikely(is_migration_disabled(p)))
+		return task_cpu(p);
+
+	/*
+	 * Kthreads may be bound to a CPU that is not yet online (per-CPU
+	 * hotplug threads created during smp_init). Placement for running
+	 * must still pick an online CPU — sched_other_cpu() allows them to
+	 * be selected despite the affinity mismatch until their CPU is up.
+	 * Userspace tasks are restricted to the active mask as usual.
+	 */
+	if (p->flags & PF_KTHREAD) {
+		cpumask_and(&valid_mask, p->cpus_ptr, cpu_online_mask);
+		/*
+		 * Only KTHREAD_IS_PER_CPU gets to stay on a CPU that is on
+		 * its way down, so do not hand a regular kthread back the
+		 * CPU bind_zero() just moved it off. Mirrors is_cpu_allowed().
+		 */
+		if (!kthread_is_per_cpu(p))
+			cpumask_andnot(&valid_mask, &valid_mask, cpu_dying_mask);
+	} else
+		cpumask_and(&valid_mask, p->cpus_ptr, cpu_active_mask);
+
+	if (unlikely(!cpumask_weight(&valid_mask))) {
+		if ((p->flags & PF_KTHREAD) && num_online_cpus())
+			return cpumask_any(cpu_online_mask);
+		/* Defer console access while holding the task's PI lock. */
+		printk_deferred(KERN_WARNING "SCHED: No cpumask for %s/%d weight %d\n",
+				p->comm, p->pid, cpumask_weight(p->cpus_ptr));
+		return cpumask_any(p->cpus_ptr);
+	}
+	return cpumask_any(&valid_mask);
+}
+
+/*
+ * For a task that's just being woken up we have a valuable balancing
+ * opportunity so choose the nearest cache most lightly loaded runqueue.
+ * Entered with rq locked and returns with the chosen runqueue locked.
+ */
+static inline int select_best_cpu(struct task_struct *p)
+{
+	unsigned int idlest = ~0U;
+	struct rq *rq = NULL;
+	int i;
+
+	if (suitable_idle_cpus(p)) {
+		int cpu = task_cpu(p);
+
+		if (unlikely(needs_other_cpu(p, cpu)))
+			cpu = valid_task_cpu(p);
+		rq = resched_best_idle(p, cpu);
+		if (likely(rq))
+			return rq->cpu;
+	}
+
+	for (i = 0; i < num_online_cpus(); i++) {
+		struct rq *other_rq = task_rq(p)->cpu_order[i];
+		int entries;
+
+		if (!other_rq->online)
+			continue;
+		if (needs_other_cpu(p, other_rq->cpu))
+			continue;
+		entries = rq_load(other_rq);
+		if (entries >= idlest)
+			continue;
+		idlest = entries;
+		rq = other_rq;
+	}
+	if (unlikely(!rq)) {
+		/*
+		 * The walk found nowhere to put @p. Its affinity may contain
+		 * only offline CPUs (hotplug kthreads bound before their CPU
+		 * is up), or the walk may simply not have reached an allowed
+		 * one: cpu_order[] is built once at boot, so once any CPU is
+		 * offline the num_online_cpus() bound above stops short of its
+		 * tail and never examines what is there.
+		 *
+		 * Staying put is only an answer if this CPU is one @p is
+		 * allowed to run on. Otherwise defer to valid_task_cpu(),
+		 * which intersects the mask directly and so does not depend on
+		 * that ordering at all. Never place a wakeup on a CPU the task
+		 * is not allowed on, whether because it is offline or because
+		 * it is not in the mask; a blocked task keeps a stale
+		 * task_cpu() across an affinity change, so the latter is the
+		 * common case here.
+		 */
+		if (unlikely(needs_other_cpu(p, task_cpu(p))))
+			return valid_task_cpu(p);
+		return task_cpu(p);
+	}
+	return rq->cpu;
+}
+#else /* CONFIG_SMP */
+
+static inline bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
+{
+	return false;
+}
+
+static int valid_task_cpu(struct task_struct *p)
+{
+	return 0;
+}
+
+static inline int select_best_cpu(struct task_struct *p)
+{
+	return 0;
+}
+
+static struct rq *resched_best_idle(struct task_struct *p, int cpu)
+{
+	return NULL;
+}
+#endif /* CONFIG_SMP */
+
+static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (ttwu_queue_wakelist(p, cpu, wake_flags))
+		return;
+
+	rq_lock(rq);
+	update_rq_clock(rq);
+	ttwu_do_activate(rq, p, wake_flags);
+	rq_unlock(rq);
+}
+
+/*
+ * Consider @state matched against @p, taking p->saved_state into account.
+ *
+ * The caller holds p::pi_lock if p != current or has preemption disabled
+ * when p == current.
+ *
+ * The rules of saved_state:
+ *
+ *   The related locking code always holds p::pi_lock when updating
+ *   p::saved_state, which means the code is fully serialized in both cases.
+ *
+ *   For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT.
+ *   No other bits set. This allows to distinguish all wakeup scenarios.
+ *
+ *   For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This
+ *   allows us to prevent early wakeup of tasks before they can be run on
+ *   asymmetric ISA architectures (eg ARMv9).
+ */
+static __always_inline
+bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success)
+{
+	int match;
+
+	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
+		WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) &&
+			     state != TASK_RTLOCK_WAIT);
+	}
+
+	*success = !!(match = __task_state_match(p, state));
+
+	/*
+	 * Saved state preserves the task state across blocking on
+	 * an RT lock or TASK_FREEZABLE tasks.  If the state matches,
+	 * set p::saved_state to TASK_RUNNING, but do not wake the task
+	 * because it waits for a lock wakeup or __thaw_task(). Also
+	 * indicate success because from the regular waker's point of
+	 * view this has succeeded.
+	 *
+	 * After acquiring the lock the task will restore p::__state
+	 * from p::saved_state which ensures that the regular
+	 * wakeup is not lost. The restore will also set
+	 * p::saved_state to TASK_RUNNING so any further tests will
+	 * not result in false positives vs. @success
+	 */
+	if (match < 0)
+		p->saved_state = TASK_RUNNING;
+
+	return match > 0;
+}
+
+/***
+ * try_to_wake_up - wake up a thread
+ * @p: the thread to be awakened
+ * @state: the mask of task states that can be woken
+ * @wake_flags: wake modifier flags (WF_*)
+ *
+ * Put it on the run-queue if it's not already there. The "current"
+ * thread is always on the run-queue (except when the actual
+ * re-schedule is in progress), and as such you're allowed to do
+ * the simpler "current->__state = TASK_RUNNING" to mark yourself
+ * runnable without the overhead of this.
+ *
+ * Return: %true if @p was woken up, %false if it was already running.
+ * or @state didn't match @p's state.
+ */
+int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
+{
+	unsigned long flags;
+	int cpu, success = 0;
+
+	preempt_disable();
+	if (p == current) {
+		/*
+		 * We're waking current, this means 'p->on_rq' and 'task_cpu(p)
+		 * == smp_processor_id()'. Together this means we can special
+		 * case the whole 'p->on_rq && ttwu_runnable()' case below
+		 * without taking any locks.
+		 *
+		 * In particular:
+		 *  - we rely on Program-Order guarantees for all the ordering,
+		 *  - we're serialized against set_special_state() by virtue of
+		 *    it disabling IRQs (this allows not taking ->pi_lock).
+		 */
+		if (!ttwu_state_match(p, state, &success))
+			goto out;
+
+		trace_sched_waking(p);
+		p->__state = TASK_RUNNING;
+		trace_sched_wakeup(p);
+		goto out;
+	}
+
+	/*
+	 * If we are going to wake up a thread waiting for CONDITION we
+	 * need to ensure that CONDITION=1 done by the caller can not be
+	 * reordered with p->__state check below. This pairs with smp_store_mb()
+	 * in set_current_state() that the waiting thread does.
+	 */
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	smp_mb__after_spinlock();
+	if (!ttwu_state_match(p, state, &success))
+		goto unlock;
+
+	trace_sched_waking(p);
+
+	/*
+	 * Ensure we load p->on_rq _after_ p->__state, otherwise it would
+	 * be possible to, falsely, observe p->on_rq == 0 and get stuck
+	 * in smp_cond_load_acquire() below.
+	 *
+	 * sched_ttwu_pending()			try_to_wake_up()
+	 *   STORE p->on_rq = 1			  LOAD p->__state
+	 *   UNLOCK rq->lock
+	 *
+	 * __schedule() (switch to task 'p')
+	 *   LOCK rq->lock			  smp_rmb();
+	 *   smp_mb__after_spinlock();
+	 *   UNLOCK rq->lock
+	 *
+	 * [task p]
+	 *   STORE p->__state = UNINTERRUPTIBLE	  LOAD p->on_rq
+	 *
+	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
+	 * __schedule().  See the comment for smp_mb__after_spinlock().
+	 */
+	smp_rmb();
+	if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
+		goto unlock;
+
+#ifdef CONFIG_SMP
+	/*
+	 * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
+	 * possible to, falsely, observe p->on_cpu == 0.
+	 *
+	 * One must be running (->on_cpu == 1) in order to remove oneself
+	 * from the runqueue.
+	 *
+	 * __schedule() (switch to task 'p')	try_to_wake_up()
+	 *   STORE p->on_cpu = 1		  LOAD p->on_rq
+	 *   UNLOCK rq->lock
+	 *
+	 * __schedule() (put 'p' to sleep)
+	 *   LOCK rq->lock			  smp_rmb();
+	 *   smp_mb__after_spinlock();
+	 *   STORE p->on_rq = 0			  LOAD p->on_cpu
+	 *
+	 * Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
+	 * __schedule().  See the comment for smp_mb__after_spinlock().
+	 *
+	 * Form a control-dep-acquire with p->on_rq == 0 above, to ensure
+	 * schedule()'s deactivate_task() has 'happened' and p will no longer
+	 * care about it's own p->__state. See the comment in __schedule().
+	 */
+	smp_acquire__after_ctrl_dep();
+
+	/*
+	 * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
+	 * == 0), which means we need to do an enqueue, change p->__state to
+	 * TASK_WAKING such that we can unlock p->pi_lock before doing the
+	 * enqueue, such as ttwu_queue_wakelist().
+	 */
+	p->__state = TASK_WAKING;
+
+	/*
+	 * If the owning (remote) CPU is still in the middle of schedule() with
+	 * this task as prev, considering queueing p on the remote CPUs wake_list
+	 * which potentially sends an IPI instead of spinning on p->on_cpu to
+	 * let the waker make forward progress. This is safe because IRQs are
+	 * disabled and the IPI will deliver after on_cpu is cleared.
+	 *
+	 * Ensure we load task_cpu(p) after p->on_cpu:
+	 *
+	 * set_task_cpu(p, cpu);
+	 *   STORE task_cpu(p) = @cpu
+	 * __schedule() (switch to task 'p')
+	 *   LOCK rq->lock
+	 *   smp_mb__after_spin_lock()		smp_cond_load_acquire(&p->on_cpu)
+	 *   STORE p->on_cpu = 1		LOAD task_cpu(p)
+	 *
+	 * to ensure we observe the correct CPU on which the task is currently
+	 * scheduling.
+	 */
+	if (smp_load_acquire(&p->on_cpu) &&
+	    ttwu_queue_wakelist(p, task_cpu(p), wake_flags | WF_ON_CPU))
+		goto unlock;
+
+	/*
+	 * If the owning (remote) CPU is still in the middle of schedule() with
+	 * this task as prev, wait until it's done referencing the task.
+	 *
+	 * Pairs with the smp_store_release() in finish_task().
+	 *
+	 * This ensures that tasks getting woken will be fully ordered against
+	 * their previous state and preserve Program Order.
+	 */
+	smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+	cpu = select_best_cpu(p);
+	if (task_cpu(p) != cpu) {
+		if (p->in_iowait) {
+			delayacct_blkio_end(p);
+			atomic_dec(&task_rq(p)->nr_iowait);
+		}
+
+		wake_flags |= WF_MIGRATED;
+		psi_ttwu_dequeue(p);
+		set_task_cpu(p, cpu);
+	}
+
+#else
+	cpu = task_cpu(p);
+#endif /* CONFIG_SMP */
+
+	ttwu_queue(p, cpu, wake_flags);
+unlock:
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+out:
+	if (success)
+		ttwu_stat(p, task_cpu(p), wake_flags);
+	preempt_enable();
+
+	return success;
+}
+
+static bool __task_needs_rq_lock(struct task_struct *p)
+{
+	unsigned int state = READ_ONCE(p->__state);
+
+	/*
+	 * Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when
+	 * the task is blocked. Make sure to check @state since ttwu() can drop
+	 * locks at the end, see ttwu_queue_wakelist().
+	 */
+	if (state == TASK_RUNNING || state == TASK_WAKING)
+		return true;
+
+	/*
+	 * Ensure we load p->on_rq after p->__state, otherwise it would be
+	 * possible to, falsely, observe p->on_rq == 0.
+	 *
+	 * See try_to_wake_up() for a longer comment.
+	 */
+	smp_rmb();
+	if (p->on_rq)
+		return true;
+
+	/*
+	 * Ensure the task has finished __schedule() and will not be referenced
+	 * anymore. Again, see try_to_wake_up() for a longer comment.
+	 */
+	smp_rmb();
+	smp_cond_load_acquire(&p->on_cpu, !VAL);
+
+	return false;
+}
+
+/**
+ * task_call_func - Invoke a function on task in fixed state
+ * @p: Process for which the function is to be invoked, can be @current.
+ * @func: Function to invoke.
+ * @arg: Argument to function.
+ *
+ * Fix the task in it's current state by avoiding wakeups and or rq operations
+ * and call @func(@arg) on it.  This function can use task_is_runnable() and
+ * task_curr() to work out what the state is, if required.  Given that @func
+ * can be invoked with a runqueue lock held, it had better be quite
+ * lightweight.
+ *
+ * Returns:
+ *   Whatever @func returns
+ */
+int task_call_func(struct task_struct *p, task_call_f func, void *arg)
+{
+	struct rq_flags rf;
+	int ret;
+
+	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
+
+	if (__task_needs_rq_lock(p)) {
+		struct rq *rq = __task_rq_lock(p, &rf);
+
+		/*
+		 * At this point the task is pinned; either:
+		 *  - blocked and we're holding off wakeups	 (pi->lock)
+		 *  - woken, and we're holding off enqueue	 (rq->lock)
+		 *  - queued, and we're holding off schedule	 (rq->lock)
+		 *  - running, and we're holding off de-schedule (rq->lock)
+		 *
+		 * The called function (@func) can use: task_curr(), p->on_rq and
+		 * p->__state to differentiate between these states.
+		 */
+		ret = func(p, arg);
+
+		__task_rq_unlock(rq, p, &rf);
+	} else {
+		ret = func(p, arg);
+	}
+
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+	return ret;
+}
+
+/**
+ * wake_up_process - Wake up a specific process
+ * @p: The process to be woken up.
+ *
+ * Attempt to wake up the nominated process and move it to the set of runnable
+ * processes.
+ *
+ * Return: 1 if the process was woken up, 0 if it was already running.
+ *
+ * This function executes a full memory barrier before accessing the task state.
+ */
+int wake_up_process(struct task_struct *p)
+{
+	return try_to_wake_up(p, TASK_NORMAL, 0);
+}
+EXPORT_SYMBOL(wake_up_process);
+
+int wake_up_state(struct task_struct *p, unsigned int state)
+{
+	return try_to_wake_up(p, state, 0);
+}
+
+static void time_slice_expired(struct task_struct *p, struct rq *rq);
+
+/*
+ * Perform scheduler related setup for a newly forked process p.
+ * p is forked by current.
+ */
+int sched_fork(u64 __maybe_unused clone_flags, struct task_struct *p)
+{
+	unsigned long flags;
+
+#ifdef CONFIG_PREEMPT_NOTIFIERS
+	INIT_HLIST_HEAD(&p->preempt_notifiers);
+#endif
+
+#ifdef CONFIG_COMPACTION
+	p->capture_control = NULL;
+#endif
+
+#ifdef CONFIG_SMP
+	p->wake_entry.u_flags = CSD_TYPE_TTWU;
+#endif
+	/* A new task starts with no I/O history of its own. */
+	muqss_iotime_task_init(p);
+
+	/*
+	 * We mark the process as NEW here. This guarantees that
+	 * nobody will actually run it, and a signal or other external
+	 * event cannot wake it up and insert it on the runqueue either.
+	 */
+	p->__state = TASK_NEW;
+
+	/*
+	 * The process state is set to the same value of the process executing
+	 * do_fork() code. That is running. This guarantees that nobody will
+	 * actually run it, and a signal or other external event cannot wake
+	 * it up and insert it on the runqueue either.
+	 */
+
+	/* Should be reset in fork.c but done here for ease of MuQSS patching */
+	p->on_cpu =
+	p->on_rq =
+	p->utime =
+	p->stime =
+	p->sched_time =
+	p->stime_ns =
+	p->utime_ns = 0;
+	skiplist_node_init(&p->node);
+
+	/*
+	 * Revert to default priority/policy on fork if requested.
+	 */
+	if (unlikely(p->sched_reset_on_fork)) {
+		if (p->policy == SCHED_FIFO || p->policy == SCHED_RR || p-> policy == SCHED_ISO) {
+			/*
+			 * __setscheduler() zeroes timer_slack_ns for rt tasks,
+			 * so restore it when demoting back to SCHED_NORMAL,
+			 * otherwise the child inherits zero slack forever.
+			 */
+			if (has_rt_policy(p))
+				p->timer_slack_ns = p->default_timer_slack_ns;
+			p->policy = SCHED_NORMAL;
+			p->normal_prio = normal_prio(p);
+		}
+
+		if (PRIO_TO_NICE(p->static_prio) < 0) {
+			p->static_prio = NICE_TO_PRIO(0);
+			p->normal_prio = p->static_prio;
+		}
+
+		/*
+		 * We don't need the reset flag anymore after the fork. It has
+		 * fulfilled its duty:
+		 */
+		p->sched_reset_on_fork = 0;
+	}
+
+	/*
+	 * Silence PROVE_RCU.
+	 */
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	rseq_sched_set_ids_changed(p);
+	set_task_cpu(p, smp_processor_id());
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+#ifdef CONFIG_SCHED_INFO
+	if (unlikely(sched_info_on()))
+		memset(&p->sched_info, 0, sizeof(p->sched_info));
+#endif
+	init_task_preempt_count(p);
+
+	return 0;
+}
+
+void sched_post_fork(struct task_struct *p)
+{
+}
+
+#ifdef CONFIG_SCHEDSTATS
+
+DEFINE_STATIC_KEY_FALSE(sched_schedstats);
+static bool __initdata __sched_schedstats = false;
+
+static void set_schedstats(bool enabled)
+{
+	if (enabled)
+		static_branch_enable(&sched_schedstats);
+	else
+		static_branch_disable(&sched_schedstats);
+}
+
+void force_schedstat_enabled(void)
+{
+	if (!schedstat_enabled()) {
+		pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
+		static_branch_enable(&sched_schedstats);
+	}
+}
+
+static int __init setup_schedstats(char *str)
+{
+	int ret = 0;
+	if (!str)
+		goto out;
+
+	/*
+	 * This code is called before jump labels have been set up, so we can't
+	 * change the static branch directly just yet.  Instead set a temporary
+	 * variable so init_schedstats() can do it later.
+	 */
+	if (!strcmp(str, "enable")) {
+		__sched_schedstats = true;
+		ret = 1;
+	} else if (!strcmp(str, "disable")) {
+		__sched_schedstats = false;
+		ret = 1;
+	}
+out:
+	if (!ret)
+		pr_warn("Unable to parse schedstats=\n");
+
+	return ret;
+}
+__setup("schedstats=", setup_schedstats);
+
+static void __init init_schedstats(void)
+{
+	set_schedstats(__sched_schedstats);
+}
+
+#ifdef CONFIG_SYSCTL
+static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer,
+		size_t *lenp, loff_t *ppos)
+{
+	struct ctl_table t;
+	int err;
+	int state = static_branch_likely(&sched_schedstats);
+
+	if (write && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	t = *table;
+	t.data = &state;
+	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
+	if (err < 0)
+		return err;
+	if (write)
+		set_schedstats(state);
+	return err;
+}
+#endif /* CONFIG_SYSCTL */
+#else  /* !CONFIG_SCHEDSTATS */
+static inline void init_schedstats(void) {}
+#endif /* CONFIG_SCHEDSTATS */
+
+#ifdef CONFIG_SYSCTL
+/*
+ * MuQSS tunables used to live in kernel/sysctl.c.  Mainline moved scheduler
+ * sysctls next to their implementation via register_sysctl_init(), so register
+ * ours the same way.  CFS/rt/fair knobs are not present under MuQSS.
+ */
+static const struct ctl_table muqss_sysctls[] = {
+	{
+		.procname	= "rr_interval",
+		.data		= &rr_interval,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ONE,
+		.extra2		= SYSCTL_ONE_THOUSAND,
+	},
+	{
+		.procname	= "interactive",
+		.data		= &sched_interactive,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
+	{
+		.procname	= "iso_cpu",
+		.data		= &sched_iso_cpu,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE_HUNDRED,
+	},
+	{
+		.procname	= "yield_type",
+		.data		= &sched_yield_type,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_TWO,
+	},
+#ifdef CONFIG_SCHEDSTATS
+	{
+		.procname	= "sched_schedstats",
+		.data		= NULL,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= sysctl_schedstats,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
+#endif
+};
+
+static int __init muqss_sysctl_init(void)
+{
+	register_sysctl_init("kernel", muqss_sysctls);
+	return 0;
+}
+late_initcall(muqss_sysctl_init);
+#endif /* CONFIG_SYSCTL */
+
+static void update_cpu_clock_switch(struct rq *rq, struct task_struct *p);
+
+static void account_task_cpu(struct rq *rq, struct task_struct *p)
+{
+	update_clocks(rq);
+	/* This isn't really a context switch but accounting is the same */
+	update_cpu_clock_switch(rq, p);
+	p->last_ran = rq->niffies;
+}
+
+bool sched_smp_initialized __read_mostly;
+
+/*
+ * High-resolution timeslice expiry (MuQSS counterpart of mainline hrtick).
+ *
+ * Reprogramming a oneshot clockevent from set_rq_task() on every context
+ * switch under the rq lock races with the tick/timer softirq path on SMP
+ * and starves TIMER_SOFTIRQ (RCU "timer wakeup didn't happen").  Mirror
+ * mainline: HARD + LAZY_REARM setup, needs_rearm 5us threshold, and defer
+ * the actual start/cancel until hrexpiry_schedule_exit() after the pick.
+ */
+#ifdef CONFIG_HIGH_RES_TIMERS
+
+enum {
+	HREXPIRE_SCHED_NONE		= 0,
+	HREXPIRE_SCHED_DEFER		= BIT(1),
+	HREXPIRE_SCHED_START		= BIT(2),
+	HREXPIRE_SCHED_REARM_HRTIMER	= BIT(3),
+};
+
+static inline int hrexpiry_enabled(struct rq *rq)
+{
+	/*
+	 * 5.12 used hrtimer_is_hres_active(); 7.1 exposes the same idea as
+	 * hrtimer_resolution != LOW_RES_NSEC once highres has switched on.
+	 */
+	return hrtimer_resolution != LOW_RES_NSEC;
+}
+
+static inline bool hrexpiry_needs_rearm(struct hrtimer *timer, ktime_t expires)
+{
+	/*
+	 * Queued is false when not started or the callback is running.  If
+	 * already queued, only reprogram when the expiry moves substantially.
+	 */
+	return !hrtimer_is_queued(timer) ||
+		abs(expires - hrtimer_get_expires(timer)) > 5000;
+}
+
+static void hrexpiry_cond_restart(struct rq *rq)
+{
+	struct hrtimer *timer = &rq->hrexpiry_timer;
+	ktime_t time = rq->hrexpiry_time;
+
+	if (hrexpiry_needs_rearm(timer, time))
+		hrtimer_start(timer, time, HRTIMER_MODE_ABS_PINNED_HARD);
+}
+
+/*
+ * Remote start IPI — wake_up_new_task may shorten a parent on another CPU.
+ * Runs hardirq/IPI context; take the rq lock like mainline __hrtick_start.
+ */
+static void __hrexpiry_start(void *arg)
+{
+	struct rq *rq = arg;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(rq->lock, flags);
+	hrexpiry_cond_restart(rq);
+	raw_spin_unlock_irqrestore(rq->lock, flags);
+}
+
+static inline void hrexpiry_clear(struct rq *rq)
+{
+	if (!hrexpiry_enabled(rq))
+		return;
+
+	/*
+	 * Inside __schedule() only drop a pending deferred start; the actual
+	 * cancel happens once in hrexpiry_schedule_exit().
+	 */
+	if (rq->hrexpiry_sched) {
+		rq->hrexpiry_sched &= ~HREXPIRE_SCHED_START;
+		return;
+	}
+
+	hrtimer_try_to_cancel(&rq->hrexpiry_timer);
+}
+
+/*
+ * High-resolution time_slice expiry.
+ * Runs from hardirq context with interrupts disabled.
+ */
+static enum hrtimer_restart hrexpiry(struct hrtimer *timer)
+{
+	struct rq *rq = container_of(timer, struct rq, hrexpiry_timer);
+	struct task_struct *p;
+
+	/* This can happen during CPU hotplug / resume */
+	if (unlikely(cpu_of(rq) != smp_processor_id()))
+		goto out;
+
+	/*
+	 * Local CPU only; no rq lock.  Force a reschedule when the slice
+	 * expires — __schedule() will pick the next deadline task.
+	 */
+	p = rq->curr;
+	p->time_slice = 0;
+	__set_tsk_resched(p);
+out:
+	return HRTIMER_NORESTART;
+}
+
+/*
+ * Called with irqs disabled under the rq lock (set_rq_task / fork path).
+ * May target a remote rq — then arm via CSD like mainline hrtick_start.
+ */
+static void hrexpiry_start(struct rq *rq, u64 delay)
+{
+	s64 delta;
+
+	if (!hrexpiry_enabled(rq))
+		return;
+
+	/* Slices < 10us are not useful and can DoS the timer subsystem. */
+	delta = max_t(s64, delay, 10000LL);
+
+	/*
+	 * Mid-schedule: note the delay and let hrexpiry_schedule_exit()
+	 * program the clockevent once.
+	 */
+	if (rq->hrexpiry_sched) {
+		rq->hrexpiry_sched |= HREXPIRE_SCHED_START;
+		rq->hrexpiry_delay = delta;
+		return;
+	}
+
+	rq->hrexpiry_time = ktime_add_ns(ktime_get(), delta);
+	if (!hrexpiry_needs_rearm(&rq->hrexpiry_timer, rq->hrexpiry_time))
+		return;
+
+	if (rq == this_rq())
+		hrtimer_start(&rq->hrexpiry_timer, rq->hrexpiry_time,
+			      HRTIMER_MODE_ABS_PINNED_HARD);
+	else
+		smp_call_function_single_async(cpu_of(rq), &rq->hrexpiry_csd);
+}
+
+static inline void hrexpiry_schedule_enter(struct rq *rq)
+{
+	rq->hrexpiry_sched = HREXPIRE_SCHED_DEFER;
+	if (hrtimer_test_and_clear_rearm_deferred())
+		rq->hrexpiry_sched |= HREXPIRE_SCHED_REARM_HRTIMER;
+}
+
+static inline void hrexpiry_schedule_exit(struct rq *rq)
+{
+	if (rq->hrexpiry_sched & HREXPIRE_SCHED_START) {
+		rq->hrexpiry_time = ktime_add_ns(ktime_get(), rq->hrexpiry_delay);
+		hrexpiry_cond_restart(rq);
+	} else if (rq->curr == rq->idle || rq->curr->policy == SCHED_FIFO) {
+		/*
+		 * No slice timer needed.  Local CPU, IRQs off: the HARD
+		 * callback cannot be running, so cancel is safe.
+		 */
+		if (hrtimer_is_queued(&rq->hrexpiry_timer))
+			hrtimer_cancel(&rq->hrexpiry_timer);
+	}
+
+	if (rq->hrexpiry_sched & HREXPIRE_SCHED_REARM_HRTIMER)
+		__hrtimer_rearm_deferred();
+
+	rq->hrexpiry_sched = HREXPIRE_SCHED_NONE;
+}
+
+static void init_rq_hrexpiry(struct rq *rq)
+{
+	INIT_CSD(&rq->hrexpiry_csd, __hrexpiry_start, rq);
+	rq->hrexpiry_sched = HREXPIRE_SCHED_NONE;
+	hrtimer_setup(&rq->hrexpiry_timer, hrexpiry, CLOCK_MONOTONIC,
+		      HRTIMER_MODE_REL_HARD | HRTIMER_MODE_LAZY_REARM);
+}
+
+#else /* !CONFIG_HIGH_RES_TIMERS */
+
+static inline int hrexpiry_enabled(struct rq *rq)
+{
+	return 0;
+}
+static inline void hrexpiry_clear(struct rq *rq) { }
+static inline void hrexpiry_start(struct rq *rq, u64 delay) { }
+static inline void hrexpiry_schedule_enter(struct rq *rq) { }
+static inline void hrexpiry_schedule_exit(struct rq *rq) { }
+static inline void init_rq_hrexpiry(struct rq *rq) { }
+
+#endif /* CONFIG_HIGH_RES_TIMERS */
+
+static inline int rq_dither(struct rq *rq)
+{
+	if (!hrexpiry_enabled(rq))
+		return HALF_JIFFY_US;
+	return 0;
+}
+
+/*
+ * wake_up_new_task - wake up a newly created task for the first time.
+ *
+ * This function will do some initial scheduler statistics housekeeping
+ * that must be done for every newly created context, then puts the task
+ * on the runqueue and wakes it.
+ */
+void wake_up_new_task(struct task_struct *p)
+{
+	struct task_struct *parent, *rq_curr;
+	struct rq *rq, *new_rq;
+	unsigned long flags;
+
+	parent = p->parent;
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	p->__state = TASK_RUNNING;
+	/* Task_rq can't change yet on a new task */
+	new_rq = rq = task_rq(p);
+	if (unlikely(needs_other_cpu(p, task_cpu(p)))) {
+		set_task_cpu(p, valid_task_cpu(p));
+		new_rq = task_rq(p);
+	}
+
+	double_rq_lock(rq, new_rq);
+	rq_curr = rq->curr;
+
+	/*
+	 * Make sure we do not leak PI boosting priority to the child.
+	 */
+	p->prio = rq_curr->normal_prio;
+
+	trace_sched_wakeup_new(p);
+
+	/*
+	 * Share the timeslice between parent and child, thus the
+	 * total amount of pending timeslices in the system doesn't change,
+	 * resulting in more scheduling fairness. If it's negative, it won't
+	 * matter since that's the same as being 0. rq->rq_deadline is only
+	 * modified within schedule() so it is always equal to
+	 * current->deadline.
+	 */
+	account_task_cpu(rq, rq_curr);
+	p->last_ran = rq_curr->last_ran;
+	if (likely(rq_curr->policy != SCHED_FIFO)) {
+		rq_curr->time_slice /= 2;
+		if (rq_curr->time_slice < RESCHED_US) {
+			/*
+			 * Forking task has run out of timeslice. Reschedule it and
+			 * start its child with a new time slice and deadline. The
+			 * child will end up running first because its deadline will
+			 * be slightly earlier.
+			 */
+			__set_tsk_resched(rq_curr);
+			time_slice_expired(p, new_rq);
+			if (suitable_idle_cpus(p))
+				resched_best_idle(p, task_cpu(p));
+			else if (unlikely(rq != new_rq))
+				try_preempt(p, new_rq);
+		} else {
+			p->time_slice = rq_curr->time_slice;
+			if (rq_curr == parent && rq == new_rq && !suitable_idle_cpus(p)) {
+				/*
+				 * The VM isn't cloned, so we're in a good position to
+				 * do child-runs-first in anticipation of an exec. This
+				 * usually avoids a lot of COW overhead.
+				 */
+				__set_tsk_resched(rq_curr);
+			} else {
+				/*
+				 * Adjust the hrexpiry since rq_curr will keep
+				 * running and its timeslice has been shortened.
+				 */
+				hrexpiry_start(rq, US_TO_NS(rq_curr->time_slice));
+				try_preempt(p, new_rq);
+			}
+		}
+	} else {
+		time_slice_expired(p, new_rq);
+		try_preempt(p, new_rq);
+	}
+	activate_task(new_rq, p, 0);
+	double_rq_unlock(rq, new_rq);
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+}
+
+#ifdef CONFIG_PREEMPT_NOTIFIERS
+
+static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
+
+void preempt_notifier_inc(void)
+{
+	static_branch_inc(&preempt_notifier_key);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_inc);
+
+void preempt_notifier_dec(void)
+{
+	static_branch_dec(&preempt_notifier_key);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_dec);
+
+/**
+ * preempt_notifier_register - tell me when current is being preempted & rescheduled
+ * @notifier: notifier struct to register
+ */
+void preempt_notifier_register(struct preempt_notifier *notifier)
+{
+	if (!static_branch_unlikely(&preempt_notifier_key))
+		WARN(1, "registering preempt_notifier while notifiers disabled\n");
+
+	hlist_add_head(&notifier->link, &current->preempt_notifiers);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_register);
+
+/**
+ * preempt_notifier_unregister - no longer interested in preemption notifications
+ * @notifier: notifier struct to unregister
+ *
+ * This is *not* safe to call from within a preemption notifier.
+ */
+void preempt_notifier_unregister(struct preempt_notifier *notifier)
+{
+	hlist_del(&notifier->link);
+}
+EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
+
+static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+	struct preempt_notifier *notifier;
+
+	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
+		notifier->ops->sched_in(notifier, raw_smp_processor_id());
+}
+
+static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+	if (static_branch_unlikely(&preempt_notifier_key))
+		__fire_sched_in_preempt_notifiers(curr);
+}
+
+static void
+__fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				 struct task_struct *next)
+{
+	struct preempt_notifier *notifier;
+
+	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
+		notifier->ops->sched_out(notifier, next);
+}
+
+static __always_inline void
+fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				 struct task_struct *next)
+{
+	if (static_branch_unlikely(&preempt_notifier_key))
+		__fire_sched_out_preempt_notifiers(curr, next);
+}
+
+#else /* !CONFIG_PREEMPT_NOTIFIERS */
+
+static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
+{
+}
+
+static inline void
+fire_sched_out_preempt_notifiers(struct task_struct *curr,
+				 struct task_struct *next)
+{
+}
+
+#endif /* CONFIG_PREEMPT_NOTIFIERS */
+
+static inline void prepare_task(struct task_struct *next)
+{
+	/*
+	 * Claim the task as running, we do this before switching to it
+	 * such that any running task will have this set.
+	 *
+	 * See the ttwu() WF_ON_CPU case and its ordering comment.
+	 */
+	WRITE_ONCE(next->on_cpu, 1);
+}
+
+static inline void finish_task(struct task_struct *prev)
+{
+	/*
+	 * This must be the very last reference to @prev from this CPU. After
+	 * p->on_cpu is cleared, the task can be moved to a different CPU. We
+	 * must ensure this doesn't happen until the switch is completely
+	 * finished.
+	 *
+	 * In particular, the load of prev->__state in finish_task_switch() must
+	 * happen before this.
+	 *
+	 * Pairs with the smp_cond_load_acquire() in try_to_wake_up() and
+	 * __task_needs_rq_lock().
+	 */
+	smp_store_release(&prev->on_cpu, 0);
+}
+
+static inline void
+prepare_lock_switch(struct rq *rq, struct task_struct *next)
+{
+	/*
+	 * Since the runqueue lock will be released by the next
+	 * task (which is an invalid locking op but in the case
+	 * of the scheduler it's an obvious special-case), so we
+	 * do an early lockdep release here:
+	 */
+	spin_release(&rq->lock->dep_map, _THIS_IP_);
+#ifdef CONFIG_DEBUG_SPINLOCK
+	/* this is a valid case when another task releases the spinlock */
+	rq->lock->owner = next;
+#endif
+}
+
+static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev)
+{
+	/*
+	 * If we are tracking spinlock dependencies then we have to
+	 * fix up the runqueue lock - which gets 'carried over' from
+	 * prev into current:
+	 */
+	spin_acquire(&rq->lock->dep_map, 0, 0, _THIS_IP_);
+
+#ifdef CONFIG_SMP
+	/*
+	 * If prev was marked as migrating to another CPU in return_task, drop
+	 * the local runqueue lock but leave interrupts disabled and grab the
+	 * remote lock we're migrating it to before enabling them.
+	 */
+	if (unlikely(task_on_rq_migrating(prev))) {
+		unsigned int migrate;
+
+		/*
+		 * Program/cancel hrexpiry on this CPU before dropping its
+		 * rq lock; after the unlock `rq` may become the remote one.
+		 */
+		hrexpiry_schedule_exit(rq);
+		sched_info_dequeue(rq, prev);
+		/*
+		 * We move the ownership of prev to the new cpu now. Note that
+		 * this does not lock ttwu out: pointing task_cpu() at wake_cpu
+		 * below sends a concurrent ttwu_runnable() to the *new*
+		 * runqueue's lock rather than the one dropped here, and
+		 * __task_rq_lock() does not spin on task_on_rq_migrating() the
+		 * way mainline's does. See the re-check before enqueueing.
+		 *
+		 * This bypasses set_task_cpu(), so any PSI state prev is still
+		 * counted for has to be moved by hand, exactly as that does.
+		 * psi_sched_switch() saw prev off the CPU as a sleep, since it
+		 * is no longer queued, so a task in iowait is carrying
+		 * TSK_IOWAIT here and enqueue_task() below would clear it on
+		 * the new CPU that never counted it.
+		 */
+		migrate = psi_migrate_begin(prev);
+		task_thread_info(prev)->cpu = prev->wake_cpu;
+		psi_migrate_end(prev, migrate);
+		raw_spin_unlock(rq->lock);
+
+		raw_spin_lock(&prev->pi_lock);
+		rq = __task_rq_lock(prev, NULL);
+		/*
+		 * Complete the handover only while it is still ours to
+		 * complete. ttwu() reaches prev here despite the comment
+		 * above: ttwu_runnable() only accepts TASK_ON_RQ_QUEUED, so a
+		 * wakeup that finds prev still TASK_ON_RQ_MIGRATING falls
+		 * through it and enqueues prev itself. Once it has, prev can be
+		 * picked and run on another CPU, and take_task() empties its
+		 * skiplist node again.
+		 *
+		 * task_queued() only asks whether that node is linked, so at
+		 * that point it reads "prev still needs enqueueing" when it
+		 * means "prev is running elsewhere", and puts a task that is on
+		 * a CPU back on a runqueue for a second CPU to pick up. One
+		 * task then runs on two CPUs off one stack, which shows up
+		 * downstream as skiplist corruption, PSI counting the task
+		 * twice, and a scribbled kernel stack.
+		 *
+		 * on_rq is what actually tracks the handover - return_task()
+		 * set TASK_ON_RQ_MIGRATING and whoever takes prev over clears
+		 * it - and pi_lock, held here and taken by ttwu() before it
+		 * does anything, serialises the two.
+		 */
+		if (likely(task_on_rq_migrating(prev))) {
+			enqueue_task(rq, prev, 0);
+			prev->on_rq = TASK_ON_RQ_QUEUED;
+			/* Wake up the CPU if it's not already running */
+			resched_if_idle(rq);
+		}
+		raw_spin_unlock(&prev->pi_lock);
+		raw_spin_unlock_irq(rq->lock);
+		return;
+	}
+#endif
+	hrexpiry_schedule_exit(rq);
+	raw_spin_unlock_irq(rq->lock);
+}
+
+#ifndef prepare_arch_switch
+# define prepare_arch_switch(next)	do { } while (0)
+#endif
+#ifndef finish_arch_switch
+# define finish_arch_switch(prev)	do { } while (0)
+#endif
+#ifndef finish_arch_post_lock_switch
+# define finish_arch_post_lock_switch()	do { } while (0)
+#endif
+
+static inline void kmap_local_sched_out(void)
+{
+#ifdef CONFIG_KMAP_LOCAL
+	if (unlikely(current->kmap_ctrl.idx))
+		__kmap_local_sched_out();
+#endif
+}
+
+static inline void kmap_local_sched_in(void)
+{
+#ifdef CONFIG_KMAP_LOCAL
+	if (unlikely(current->kmap_ctrl.idx))
+		__kmap_local_sched_in();
+#endif
+}
+
+/**
+ * prepare_task_switch - prepare to switch tasks
+ * @rq: the runqueue preparing to switch
+ * @next: the task we are going to switch to.
+ *
+ * This is called with the rq lock held and interrupts off. It must
+ * be paired with a subsequent finish_task_switch after the context
+ * switch.
+ *
+ * prepare_task_switch sets up locking and calls architecture specific
+ * hooks.
+ */
+static inline void
+prepare_task_switch(struct rq *rq, struct task_struct *prev,
+		    struct task_struct *next)
+{
+	kcov_prepare_switch(prev);
+	sched_info_switch(rq, prev, next);
+	perf_event_task_sched_out(prev, next);
+	/*
+	 * rseq_preempt(prev) became rseq_sched_switch_event(next): rseq is now
+	 * told about the task being scheduled *in*.
+	 */
+	rseq_sched_switch_event(next);
+	fire_sched_out_preempt_notifiers(prev, next);
+	kmap_local_sched_out();
+	prepare_task(next);
+	prepare_arch_switch(next);
+}
+
+/**
+ * finish_task_switch - clean up after a task-switch
+ * @rq: runqueue associated with task-switch
+ * @prev: the thread we just switched away from.
+ *
+ * finish_task_switch must be called after the context switch, paired
+ * with a prepare_task_switch call before the context switch.
+ * finish_task_switch will reconcile locking set up by prepare_task_switch,
+ * and do any other architecture-specific cleanup actions.
+ *
+ * Note that we may have delayed dropping an mm in context_switch(). If
+ * so, we finish that here outside of the runqueue lock.  (Doing it
+ * with the lock held can cause deadlocks; see schedule() for
+ * details.)
+ *
+ * The context switch have flipped the stack from under us and restored the
+ * local variables which were saved when this task called schedule() in the
+ * past. prev == current is still correct but we need to recalculate this_rq
+ * because prev may have moved to another CPU.
+ */
+static void finish_task_switch(struct task_struct *prev)
+	__releases(rq->lock)
+{
+	struct rq *rq = this_rq();
+	struct mm_struct *mm = rq->prev_mm;
+	long prev_state;
+
+	/*
+	 * The previous task will have left us with a preempt_count of 2
+	 * because it left us after:
+	 *
+	 *	schedule()
+	 *	  preempt_disable();			// 1
+	 *	  __schedule()
+	 *	    raw_spin_lock_irq(rq->lock)	// 2
+	 *
+	 * Also, see FORK_PREEMPT_COUNT.
+	 */
+	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
+		      "corrupted preempt_count: %s/%d/0x%x\n",
+		      current->comm, current->pid, preempt_count()))
+		preempt_count_set(FORK_PREEMPT_COUNT);
+
+	rq->prev_mm = NULL;
+
+	/*
+	 * A task struct has one reference for the use as "current".
+	 * If a task dies, then it sets TASK_DEAD in tsk->__state and calls
+	 * schedule one last time. The schedule call will never return, and
+	 * the scheduled task must drop that reference.
+	 *
+	 * We must observe prev->__state before clearing prev->on_cpu (in
+	 * finish_task), otherwise a concurrent wakeup can get prev
+	 * running on another CPU and we could rave with its RUNNING -> DEAD
+	 * transition, resulting in a double drop.
+	 */
+	prev_state = prev->__state;
+	vtime_task_switch(prev);
+	perf_event_task_sched_in(prev, current);
+	finish_task(prev);
+	finish_lock_switch(rq, prev);
+	finish_arch_post_lock_switch();
+	kcov_finish_switch(current);
+	/*
+	 * kmap_local_sched_out() is invoked with rq::lock held and
+	 * interrupts disabled. There is no requirement for that, but the
+	 * sched out code does not have an interrupt enabled section.
+	 * Restoring the maps on sched in does not require interrupts being
+	 * disabled either.
+	 */
+	kmap_local_sched_in();
+
+	/*
+	 * Any cached block-layer timestamp (plug->cur_ktime) is stale now,
+	 * invalidate it.
+	 */
+	blk_plug_invalidate_ts();
+
+	fire_sched_in_preempt_notifiers(current);
+	/*
+	 * When switching through a kernel thread, the loop in
+	 * membarrier_{private,global}_expedited() may have observed that
+	 * kernel thread and not issued an IPI. It is therefore possible to
+	 * schedule between user->kernel->user threads without passing though
+	 * switch_mm(). Membarrier requires a barrier after storing to
+	 * rq->curr, before returning to userspace, so provide them here:
+	 *
+	 * - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
+	 *   provided by mmdrop_lazy_tlb(),
+	 * - a sync_core for SYNC_CORE.
+	 */
+	if (mm) {
+		membarrier_mm_sync_core_before_usermode(mm);
+		mmdrop_lazy_tlb_sched(mm);
+	}
+	if (unlikely(prev_state == TASK_DEAD)) {
+		/*
+		 * Remove function-return probe instances associated with this
+		 * task and put them back on the free list.
+		 */
+		kprobe_flush_task(prev);
+
+		/*
+		 * Unlink from css_set after the final switch so rmdir can
+		 * complete. Must precede put_task_struct_rcu_user().
+		 */
+		cgroup_task_dead(prev);
+
+		/* Task is done with its stack. */
+		put_task_stack(prev);
+
+		put_task_struct_rcu_user(prev);
+	}
+}
+
+/**
+ * schedule_tail - first thing a freshly forked thread must call.
+ * @prev: the thread we just switched away from.
+ */
+asmlinkage __visible void schedule_tail(struct task_struct *prev)
+{
+	/*
+	 * New tasks start with FORK_PREEMPT_COUNT, see there and
+	 * finish_task_switch() for details.
+	 *
+	 * finish_task_switch() will drop rq->lock() and lower preempt_count
+	 * and the preempt_enable() will end up enabling preemption (on
+	 * PREEMPT_COUNT kernels).
+	 */
+
+	finish_task_switch(prev);
+	preempt_enable();
+
+	if (current->set_child_tid)
+		put_user(task_pid_vnr(current), current->set_child_tid);
+
+	calculate_sigpending();
+}
+
+/*
+ * context_switch - switch to the new MM and the new thread's register state.
+ */
+static __always_inline void
+context_switch(struct rq *rq, struct task_struct *prev,
+	       struct task_struct *next)
+{
+	prepare_task_switch(rq, prev, next);
+
+	/*
+	 * For paravirt, this is coupled with an exit in switch_to to
+	 * combine the page table reload and the switch backend into
+	 * one hypercall.
+	 */
+	arch_start_context_switch(prev);
+
+	/*
+	 * kernel -> kernel   lazy + transfer active
+	 *   user -> kernel   lazy + mmgrab_lazy_tlb() active
+	 *
+	 * kernel ->   user   switch + mmdrop_lazy_tlb() active
+	 *   user ->   user   switch
+	 */
+	if (!next->mm) {                                // to kernel
+		enter_lazy_tlb(prev->active_mm, next);
+
+		next->active_mm = prev->active_mm;
+		if (prev->mm)                           // from user
+			mmgrab_lazy_tlb(prev->active_mm);
+		else
+			prev->active_mm = NULL;
+	} else {                                        // to user
+		membarrier_switch_mm(rq, prev->active_mm, next->mm);
+		/*
+		 * sys_membarrier() requires an smp_mb() between setting
+		 * rq->curr / membarrier_switch_mm() and returning to userspace.
+		 *
+		 * The below provides this either through switch_mm(), or in
+		 * case 'prev->active_mm == next->mm' through
+		 * finish_task_switch()'s mmdrop_lazy_tlb().
+		 */
+		switch_mm_irqs_off(prev->active_mm, next->mm, next);
+		lru_gen_use_mm(next->mm);
+
+		if (!prev->mm) {                        // from kernel
+			/* will mmdrop_lazy_tlb() in finish_task_switch(). */
+			rq->prev_mm = prev->active_mm;
+			prev->active_mm = NULL;
+		}
+	}
+	prepare_lock_switch(rq, next);
+
+	/* Here we just switch the register state and the stack. */
+	switch_to(prev, next, prev);
+	barrier();
+
+	finish_task_switch(prev);
+}
+
+/*
+ * nr_running, nr_uninterruptible and nr_context_switches:
+ *
+ * externally visible scheduler statistics: current number of runnable
+ * threads, total number of context switches performed since bootup.
+ */
+unsigned int nr_running(void)
+{
+	unsigned long i, sum = 0;
+
+	for_each_online_cpu(i)
+		sum += cpu_rq(i)->nr_running;
+
+	return sum;
+}
+
+static unsigned long nr_uninterruptible(void)
+{
+	unsigned long i, sum = 0;
+
+	for_each_online_cpu(i)
+		sum += cpu_rq(i)->nr_uninterruptible;
+
+	return sum;
+}
+
+/*
+ * Check if only the current task is running on the CPU.
+ *
+ * Caution: this function does not check that the caller has disabled
+ * preemption, thus the result might have a time-of-check-to-time-of-use
+ * race.  The caller is responsible to use it correctly, for example:
+ *
+ * - from a non-preemptible section (of course)
+ *
+ * - from a thread that is bound to a single CPU
+ *
+ * - in a loop with very short iterations (e.g. a polling loop)
+ */
+bool single_task_running(void)
+{
+	if (rq_load(raw_rq()) == 1)
+		return true;
+	else
+		return false;
+}
+EXPORT_SYMBOL(single_task_running);
+
+unsigned long long nr_context_switches(void)
+{
+	int cpu;
+	unsigned long long sum = 0;
+
+	for_each_possible_cpu(cpu)
+		sum += cpu_rq(cpu)->nr_switches;
+
+	return sum;
+}
+
+/*
+ * Consumers of these two interfaces, like for example the cpufreq menu
+ * governor are using nonsensical data. Boosting frequency for a CPU that has
+ * IO-wait which might not even end up running the task when it does become
+ * runnable.
+ */
+
+unsigned int nr_iowait_cpu(int cpu)
+{
+	return atomic_read(&cpu_rq(cpu)->nr_iowait);
+}
+
+/*
+ * IO-wait accounting, and how it's mostly bollocks (on SMP).
+ *
+ * The idea behind IO-wait account is to account the idle time that we could
+ * have spend running if it were not for IO. That is, if we were to improve the
+ * storage performance, we'd have a proportional reduction in IO-wait time.
+ *
+ * This all works nicely on UP, where, when a task blocks on IO, we account
+ * idle time as IO-wait, because if the storage were faster, it could've been
+ * running and we'd not be idle.
+ *
+ * This has been extended to SMP, by doing the same for each CPU. This however
+ * is broken.
+ *
+ * Imagine for instance the case where two tasks block on one CPU, only the one
+ * CPU will have IO-wait accounted, while the other has regular idle. Even
+ * though, if the storage were faster, both could've ran at the same time,
+ * utilising both CPUs.
+ *
+ * This means, that when looking globally, the current IO-wait accounting on
+ * SMP is a lower bound, by reason of under accounting.
+ *
+ * Worse, since the numbers are provided per CPU, they are sometimes
+ * interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
+ * associated with any one particular CPU, it can wake to another CPU than it
+ * blocked on. This means the per CPU IO-wait number is meaningless.
+ *
+ * Task CPU affinities can make all that even more 'interesting'.
+ */
+
+unsigned int nr_iowait(void)
+{
+	unsigned long cpu, sum = 0;
+
+	for_each_possible_cpu(cpu)
+		sum += nr_iowait_cpu(cpu);
+
+	return sum;
+}
+
+static unsigned long nr_active(void)
+{
+	return nr_running() + nr_uninterruptible();
+}
+
+/* Variables and functions for calc_load */
+static unsigned long calc_load_update;
+unsigned long avenrun[3];
+EXPORT_SYMBOL(avenrun);
+
+/**
+ * get_avenrun - get the load average array
+ * @loads:	pointer to dest load array
+ * @offset:	offset to add
+ * @shift:	shift count to shift the result left
+ *
+ * These values are estimates at best, so no need for locking.
+ */
+void get_avenrun(unsigned long *loads, unsigned long offset, int shift)
+{
+	loads[0] = (avenrun[0] + offset) << shift;
+	loads[1] = (avenrun[1] + offset) << shift;
+	loads[2] = (avenrun[2] + offset) << shift;
+}
+
+/*
+ * calc_load - update the avenrun load estimates every LOAD_FREQ seconds.
+ */
+void calc_global_load(void)
+{
+	long active;
+
+	if (time_before(jiffies, READ_ONCE(calc_load_update)))
+		return;
+	active = nr_active() * FIXED_1;
+
+	avenrun[0] = calc_load(avenrun[0], EXP_1, active);
+	avenrun[1] = calc_load(avenrun[1], EXP_5, active);
+	avenrun[2] = calc_load(avenrun[2], EXP_15, active);
+
+	calc_load_update = jiffies + LOAD_FREQ;
+}
+
+/**
+ * fixed_power_int - compute: x^n, in O(log n) time
+ *
+ * @x:         base of the power
+ * @frac_bits: fractional bits of @x
+ * @n:         power to raise @x to.
+ *
+ * By exploiting the relation between the definition of the natural power
+ * function: x^n := x*x*...*x (x multiplied by itself for n times), and
+ * the binary encoding of numbers used by computers: n := \Sum n_i * 2^i,
+ * (where: n_i \elem {0, 1}, the binary vector representing n),
+ * we find: x^n := x^(\Sum n_i * 2^i) := \Prod x^(n_i * 2^i), which is
+ * of course trivially computable in O(log_2 n), the length of our binary
+ * vector.
+ */
+static unsigned long
+fixed_power_int(unsigned long x, unsigned int frac_bits, unsigned int n)
+{
+	unsigned long result = 1UL << frac_bits;
+
+	if (n) {
+		for (;;) {
+			if (n & 1) {
+				result *= x;
+				result += 1UL << (frac_bits - 1);
+				result >>= frac_bits;
+			}
+			n >>= 1;
+			if (!n)
+				break;
+			x *= x;
+			x += 1UL << (frac_bits - 1);
+			x >>= frac_bits;
+		}
+	}
+
+	return result;
+}
+
+/*
+ * a1 = a0 * e + a * (1 - e)
+ *
+ * a2 = a1 * e + a * (1 - e)
+ *    = (a0 * e + a * (1 - e)) * e + a * (1 - e)
+ *    = a0 * e^2 + a * (1 - e) * (1 + e)
+ *
+ * a3 = a2 * e + a * (1 - e)
+ *    = (a0 * e^2 + a * (1 - e) * (1 + e)) * e + a * (1 - e)
+ *    = a0 * e^3 + a * (1 - e) * (1 + e + e^2)
+ *
+ *  ...
+ *
+ * an = a0 * e^n + a * (1 - e) * (1 + e + ... + e^n-1) [1]
+ *    = a0 * e^n + a * (1 - e) * (1 - e^n)/(1 - e)
+ *    = a0 * e^n + a * (1 - e^n)
+ *
+ * [1] application of the geometric series:
+ *
+ *              n         1 - x^(n+1)
+ *     S_n := \Sum x^i = -------------
+ *             i=0          1 - x
+ */
+unsigned long
+calc_load_n(unsigned long load, unsigned long exp,
+	    unsigned long active, unsigned int n)
+{
+	return calc_load(load, fixed_power_int(exp, FSHIFT, n), active);
+}
+
+DEFINE_PER_CPU(struct kernel_stat, kstat);
+DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat) = {
+#ifdef CONFIG_NO_HZ_COMMON
+	.idle_sleeptime_seq = SEQCNT_ZERO(kernel_cpustat.idle_sleeptime_seq)
+#endif
+};
+
+EXPORT_PER_CPU_SYMBOL(kstat);
+EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
+
+#ifdef CONFIG_PARAVIRT
+static inline u64 steal_ticks(u64 steal)
+{
+	if (unlikely(steal > NSEC_PER_SEC))
+		return div_u64(steal, TICK_NSEC);
+
+	return __iter_div_u64_rem(steal, TICK_NSEC, &steal);
+}
+#endif
+
+#ifndef nsecs_to_cputime
+# define nsecs_to_cputime(__nsecs)	nsecs_to_jiffies(__nsecs)
+#endif
+
+/*
+ * On each tick, add the number of nanoseconds to the unbanked variables and
+ * once one tick's worth has accumulated, account it allowing for accurate
+ * sub-tick accounting and totals. Use the TICK_APPROX_NS to match the way we
+ * deduct nanoseconds.
+ */
+static void pc_idle_time(struct rq *rq, struct task_struct *idle, unsigned long ns)
+{
+	u64 *cpustat = kcpustat_this_cpu->cpustat;
+	unsigned long ticks;
+
+	if (atomic_read(&rq->nr_iowait) > 0) {
+		rq->iowait_ns += ns;
+		if (rq->iowait_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->iowait_ns);
+			cpustat[CPUTIME_IOWAIT] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->iowait_ns %= JIFFY_NS;
+		}
+	} else {
+		rq->idle_ns += ns;
+		if (rq->idle_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->idle_ns);
+			cpustat[CPUTIME_IDLE] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->idle_ns %= JIFFY_NS;
+		}
+	}
+	acct_update_integrals(idle);
+}
+
+static void pc_system_time(struct rq *rq, struct task_struct *p,
+			   int hardirq_offset, unsigned long ns)
+{
+	u64 *cpustat = kcpustat_this_cpu->cpustat;
+	unsigned long ticks;
+
+	p->stime_ns += ns;
+	if (p->stime_ns >= JIFFY_NS) {
+		ticks = NS_TO_JIFFIES(p->stime_ns);
+		p->stime_ns %= JIFFY_NS;
+		p->stime += (__force u64)TICK_APPROX_NS * ticks;
+		account_group_system_time(p, TICK_APPROX_NS * ticks);
+	}
+	p->sched_time += ns;
+	account_group_exec_runtime(p, ns);
+
+	if (hardirq_count() - hardirq_offset) {
+		rq->irq_ns += ns;
+		if (rq->irq_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->irq_ns);
+			cpustat[CPUTIME_IRQ] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->irq_ns %= JIFFY_NS;
+		}
+	} else if (in_serving_softirq() || this_cpu_ksoftirqd() == p) {
+		/*
+		 * ksoftirqd time does not get accounted in cpu_softirq_time, so
+		 * it has to be handled separately here. Naming it explicitly
+		 * also catches it between batches, where it has reenabled bh
+		 * and in_serving_softirq() no longer holds - which is where the
+		 * context switch that lands us here happens.
+		 */
+		rq->softirq_ns += ns;
+		if (rq->softirq_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->softirq_ns);
+			cpustat[CPUTIME_SOFTIRQ] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->softirq_ns %= JIFFY_NS;
+		}
+	} else {
+		rq->system_ns += ns;
+		if (rq->system_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->system_ns);
+			cpustat[CPUTIME_SYSTEM] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->system_ns %= JIFFY_NS;
+		}
+	}
+	acct_update_integrals(p);
+}
+
+static void pc_user_time(struct rq *rq, struct task_struct *p, unsigned long ns)
+{
+	u64 *cpustat = kcpustat_this_cpu->cpustat;
+	unsigned long ticks;
+
+	p->utime_ns += ns;
+	if (p->utime_ns >= JIFFY_NS) {
+		ticks = NS_TO_JIFFIES(p->utime_ns);
+		p->utime_ns %= JIFFY_NS;
+		p->utime += (__force u64)TICK_APPROX_NS * ticks;
+		account_group_user_time(p, TICK_APPROX_NS * ticks);
+	}
+	p->sched_time += ns;
+	account_group_exec_runtime(p, ns);
+
+	if (task_nice(p) > 0 || idleprio_task(p)) {
+		rq->nice_ns += ns;
+		if (rq->nice_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->nice_ns);
+			cpustat[CPUTIME_NICE] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->nice_ns %= JIFFY_NS;
+		}
+	} else {
+		rq->user_ns += ns;
+		if (rq->user_ns >= JIFFY_NS) {
+			ticks = NS_TO_JIFFIES(rq->user_ns);
+			cpustat[CPUTIME_USER] += (__force u64)TICK_APPROX_NS * ticks;
+			rq->user_ns %= JIFFY_NS;
+		}
+	}
+	acct_update_integrals(p);
+}
+
+/*
+ * This is called on clock ticks.
+ * Bank in p->sched_time the ns elapsed since the last tick or switch.
+ * CPU scheduler quota accounting is also performed here in microseconds.
+ */
+static void update_cpu_clock_tick(struct rq *rq, struct task_struct *p)
+{
+	s64 account_ns = rq->niffies - p->last_ran;
+	struct task_struct *idle = rq->idle;
+
+	/* Accurate tick timekeeping */
+	if (user_mode(get_irq_regs()))
+		pc_user_time(rq, p, account_ns);
+	else if (p != idle || (irq_count() != HARDIRQ_OFFSET)) {
+		pc_system_time(rq, p, HARDIRQ_OFFSET, account_ns);
+	} else
+		pc_idle_time(rq, idle, account_ns);
+
+	/* time_slice accounting is done in usecs to avoid overflow on 32bit */
+	if (p->policy != SCHED_FIFO && p != idle)
+		p->time_slice -= NS_TO_US(account_ns);
+
+	p->last_ran = rq->niffies;
+}
+
+/*
+ * This is called on context switches.
+ * Bank in p->sched_time the ns elapsed since the last tick or switch.
+ * CPU scheduler quota accounting is also performed here in microseconds.
+ */
+static void update_cpu_clock_switch(struct rq *rq, struct task_struct *p)
+{
+	s64 account_ns = rq->niffies - p->last_ran;
+	struct task_struct *idle = rq->idle;
+
+	/*
+	 * Accurate subtick timekeeping. There is no interrupt frame to sample
+	 * here the way update_cpu_clock_tick() samples one: the entry code has
+	 * already restored the previous value by the time irqentry_exit()
+	 * reaches preempt_schedule_irq(), and a task that called schedule()
+	 * itself never had a frame at all. What can be said for certain is that
+	 * a task which never executes user code cannot have spent this interval
+	 * in userspace, so charge all of those as system time. Without this the
+	 * whole of every kernel thread's runtime that lands between two ticks
+	 * is booked as user time, which is how irq threads, kworkers and
+	 * ksoftirqd end up carrying utime they cannot possibly have accrued.
+	 *
+	 * A user task that blocks in a syscall genuinely does split its
+	 * interval between the two modes, and knowing where it crossed over
+	 * needs timestamped user/kernel transitions - vtime - that MuQSS
+	 * deliberately does not take. Those keep the historical assumption.
+	 */
+	if (p == idle)
+		pc_idle_time(rq, idle, account_ns);
+	else if (is_user_task(p))
+		pc_user_time(rq, p, account_ns);
+	else
+		pc_system_time(rq, p, 0, account_ns);
+
+	/* time_slice accounting is done in usecs to avoid overflow on 32bit */
+	if (p->policy != SCHED_FIFO && p != idle)
+		p->time_slice -= NS_TO_US(account_ns);
+}
+
+#ifdef CONFIG_MUQSS_IOTIME
+/*
+ * The CPU time current has consumed so far, for measuring how long a stretch
+ * of work took the thread doing it.
+ *
+ * ->sched_time alone is not enough. It is banked at ticks and at context
+ * switches, and a kworker's switches happen at the ends of a whole batch of
+ * work items, so the delta across any one of them is usually a flat zero. The
+ * unbanked remainder has to be added in, and that means reading a clock:
+ * sched_clock_cpu() rather than rq->niffies, which is only refreshed under
+ * the rq lock and would be stale by up to a tick here.
+ *
+ * niffies is monotonised forward of the raw clock, so last_ran can be ahead
+ * of what we read and the remainder can come out negative. Drop it in that
+ * case; it is bounded by the skew between the two and the banked figure is
+ * still right.
+ *
+ * Interrupts are off across the read so the tick cannot land between taking
+ * ->sched_time and taking ->last_ran and have the interval counted in both.
+ * That also pins us to this CPU, which is what makes the subtraction a
+ * same-clock one.
+ */
+u64 muqss_task_runtime_live(void)
+{
+	struct task_struct *p = current;
+	unsigned long flags;
+	s64 remainder;
+	u64 ns, ran;
+
+	local_irq_save(flags);
+	ns = p->sched_time;
+	ran = p->last_ran;
+	remainder = sched_clock_cpu(smp_processor_id()) - ran;
+	local_irq_restore(flags);
+
+	if (likely(remainder > 0))
+		ns += remainder;
+
+	return ns;
+}
+#endif
+
+/*
+ * Return any ns on the sched_clock that have not yet been accounted in
+ * @p in case that task is currently running.
+ *
+ * Called with task_rq_lock(p) held.
+ */
+static inline u64 do_task_delta_exec(struct task_struct *p, struct rq *rq)
+{
+	u64 ns = 0;
+
+	/*
+	 * Must be ->curr _and_ ->on_rq.  If dequeued, we would
+	 * project cycles that may never be accounted to this
+	 * thread, breaking clock_gettime().
+	 */
+	if (p == rq->curr && task_on_rq_queued(p)) {
+		update_clocks(rq);
+		ns = rq->niffies - p->last_ran;
+	}
+
+	return ns;
+}
+
+/*
+ * Return accounted runtime for the task.
+ * Return separately the current's pending runtime that have not been
+ * accounted yet.
+ */
+unsigned long long task_sched_runtime(struct task_struct *p)
+{
+	struct rq_flags rf;
+	struct rq *rq;
+	u64 ns;
+
+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
+	/*
+	 * 64-bit doesn't need locks to atomically read a 64-bit value.
+	 * So we have a optimisation chance when the task's delta_exec is 0.
+	 * Reading ->on_cpu is racy, but this is ok.
+	 *
+	 * If we race with it leaving CPU, we'll take a lock. So we're correct.
+	 * If we race with it entering CPU, unaccounted time is 0. This is
+	 * indistinguishable from the read occurring a few cycles earlier.
+	 * If we see ->on_cpu without ->on_rq, the task is leaving, and has
+	 * been accounted, so we're correct here as well.
+	 */
+	if (!p->on_cpu || !task_on_rq_queued(p))
+		return tsk_seruntime(p);
+#endif
+
+	rq = task_rq_lock(p, &rf);
+	ns = p->sched_time + do_task_delta_exec(p, rq);
+	task_rq_unlock(rq, p, &rf);
+
+	return ns;
+}
+
+/*
+ * Functions to test for when SCHED_ISO tasks have used their allocated
+ * quota as real time scheduling and convert them back to SCHED_NORMAL. All
+ * data is modified only by the local runqueue during sched_tick with
+ * interrupts disabled.
+ */
+
+/*
+ * Test if SCHED_ISO tasks have run longer than their alloted period as RT
+ * tasks and set the refractory flag if necessary. There is 10% hysteresis
+ * for unsetting the flag. 115/128 is ~90/100 as a fast shift instead of a
+ * slow division.
+ */
+static inline void iso_tick(struct rq *rq)
+{
+	rq->iso_ticks = rq->iso_ticks * (ISO_PERIOD - 1) / ISO_PERIOD;
+	rq->iso_ticks += 100;
+	if (rq->iso_ticks > ISO_PERIOD * sched_iso_cpu) {
+		rq->iso_refractory = true;
+		if (unlikely(rq->iso_ticks > ISO_PERIOD * 100))
+			rq->iso_ticks = ISO_PERIOD * 100;
+	}
+}
+
+/* No SCHED_ISO task was running so decrease rq->iso_ticks */
+static inline void no_iso_tick(struct rq *rq, int ticks)
+{
+	if (rq->iso_ticks > 0 || rq->iso_refractory) {
+		rq->iso_ticks = rq->iso_ticks * (ISO_PERIOD - ticks) / ISO_PERIOD;
+		if (rq->iso_ticks < ISO_PERIOD * (sched_iso_cpu * 115 / 128)) {
+			rq->iso_refractory = false;
+			if (unlikely(rq->iso_ticks < 0))
+				rq->iso_ticks = 0;
+		}
+	}
+}
+
+/* This manages tasks that have run out of timeslice during a sched_tick */
+static void task_running_tick(struct rq *rq)
+{
+	struct task_struct *p = rq->curr;
+
+	/*
+	 * If a SCHED_ISO task is running we increment the iso_ticks. In
+	 * order to prevent SCHED_ISO tasks from causing starvation in the
+	 * presence of true RT tasks we account those as iso_ticks as well.
+	 */
+	if (rt_task(p) || task_running_iso(p))
+		iso_tick(rq);
+	else
+		no_iso_tick(rq, 1);
+
+	/* SCHED_FIFO tasks never run out of timeslice. */
+	if (p->policy == SCHED_FIFO)
+		return;
+
+	if (iso_task(p)) {
+		if (task_running_iso(p)) {
+			if (rq->iso_refractory) {
+				/*
+				 * SCHED_ISO task is running as RT and limit
+				 * has been hit. Force it to reschedule as
+				 * SCHED_NORMAL by zeroing its time_slice
+				 */
+				p->time_slice = 0;
+			}
+		} else if (!rq->iso_refractory) {
+			/* Can now run again ISO. Reschedule to pick up prio */
+			goto out_resched;
+		}
+	}
+
+	/*
+	 * Tasks that were scheduled in the first half of a tick are not
+	 * allowed to run into the 2nd half of the next tick if they will
+	 * run out of time slice in the interim. Otherwise, if they have
+	 * less than RESCHED_US μs of time slice left they will be rescheduled.
+	 * Dither is used as a backup for when hrexpiry is disabled or high res
+	 * timers not configured in.
+	 */
+	if (p->time_slice - rq->dither >= RESCHED_US)
+		return;
+out_resched:
+	rq_lock(rq);
+	__set_tsk_resched(p);
+	rq_unlock(rq);
+}
+
+static inline void task_tick(struct rq *rq)
+{
+	if (!rq_idle(rq))
+		task_running_tick(rq);
+	else if (rq->last_jiffy > rq->last_scheduler_tick)
+		no_iso_tick(rq, rq->last_jiffy - rq->last_scheduler_tick);
+}
+
+#ifdef CONFIG_NO_HZ_FULL
+/*
+ * We can stop the timer tick any time highres timers are active since
+ * we rely entirely on highres timeouts for task expiry rescheduling.
+ */
+static void sched_stop_tick(struct rq *rq, int cpu)
+{
+	if (!hrexpiry_enabled(rq))
+		return;
+	if (!tick_nohz_full_enabled())
+		return;
+	if (!tick_nohz_full_cpu(cpu))
+		return;
+	tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED);
+}
+
+static inline void sched_start_tick(struct rq *rq, int cpu)
+{
+	tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
+}
+
+struct tick_work {
+	int			cpu;
+	atomic_t		state;
+	struct delayed_work	work;
+};
+/* Values for ->state, see diagram below. */
+#define TICK_SCHED_REMOTE_OFFLINE	0
+#define TICK_SCHED_REMOTE_OFFLINING	1
+#define TICK_SCHED_REMOTE_RUNNING	2
+
+/*
+ * State diagram for ->state:
+ *
+ *
+ *          TICK_SCHED_REMOTE_OFFLINE
+ *                    |   ^
+ *                    |   |
+ *                    |   | sched_tick_remote()
+ *                    |   |
+ *                    |   |
+ *                    +--TICK_SCHED_REMOTE_OFFLINING
+ *                    |   ^
+ *                    |   |
+ * sched_tick_start() |   | sched_tick_stop()
+ *                    |   |
+ *                    V   |
+ *          TICK_SCHED_REMOTE_RUNNING
+ *
+ *
+ * Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
+ * and sched_tick_start() are happy to leave the state in RUNNING.
+ */
+
+static struct tick_work __percpu *tick_work_cpu;
+
+static void sched_tick_remote(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct tick_work *twork = container_of(dwork, struct tick_work, work);
+	int cpu = twork->cpu;
+	struct rq *rq = cpu_rq(cpu);
+	struct task_struct *curr;
+	u64 delta;
+	int os;
+
+	/*
+	 * Handle the tick only if it appears the remote CPU is running in full
+	 * dynticks mode. The check is racy by nature, but missing a tick or
+	 * having one too much is no big deal because the scheduler tick updates
+	 * statistics and checks timeslices in a time-independent way, regardless
+	 * of when exactly it is running.
+	 */
+	if (!tick_nohz_tick_stopped_cpu(cpu))
+		goto out_requeue;
+
+	rq_lock_irq(rq);
+	if (cpu_is_offline(cpu))
+		goto out_unlock;
+
+	curr = rq->curr;
+	update_rq_clock(rq);
+
+	if (!is_idle_task(curr)) {
+		/*
+		 * Make sure the next tick runs within a reasonable
+		 * amount of time.
+		 */
+		delta = rq_clock_task(rq) - curr->last_ran;
+		WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 3);
+	}
+	/*
+	 * task_tick() takes the rq lock itself when it needs to force a
+	 * reschedule so we must drop it here, keeping interrupts disabled
+	 * to match the context it is called in from sched_tick().
+	 */
+	rq_unlock(rq);
+	task_tick(rq);
+	local_irq_enable();
+	goto out_requeue;
+
+out_unlock:
+	rq_unlock_irq(rq, NULL);
+
+out_requeue:
+
+	/*
+	 * Run the remote tick once per second (1Hz). This arbitrary
+	 * frequency is large enough to avoid overload but short enough
+	 * to keep scheduler internal stats reasonably up to date.  But
+	 * first update state to reflect hotplug activity if required.
+	 */
+	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
+	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
+	if (os == TICK_SCHED_REMOTE_RUNNING)
+		queue_delayed_work(system_unbound_wq, dwork, HZ);
+}
+
+static void sched_tick_start(int cpu)
+{
+	struct tick_work *twork;
+	int os;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_TICK))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
+	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
+	if (os == TICK_SCHED_REMOTE_OFFLINE) {
+		twork->cpu = cpu;
+		INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
+		queue_delayed_work(system_unbound_wq, &twork->work, HZ);
+	}
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void sched_tick_stop(int cpu)
+{
+	struct tick_work *twork;
+	int os;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_TICK))
+		return;
+
+	WARN_ON_ONCE(!tick_work_cpu);
+
+	twork = per_cpu_ptr(tick_work_cpu, cpu);
+	/* There cannot be competing actions, but don't rely on stop-machine. */
+	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
+	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
+	/* Don't cancel, as this would mess up the state machine. */
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+int __init sched_tick_offload_init(void)
+{
+	tick_work_cpu = alloc_percpu(struct tick_work);
+	BUG_ON(!tick_work_cpu);
+	return 0;
+}
+
+#else /* !CONFIG_NO_HZ_FULL */
+static inline void sched_stop_tick(struct rq *rq, int cpu) {}
+static inline void sched_start_tick(struct rq *rq, int cpu) {}
+static inline void sched_tick_start(int cpu) { }
+static inline void sched_tick_stop(int cpu) { }
+#endif
+
+/*
+ * This function gets called by the timer code, with HZ frequency.
+ * We call it with interrupts disabled.
+ */
+void sched_tick(void)
+{
+	int cpu __maybe_unused = smp_processor_id();
+	struct rq *rq = cpu_rq(cpu);
+
+	arch_scale_freq_tick();
+	sched_clock_tick();
+	update_clocks(rq);
+	update_load_avg(rq, 0);
+	update_cpu_clock_tick(rq, rq->curr);
+	task_tick(rq);
+	rq->last_scheduler_tick = rq->last_jiffy;
+	rq->last_tick = rq->clock;
+	perf_event_task_tick();
+	sched_stop_tick(rq, cpu);
+}
+
+#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
+				defined(CONFIG_TRACE_PREEMPT_TOGGLE))
+/*
+ * If the value passed in is equal to the current preempt count
+ * then we just disabled preemption. Start timing the latency.
+ */
+static inline void preempt_latency_start(int val)
+{
+	if (preempt_count() == val) {
+		unsigned long ip = get_lock_parent_ip();
+#ifdef CONFIG_DEBUG_PREEMPT
+		current->preempt_disable_ip = ip;
+#endif
+		trace_preempt_off(CALLER_ADDR0, ip);
+	}
+}
+
+void preempt_count_add(int val)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Underflow?
+	 */
+	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+		return;
+#endif
+	__preempt_count_add(val);
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Spinlock count overflowing soon?
+	 */
+	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
+				PREEMPT_MASK - 10);
+#endif
+	preempt_latency_start(val);
+}
+EXPORT_SYMBOL(preempt_count_add);
+NOKPROBE_SYMBOL(preempt_count_add);
+
+/*
+ * If the value passed in equals to the current preempt count
+ * then we just enabled preemption. Stop timing the latency.
+ */
+static inline void preempt_latency_stop(int val)
+{
+	if (preempt_count() == val)
+		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
+}
+
+void preempt_count_sub(int val)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	/*
+	 * Underflow?
+	 */
+	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+		return;
+	/*
+	 * Is the spinlock portion underflowing?
+	 */
+	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
+			!(preempt_count() & PREEMPT_MASK)))
+		return;
+#endif
+
+	preempt_latency_stop(val);
+	__preempt_count_sub(val);
+}
+EXPORT_SYMBOL(preempt_count_sub);
+NOKPROBE_SYMBOL(preempt_count_sub);
+
+#else
+static inline void preempt_latency_start(int val) { }
+static inline void preempt_latency_stop(int val) { }
+#endif
+
+static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
+{
+#ifdef CONFIG_DEBUG_PREEMPT
+	return p->preempt_disable_ip;
+#else
+	return 0;
+#endif
+}
+
+/*
+ * The time_slice is only refilled when it is empty and that is when we set a
+ * new deadline. Make sure update_clocks has been called recently to update
+ * rq->niffies.
+ */
+static void time_slice_expired(struct task_struct *p, struct rq *rq)
+{
+	p->time_slice = timeslice();
+	/*
+	 * This assignment is absolute, so the charge has to be folded in here
+	 * rather than left to enqueue_task(), or expiry would reset the task
+	 * to an uncharged baseline.
+	 */
+	p->deadline = rq->niffies + task_deadline_diff(p) +
+		      consume_task_penalty(p);
+#ifdef CONFIG_SMT_NICE
+	if (!p->mm)
+		p->smt_bias = 0;
+	else if (rt_task(p))
+		p->smt_bias = 1 << 30;
+	else if (task_running_iso(p))
+		p->smt_bias = 1 << 29;
+	else if (idleprio_task(p)) {
+		if (task_running_idle(p))
+			p->smt_bias = 0;
+		else
+			p->smt_bias = 1;
+	} else if (--p->smt_bias < 1)
+		p->smt_bias = MAX_PRIO - p->static_prio;
+#endif
+}
+
+/*
+ * Timeslices below RESCHED_US are considered as good as expired as there's no
+ * point rescheduling when there's so little time left. SCHED_BATCH tasks
+ * have been flagged be not latency sensitive and likely to be fully CPU
+ * bound so every time they're rescheduled they have their time_slice
+ * refilled, but get a new later deadline to have little effect on
+ * SCHED_NORMAL tasks.
+
+ */
+static inline void check_deadline(struct task_struct *p, struct rq *rq)
+{
+	if (p->time_slice < RESCHED_US || batch_task(p))
+		time_slice_expired(p, rq);
+}
+
+/*
+ * Task selection with skiplists is a simple matter of picking off the first
+ * task in the sorted list, an O(1) operation. The lookup is amortised O(1)
+ * being bound to the number of processors.
+ *
+ * Runqueues are selectively locked based on their unlocked data and then
+ * unlocked if not needed. At most 3 locks will be held at any time and are
+ * released as soon as they're no longer needed. All balancing between CPUs
+ * is thus done here in an extremely simple first come best fit manner.
+ *
+ * This iterates over runqueues in cache locality order. In interactive mode
+ * it iterates over all CPUs and finds the task with the best key/deadline.
+ * In non-interactive mode it will only take a task if it's from the current
+ * runqueue or a runqueue with more tasks than the current one with a better
+ * key/deadline.
+ */
+#ifdef CONFIG_SMP
+static inline struct task_struct
+*earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle)
+{
+	struct rq *locked = NULL, *chosen = NULL;
+	struct task_struct *edt = idle;
+	int i, best_entries = 0;
+	u64 best_key = ~0ULL;
+
+	for (i = 0; i < total_runqueues; i++) {
+		skiplist *sl = rq->sl_order[i];
+		struct rq *other_rq = NULL;
+		skiplist_node *next;
+		int entries;
+
+		entries = READ_ONCE(sl->entries);
+		/*
+		 * Check for queued entres lockless first. The local runqueue
+		 * is locked so entries will always be accurate.
+		 */
+		if (!sched_interactive) {
+			/*
+			 * Don't reschedule balance across nodes unless the CPU
+			 * is idle. Non-interactive needs the rq: cpu is read
+			 * before the entries filter.
+			 */
+			other_rq = rq_order(rq, i);
+			if (edt != idle && rq->cpu_locality[other_rq->cpu] > LOCALITY_SMP)
+				break;
+			if (entries <= best_entries)
+				continue;
+		} else if (!entries)
+			continue;
+
+		/* if (i) implies other_rq != rq */
+		if (i) {
+			/* Check for best id queued lockless first */
+			if (READ_ONCE(sl->best_key) >= best_key)
+				continue;
+
+			other_rq = rq_order(rq, i);
+			if (unlikely(!trylock_rq(rq, other_rq)))
+				continue;
+
+			/* Need to reevaluate entries after locking */
+			entries = sl->entries;
+			if (unlikely(!entries)) {
+				unlock_rq(other_rq);
+				continue;
+			}
+		}
+
+		next = sl->header;
+		/*
+		 * In interactive mode we check beyond the best entry on other
+		 * runqueues if we can't get the best for smt or affinity
+		 * reasons.
+		 */
+		while ((next = next->next[0]) != sl->header) {
+			struct task_struct *p;
+			u64 key = next->key;
+
+			/* Reevaluate key after locking */
+			if (key >= best_key)
+				break;
+
+			p = container_of(next, struct task_struct, node);
+			if (!smt_schedule(p, rq)) {
+				if (i && !sched_interactive)
+					break;
+				continue;
+			}
+
+			if (sched_other_cpu(p, cpu)) {
+				if (sched_interactive || !i)
+					continue;
+				break;
+			}
+			/* Make sure affinity is ok */
+			if (i) {
+				/* From this point on p is the best so far */
+				if (locked)
+					unlock_rq(locked);
+				chosen = locked = other_rq;
+			}
+			best_entries = entries;
+			best_key = key;
+			edt = p;
+			break;
+		}
+		/* rq->preempting is a hint only as the state may have changed
+		 * since it was set with the resched call but if we have met
+		 * the condition we can break out here. */
+		if (edt == rq->preempting)
+			break;
+		if (i && other_rq != chosen)
+			unlock_rq(other_rq);
+	}
+
+	if (likely(edt != idle))
+		take_task(rq, cpu, edt);
+
+	if (locked)
+		unlock_rq(locked);
+
+	rq->preempting = NULL;
+
+	return edt;
+}
+#else /* CONFIG_SMP */
+static inline struct task_struct
+*earliest_deadline_task(struct rq *rq, int cpu, struct task_struct *idle)
+{
+	struct task_struct *edt;
+
+	if (unlikely(!rq->sl->entries))
+		return idle;
+	edt = container_of(rq->node->next[0], struct task_struct, node);
+	take_task(rq, cpu, edt);
+	return edt;
+}
+#endif /* CONFIG_SMP */
+
+/*
+ * Print scheduling while atomic bug:
+ */
+static noinline void __schedule_bug(struct task_struct *prev)
+{
+	/* Save this before calling printk(), since that will clobber it */
+	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
+
+	if (oops_in_progress)
+		return;
+
+	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
+		prev->comm, prev->pid, preempt_count());
+
+	debug_show_held_locks(prev);
+	print_modules();
+	if (irqs_disabled())
+		print_irqtrace_events(prev);
+	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
+	    && in_atomic_preempt_off()) {
+		pr_err("Preemption disabled at:");
+		print_ip_sym(KERN_ERR, preempt_disable_ip);
+	}
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+
+/*
+ * Various schedule()-time debugging checks and statistics:
+ */
+static inline void schedule_debug(struct task_struct *prev, bool preempt)
+{
+#ifdef CONFIG_SCHED_STACK_END_CHECK
+	if (task_stack_end_corrupted(prev))
+		panic("corrupted stack end detected inside scheduler\n");
+
+	if (task_scs_end_corrupted(prev))
+		panic("corrupted shadow stack detected inside scheduler\n");
+#endif
+
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+	if (!preempt && prev->__state && prev->non_block_count) {
+		printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
+			prev->comm, prev->pid, prev->non_block_count);
+		dump_stack();
+		add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+	}
+#endif
+
+	if (unlikely(in_atomic_preempt_off())) {
+		__schedule_bug(prev);
+		preempt_count_set(PREEMPT_DISABLED);
+	}
+	rcu_sleep_check();
+	SCHED_WARN_ON(ct_state() == CT_STATE_USER);
+
+	profile_hit(SCHED_PROFILING, __builtin_return_address(0));
+
+	schedstat_inc(this_rq()->sched_count);
+}
+
+/*
+ * The currently running task's information is all stored in rq local data
+ * which is only modified by the local CPU.
+ */
+static inline void set_rq_task(struct rq *rq, struct task_struct *p)
+{
+	if (p == rq->idle || p->policy == SCHED_FIFO)
+		hrexpiry_clear(rq);
+	else
+		hrexpiry_start(rq, US_TO_NS(p->time_slice));
+	if (rq->clock - rq->last_tick > HALF_JIFFY_NS)
+		rq->dither = 0;
+	else
+		rq->dither = rq_dither(rq);
+
+	rq->rq_deadline = p->deadline;
+	rq->rq_prio = p->prio;
+#ifdef CONFIG_SMT_NICE
+	rq->rq_mm = p->mm;
+	rq->rq_smt_bias = p->smt_bias;
+#endif
+}
+
+#ifdef CONFIG_SMT_NICE
+static void check_no_siblings(struct rq __maybe_unused *this_rq) {}
+static void wake_no_siblings(struct rq __maybe_unused *this_rq) {}
+static void (*check_siblings)(struct rq *this_rq) = &check_no_siblings;
+static void (*wake_siblings)(struct rq *this_rq) = &wake_no_siblings;
+
+/* Iterate over smt siblings when we've scheduled a process on cpu and decide
+ * whether they should continue running or be descheduled. */
+static void check_smt_siblings(struct rq *this_rq)
+{
+	int other_cpu;
+
+	for_each_cpu(other_cpu, &this_rq->thread_mask) {
+		struct task_struct *p;
+		struct rq *rq;
+
+		rq = cpu_rq(other_cpu);
+		if (rq_idle(rq))
+			continue;
+		p = rq->curr;
+		if (!smt_schedule(p, this_rq))
+			resched_curr(rq);
+	}
+}
+
+static void wake_smt_siblings(struct rq *this_rq)
+{
+	int other_cpu;
+
+	for_each_cpu(other_cpu, &this_rq->thread_mask) {
+		struct rq *rq;
+
+		rq = cpu_rq(other_cpu);
+		if (rq_idle(rq))
+			resched_idle(rq);
+	}
+}
+#else
+static void check_siblings(struct rq __maybe_unused *this_rq) {}
+static void wake_siblings(struct rq __maybe_unused *this_rq) {}
+#endif
+
+/*
+ * schedule() is the main scheduler function.
+ *
+ * The main means of driving the scheduler and thus entering this function are:
+ *
+ *   1. Explicit blocking: mutex, semaphore, waitqueue, etc.
+ *
+ *   2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
+ *      paths. For example, see arch/x86/entry_64.S.
+ *
+ *      To drive preemption between tasks, the scheduler sets the flag in timer
+ *      interrupt handler sched_tick().
+ *
+ *   3. Wakeups don't really cause entry into schedule(). They add a
+ *      task to the run-queue and that's it.
+ *
+ *      Now, if the new task added to the run-queue preempts the current
+ *      task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
+ *      called on the nearest possible occasion:
+ *
+ *       - If the kernel is preemptible (CONFIG_PREEMPTION=y):
+ *
+ *         - in syscall or exception context, at the next outmost
+ *           preempt_enable(). (this might be as soon as the wake_up()'s
+ *           spin_unlock()!)
+ *
+ *         - in IRQ context, return from interrupt-handler to
+ *           preemptible context
+ *
+ *       - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
+ *         then at the next:
+ *
+ *          - cond_resched() call
+ *          - explicit schedule() call
+ *          - return from syscall or exception to user-space
+ *          - return from interrupt-handler to user-space
+ *
+ * WARNING: must be called with preemption disabled!
+ */
+#define SM_IDLE			(-1)
+#define SM_NONE			0
+#define SM_PREEMPT		1
+#define SM_RTLOCK_WAIT		2
+
+static void __sched notrace __schedule(int sched_mode)
+{
+	struct task_struct *prev, *next, *idle;
+	unsigned long *switch_count;
+	unsigned long prev_state;
+	bool deactivate = false;
+	struct rq *rq;
+	u64 niffies;
+	int cpu;
+	/*
+	 * On PREEMPT_RT, SM_RTLOCK_WAIT is noted as a preemption by
+	 * schedule_debug() and RCU. Task-state changes still treat
+	 * only SM_PREEMPT as preemption so a sleeping lock wait can
+	 * deactivate.
+	 */
+	bool preempt = sched_mode > SM_NONE;
+
+	cpu = smp_processor_id();
+	rq = cpu_rq(cpu);
+	prev = rq->curr;
+	idle = rq->idle;
+
+	schedule_debug(prev, preempt);
+
+	klp_sched_try_switch(prev);
+
+	local_irq_disable();
+	rcu_note_context_switch(preempt);
+
+	/*
+	 * Make sure that signal_pending_state()->signal_pending() below
+	 * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
+	 * done by the caller to avoid the race with signal_wake_up():
+	 *
+	 * __set_current_state(@state)		signal_wake_up()
+	 * schedule()				  set_tsk_thread_flag(p, TIF_SIGPENDING)
+	 *					  wake_up_state(p, state)
+	 *   LOCK rq->lock			    LOCK p->pi_state
+	 *   smp_mb__after_spinlock()		    smp_mb__after_spinlock()
+	 *     if (signal_pending_state())	    if (p->__state & @state)
+	 *
+	 * Also, the membarrier system call requires a full memory barrier
+	 * after coming from user-space, before storing to rq->curr.
+	 */
+	rq_lock(rq);
+	smp_mb__after_spinlock();
+#ifdef CONFIG_SMP
+	if (rq->preempt) {
+		/*
+		 * Make sure resched_curr hasn't triggered a preemption
+		 * locklessly on a task that has since scheduled away. Spurious
+		 * wakeup of idle is okay though.
+		 */
+		if (unlikely(sched_mode == SM_PREEMPT && prev != idle &&
+			     !test_tsk_need_resched(prev))) {
+			rq->preempt = NULL;
+			clear_preempt_need_resched();
+			rq_unlock_irq(rq, NULL);
+			return;
+		}
+		rq->preempt = NULL;
+	}
+	migrate_disable_switch(rq, prev);
+#endif
+
+	/*
+	 * Defer hrexpiry start/cancel until we leave __schedule so we do not
+	 * thrash the oneshot clockevent under the rq lock (mainline hrtick).
+	 */
+	hrexpiry_schedule_enter(rq);
+
+	switch_count = &prev->nivcsw;
+
+	/* Task state changes only consider SM_PREEMPT as preemption */
+	preempt = sched_mode == SM_PREEMPT;
+
+	/*
+	 * We must load prev->__state once (task_struct::state is volatile), such
+	 * that:
+	 *
+	 *  - we form a control dependency vs deactivate_task() below.
+	 *  - ptrace_{,un}freeze_traced() can change ->state underneath us.
+	 */
+	prev_state = prev->__state;
+	if (!preempt && prev_state) {
+		if (signal_pending_state(prev_state, prev)) {
+			prev->__state = TASK_RUNNING;
+		} else {
+			prev->sched_contributes_to_load =
+				(prev_state & TASK_UNINTERRUPTIBLE) &&
+				!(prev_state & TASK_NOLOAD) &&
+				!(prev_state & TASK_FROZEN);
+
+			if (prev->sched_contributes_to_load)
+				rq->nr_uninterruptible++;
+
+			/*
+			 * __schedule()			ttwu()
+			 *   prev_state = prev->__state;    if (p->on_rq && ...)
+			 *   if (prev_state)		    goto out;
+			 *     p->on_rq = 0;		  smp_acquire__after_ctrl_dep();
+			 *				  p->__state = TASK_WAKING
+			 *
+			 * Where __schedule() and ttwu() have matching control dependencies.
+			 *
+			 * After this, schedule() must not care about p->__state any more.
+			 */
+			deactivate = true;
+
+			if (prev->in_iowait) {
+				atomic_inc(&rq->nr_iowait);
+				delayacct_blkio_start();
+			}
+		}
+		switch_count = &prev->nvcsw;
+	}
+
+	/*
+	 * Store the niffy value here for use by the next task's last_ran
+	 * below to avoid losing niffies due to update_clocks being called
+	 * again after this point.
+	 */
+	update_clocks(rq);
+	niffies = rq->niffies;
+	update_cpu_clock_switch(rq, prev);
+
+	clear_tsk_need_resched(prev);
+	clear_preempt_need_resched();
+
+	if (idle != prev) {
+		check_deadline(prev, rq);
+		return_task(prev, rq, cpu, deactivate);
+	}
+
+	next = earliest_deadline_task(rq, cpu, idle);
+	if (likely(next->prio != PRIO_LIMIT))
+		clear_cpuidle_map(cpu);
+	else {
+#ifdef CONFIG_SMP
+		if (prev != idle)
+			rq->idle_jiffy = jiffies;
+#endif
+		set_cpuidle_map(cpu);
+		update_load_avg(rq, 0);
+	}
+
+	set_rq_task(rq, next);
+	next->last_ran = niffies;
+
+	if (likely(prev != next)) {
+		/*
+		 * Don't reschedule an idle task or deactivated tasks
+		 */
+		if (prev == idle)
+			inc_nr_running(rq);
+		else if (!deactivate)
+			resched_suitable_idle(prev);
+		/*
+		 * The task on the CPU is not on the skiplist, so it holds a
+		 * count of its own on top of the queued ones. nr_running only
+		 * changes when that slot is created or destroyed - i.e. on the
+		 * idle transitions above and below - but rt_nr_running also
+		 * depends on *which* task holds it, so hand it over on every
+		 * switch. Leaving that to the idle transitions leaked a count
+		 * on every rt -> non-rt switch and underflowed on the reverse.
+		 */
+		if (prev != idle && rt_task(prev))
+			rq->rt_nr_running--;
+		if (next != idle && rt_task(next))
+			rq->rt_nr_running++;
+		if (unlikely(next == idle)) {
+			dec_nr_running(rq);
+			wake_siblings(rq);
+		} else
+			check_siblings(rq);
+		rq->nr_switches++;
+		/*
+		 * RCU users of rcu_dereference(rq->curr) may not see
+		 * changes to task_struct made by pick_next_task().
+		 */
+		RCU_INIT_POINTER(rq->curr, next);
+		/*
+		 * The membarrier system call requires each architecture
+		 * to have a full memory barrier after updating
+		 * rq->curr, before returning to user-space.
+		 *
+		 * Here are the schemes providing that barrier on the
+		 * various architectures:
+		 * - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC.
+		 *   switch_mm() rely on membarrier_arch_switch_mm() on PowerPC.
+		 * - finish_lock_switch() for weakly-ordered
+		 *   architectures where spin_unlock is a full barrier,
+		 * - switch_to() for arm64 (weakly-ordered, spin_unlock
+		 *   is a RELEASE barrier),
+		 */
+		++*switch_count;
+
+		psi_sched_switch(prev, next, !task_on_rq_queued(prev));
+
+		trace_sched_switch(preempt, prev, next, prev->__state);
+		context_switch(rq, prev, next); /* unlocks the rq via finish_lock_switch */
+	} else {
+		check_siblings(rq);
+		hrexpiry_schedule_exit(rq);
+		rq_unlock(rq);
+		local_irq_enable();
+	}
+}
+
+void __noreturn do_task_dead(void)
+{
+	/* Causes final put_task_struct in finish_task_switch(). */
+	set_special_state(TASK_DEAD);
+
+	/* Tell freezer to ignore us: */
+	current->flags |= PF_NOFREEZE;
+	__schedule(SM_NONE);
+	BUG();
+
+	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
+	for (;;)
+		cpu_relax();
+}
+
+static inline void sched_submit_work(struct task_struct *tsk)
+{
+	unsigned int task_flags;
+
+	if (!tsk->__state)
+		return;
+
+	task_flags = tsk->flags;
+	/*
+	 * If a worker went to sleep, notify and ask workqueue whether
+	 * it wants to wake up a task to maintain concurrency.
+	 * As this function is called inside the schedule() context,
+	 * we disable preemption to avoid it calling schedule() again
+	 * in the possible wakeup of a kworker and because wq_worker_sleeping()
+	 * requires it.
+	 */
+	if (task_flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+		preempt_disable();
+		if (task_flags & PF_WQ_WORKER)
+			wq_worker_sleeping(tsk);
+		else
+			io_wq_worker_sleeping(tsk);
+		preempt_enable_no_resched();
+	}
+
+	if (tsk->pi_blocked_on)
+		return;
+
+	/*
+	 * If we are going to sleep and we have plugged IO queued,
+	 * make sure to submit it to avoid deadlocks.
+	 */
+	blk_flush_plug(tsk->plug, true);
+}
+
+static inline void sched_update_worker(struct task_struct *tsk)
+{
+	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) {
+		if (tsk->flags & PF_WQ_WORKER)
+			wq_worker_running(tsk);
+		else
+			io_wq_worker_running(tsk);
+	}
+}
+
+static __always_inline void __schedule_loop(int sched_mode)
+{
+	do {
+		preempt_disable();
+		__schedule(sched_mode);
+		sched_preempt_enable_no_resched();
+	} while (need_resched());
+}
+
+asmlinkage __visible void __sched schedule(void)
+{
+	struct task_struct *tsk = current;
+
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(!tsk->sched_rt_mutex);
+#endif
+	sched_submit_work(tsk);
+	__schedule_loop(SM_NONE);
+	sched_update_worker(tsk);
+}
+
+EXPORT_SYMBOL(schedule);
+
+/*
+ * synchronize_rcu_tasks() makes sure that no task is stuck in preempted
+ * state (have scheduled out non-voluntarily) by making sure that all
+ * tasks have either left the run queue or have gone into user space.
+ * As idle tasks do not do either, they must not ever be preempted
+ * (schedule out non-voluntarily).
+ *
+ * schedule_idle() is similar to schedule_preempt_disable() except that it
+ * never enables preemption because it does not call sched_submit_work().
+ */
+void __sched schedule_idle(void)
+{
+	/*
+	 * As this skips calling sched_submit_work(), which the idle task does
+	 * regardless because that function is a nop when the task is in a
+	 * TASK_RUNNING state, make sure this isn't used someplace that the
+	 * current task can be in any other state. Note, idle is always in the
+	 * TASK_RUNNING state.
+	 */
+	WARN_ON_ONCE(current->__state);
+	do {
+		__schedule(SM_IDLE);
+	} while (need_resched());
+}
+
+#if defined(CONFIG_CONTEXT_TRACKING) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_OFFSTACK)
+asmlinkage __visible void __sched schedule_user(void)
+{
+	/*
+	 * If we come here after a random call to set_need_resched(),
+	 * or we have been woken up remotely but the IPI has not yet arrived,
+	 * we haven't yet exited the RCU idle mode. Do it here manually until
+	 * we find a better solution.
+	 *
+	 * NB: There are buggy callers of this function.  Ideally we
+	 * should warn if prev_state != IN_USER, but that will trigger
+	 * too frequently to make sense yet.
+	 */
+	enum ctx_state prev_state = exception_enter();
+	schedule();
+	exception_exit(prev_state);
+}
+#endif
+
+/**
+ * schedule_preempt_disabled - called with preemption disabled
+ *
+ * Returns with preemption disabled. Note: preempt_count must be 1
+ */
+void __sched schedule_preempt_disabled(void)
+{
+	sched_preempt_enable_no_resched();
+	schedule();
+	preempt_disable();
+}
+
+#ifdef CONFIG_PREEMPT_RT
+void __sched notrace schedule_rtlock(void)
+{
+	__schedule_loop(SM_RTLOCK_WAIT);
+}
+NOKPROBE_SYMBOL(schedule_rtlock);
+#endif
+
+static void __sched notrace preempt_schedule_common(void)
+{
+	do {
+		/*
+		 * Because the function tracer can trace preempt_count_sub()
+		 * and it also uses preempt_enable/disable_notrace(), if
+		 * NEED_RESCHED is set, the preempt_enable_notrace() called
+		 * by the function tracer will call this function again and
+		 * cause infinite recursion.
+		 *
+		 * Preemption must be disabled here before the function
+		 * tracer can trace. Break up preempt_disable() into two
+		 * calls. One to disable preemption without fear of being
+		 * traced. The other to still record the preemption latency,
+		 * which can also be traced by the function tracer.
+		 */
+		preempt_disable_notrace();
+		preempt_latency_start(1);
+		__schedule(SM_PREEMPT);
+		preempt_latency_stop(1);
+		preempt_enable_no_resched_notrace();
+
+		/*
+		 * Check again in case we missed a preemption opportunity
+		 * between schedule and now.
+		 */
+	} while (need_resched());
+}
+
+#ifdef CONFIG_PREEMPTION
+/*
+ * This is the entry point to schedule() from in-kernel preemption
+ * off of preempt_enable.
+ */
+asmlinkage __visible void __sched notrace preempt_schedule(void)
+{
+	/*
+	 * If there is a non-zero preempt_count or interrupts are disabled,
+	 * we do not want to preempt the current task. Just return..
+	 */
+	if (likely(!preemptible()))
+		return;
+
+	preempt_schedule_common();
+}
+NOKPROBE_SYMBOL(preempt_schedule);
+EXPORT_SYMBOL(preempt_schedule);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+#  ifndef preempt_schedule_dynamic_enabled
+#   define preempt_schedule_dynamic_enabled	preempt_schedule
+#   define preempt_schedule_dynamic_disabled	NULL
+#  endif
+DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule);
+void __sched notrace dynamic_preempt_schedule(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule))
+		return;
+	preempt_schedule();
+}
+NOKPROBE_SYMBOL(dynamic_preempt_schedule);
+EXPORT_SYMBOL(dynamic_preempt_schedule);
+# endif
+#endif
+
+
+/**
+ * preempt_schedule_notrace - preempt_schedule called by tracing
+ *
+ * The tracing infrastructure uses preempt_enable_notrace to prevent
+ * recursion and tracing preempt enabling caused by the tracing
+ * infrastructure itself. But as tracing can happen in areas coming
+ * from userspace or just about to enter userspace, a preempt enable
+ * can occur before user_exit() is called. This will cause the scheduler
+ * to be called when the system is still in usermode.
+ *
+ * To prevent this, the preempt_enable_notrace will use this function
+ * instead of preempt_schedule() to exit user context if needed before
+ * calling the scheduler.
+ */
+asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
+{
+	enum ctx_state prev_ctx;
+
+	if (likely(!preemptible()))
+		return;
+
+	do {
+		/*
+		 * Because the function tracer can trace preempt_count_sub()
+		 * and it also uses preempt_enable/disable_notrace(), if
+		 * NEED_RESCHED is set, the preempt_enable_notrace() called
+		 * by the function tracer will call this function again and
+		 * cause infinite recursion.
+		 *
+		 * Preemption must be disabled here before the function
+		 * tracer can trace. Break up preempt_disable() into two
+		 * calls. One to disable preemption without fear of being
+		 * traced. The other to still record the preemption latency,
+		 * which can also be traced by the function tracer.
+		 */
+		preempt_disable_notrace();
+		preempt_latency_start(1);
+		/*
+		 * Needs preempt disabled in case user_exit() is traced
+		 * and the tracer calls preempt_enable_notrace() causing
+		 * an infinite recursion.
+		 */
+		prev_ctx = exception_enter();
+		__schedule(SM_PREEMPT);
+		exception_exit(prev_ctx);
+
+		preempt_latency_stop(1);
+		preempt_enable_no_resched_notrace();
+	} while (need_resched());
+}
+EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
+#  ifndef preempt_schedule_notrace_dynamic_enabled
+#   define preempt_schedule_notrace_dynamic_enabled	preempt_schedule_notrace
+#   define preempt_schedule_notrace_dynamic_disabled	NULL
+#  endif
+DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
+EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace);
+void __sched notrace dynamic_preempt_schedule_notrace(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace))
+		return;
+	preempt_schedule_notrace();
+}
+NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace);
+EXPORT_SYMBOL(dynamic_preempt_schedule_notrace);
+# endif
+#endif
+
+#endif /* CONFIG_PREEMPTION */
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+/*
+ * These aliases sit beside __cond_resched() in core.c, which MuQSS.c defines
+ * much further down; declare them here so sched_dynamic_update() can see them.
+ */
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+#  define cond_resched_dynamic_enabled		__cond_resched
+#  define cond_resched_dynamic_disabled		((void *)&__static_call_return0)
+#  define might_resched_dynamic_enabled		__cond_resched
+#  define might_resched_dynamic_disabled	((void *)&__static_call_return0)
+# endif
+#endif
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+
+# ifdef CONFIG_GENERIC_IRQ_ENTRY
+#  include <linux/irq-entry-common.h>
+# endif
+
+/*
+ * SC:cond_resched
+ * SC:might_resched
+ * SC:preempt_schedule
+ * SC:preempt_schedule_notrace
+ * SC:irqentry_exit_cond_resched
+ *
+ *
+ * NONE:
+ *   cond_resched               <- __cond_resched
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- NOP
+ *   preempt_schedule_notrace   <- NOP
+ *   irqentry_exit_cond_resched <- NOP
+ *   dynamic_preempt_lazy       <- false
+ *
+ * VOLUNTARY:
+ *   cond_resched               <- __cond_resched
+ *   might_resched              <- __cond_resched
+ *   preempt_schedule           <- NOP
+ *   preempt_schedule_notrace   <- NOP
+ *   irqentry_exit_cond_resched <- NOP
+ *   dynamic_preempt_lazy       <- false
+ *
+ * FULL:
+ *   cond_resched               <- RET0
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- preempt_schedule
+ *   preempt_schedule_notrace   <- preempt_schedule_notrace
+ *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
+ *   dynamic_preempt_lazy       <- false
+ *
+ * LAZY:
+ *   cond_resched               <- RET0
+ *   might_resched              <- RET0
+ *   preempt_schedule           <- preempt_schedule
+ *   preempt_schedule_notrace   <- preempt_schedule_notrace
+ *   irqentry_exit_cond_resched <- irqentry_exit_cond_resched
+ *   dynamic_preempt_lazy       <- true
+ */
+
+enum {
+	preempt_dynamic_undefined = -1,
+	preempt_dynamic_none,
+	preempt_dynamic_voluntary,
+	preempt_dynamic_full,
+	preempt_dynamic_lazy,
+};
+
+int preempt_dynamic_mode = preempt_dynamic_undefined;
+
+int sched_dynamic_mode(const char *str)
+{
+# if !(defined(CONFIG_PREEMPT_RT) || defined(CONFIG_ARCH_HAS_PREEMPT_LAZY))
+	if (!strcmp(str, "none"))
+		return preempt_dynamic_none;
+
+	if (!strcmp(str, "voluntary"))
+		return preempt_dynamic_voluntary;
+# endif
+
+	if (!strcmp(str, "full"))
+		return preempt_dynamic_full;
+
+# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY
+	if (!strcmp(str, "lazy"))
+		return preempt_dynamic_lazy;
+# endif
+
+	return -EINVAL;
+}
+
+# define preempt_dynamic_key_enable(f)	static_key_enable(&sk_dynamic_##f.key)
+# define preempt_dynamic_key_disable(f)	static_key_disable(&sk_dynamic_##f.key)
+
+# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
+#  define preempt_dynamic_enable(f)	static_call_update(f, f##_dynamic_enabled)
+#  define preempt_dynamic_disable(f)	static_call_update(f, f##_dynamic_disabled)
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+#  define preempt_dynamic_enable(f)	preempt_dynamic_key_enable(f)
+#  define preempt_dynamic_disable(f)	preempt_dynamic_key_disable(f)
+# else
+#  error "Unsupported PREEMPT_DYNAMIC mechanism"
+# endif
+
+static DEFINE_MUTEX(sched_dynamic_mutex);
+
+static void __sched_dynamic_update(int mode)
+{
+	/*
+	 * Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
+	 * the ZERO state, which is invalid.
+	 */
+	preempt_dynamic_enable(cond_resched);
+	preempt_dynamic_enable(might_resched);
+	preempt_dynamic_enable(preempt_schedule);
+	preempt_dynamic_enable(preempt_schedule_notrace);
+	preempt_dynamic_enable(irqentry_exit_cond_resched);
+	preempt_dynamic_key_disable(preempt_lazy);
+
+	switch (mode) {
+	case preempt_dynamic_none:
+		preempt_dynamic_enable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_disable(preempt_schedule);
+		preempt_dynamic_disable(preempt_schedule_notrace);
+		preempt_dynamic_disable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: none\n");
+		break;
+
+	case preempt_dynamic_voluntary:
+		preempt_dynamic_enable(cond_resched);
+		preempt_dynamic_enable(might_resched);
+		preempt_dynamic_disable(preempt_schedule);
+		preempt_dynamic_disable(preempt_schedule_notrace);
+		preempt_dynamic_disable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: voluntary\n");
+		break;
+
+	case preempt_dynamic_full:
+		preempt_dynamic_disable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_enable(preempt_schedule);
+		preempt_dynamic_enable(preempt_schedule_notrace);
+		preempt_dynamic_enable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_disable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: full\n");
+		break;
+
+	case preempt_dynamic_lazy:
+		preempt_dynamic_disable(cond_resched);
+		preempt_dynamic_disable(might_resched);
+		preempt_dynamic_enable(preempt_schedule);
+		preempt_dynamic_enable(preempt_schedule_notrace);
+		preempt_dynamic_enable(irqentry_exit_cond_resched);
+		preempt_dynamic_key_enable(preempt_lazy);
+		if (mode != preempt_dynamic_mode)
+			pr_info("Dynamic Preempt: lazy\n");
+		break;
+	}
+
+	WRITE_ONCE(preempt_dynamic_mode, mode);
+}
+
+void sched_dynamic_update(int mode)
+{
+	mutex_lock(&sched_dynamic_mutex);
+	__sched_dynamic_update(mode);
+	mutex_unlock(&sched_dynamic_mutex);
+}
+
+static int __init setup_preempt_mode(char *str)
+{
+	int mode = sched_dynamic_mode(str);
+	if (mode < 0) {
+		pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
+		return 0;
+	}
+
+	sched_dynamic_update(mode);
+	return 1;
+}
+__setup("preempt=", setup_preempt_mode);
+
+static void __init preempt_dynamic_init(void)
+{
+	if (preempt_dynamic_mode == preempt_dynamic_undefined) {
+		if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
+			sched_dynamic_update(preempt_dynamic_none);
+		} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
+			sched_dynamic_update(preempt_dynamic_voluntary);
+		} else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
+			sched_dynamic_update(preempt_dynamic_lazy);
+		} else {
+			/* Default static call setting, nothing to do */
+			WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
+			preempt_dynamic_mode = preempt_dynamic_full;
+			pr_info("Dynamic Preempt: full\n");
+		}
+	}
+}
+
+# define PREEMPT_MODEL_ACCESSOR(mode)					\
+	bool preempt_model_##mode(void)					\
+	{								\
+		int mode = READ_ONCE(preempt_dynamic_mode);		\
+		WARN_ON_ONCE(mode == preempt_dynamic_undefined);	\
+		return mode == preempt_dynamic_##mode;			\
+	}								\
+	EXPORT_SYMBOL_GPL(preempt_model_##mode)
+
+PREEMPT_MODEL_ACCESSOR(none);
+PREEMPT_MODEL_ACCESSOR(voluntary);
+PREEMPT_MODEL_ACCESSOR(full);
+PREEMPT_MODEL_ACCESSOR(lazy);
+
+#else /* !CONFIG_PREEMPT_DYNAMIC: */
+
+#define preempt_dynamic_mode -1
+
+static inline void preempt_dynamic_init(void) { }
+
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+const char *preempt_modes[] = {
+	"none", "voluntary", "full", "lazy", NULL,
+};
+/*
+ * This is the entry point to schedule() from kernel preemption
+ * off of irq context.
+ * Note, that this is called and return with irqs disabled. This will
+ * protect us against recursive calling from irq.
+ */
+asmlinkage __visible void __sched preempt_schedule_irq(void)
+{
+	enum ctx_state prev_state;
+
+	/* Catch callers which need to be fixed */
+	BUG_ON(preempt_count() || !irqs_disabled());
+
+	prev_state = exception_enter();
+
+	do {
+		preempt_disable();
+		local_irq_enable();
+		__schedule(SM_PREEMPT);
+		local_irq_disable();
+		sched_preempt_enable_no_resched();
+	} while (need_resched());
+
+	exception_exit(prev_state);
+}
+
+int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
+			  void *key)
+{
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_SCHED_DEBUG) && wake_flags & ~WF_SYNC);
+	return try_to_wake_up(curr->private, mode, wake_flags);
+}
+EXPORT_SYMBOL(default_wake_function);
+
+#ifdef CONFIG_RT_MUTEXES
+
+static inline int __rt_effective_prio(struct task_struct *pi_task, int prio)
+{
+	if (pi_task)
+		prio = min(prio, pi_task->prio);
+
+	return prio;
+}
+
+static inline int rt_effective_prio(struct task_struct *p, int prio)
+{
+	struct task_struct *pi_task = rt_mutex_get_top_task(p);
+
+	return __rt_effective_prio(pi_task, prio);
+}
+
+/*
+ * rt_mutex_setprio - set the current priority of a task
+ * @p: task to boost
+ * @pi_task: donor task
+ *
+ * This function changes the 'effective' priority of a task. It does
+ * not touch ->normal_prio like __setscheduler().
+ *
+ * Used by the rt_mutex code to implement priority inheritance
+ * logic. Call site only calls if the priority of the task changed.
+ */
+void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
+{
+	int prio, oldprio;
+	struct rq *rq;
+
+	/* XXX used to be waiter->prio, not waiter->task->prio */
+	prio = __rt_effective_prio(pi_task, p->normal_prio);
+
+	/*
+	 * If nothing changed; bail early.
+	 */
+	if (p->pi_top_task == pi_task && prio == p->prio)
+		return;
+
+	rq = __task_rq_lock(p, NULL);
+	update_rq_clock(rq);
+	/*
+	 * Set under pi_lock && rq->lock, such that the value can be used under
+	 * either lock.
+	 *
+	 * Note that there is loads of tricky to make this pointer cache work
+	 * right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
+	 * ensure a task is de-boosted (pi_task is set to NULL) before the
+	 * task is allowed to run again (and can exit). This ensures the pointer
+	 * points to a blocked task -- which guarantees the task is present.
+	 */
+	p->pi_top_task = pi_task;
+
+	/*
+	 * For FIFO/RR we only need to set prio, if that matches we're done.
+	 */
+	if (prio == p->prio)
+		goto out_unlock;
+
+	/*
+	 * Idle task boosting is a nono in general. There is one
+	 * exception, when PREEMPT_RT and NOHZ is active:
+	 *
+	 * The idle task calls get_next_timer_interrupt() and holds
+	 * the timer wheel base->lock on the CPU and another CPU wants
+	 * to access the timer (probably to cancel it). We can safely
+	 * ignore the boosting request, as the idle CPU runs this code
+	 * with interrupts disabled and will complete the lock
+	 * protected section without being interrupted. So there is no
+	 * real need to boost.
+	 */
+	if (unlikely(p == rq->idle)) {
+		WARN_ON(p != rq->curr);
+		WARN_ON(p->pi_blocked_on);
+		goto out_unlock;
+	}
+
+	trace_sched_pi_setprio(p, pi_task);
+	oldprio = p->prio;
+	p->prio = prio;
+	if (task_running(rq, p)){
+		rt_running_reprio(rq, oldprio, prio);
+		if (prio > oldprio)
+			resched_task(p);
+	} else if (task_queued(p)) {
+		dequeue_task(rq, p, DEQUEUE_SAVE);
+		enqueue_task(rq, p, ENQUEUE_RESTORE);
+		if (prio < oldprio)
+			try_preempt(p, rq);
+	}
+out_unlock:
+	/* Avoid rq from going away on us: */
+	preempt_disable();
+	__task_rq_unlock(rq, p, NULL);
+
+	preempt_enable();
+}
+#else
+static inline int rt_effective_prio(struct task_struct *p, int prio)
+{
+	return prio;
+}
+#endif
+
+/*
+ * Adjust the deadline for when the priority is to change, before it's
+ * changed.
+ */
+static inline void adjust_deadline(struct task_struct *p, int new_prio)
+{
+	p->deadline += static_deadline_diff(new_prio) - task_deadline_diff(p);
+}
+
+void set_user_nice(struct task_struct *p, long nice)
+{
+	int new_static, old_static;
+	struct rq_flags rf;
+	struct rq *rq;
+
+	if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE)
+		return;
+	new_static = NICE_TO_PRIO(nice);
+	/*
+	 * We have to be careful, if called from sys_setpriority(),
+	 * the task might be in the middle of scheduling on another CPU.
+	 */
+	rq = task_rq_lock(p, &rf);
+	update_rq_clock(rq);
+
+	/*
+	 * The RT priorities are set via sched_setscheduler(), but we still
+	 * allow the 'normal' nice value to be set - but as expected
+	 * it won't have any effect on scheduling until the task is
+	 * not SCHED_NORMAL/SCHED_BATCH:
+	 */
+	if (has_rt_policy(p)) {
+		p->static_prio = new_static;
+		goto out_unlock;
+	}
+
+	adjust_deadline(p, new_static);
+	old_static = p->static_prio;
+	p->static_prio = new_static;
+	p->prio = effective_prio(p);
+
+	if (task_queued(p)) {
+		dequeue_task(rq, p, DEQUEUE_SAVE);
+		enqueue_task(rq, p, ENQUEUE_RESTORE);
+		if (new_static < old_static)
+			try_preempt(p, rq);
+	} else if (task_running(rq, p)) {
+		set_rq_task(rq, p);
+		if (old_static < new_static)
+			resched_task(p);
+	}
+out_unlock:
+	task_rq_unlock(rq, p, &rf);
+}
+EXPORT_SYMBOL(set_user_nice);
+
+/*
+ * can_nice - check if a task can reduce its nice value
+ * @p: task
+ * @nice: nice value
+ */
+int can_nice(const struct task_struct *p, const int nice)
+{
+	/* Convert nice value [19,-20] to rlimit style value [1,40] */
+	int nice_rlim = nice_to_rlimit(nice);
+
+	return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
+		capable(CAP_SYS_NICE));
+}
+
+#ifdef __ARCH_WANT_SYS_NICE
+
+/*
+ * sys_nice - change the priority of the current process.
+ * @increment: priority increment
+ *
+ * sys_setpriority is a more generic, but much slower function that
+ * does similar things.
+ */
+SYSCALL_DEFINE1(nice, int, increment)
+{
+	long nice, retval;
+
+	/*
+	 * Setpriority might change our priority at the same moment.
+	 * We don't have to worry. Conceptually one call occurs first
+	 * and we have a single winner.
+	 */
+
+	increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH);
+	nice = task_nice(current) + increment;
+
+	nice = clamp_val(nice, MIN_NICE, MAX_NICE);
+	if (increment < 0 && !can_nice(current, nice))
+		return -EPERM;
+
+	retval = security_task_setnice(current, nice);
+	if (retval)
+		return retval;
+
+	set_user_nice(current, nice);
+	return 0;
+}
+
+#endif
+
+/**
+ * task_prio - return the priority value of a given task.
+ * @p: the task in question.
+ *
+ * Return: The priority value as seen by users in /proc.
+ *
+ * sched policy         return value   kernel prio    user prio/nice
+ *
+ * normal, batch,          [1 ... 41]     101                 0/[-20 ... 19]
+ * idle                   [42 ... 81]     102                 0/[-20 ... 19]
+ * iso                     [0 ... 41]     100                 0/[-20 ... 19]
+ * fifo, rr             [-2 ... -100]     [98 ... 0]          [1 ... 99]
+ */
+int task_prio(const struct task_struct *p)
+{
+	int delta, prio = p->prio - MAX_RT_PRIO;
+
+	/* rt tasks and iso tasks */
+	if (prio <= 0)
+		goto out;
+
+	/* Convert to ms to avoid overflows */
+	delta = NS_TO_MS(p->deadline - task_rq(p)->niffies);
+	if (unlikely(delta < 0))
+		delta = 0;
+	delta = delta * 40 / ms_longest_deadline_diff();
+	if (delta <= 80)
+		prio += delta;
+	if (idleprio_task(p))
+		prio += 40;
+out:
+	return prio;
+}
+
+#ifdef CONFIG_SMP
+static inline bool rt_rq_is_runnable(struct rq *rt_rq)
+{
+	return rt_rq->rt_nr_running;
+}
+
+/*
+ * This function computes an effective utilization for the given CPU, to be
+ * used for frequency selection given the linear relation: f = u * f_max.
+ *
+ * The scheduler tracks the following metrics:
+ *
+ *   cpu_util_{cfs,rt,dl,irq}()
+ *   cpu_bw_dl()
+ *
+ * Where the cfs,rt and dl util numbers are tracked with the same metric and
+ * synchronized windows and are thus directly comparable.
+ *
+ * The cfs,rt,dl utilization are the running times measured with rq->clock_task
+ * which excludes things like IRQ and steal-time. These latter are then accrued
+ * in the irq utilization.
+ *
+ * The DL bandwidth number otoh is not a measured metric but a value computed
+ * based on the task model parameters and gives the minimal utilization
+ * required to meet deadlines.
+ */
+unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
+				 unsigned long *min, unsigned long *max)
+{
+	unsigned long util, irq, scale;
+	struct rq *rq = cpu_rq(cpu);
+
+	scale = arch_scale_cpu_capacity(cpu);
+
+	/*
+	 * Early check to see if IRQ/steal time saturates the CPU, can be
+	 * because of inaccuracies in how we track these.
+	 */
+	irq = cpu_util_irq(rq);
+	if (unlikely(irq >= scale)) {
+		if (min)
+			*min = scale;
+		if (max)
+			*max = scale;
+		return scale;
+	}
+
+	/*
+	 * MuQSS has no utilisation clamping and no deadline bandwidth, so the
+	 * usable range is simply the whole capacity. A runnable realtime task
+	 * asks for the maximum.
+	 */
+	if (min)
+		*min = rt_rq_is_runnable(rq) ? scale : 0;
+	if (max)
+		*max = scale;
+
+	util = util_cfs + cpu_util_rt(rq);
+	util = scale_irq_capacity(util, irq, scale);
+
+	return min_t(unsigned long, scale, util);
+}
+
+/*
+ * Available idle CPU capacity, used by the energy model. MuQSS does not
+ * implement energy aware scheduling, but the interface is still called.
+ */
+unsigned long sched_cpu_util(int cpu)
+{
+	unsigned long min, max;
+
+	return effective_cpu_util(cpu, cpu_util_cfs(cpu_rq(cpu)), &min, &max);
+}
+#endif /* CONFIG_SMP */
+
+/**
+ * idle_cpu - is a given CPU idle currently?
+ * @cpu: the processor in question.
+ *
+ * Return: 1 if the CPU is currently idle. 0 otherwise.
+ */
+int idle_cpu(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	if (rq->curr != rq->idle)
+		return 0;
+
+	if (rq->nr_running)
+		return 0;
+
+#ifdef CONFIG_SMP
+	if (rq->ttwu_pending)
+		return 0;
+#endif
+
+	return 1;
+}
+
+/**
+ * available_idle_cpu - is a given CPU idle for enqueuing work.
+ * @cpu: the CPU in question.
+ *
+ * Return: 1 if the CPU is currently idle. 0 otherwise.
+ */
+bool available_idle_cpu(int cpu)
+{
+	if (!idle_cpu(cpu))
+		return 0;
+
+	if (vcpu_is_preempted(cpu))
+		return 0;
+
+	return 1;
+}
+
+/**
+ * idle_task - return the idle task for a given CPU.
+ * @cpu: the processor in question.
+ *
+ * Return: The idle task for the CPU @cpu.
+ */
+struct task_struct *idle_task(int cpu)
+{
+	return cpu_rq(cpu)->idle;
+}
+
+/**
+ * find_process_by_pid - find a process with a matching PID value.
+ * @pid: the pid in question.
+ *
+ * The task of @pid, if found. %NULL otherwise.
+ */
+static inline struct task_struct *find_process_by_pid(pid_t pid)
+{
+	return pid ? find_task_by_vpid(pid) : current;
+}
+
+/* Actually do priority change: must hold rq lock. */
+static void __setscheduler(struct task_struct *p, struct rq *rq, int policy,
+			   int prio, const struct sched_attr *attr,
+			   bool keep_boost)
+{
+	int oldrtprio, oldprio;
+
+	/*
+	 * If params can't change scheduling class changes aren't allowed
+	 * either.
+	 */
+	if (attr->sched_flags & SCHED_FLAG_KEEP_PARAMS)
+		return;
+
+	p->policy = policy;
+	oldrtprio = p->rt_priority;
+	p->rt_priority = prio;
+
+	/* rt-policy tasks do not have a timerslack */
+	if (has_rt_policy(p)) {
+		p->timer_slack_ns = 0;
+	} else if (p->timer_slack_ns == 0) {
+		/* when switching back to non-rt policy, restore timerslack */
+		p->timer_slack_ns = p->default_timer_slack_ns;
+	}
+
+	p->normal_prio = normal_prio(p);
+	oldprio = p->prio;
+	/*
+	 * Keep a potential priority boosting if called from
+	 * sched_setscheduler().
+	 */
+	p->prio = normal_prio(p);
+	if (keep_boost)
+		p->prio = rt_effective_prio(p, p->prio);
+
+	if (task_running(rq, p)) {
+		rt_running_reprio(rq, oldprio, p->prio);
+		set_rq_task(rq, p);
+		resched_task(p);
+	} else if (task_queued(p)) {
+		dequeue_task(rq, p, DEQUEUE_SAVE);
+		enqueue_task(rq, p, ENQUEUE_RESTORE);
+		if (p->prio < oldprio || p->rt_priority > oldrtprio)
+			try_preempt(p, rq);
+	}
+}
+
+/*
+ * Check the target process has a UID that matches the current process's
+ */
+static bool check_same_owner(struct task_struct *p)
+{
+	const struct cred *cred = current_cred(), *pcred;
+	bool match;
+
+	rcu_read_lock();
+	pcred = __task_cred(p);
+	match = (uid_eq(cred->euid, pcred->euid) ||
+		 uid_eq(cred->euid, pcred->uid));
+	rcu_read_unlock();
+	return match;
+}
+
+static int __sched_setscheduler(struct task_struct *p,
+				const struct sched_attr *attr,
+				bool user, bool pi)
+{
+	int retval, policy = attr->sched_policy, oldpolicy = -1, priority = attr->sched_priority;
+	unsigned long rlim_rtprio = 0;
+	struct rq_flags rf;
+	int reset_on_fork;
+	struct rq *rq;
+
+	/* The pi code expects interrupts enabled */
+	BUG_ON(pi && in_interrupt());
+
+	if (is_rt_policy(policy) && !capable(CAP_SYS_NICE)) {
+		unsigned long lflags;
+
+		if (!lock_task_sighand(p, &lflags))
+			return -ESRCH;
+		rlim_rtprio = task_rlimit(p, RLIMIT_RTPRIO);
+		unlock_task_sighand(p, &lflags);
+		if (rlim_rtprio)
+			goto recheck;
+		/*
+		 * If the caller requested an RT policy without having the
+		 * necessary rights, we downgrade the policy to SCHED_ISO.
+		 * We also set the parameter to zero to pass the checks.
+		 */
+		policy = SCHED_ISO;
+		priority = 0;
+	}
+recheck:
+	/* Double check policy once rq lock held */
+	if (policy < 0) {
+		reset_on_fork = p->sched_reset_on_fork;
+		policy = oldpolicy = p->policy;
+	} else {
+		reset_on_fork = !!(policy & SCHED_RESET_ON_FORK);
+		policy &= ~SCHED_RESET_ON_FORK;
+
+		if (!SCHED_RANGE(policy))
+			return -EINVAL;
+	}
+
+	if (attr->sched_flags & ~(SCHED_FLAG_ALL | SCHED_FLAG_SUGOV))
+		return -EINVAL;
+
+	/*
+	 * Valid priorities for SCHED_FIFO and SCHED_RR are
+	 * 1..MAX_RT_PRIO-1, valid priority for SCHED_NORMAL and
+	 * SCHED_BATCH is 0.
+	 */
+	if (priority > MAX_RT_PRIO-1)
+		return -EINVAL;
+	if (is_rt_policy(policy) != (priority != 0))
+		return -EINVAL;
+
+	/*
+	 * Allow unprivileged RT tasks to decrease priority:
+	 */
+	if (user && !capable(CAP_SYS_NICE)) {
+		if (is_rt_policy(policy)) {
+			unsigned long rlim_rtprio =
+					task_rlimit(p, RLIMIT_RTPRIO);
+
+			/* Can't set/change the rt policy */
+			if (policy != p->policy && !rlim_rtprio)
+				return -EPERM;
+
+			/* Can't increase priority */
+			if (priority > p->rt_priority &&
+			    priority > rlim_rtprio)
+				return -EPERM;
+		} else {
+			switch (p->policy) {
+				/*
+				 * Can only downgrade policies but not back to
+				 * SCHED_NORMAL
+				 */
+				case SCHED_ISO:
+					if (policy == SCHED_ISO)
+						goto out;
+					if (policy != SCHED_NORMAL)
+						return -EPERM;
+					break;
+				case SCHED_BATCH:
+					if (policy == SCHED_BATCH)
+						goto out;
+					if (policy != SCHED_IDLEPRIO)
+						return -EPERM;
+					break;
+				case SCHED_IDLEPRIO:
+					if (policy == SCHED_IDLEPRIO)
+						goto out;
+					return -EPERM;
+				default:
+					break;
+			}
+		}
+
+		/* Can't change other user's priorities */
+		if (!check_same_owner(p))
+			return -EPERM;
+
+		/* Normal users shall not reset the sched_reset_on_fork flag: */
+		if (p->sched_reset_on_fork && !reset_on_fork)
+			return -EPERM;
+	}
+
+	if (user) {
+		retval = security_task_setscheduler(p);
+		if (retval)
+			return retval;
+	}
+
+	if (pi)
+		cpuset_lock();
+
+	/*
+	 * Make sure no PI-waiters arrive (or leave) while we are
+	 * changing the priority of the task:
+	 *
+	 * To be able to change p->policy safely, the runqueue lock must be
+	 * held.
+	 */
+	rq = task_rq_lock(p, &rf);
+	update_rq_clock(rq);
+
+	/*
+	 * Changing the policy of the stop threads its a very bad idea:
+	 */
+	if (p == rq->stop) {
+		retval = -EINVAL;
+		goto unlock;
+	}
+
+	/*
+	 * If not changing anything there's no need to proceed further,
+	 * but store a possible modification of reset_on_fork.
+	 */
+	if (unlikely(policy == p->policy && (!is_rt_policy(policy) ||
+	    priority == p->rt_priority))) {
+		p->sched_reset_on_fork = reset_on_fork;
+		retval = 0;
+		goto unlock;
+	}
+
+	/* Re-check policy now with rq lock held */
+	if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
+		policy = oldpolicy = -1;
+		task_rq_unlock(rq, p, &rf);
+		if (pi)
+			cpuset_unlock();
+		goto recheck;
+	}
+	p->sched_reset_on_fork = reset_on_fork;
+
+	__setscheduler(p, rq, policy, priority, attr, pi);
+
+	/* Avoid rq from going away on us: */
+	preempt_disable();
+	task_rq_unlock(rq, p, &rf);
+
+	if (pi) {
+		cpuset_unlock();
+		rt_mutex_adjust_pi(p);
+	}
+	preempt_enable();
+out:
+	return 0;
+
+unlock:
+	task_rq_unlock(rq, p, &rf);
+	if (pi)
+		cpuset_unlock();
+	return retval;
+}
+
+static int _sched_setscheduler(struct task_struct *p, int policy,
+			       const struct sched_param *param, bool check)
+{
+	struct sched_attr attr = {
+		.sched_policy   = policy,
+		.sched_priority = param->sched_priority,
+		.sched_nice	= PRIO_TO_NICE(p->static_prio),
+	};
+
+	return __sched_setscheduler(p, &attr, check, true);
+}
+/**
+ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * Use sched_set_fifo(), read its comment.
+ *
+ * Return: 0 on success. An error code otherwise.
+ *
+ * NOTE that the task may be already dead.
+ */
+int sched_setscheduler(struct task_struct *p, int policy,
+		       const struct sched_param *param)
+{
+	return _sched_setscheduler(p, policy, param, true);
+}
+
+
+int sched_setattr(struct task_struct *p, const struct sched_attr *attr)
+{
+	return __sched_setscheduler(p, attr, true, true);
+}
+
+int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
+{
+	return __sched_setscheduler(p, attr, false, true);
+}
+
+/**
+ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
+ * @p: the task in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * Just like sched_setscheduler, only don't bother checking if the
+ * current context has permission.  For example, this is needed in
+ * stop_machine(): we create temporary high priority worker threads,
+ * but our caller might not have that capability.
+ *
+ * Return: 0 on success. An error code otherwise.
+ */
+int sched_setscheduler_nocheck(struct task_struct *p, int policy,
+			       const struct sched_param *param)
+{
+	return _sched_setscheduler(p, policy, param, false);
+}
+
+/*
+ * SCHED_FIFO is a broken scheduler model; that is, it is fundamentally
+ * incapable of resource management, which is the one thing an OS really should
+ * be doing.
+ *
+ * This is of course the reason it is limited to privileged users only.
+ *
+ * Worse still; it is fundamentally impossible to compose static priority
+ * workloads. You cannot take two correctly working static prio workloads
+ * and smash them together and still expect them to work.
+ *
+ * For this reason 'all' FIFO tasks the kernel creates are basically at:
+ *
+ *   MAX_RT_PRIO / 2
+ *
+ * The administrator _MUST_ configure the system, the kernel simply doesn't
+ * know enough information to make a sensible choice.
+ */
+void sched_set_fifo(struct task_struct *p)
+{
+	struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 };
+	WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0);
+}
+EXPORT_SYMBOL_GPL(sched_set_fifo);
+
+/*
+ * For when you don't much care about FIFO, but want to be above SCHED_NORMAL.
+ */
+void sched_set_fifo_low(struct task_struct *p)
+{
+	struct sched_param sp = { .sched_priority = 1 };
+	WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0);
+}
+EXPORT_SYMBOL_GPL(sched_set_fifo_low);
+
+void sched_set_normal(struct task_struct *p, int nice)
+{
+	struct sched_attr attr = {
+		.sched_policy = SCHED_NORMAL,
+		.sched_nice = nice,
+	};
+	WARN_ON_ONCE(sched_setattr_nocheck(p, &attr) != 0);
+}
+EXPORT_SYMBOL_GPL(sched_set_normal);
+
+static int
+do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
+{
+	struct sched_param lparam;
+	struct task_struct *p;
+	int retval;
+
+	if (!param || pid < 0)
+		return -EINVAL;
+	if (copy_from_user(&lparam, param, sizeof(struct sched_param)))
+		return -EFAULT;
+
+	rcu_read_lock();
+	retval = -ESRCH;
+	p = find_process_by_pid(pid);
+	if (likely(p))
+		get_task_struct(p);
+	rcu_read_unlock();
+
+	if (likely(p)) {
+		retval = sched_setscheduler(p, policy, &lparam);
+		put_task_struct(p);
+	}
+
+	return retval;
+}
+
+/*
+ * Mimics kernel/events/core.c perf_copy_attr().
+ */
+static int sched_copy_attr(struct sched_attr __user *uattr,
+			   struct sched_attr *attr)
+{
+	u32 size;
+	int ret;
+
+	/* Zero the full structure, so that a short copy will be nice: */
+	memset(attr, 0, sizeof(*attr));
+
+	ret = get_user(size, &uattr->size);
+	if (ret)
+		return ret;
+
+	/* ABI compatibility quirk: */
+	if (!size)
+		size = SCHED_ATTR_SIZE_VER0;
+
+	if (size < SCHED_ATTR_SIZE_VER0 || size > PAGE_SIZE)
+		goto err_size;
+
+	ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size);
+	if (ret) {
+		if (ret == -E2BIG)
+			goto err_size;
+		return ret;
+	}
+
+	/*
+	 * XXX: Do we want to be lenient like existing syscalls; or do we want
+	 * to be strict and return an error on out-of-bounds values?
+	 */
+	attr->sched_nice = clamp(attr->sched_nice, -20, 19);
+
+	/* sched/core.c uses zero here but we already know ret is zero */
+	return 0;
+
+err_size:
+	put_user(sizeof(*attr), &uattr->size);
+	return -E2BIG;
+}
+
+/*
+ * sched_setparam() passes in -1 for its policy, to let the functions
+ * it calls know not to change it.
+ */
+#define SETPARAM_POLICY	-1
+
+/**
+ * sys_sched_setscheduler - set/change the scheduler policy and RT priority
+ * @pid: the pid in question.
+ * @policy: new policy.
+ * @param: structure containing the new RT priority.
+ *
+ * Return: 0 on success. An error code otherwise.
+ */
+SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, struct sched_param __user *, param)
+{
+	if (policy < 0)
+		return -EINVAL;
+
+	return do_sched_setscheduler(pid, policy, param);
+}
+
+/**
+ * sys_sched_setparam - set/change the RT priority of a thread
+ * @pid: the pid in question.
+ * @param: structure containing the new RT priority.
+ *
+ * Return: 0 on success. An error code otherwise.
+ */
+SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
+{
+	return do_sched_setscheduler(pid, SETPARAM_POLICY, param);
+}
+
+/**
+ * sys_sched_setattr - same as above, but with extended sched_attr
+ * @pid: the pid in question.
+ * @uattr: structure containing the extended parameters.
+ */
+SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
+			       unsigned int, flags)
+{
+	struct sched_attr attr;
+	struct task_struct *p;
+	int retval;
+
+	if (!uattr || pid < 0 || flags)
+		return -EINVAL;
+
+	retval = sched_copy_attr(uattr, &attr);
+	if (retval)
+		return retval;
+
+	if ((int)attr.sched_policy < 0)
+		return -EINVAL;
+	if (attr.sched_flags & SCHED_FLAG_KEEP_POLICY)
+		attr.sched_policy = SETPARAM_POLICY;
+
+	rcu_read_lock();
+	retval = -ESRCH;
+	p = find_process_by_pid(pid);
+	if (likely(p))
+		get_task_struct(p);
+	rcu_read_unlock();
+
+	if (likely(p)) {
+		retval = sched_setattr(p, &attr);
+		put_task_struct(p);
+	}
+
+	return retval;
+}
+
+/**
+ * sys_sched_getscheduler - get the policy (scheduling class) of a thread
+ * @pid: the pid in question.
+ *
+ * Return: On success, the policy of the thread. Otherwise, a negative error
+ * code.
+ */
+SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
+{
+	struct task_struct *p;
+	int retval = -EINVAL;
+
+	if (pid < 0)
+		goto out_nounlock;
+
+	retval = -ESRCH;
+	rcu_read_lock();
+	p = find_process_by_pid(pid);
+	if (p) {
+		retval = security_task_getscheduler(p);
+		if (!retval)
+			retval = p->policy;
+	}
+	rcu_read_unlock();
+
+out_nounlock:
+	return retval;
+}
+
+/**
+ * sys_sched_getscheduler - get the RT priority of a thread
+ * @pid: the pid in question.
+ * @param: structure containing the RT priority.
+ *
+ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error
+ * code.
+ */
+SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
+{
+	struct sched_param lp = { .sched_priority = 0 };
+	struct task_struct *p;
+	int retval = -EINVAL;
+
+	if (!param || pid < 0)
+		goto out_nounlock;
+
+	rcu_read_lock();
+	p = find_process_by_pid(pid);
+	retval = -ESRCH;
+	if (!p)
+		goto out_unlock;
+
+	retval = security_task_getscheduler(p);
+	if (retval)
+		goto out_unlock;
+
+	if (has_rt_policy(p))
+		lp.sched_priority = p->rt_priority;
+	rcu_read_unlock();
+
+	/*
+	 * This one might sleep, we cannot do it with a spinlock held ...
+	 */
+	retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0;
+
+out_nounlock:
+	return retval;
+
+out_unlock:
+	rcu_read_unlock();
+	return retval;
+}
+
+/*
+ * Copy the kernel size attribute structure (which might be larger
+ * than what user-space knows about) to user-space.
+ *
+ * Note that all cases are valid: user-space buffer can be larger or
+ * smaller than the kernel-space buffer. The usual case is that both
+ * have the same size.
+ */
+static int
+sched_attr_copy_to_user(struct sched_attr __user *uattr,
+			struct sched_attr *kattr,
+			unsigned int usize)
+{
+	unsigned int ksize = sizeof(*kattr);
+
+	if (!access_ok(uattr, usize))
+		return -EFAULT;
+
+	/*
+	 * sched_getattr() ABI forwards and backwards compatibility:
+	 *
+	 * If usize == ksize then we just copy everything to user-space and all is good.
+	 *
+	 * If usize < ksize then we only copy as much as user-space has space for,
+	 * this keeps ABI compatibility as well. We skip the rest.
+	 *
+	 * If usize > ksize then user-space is using a newer version of the ABI,
+	 * which part the kernel doesn't know about. Just ignore it - tooling can
+	 * detect the kernel's knowledge of attributes from the attr->size value
+	 * which is set to ksize in this case.
+	 */
+	kattr->size = min(usize, ksize);
+
+	if (copy_to_user(uattr, kattr, kattr->size))
+		return -EFAULT;
+
+	return 0;
+}
+
+/**
+ * sys_sched_getattr - similar to sched_getparam, but with sched_attr
+ * @pid: the pid in question.
+ * @uattr: structure containing the extended parameters.
+ * @usize: sizeof(attr) for fwd/bwd comp.
+ * @flags: for future extension.
+ */
+SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
+		unsigned int, usize, unsigned int, flags)
+{
+	struct sched_attr kattr = { };
+	struct task_struct *p;
+	int retval;
+
+	if (!uattr || pid < 0 || usize > PAGE_SIZE ||
+	    usize < SCHED_ATTR_SIZE_VER0 || flags)
+		return -EINVAL;
+
+	rcu_read_lock();
+	p = find_process_by_pid(pid);
+	retval = -ESRCH;
+	if (!p)
+		goto out_unlock;
+
+	retval = security_task_getscheduler(p);
+	if (retval)
+		goto out_unlock;
+
+	kattr.sched_policy = p->policy;
+	if (rt_task(p))
+		kattr.sched_priority = p->rt_priority;
+	else
+		kattr.sched_nice = task_nice(p);
+
+	rcu_read_unlock();
+
+	return sched_attr_copy_to_user(uattr, &kattr, usize);
+
+out_unlock:
+	rcu_read_unlock();
+	return retval;
+}
+
+long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
+{
+	cpumask_var_t cpus_allowed, new_mask;
+	struct task_struct *p;
+	int retval;
+
+	rcu_read_lock();
+
+	p = find_process_by_pid(pid);
+	if (!p) {
+		rcu_read_unlock();
+		return -ESRCH;
+	}
+
+	/* Prevent p going away */
+	get_task_struct(p);
+	rcu_read_unlock();
+
+	if (p->flags & PF_NO_SETAFFINITY) {
+		retval = -EINVAL;
+		goto out_put_task;
+	}
+	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
+		retval = -ENOMEM;
+		goto out_put_task;
+	}
+	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
+		retval = -ENOMEM;
+		goto out_free_cpus_allowed;
+	}
+	retval = -EPERM;
+	if (!check_same_owner(p)) {
+		rcu_read_lock();
+		if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) {
+			rcu_read_unlock();
+			goto out_unlock;
+		}
+		rcu_read_unlock();
+	}
+
+	retval = security_task_setscheduler(p);
+	if (retval)
+		goto out_unlock;
+
+	cpuset_cpus_allowed(p, cpus_allowed);
+	cpumask_and(new_mask, in_mask, cpus_allowed);
+again:
+	retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK | SCA_USER);
+
+	if (!retval) {
+		cpuset_cpus_allowed(p, cpus_allowed);
+		if (!cpumask_subset(new_mask, cpus_allowed)) {
+			/*
+			 * We must have raced with a concurrent cpuset
+			 * update. Just reset the cpus_allowed to the
+			 * cpuset's cpus_allowed
+			 */
+			cpumask_copy(new_mask, cpus_allowed);
+			goto again;
+		}
+	}
+out_unlock:
+	free_cpumask_var(new_mask);
+out_free_cpus_allowed:
+	free_cpumask_var(cpus_allowed);
+out_put_task:
+	put_task_struct(p);
+	return retval;
+}
+
+static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
+			     cpumask_t *new_mask)
+{
+	if (len < cpumask_size())
+		cpumask_clear(new_mask);
+	else if (len > cpumask_size())
+		len = cpumask_size();
+
+	return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
+}
+
+
+/**
+ * sys_sched_setaffinity - set the CPU affinity of a process
+ * @pid: pid of the process
+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
+ * @user_mask_ptr: user-space pointer to the new CPU mask
+ *
+ * Return: 0 on success. An error code otherwise.
+ */
+SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
+		unsigned long __user *, user_mask_ptr)
+{
+	cpumask_var_t new_mask;
+	int retval;
+
+	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
+		return -ENOMEM;
+
+	retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
+	if (retval == 0)
+		retval = sched_setaffinity(pid, new_mask);
+	free_cpumask_var(new_mask);
+	return retval;
+}
+
+long sched_getaffinity(pid_t pid, cpumask_t *mask)
+{
+	struct task_struct *p;
+	unsigned long flags;
+	int retval;
+
+	cpus_read_lock();
+	rcu_read_lock();
+
+	retval = -ESRCH;
+	p = find_process_by_pid(pid);
+	if (!p)
+		goto out_unlock;
+
+	retval = security_task_getscheduler(p);
+	if (retval)
+		goto out_unlock;
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	cpumask_and(mask, &p->cpus_mask, cpu_active_mask);
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+out_unlock:
+	rcu_read_unlock();
+	cpus_read_unlock();
+
+	return retval;
+}
+
+/**
+ * sys_sched_getaffinity - get the CPU affinity of a process
+ * @pid: pid of the process
+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr
+ * @user_mask_ptr: user-space pointer to hold the current CPU mask
+ *
+ * Return: 0 on success. An error code otherwise.
+ */
+SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
+		unsigned long __user *, user_mask_ptr)
+{
+	int ret;
+	cpumask_var_t mask;
+
+	if ((len * BITS_PER_BYTE) < nr_cpu_ids)
+		return -EINVAL;
+	if (len & (sizeof(unsigned long)-1))
+		return -EINVAL;
+
+	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
+		return -ENOMEM;
+
+	ret = sched_getaffinity(pid, mask);
+	if (ret == 0) {
+		unsigned int retlen = min(len, cpumask_size());
+
+		if (copy_to_user(user_mask_ptr, mask, retlen))
+			ret = -EFAULT;
+		else
+			ret = retlen;
+	}
+	free_cpumask_var(mask);
+
+	return ret;
+}
+
+static void do_sched_yield(void)
+{
+	struct rq_flags rf;
+	struct rq *rq;
+
+	if (!sched_yield_type)
+		return;
+
+	rq = this_rq_lock_irq(&rf);
+
+	if (sched_yield_type > 1)
+		time_slice_expired(current, rq);
+	schedstat_inc(rq->yld_count);
+
+	preempt_disable();
+	rq_unlock_irq(rq, &rf);
+	sched_preempt_enable_no_resched();
+
+	schedule();
+}
+
+/**
+ * sys_sched_yield - yield the current processor to other threads.
+ *
+ * This function yields the current CPU to other tasks. If there are no
+ * other threads running on this CPU then this function will return.
+ *
+ * Return: 0.
+ */
+SYSCALL_DEFINE0(sched_yield)
+{
+	do_sched_yield();
+	return 0;
+}
+
+#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
+int __sched __cond_resched(void)
+{
+	if (should_resched(0)) {
+		preempt_schedule_common();
+		return 1;
+	}
+#ifndef CONFIG_PREEMPT_RCU
+	rcu_all_qs();
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(__cond_resched);
+#endif
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
+DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
+EXPORT_STATIC_CALL_TRAMP(cond_resched);
+
+DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
+EXPORT_STATIC_CALL_TRAMP(might_resched);
+# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
+int __sched dynamic_cond_resched(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_cond_resched))
+		return 0;
+	return __cond_resched();
+}
+EXPORT_SYMBOL(dynamic_cond_resched);
+
+static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched);
+int __sched dynamic_might_resched(void)
+{
+	if (!static_branch_unlikely(&sk_dynamic_might_resched))
+		return 0;
+	return __cond_resched();
+}
+EXPORT_SYMBOL(dynamic_might_resched);
+# endif
+#endif /* CONFIG_PREEMPT_DYNAMIC */
+
+/*
+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock,
+ * call schedule, and on return reacquire the lock.
+ *
+ * This works OK both with and without CONFIG_PREEMPTION.  We do strange low-level
+ * operations here to prevent schedule() from being called twice (once via
+ * spin_unlock(), once by hand).
+ */
+int __cond_resched_lock(spinlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held(lock);
+
+	if (spin_needbreak(lock) || resched) {
+		spin_unlock(lock);
+		if (resched)
+			preempt_schedule_common();
+		else
+			cpu_relax();
+		ret = 1;
+		spin_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_lock);
+
+int __cond_resched_rwlock_read(rwlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held_read(lock);
+
+	if (rwlock_needbreak(lock) || resched) {
+		read_unlock(lock);
+		if (resched)
+			preempt_schedule_common();
+		else
+			cpu_relax();
+		ret = 1;
+		read_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_rwlock_read);
+
+int __cond_resched_rwlock_write(rwlock_t *lock)
+{
+	int resched = should_resched(PREEMPT_LOCK_OFFSET);
+	int ret = 0;
+
+	lockdep_assert_held_write(lock);
+
+	if (rwlock_needbreak(lock) || resched) {
+		write_unlock(lock);
+		if (resched)
+			preempt_schedule_common();
+		else
+			cpu_relax();
+		ret = 1;
+		write_lock(lock);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(__cond_resched_rwlock_write);
+
+/**
+ * yield - yield the current processor to other threads.
+ *
+ * Do not ever use this function, there's a 99% chance you're doing it wrong.
+ *
+ * The scheduler is at all times free to pick the calling task as the most
+ * eligible task to run, if removing the yield() call from your code breaks
+ * it, it's already broken.
+ *
+ * Typical broken usage is:
+ *
+ * while (!event)
+ *	yield();
+ *
+ * where one assumes that yield() will let 'the other' process run that will
+ * make event true. If the current task is a SCHED_FIFO task that will never
+ * happen. Never use yield() as a progress guarantee!!
+ *
+ * If you want to use yield() to wait for something, use wait_event().
+ * If you want to use yield() to be 'nice' for others, use cond_resched().
+ * If you still want to use yield(), do not!
+ */
+void __sched yield(void)
+{
+	set_current_state(TASK_RUNNING);
+	do_sched_yield();
+}
+EXPORT_SYMBOL(yield);
+
+/**
+ * yield_to - yield the current processor to another thread in
+ * your thread group, or accelerate that thread toward the
+ * processor it's on.
+ * @p: target task
+ * @preempt: whether task preemption is allowed or not
+ *
+ * It's the caller's job to ensure that the target task struct
+ * can't go away on us before we can do any checks.
+ *
+ * Return:
+ *	true (>0) if we indeed boosted the target task.
+ *	false (0) if we failed to boost the target.
+ *	-ESRCH if there's no task to yield to.
+ */
+int __sched yield_to(struct task_struct *p, bool preempt)
+{
+	struct task_struct *rq_p;
+	struct rq *rq, *p_rq;
+	unsigned long flags;
+	int yielded = 0;
+
+	local_irq_save(flags);
+	rq = this_rq();
+
+again:
+	p_rq = task_rq(p);
+	/*
+	 * If we're the only runnable task on the rq and target rq also
+	 * has only one task, there's absolutely no point in yielding.
+	 */
+	if (task_running(p_rq, p) || p->__state) {
+		yielded = -ESRCH;
+		goto out_irq;
+	}
+
+	double_rq_lock(rq, p_rq);
+	if (unlikely(task_rq(p) != p_rq)) {
+		double_rq_unlock(rq, p_rq);
+		goto again;
+	}
+
+	yielded = 1;
+	schedstat_inc(rq->yld_count);
+	rq_p = rq->curr;
+	if (p->deadline > rq_p->deadline)
+		p->deadline = rq_p->deadline;
+	p->time_slice += rq_p->time_slice;
+	if (p->time_slice > timeslice())
+		p->time_slice = timeslice();
+	time_slice_expired(rq_p, rq);
+	if (preempt && rq != p_rq)
+		resched_task(p_rq->curr);
+	double_rq_unlock(rq, p_rq);
+out_irq:
+	local_irq_restore(flags);
+
+	if (yielded > 0)
+		schedule();
+	return yielded;
+}
+EXPORT_SYMBOL_GPL(yield_to);
+
+int io_schedule_prepare(void)
+{
+	int old_iowait = current->in_iowait;
+
+	current->in_iowait = 1;
+	blk_flush_plug(current->plug, true);
+
+	return old_iowait;
+}
+
+void io_schedule_finish(int token)
+{
+	current->in_iowait = token;
+}
+
+/*
+ * This task is about to go to sleep on IO.  Increment rq->nr_iowait so
+ * that process accounting knows that this is a task in IO wait state.
+ *
+ * But don't do that if it is a deliberate, throttling IO wait (this task
+ * has set its backing_dev_info: the queue against which it should throttle)
+ */
+
+long __sched io_schedule_timeout(long timeout)
+{
+	int token;
+	long ret;
+
+	token = io_schedule_prepare();
+	ret = schedule_timeout(timeout);
+	io_schedule_finish(token);
+
+	return ret;
+}
+EXPORT_SYMBOL(io_schedule_timeout);
+
+void __sched io_schedule(void)
+{
+	int token;
+
+	token = io_schedule_prepare();
+	schedule();
+	io_schedule_finish(token);
+}
+EXPORT_SYMBOL(io_schedule);
+
+/**
+ * sys_sched_get_priority_max - return maximum RT priority.
+ * @policy: scheduling class.
+ *
+ * Return: On success, this syscall returns the maximum
+ * rt_priority that can be used by a given scheduling class.
+ * On failure, a negative error code is returned.
+ */
+SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
+{
+	int ret = -EINVAL;
+
+	switch (policy) {
+	case SCHED_FIFO:
+	case SCHED_RR:
+		ret = MAX_RT_PRIO-1;
+		break;
+	case SCHED_NORMAL:
+	case SCHED_BATCH:
+	case SCHED_ISO:
+	case SCHED_IDLEPRIO:
+		ret = 0;
+		break;
+	}
+	return ret;
+}
+
+/**
+ * sys_sched_get_priority_min - return minimum RT priority.
+ * @policy: scheduling class.
+ *
+ * Return: On success, this syscall returns the minimum
+ * rt_priority that can be used by a given scheduling class.
+ * On failure, a negative error code is returned.
+ */
+SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
+{
+	int ret = -EINVAL;
+
+	switch (policy) {
+	case SCHED_FIFO:
+	case SCHED_RR:
+		ret = 1;
+		break;
+	case SCHED_NORMAL:
+	case SCHED_BATCH:
+	case SCHED_ISO:
+	case SCHED_IDLEPRIO:
+		ret = 0;
+		break;
+	}
+	return ret;
+}
+
+static int sched_rr_get_interval(pid_t pid, struct timespec64 *t)
+{
+	struct task_struct *p;
+	unsigned int time_slice;
+	struct rq_flags rf;
+	struct rq *rq;
+	int retval;
+
+	if (pid < 0)
+		return -EINVAL;
+
+	retval = -ESRCH;
+	rcu_read_lock();
+	p = find_process_by_pid(pid);
+	if (!p)
+		goto out_unlock;
+
+	retval = security_task_getscheduler(p);
+	if (retval)
+		goto out_unlock;
+
+	rq = task_rq_lock(p, &rf);
+	time_slice = p->policy == SCHED_FIFO ? 0 : MS_TO_NS(task_timeslice(p));
+	task_rq_unlock(rq, p, &rf);
+
+	rcu_read_unlock();
+	*t = ns_to_timespec64(time_slice);
+	return 0;
+
+out_unlock:
+	rcu_read_unlock();
+	return retval;
+}
+
+/**
+ * sys_sched_rr_get_interval - return the default timeslice of a process.
+ * @pid: pid of the process.
+ * @interval: userspace pointer to the timeslice value.
+ *
+ * this syscall writes the default timeslice value of a given process
+ * into the user-space timespec buffer. A value of '0' means infinity.
+ *
+ * Return: On success, 0 and the timeslice is in @interval. Otherwise,
+ * an error code.
+ */
+SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
+		struct __kernel_timespec __user *, interval)
+{
+	struct timespec64 t;
+	int retval = sched_rr_get_interval(pid, &t);
+
+	if (retval == 0)
+		retval = put_timespec64(&t, interval);
+
+	return retval;
+}
+
+#ifdef CONFIG_COMPAT_32BIT_TIME
+SYSCALL_DEFINE2(sched_rr_get_interval_time32, pid_t, pid,
+		struct old_timespec32 __user *, interval)
+{
+	struct timespec64 t;
+	int retval = sched_rr_get_interval(pid, &t);
+
+	if (retval == 0)
+		retval = put_old_timespec32(&t, interval);
+	return retval;
+}
+#endif
+
+void sched_show_task(struct task_struct *p)
+{
+	unsigned long free = 0;
+	int ppid;
+
+	if (!try_get_task_stack(p))
+		return;
+
+	printk(KERN_INFO "%-15.15s %c", p->comm, task_state_to_char(p));
+
+	if (p->__state == TASK_RUNNING)
+		printk(KERN_CONT "  running task    ");
+#ifdef CONFIG_DEBUG_STACK_USAGE
+	free = stack_not_used(p);
+#endif
+	ppid = 0;
+	rcu_read_lock();
+	if (pid_alive(p))
+		ppid = task_pid_nr(rcu_dereference(p->real_parent));
+	rcu_read_unlock();
+	pr_cont(" stack:%5lu pid:%5d ppid:%6d flags:0x%08lx\n",
+		free, task_pid_nr(p), ppid,
+		(unsigned long)task_thread_info(p)->flags);
+
+	print_worker_info(KERN_INFO, p);
+	print_stop_info(KERN_INFO, p);
+	show_stack(p, NULL, KERN_INFO);
+	put_task_stack(p);
+}
+EXPORT_SYMBOL_GPL(sched_show_task);
+
+static inline bool
+state_filter_match(unsigned long state_filter, struct task_struct *p)
+{
+	/* no filter, everything matches */
+	if (!state_filter)
+		return true;
+
+	/* filter, but doesn't match */
+	if (!(p->__state & state_filter))
+		return false;
+
+	/*
+	 * When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
+	 * TASK_KILLABLE).
+	 */
+	if (state_filter == TASK_UNINTERRUPTIBLE && p->__state == TASK_IDLE)
+		return false;
+
+	return true;
+}
+
+void show_state_filter(unsigned int state_filter)
+{
+	struct task_struct *g, *p;
+
+	rcu_read_lock();
+	for_each_process_thread(g, p) {
+		/*
+		 * reset the NMI-timeout, listing all files on a slow
+		 * console might take a lot of time:
+		 * Also, reset softlockup watchdogs on all CPUs, because
+		 * another CPU might be blocked waiting for us to process
+		 * an IPI.
+		 */
+		touch_nmi_watchdog();
+		touch_all_softlockup_watchdogs();
+		if (state_filter_match(state_filter, p))
+			sched_show_task(p);
+	}
+
+	rcu_read_unlock();
+	/*
+	 * Only show locks if all tasks are dumped:
+	 */
+	if (!state_filter)
+		debug_show_all_locks();
+}
+
+void dump_cpu_task(int cpu)
+{
+	pr_info("Task dump for CPU %d:\n", cpu);
+	sched_show_task(cpu_curr(cpu));
+}
+
+#ifdef CONFIG_SMP
+void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
+{
+	if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
+		p->cpus_ptr = ctx->new_mask;
+		return;
+	}
+
+	cpumask_copy(&p->cpus_mask, ctx->new_mask);
+	p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+}
+
+void
+__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
+{
+	struct rq *rq = task_rq(p);
+
+	lockdep_assert_held(&p->pi_lock);
+
+	/*
+	 * Keep cpus_mask and nr_cpus_allowed in lockstep. sched_other_cpu()
+	 * relies on nr_cpus_allowed == 1 to allow hotplug kthreads bound to
+	 * offline CPUs to still be picked on an online runqueue.
+	 */
+	set_cpus_allowed_common(p, ctx);
+
+	if (task_queued(p)) {
+		/*
+		 * Because __kthread_bind() calls this on blocked tasks without
+		 * holding rq->lock.
+		 */
+		lockdep_assert_held(rq->lock);
+	}
+}
+
+static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
+				   struct task_struct *p, int new_cpu);
+
+/*
+ * Calling do_set_cpus_allowed from outside the scheduler code should not be
+ * called on a running or queued task. We should be holding pi_lock.
+ */
+void do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
+{
+	__do_set_cpus_allowed(p, ctx);
+
+	/*
+	 * A mask forced by cpuset or kthread_bind is a change of what the
+	 * task wants, so an override bind_zero() had to make must not be
+	 * restored over the top of it later. What was saved is left in place
+	 * rather than freed, which here would be under pi_lock.
+	 */
+	if (ctx->flags & SCA_USER)
+		p->zerobound = false;
+
+	if (needs_other_cpu(p, task_cpu(p))) {
+		struct rq *rq;
+
+		rq = __task_rq_lock(p, NULL);
+		/*
+		 * A blocked task is on no runqueue, so task_cpu() only says
+		 * which rq lock protects it and which CPU PSI counts its state
+		 * on. It is left exactly as it is.
+		 *
+		 * Moving it would leave the TSK_IOWAIT it is carrying counted
+		 * on the CPU it came from, and the wakeup would then clear it
+		 * on a CPU that never counted it and underflow that counter.
+		 * Nor may the destination be recorded in wake_cpu: that is not
+		 * a hint, it is the bit that arms the lazy migration handshake.
+		 * return_task() reads wake_cpu != task_cpu() as "set_task_cpu()
+		 * moved this running task, hand it over in
+		 * finish_lock_switch()", so stamping it here arms a migration
+		 * no one arranged. Widening the mask again before the task
+		 * wakes is enough to leave it armed, since ttwu only calls
+		 * set_task_cpu() - the one place that resyncs wake_cpu - when
+		 * it picks a different CPU, and a wakeup back onto task_cpu()
+		 * therefore keeps the stale value until the next deschedule
+		 * migrates the task out from under the switch.
+		 *
+		 * Nothing needs either. select_best_cpu() honours the new mask
+		 * when the task wakes and set_task_cpu() takes the PSI state
+		 * with it, which is why affine_move_task() leaves a blocked
+		 * task alone here as well.
+		 */
+		if (task_running(rq, p)) {
+			/*
+			 * A running task is not on any skiplist, so
+			 * set_task_cpu() only tags wake_cpu for it; resched_task()
+			 * gets it off the CPU so finish_lock_switch() can
+			 * complete the move.
+			 */
+			set_task_cpu(p, valid_task_cpu(p));
+			resched_task(p);
+		} else if (task_queued(p)) {
+			/*
+			 * A queued task's skiplist node is linked into this
+			 * rq's list, so its CPU must not be rewritten
+			 * underneath it: task_rq(p) would then name a runqueue
+			 * the node is not in, and the next dequeue_task() would
+			 * try to unlink it from there and trip
+			 * skiplist_delete()'s "m < 0" while leaving the node
+			 * linked in the list it really is on. Unlink it first,
+			 * exactly as every other migration of a queued task
+			 * does.
+			 */
+			rq = move_queued_task(rq, NULL, p, valid_task_cpu(p));
+		}
+		__task_rq_unlock(rq, p, NULL);
+	}
+}
+
+/* migrate_disable()/migrate_enable() are inline in <linux/sched.h> now. */
+#endif
+
+/**
+ * init_idle - set up an idle thread for a given CPU
+ * @idle: task in question
+ * @cpu: cpu the idle task belongs to
+ *
+ * NOTE: this function does not set the idle thread's NEED_RESCHED
+ * flag, to make booting more robust.
+ */
+void init_idle(struct task_struct *idle, int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&idle->pi_lock, flags);
+	raw_spin_lock(rq->lock);
+	idle->last_ran = rq->niffies;
+	time_slice_expired(idle, rq);
+	idle->__state = TASK_RUNNING;
+	/* Setting prio to illegal value shouldn't matter when never queued */
+	idle->prio = PRIO_LIMIT;
+	idle->flags |= PF_IDLE;
+
+	scs_task_reset(idle);
+	kasan_unpoison_task_stack(idle);
+
+#ifdef CONFIG_SMP
+	/*
+	 * It's possible that init_idle() gets called multiple times on a task,
+	 * in that case do_set_cpus_allowed() will not do the right thing.
+	 *
+	 * And since this is boot we can forgo the serialisation.
+	 */
+	{
+		struct affinity_context ac = { .new_mask = cpumask_of(cpu) };
+
+		set_cpus_allowed_common(idle, &ac);
+	}
+#ifdef CONFIG_SMT_NICE
+	idle->smt_bias = 0;
+#endif
+#endif
+	set_rq_task(rq, idle);
+
+	/* Silence PROVE_RCU */
+	rcu_read_lock();
+	set_task_cpu(idle, cpu);
+	rcu_read_unlock();
+
+	rq->idle = idle;
+	rcu_assign_pointer(rq->curr, idle);
+	idle->on_rq = TASK_ON_RQ_QUEUED;
+	raw_spin_unlock(rq->lock);
+	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
+
+	/* Set the preempt count _outside_ the spinlocks! */
+	init_idle_preempt_count(idle, cpu);
+
+	ftrace_graph_init_idle_task(idle, cpu);
+	vtime_init_idle(idle, cpu);
+#ifdef CONFIG_SMP
+	sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu);
+#endif
+}
+
+int cpuset_cpumask_can_shrink(const struct cpumask __maybe_unused *cur,
+			      const struct cpumask __maybe_unused *trial)
+{
+	return 1;
+}
+
+int task_can_attach(struct task_struct *p)
+{
+	int ret = 0;
+
+	/*
+	 * Kthreads which disallow setaffinity shouldn't be moved
+	 * to a new cpuset; we don't want to change their CPU
+	 * affinity and isolating such threads by their set of
+	 * allowed nodes is unnecessary.  Thus, cpusets are not
+	 * applicable for such threads.  This prevents checking for
+	 * success of set_cpus_allowed_ptr() on all attached tasks
+	 * before cpus_mask may be changed.
+	 */
+	if (p->flags & PF_NO_SETAFFINITY)
+		ret = -EINVAL;
+
+	return ret;
+}
+
+void resched_cpu(int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	rq_lock_irqsave(rq, &rf);
+	if (cpu_online(cpu) || cpu == smp_processor_id())
+		resched_curr(rq);
+	rq_unlock_irqrestore(rq, &rf);
+}
+
+#ifdef CONFIG_SMP
+#ifdef CONFIG_NO_HZ_COMMON
+void nohz_balance_enter_idle(int cpu) {}
+
+/*
+ * In the semi idle case, use the nearest busy CPU for migrating timers
+ * from an idle CPU.  This is good for power-savings.
+ *
+ * We don't do similar optimization for completely idle system, as
+ * selecting an idle CPU will add more delays to the timers than intended
+ * (as that CPU's timer base may not be uptodate wrt jiffies etc).
+ */
+int get_nohz_timer_target(void)
+{
+	int i, cpu = smp_processor_id(), default_cpu = -1;
+	struct sched_domain *sd;
+
+	if (housekeeping_cpu(cpu, HK_TYPE_TIMER)) {
+		if (!idle_cpu(cpu))
+			return cpu;
+		default_cpu = cpu;
+	}
+
+	rcu_read_lock();
+	for_each_domain(cpu, sd) {
+		for_each_cpu_and(i, sched_domain_span(sd),
+			housekeeping_cpumask(HK_TYPE_TIMER)) {
+			if (cpu == i)
+				continue;
+
+			if (!idle_cpu(i)) {
+				cpu = i;
+				goto unlock;
+			}
+		}
+	}
+
+	if (default_cpu == -1)
+		default_cpu = housekeeping_any_cpu(HK_TYPE_TIMER);
+	cpu = default_cpu;
+unlock:
+	rcu_read_unlock();
+	return cpu;
+}
+
+/*
+ * When add_timer_on() enqueues a timer into the timer wheel of an
+ * idle CPU then this timer might expire before the next timer event
+ * which is scheduled to wake up that CPU. In case of a completely
+ * idle system the next event might even be infinite time into the
+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and
+ * leaves the inner idle loop so the newly added timer is taken into
+ * account when the CPU goes back to idle and evaluates the timer
+ * wheel for the next timer event.
+ */
+static void wake_up_idle_cpu(int cpu)
+{
+	if (cpu == smp_processor_id())
+		return;
+
+	if (set_nr_and_not_polling(cpu_rq(cpu)->idle))
+		smp_sched_reschedule(cpu);
+	else
+		trace_sched_wake_idle_without_ipi(cpu);
+}
+
+static bool wake_up_full_nohz_cpu(int cpu)
+{
+	/*
+	 * We just need the target to call irq_exit() and re-evaluate
+	 * the next tick. The nohz full kick at least implies that.
+	 * If needed we can still optimize that later with an
+	 * empty IRQ.
+	 */
+	if (cpu_is_offline(cpu))
+		return true;  /* Don't try to wake offline CPUs. */
+	if (tick_nohz_full_cpu(cpu)) {
+		if (cpu != smp_processor_id() ||
+		    tick_nohz_tick_stopped())
+			tick_nohz_full_kick_cpu(cpu);
+		return true;
+	}
+
+	return false;
+}
+
+/*
+ * Wake up the specified CPU.  If the CPU is going offline, it is the
+ * caller's responsibility to deal with the lost wakeup, for example,
+ * by hooking into the CPU_DEAD notifier like timers and hrtimers do.
+ */
+void wake_up_nohz_cpu(int cpu)
+{
+	if (!wake_up_full_nohz_cpu(cpu))
+		wake_up_idle_cpu(cpu);
+}
+#endif /* CONFIG_NO_HZ_COMMON */
+
+/*
+ * This is how migration works:
+ *
+ * 1) we invoke migration_cpu_stop() on the target CPU using
+ *    stop_one_cpu().
+ * 2) stopper starts to run (implicitly forcing the migrated thread
+ *    off the CPU)
+ * 3) it checks whether the migrated task is still in the wrong runqueue.
+ * 4) if it's in the wrong runqueue then the migration thread removes
+ *    it and puts it into the right queue.
+ * 5) stopper completes and stop_one_cpu() returns and the migration
+ *    is done.
+ */
+
+/*
+ * move_queued_task - move a queued task to new rq.
+ *
+ * Returns (locked) new rq. Old rq's lock is released. Running tasks
+ * are not on the skiplist, so callers must use task_queued().
+ */
+static struct rq *move_queued_task(struct rq *rq,
+				   struct rq_flags __always_unused *rf,
+				   struct task_struct *p, int new_cpu)
+{
+	lockdep_assert_held(rq->lock);
+
+	dequeue_task(rq, p, 0);
+	set_task_cpu(p, new_cpu);
+	rq_unlock(rq);
+
+	rq = cpu_rq(new_cpu);
+
+	rq_lock(rq);
+	WARN_ON_ONCE(task_cpu(p) != new_cpu);
+	enqueue_task(rq, p, ENQUEUE_MIGRATED);
+	try_preempt(p, rq);
+
+	return rq;
+}
+
+struct set_affinity_pending;
+
+struct migration_arg {
+	struct task_struct		*task;
+	int				dest_cpu;
+	struct set_affinity_pending	*pending;
+};
+
+/*
+ * @refs: number of wait_for_completion()
+ * @stop_pending: is @stop_work in use
+ */
+struct set_affinity_pending {
+	refcount_t		refs;
+	unsigned int		stop_pending;
+	struct completion	done;
+	struct cpu_stop_work	stop_work;
+	struct migration_arg	arg;
+};
+
+static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
+				 struct task_struct *p, int dest_cpu)
+{
+	/* Affinity changed (again). */
+	if (!is_cpu_allowed(p, dest_cpu))
+		return rq;
+
+	return move_queued_task(rq, rf, p, dest_cpu);
+}
+
+static int migration_cpu_stop(void *data)
+{
+	struct migration_arg *arg = data;
+	struct set_affinity_pending *pending = arg->pending;
+	struct task_struct *p = arg->task;
+	struct rq *rq = this_rq();
+	bool complete = false;
+	struct rq_flags rf;
+
+	/*
+	 * The original target CPU might have gone down and we might
+	 * be on another CPU but it doesn't matter.
+	 */
+	local_irq_save(rf.flags);
+	/*
+	 * Flush pending wakeups so we do not miss enforcing cpus_ptr,
+	 * see set_cpus_allowed_ptr()'s TASK_WAKING test.
+	 */
+	flush_smp_call_function_queue();
+
+	raw_spin_lock(&p->pi_lock);
+	rq_lock(rq);
+
+	/*
+	 * If we were passed a pending, then ->stop_pending was set, thus
+	 * p->migration_pending must have remained stable.
+	 */
+	WARN_ON_ONCE(pending && pending != p->migration_pending);
+
+	/*
+	 * If task_rq(p) != rq, it cannot be migrated here, because we're
+	 * holding rq->lock. If p is not queued it cannot get enqueued
+	 * because we're holding p->pi_lock.
+	 */
+	if (task_rq(p) == rq) {
+		if (is_migration_disabled(p))
+			goto out;
+
+		if (pending) {
+			p->migration_pending = NULL;
+			complete = true;
+
+			if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask))
+				goto out;
+		}
+
+		/*
+		 * Mainline updates the rq clock before __migrate_task() here.
+		 * MuQSS has no need to: dequeue_task() and enqueue_task() each
+		 * call update_clocks() on the runqueue they touch, so niffies,
+		 * which deadlines are based on, is current on both sides of
+		 * the migration. Adding one here would only be redundant.
+		 */
+		if (task_queued(p))
+			rq = __migrate_task(rq, &rf, p, arg->dest_cpu);
+		else
+			p->wake_cpu = arg->dest_cpu;
+	} else if (pending) {
+		/*
+		 * The task moved before the stopper got to run. We're
+		 * holding ->pi_lock, so the allowed mask is stable - if
+		 * it got somewhere allowed, we're done.
+		 */
+		if (cpumask_test_cpu(task_cpu(p), p->cpus_ptr)) {
+			p->migration_pending = NULL;
+			complete = true;
+			goto out;
+		}
+
+		/*
+		 * When migrate_enable() hits a rq mis-match we can't
+		 * reliably determine is_migration_disabled() and so
+		 * have to chase after it.
+		 */
+		WARN_ON_ONCE(!pending->stop_pending);
+		preempt_disable();
+		rq_unlock(rq);
+		raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+		stop_one_cpu_nowait(task_cpu(p), migration_cpu_stop,
+				    &pending->arg, &pending->stop_work);
+		preempt_enable();
+		return 0;
+	}
+out:
+	if (pending)
+		pending->stop_pending = false;
+	rq_unlock(rq);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
+
+	if (complete)
+		complete_all(&pending->done);
+
+	return 0;
+}
+
+/*
+ * When given a valid mask, __set_cpus_allowed_ptr() must block until
+ * the designated task is enqueued on an allowed CPU. A running task
+ * is kicked off with the CPU stopper. A migrate_disable() region on
+ * the target delays that move until the outermost migrate_enable();
+ * concurrent waiters share one set_affinity_pending.
+ */
+static int affine_move_task(struct rq *rq, struct task_struct *p,
+			    struct rq_flags *rf, int dest_cpu, unsigned int flags)
+{
+	struct set_affinity_pending my_pending = { }, *pending = NULL;
+	bool stop_pending, complete = false;
+
+	/* Can the task run on the task's current CPU? If so, we're done */
+	if (cpumask_test_cpu(task_cpu(p), &p->cpus_mask)) {
+		/*
+		 * If there are pending waiters, but no pending stop_work,
+		 * then complete now.
+		 */
+		pending = p->migration_pending;
+		if (pending && !pending->stop_pending) {
+			p->migration_pending = NULL;
+			complete = true;
+		}
+
+		task_rq_unlock(rq, p, rf);
+
+		if (complete)
+			complete_all(&pending->done);
+
+		return 0;
+	}
+
+	if (!(flags & SCA_MIGRATE_ENABLE)) {
+		/* serialized by p->pi_lock */
+		if (!p->migration_pending) {
+			refcount_set(&my_pending.refs, 1);
+			init_completion(&my_pending.done);
+			my_pending.arg = (struct migration_arg) {
+				.task = p,
+				.dest_cpu = dest_cpu,
+				.pending = &my_pending,
+			};
+
+			p->migration_pending = &my_pending;
+		} else {
+			pending = p->migration_pending;
+			refcount_inc(&pending->refs);
+			/*
+			 * Affinity has changed, but we've already installed a
+			 * pending. migration_cpu_stop() *must* see this, else
+			 * we risk completing despite a task on a disallowed
+			 * CPU. Serialized by p->pi_lock.
+			 */
+			pending->arg.dest_cpu = dest_cpu;
+		}
+	}
+	pending = p->migration_pending;
+	/*
+	 * !MIGRATE_ENABLE installs a pending if there wasn't one.
+	 * MIGRATE_ENABLE only gets here because the current CPU no
+	 * longer matches, so a concurrent set_cpus_allowed_ptr()
+	 * should still be pending completion.
+	 */
+	if (WARN_ON_ONCE(!pending)) {
+		task_rq_unlock(rq, p, rf);
+		return -EINVAL;
+	}
+
+	if (task_running(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) {
+		/*
+		 * MIGRATE_ENABLE gets here because 'p == current'. For
+		 * anything else we cannot do is_migration_disabled();
+		 * punt and have the stopper handle it race-free.
+		 */
+		stop_pending = pending->stop_pending;
+		if (!stop_pending)
+			pending->stop_pending = true;
+
+		preempt_disable();
+		task_rq_unlock(rq, p, rf);
+		if (!stop_pending) {
+			stop_one_cpu_nowait(cpu_of(rq), migration_cpu_stop,
+					    &pending->arg, &pending->stop_work);
+		}
+		preempt_enable();
+
+		if (flags & SCA_MIGRATE_ENABLE)
+			return 0;
+	} else {
+		if (!is_migration_disabled(p)) {
+			if (task_queued(p))
+				rq = move_queued_task(rq, rf, p, dest_cpu);
+
+			if (!pending->stop_pending) {
+				p->migration_pending = NULL;
+				complete = true;
+			}
+		}
+		task_rq_unlock(rq, p, rf);
+
+		if (complete)
+			complete_all(&pending->done);
+	}
+
+	wait_for_completion(&pending->done);
+
+	if (refcount_dec_and_test(&pending->refs))
+		wake_up_var(&pending->refs); /* No UaF, just an address */
+
+	/*
+	 * Block the original owner of &pending until all subsequent
+	 * callers have seen the completion and decremented the refcount
+	 */
+	wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
+
+	WARN_ON_ONCE(my_pending.stop_pending);
+
+	return 0;
+}
+
+/*
+ * Record @new_mask as the affinity userspace asked for, handing back the
+ * allocation it displaced for the caller to free once the locks are
+ * dropped. An explicit request also supersedes any override bind_zero()
+ * had to force, since this mask is the one that is wanted now.
+ */
+static cpumask_t *set_user_cpus_ptr(struct task_struct *p,
+				    const struct cpumask *new_mask,
+				    cpumask_t *user_mask)
+{
+	lockdep_assert_held(&p->pi_lock);
+
+	if (user_mask) {
+		cpumask_copy(user_mask, new_mask);
+		swap(p->user_cpus_ptr, user_mask);
+	}
+	p->zerobound = false;
+
+	return user_mask;
+}
+
+/*
+ * Change a given task's CPU affinity. Migrate the thread to a
+ * proper CPU and schedule it away if the CPU it's executing on
+ * is removed from the allowed bitmask.
+ *
+ * NOTE: the caller must have a valid reference to the task, the
+ * task must not exit() & deallocate itself prematurely. The
+ * call is not atomic; no spinlocks may be held.
+ */
+static int __set_cpus_allowed_ptr(struct task_struct *p,
+				  const struct cpumask *new_mask,
+				  u32 flags)
+{
+	const struct cpumask *cpu_valid_mask = cpu_active_mask;
+	cpumask_t *user_mask = NULL;
+	unsigned int dest_cpu;
+	struct rq_flags rf;
+	struct rq *rq;
+	int ret = 0;
+
+	/*
+	 * user_cpus_ptr keeps what userspace asked for so that
+	 * relax_compatible_cpus_allowed_ptr() and unbind_zero() have
+	 * something to put back. Only sched_setaffinity() passes SCA_USER
+	 * and it is always sleepable, so make room before taking any lock;
+	 * failing here only costs the task that restore, not the syscall.
+	 */
+	if (flags & SCA_USER)
+		user_mask = kmalloc(cpumask_size(), GFP_KERNEL);
+
+	rq = task_rq_lock(p, &rf);
+	update_rq_clock(rq);
+
+	if ((p->flags & PF_KTHREAD) || is_migration_disabled(p)) {
+		/*
+		 * Kernel threads are allowed on online && !active CPUs.
+		 * migrate_disabled() tasks must not fail the dest pick
+		 * on SCA_MIGRATE_ENABLE or we skip set_cpus_allowed_common()
+		 * and never reset p->cpus_ptr.
+		 */
+		cpu_valid_mask = cpu_online_mask;
+	}
+
+	/*
+	 * Must re-check here, to close a race against __kthread_bind(),
+	 * sched_setaffinity() is not guaranteed to observe the flag.
+	 */
+	if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	if (!(flags & SCA_MIGRATE_ENABLE)) {
+		if (cpumask_equal(&p->cpus_mask, new_mask)) {
+			/*
+			 * Nothing to change, but asking for exactly the mask
+			 * bind_zero() forced is still userspace choosing it,
+			 * and takes over from the restore. An internal caller
+			 * changing nothing must not cost the task that.
+			 */
+			if (flags & SCA_USER)
+				user_mask = set_user_cpus_ptr(p, new_mask,
+							      user_mask);
+			goto out;
+		}
+
+		if (WARN_ON_ONCE(p == current &&
+				 is_migration_disabled(p) &&
+				 !cpumask_test_cpu(task_cpu(p), new_mask))) {
+			ret = -EBUSY;
+			goto out;
+		}
+	}
+
+	/*
+	 * Picking a ~random cpu helps in cases where we are changing affinity
+	 * for groups of tasks (ie. cpuset), so that load balancing is not
+	 * immediately required to distribute the tasks within their new mask.
+	 */
+	dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask);
+	if (dest_cpu >= nr_cpu_ids) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	{
+		struct affinity_context ac = {
+			.new_mask = new_mask,
+			.flags = flags,
+		};
+
+		__do_set_cpus_allowed(p, &ac);
+	}
+
+	/*
+	 * migrate_enable() only reinstates cpus_mask, so it is not a change
+	 * of what the task wants and must not disturb what is saved.
+	 */
+	if (!(flags & SCA_MIGRATE_ENABLE))
+		user_mask = set_user_cpus_ptr(p, new_mask, user_mask);
+
+	if (p->flags & PF_KTHREAD) {
+		/*
+		 * For kernel threads that do indeed end up on online &&
+		 * !active we want to ensure they are strict per-CPU threads.
+		 */
+		WARN_ON(cpumask_intersects(new_mask, cpu_online_mask) &&
+			!cpumask_intersects(new_mask, cpu_active_mask) &&
+			p->nr_cpus_allowed != 1);
+	}
+
+	ret = affine_move_task(rq, p, &rf, dest_cpu, flags);
+	kfree(user_mask);
+
+	return ret;
+
+out:
+	task_rq_unlock(rq, p, &rf);
+	kfree(user_mask);
+
+	return ret;
+}
+
+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
+{
+	return __set_cpus_allowed_ptr(p, new_mask, 0);
+}
+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
+
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * Skip affinity rewrite for tasks whose mask is managed elsewhere: the
+ * hotplug/stopper threads, idle, and kthreads that forbid setaffinity
+ * (including KTHREAD_IS_PER_CPU, rebound on unpark).
+ */
+static bool bind_zero_skip_affinity(struct task_struct *p, struct rq *rq)
+{
+	return p == current || is_idle_task(p) || p == rq->stop ||
+	       (p->flags & PF_NO_SETAFFINITY) || kthread_is_per_cpu(p);
+}
+
+/*
+ * Move @p off @src_cpu under pi+rq lock, and only override its affinity
+ * if losing @src_cpu would leave it with no active CPU at all.
+ *
+ * Like mainline, the common case does not touch cpus_mask: an offline or
+ * deactivated CPU is already refused by is_cpu_allowed(), sched_other_cpu(),
+ * needs_other_cpu() and valid_task_cpu(), so the mask needs no editing and
+ * the task keeps the affinity userspace asked for across a hotplug cycle.
+ * Narrowing here would be permanent, as unbind_zero() only restores tasks
+ * that were forced.
+ *
+ * The desperate case still has to put the task somewhere: substitute an
+ * active fallback (usually CPU0) for @src_cpu and record that with
+ * zerobound, stashing the mask being replaced in user_cpus_ptr so
+ * unbind_zero() can put back exactly what userspace asked for once
+ * @src_cpu returns.
+ *
+ * migrate_disable() tasks keep their pin (cpus_ptr) and are left for
+ * wait_empty to wait out; only cpus_mask is updated so migrate_enable()
+ * will affine_move them.
+ */
+static int bind_zero_one(struct task_struct *p, int src_cpu)
+{
+	cpumask_t *user_mask, new_mask;
+	struct rq_flags rf;
+	struct rq *rq;
+	int dest, bound = 0;
+
+	/*
+	 * The override below cannot allocate under the runqueue lock, so
+	 * speculatively provide it room for the old mask here. Almost every
+	 * task takes the fast path and hands this straight back.
+	 */
+	user_mask = kmalloc(cpumask_size(), GFP_ATOMIC);
+
+	rq = task_rq_lock(p, &rf);
+
+	if (!bind_zero_skip_affinity(p, rq) &&
+	    cpumask_test_cpu(src_cpu, &p->cpus_mask)) {
+		cpumask_copy(&new_mask, &p->cpus_mask);
+		cpumask_clear_cpu(src_cpu, &new_mask);
+		if (!cpumask_intersects(&new_mask, cpu_active_mask)) {
+			dest = cpumask_any(cpu_active_mask);
+			if (dest >= nr_cpu_ids)
+				dest = cpumask_any(cpu_online_mask);
+			if (dest < nr_cpu_ids) {
+				struct affinity_context ac = {
+					.new_mask = &new_mask
+				};
+
+				/*
+				 * Only the first override saves; a second one
+				 * would only be replacing an already forced
+				 * mask with another. What is being displaced
+				 * is what has to come back, so it takes over
+				 * any mask sched_setaffinity() left here -
+				 * reusing its allocation - rather than let a
+				 * request cpusets have since narrowed be
+				 * restored wider than the cpuset allows.
+				 */
+				if (!p->zerobound) {
+					if (p->user_cpus_ptr) {
+						cpumask_copy(p->user_cpus_ptr,
+							     &p->cpus_mask);
+					} else if (user_mask) {
+						cpumask_copy(user_mask,
+							     &p->cpus_mask);
+						p->user_cpus_ptr = user_mask;
+						user_mask = NULL;
+					}
+				}
+				cpumask_set_cpu(dest, &new_mask);
+				p->zerobound = true;
+				__do_set_cpus_allowed(p, &ac);
+				bound = 1;
+			}
+		}
+	}
+
+	if (task_cpu(p) == src_cpu && p != current &&
+	    !is_idle_task(p) && p != rq->stop &&
+	    !is_migration_disabled(p)) {
+		dest = valid_task_cpu(p);
+		if (dest != src_cpu) {
+			/*
+			 * Only a queued task has to be moved. A blocked one is
+			 * on no runqueue, so it does not hold @src_cpu up, and
+			 * its destination must not be stamped into wake_cpu:
+			 * that arms return_task()'s migration handshake, which
+			 * would then fire on a task nobody is migrating. Its
+			 * wakeup picks an allowed CPU by itself.
+			 */
+			if (task_queued(p))
+				rq = move_queued_task(rq, &rf, p, dest);
+		}
+	}
+
+	task_rq_unlock(rq, p, &rf);
+	kfree(user_mask);
+	return bound;
+}
+
+/*
+ * Called from sched_cpu_wait_empty() (sleepable) on the outgoing CPU.
+ * Walk every task and move anyone still queued here. Affinity is only
+ * overridden for the few tasks @src_cpu leaving would otherwise strand,
+ * so on a normal offline this reports nothing.
+ */
+static void bind_zero(int src_cpu)
+{
+	struct task_struct *g, *p;
+	int bound = 0;
+
+	if (src_cpu == 0)
+		return;
+
+	rcu_read_lock();
+	for_each_process_thread(g, p) {
+		if (task_cpu(p) != src_cpu &&
+		    !cpumask_test_cpu(src_cpu, p->cpus_ptr) &&
+		    !cpumask_test_cpu(src_cpu, &p->cpus_mask))
+			continue;
+		bound += bind_zero_one(p, src_cpu);
+	}
+	rcu_read_unlock();
+
+	if (bound) {
+		printk(KERN_INFO "MuQSS overrode affinity for %d processes left with no active cpu by offlining cpu %d\n",
+		       bound, src_cpu);
+	}
+}
+
+/*
+ * Undo the affinity override bind_zero() had to force onto tasks that
+ * @src_cpu leaving would have stranded. Hold pi+rq so cpus_mask /
+ * nr_cpus_allowed stay in lockstep. Unbound kthreads are eligible — only
+ * per-CPU / PF_NO_SETAFFINITY threads were skipped on the way out.
+ *
+ * Restoring the saved mask can take the CPU the task is on right now away
+ * from it, so kick it off exactly like do_set_cpus_allowed() does.
+ */
+static void unbind_zero(int src_cpu)
+{
+	int restored = 0, unbound = 0;
+	struct task_struct *g, *p;
+
+	if (src_cpu == 0)
+		return;
+
+	rcu_read_lock();
+	for_each_process_thread(g, p) {
+		struct rq_flags rf;
+		struct rq *rq;
+
+		if (!p->zerobound)
+			continue;
+
+		rq = task_rq_lock(p, &rf);
+		if (!p->zerobound) {
+			task_rq_unlock(rq, p, &rf);
+			continue;
+		}
+
+		if (likely(p->user_cpus_ptr)) {
+			/*
+			 * Wait for one of the CPUs the task actually wanted,
+			 * or restoring would strand it all over again. The
+			 * saved mask stays put once copied back: it is still
+			 * what the task wants, and what a later override or
+			 * relax_compatible_cpus_allowed_ptr() restores.
+			 */
+			if (cpumask_test_cpu(src_cpu, p->user_cpus_ptr)) {
+				struct affinity_context ac = {
+					.new_mask = p->user_cpus_ptr
+				};
+
+				__do_set_cpus_allowed(p, &ac);
+				p->zerobound = false;
+				restored++;
+			}
+		} else if (!cpumask_test_cpu(src_cpu, &p->cpus_mask)) {
+			/*
+			 * bind_zero() had no room to save the old mask, so
+			 * hand @src_cpu back and stop there rather than
+			 * accumulate every CPU that ever comes online.
+			 */
+			cpumask_set_cpu(src_cpu, &p->cpus_mask);
+			p->nr_cpus_allowed = cpumask_weight(&p->cpus_mask);
+			p->zerobound = false;
+			unbound++;
+		}
+
+		if (!p->zerobound && needs_other_cpu(p, task_cpu(p))) {
+			int dest = valid_task_cpu(p);
+
+			/*
+			 * As in bind_zero_one(), a blocked task is left alone
+			 * rather than having @dest stamped into its wake_cpu,
+			 * which would arm return_task()'s migration handshake
+			 * for a migration that is not happening.
+			 */
+			if (task_queued(p))
+				rq = move_queued_task(rq, &rf, p, dest);
+			else if (task_running(rq, p)) {
+				set_task_cpu(p, dest);
+				resched_task(p);
+			}
+		}
+
+		task_rq_unlock(rq, p, &rf);
+	}
+	rcu_read_unlock();
+
+	if (restored) {
+		printk(KERN_INFO "MuQSS restored the original affinity of %d processes by onlining cpu %d\n",
+		       restored, src_cpu);
+	}
+	if (unbound) {
+		printk(KERN_INFO "MuQSS added affinity for %d processes to cpu %d\n",
+		       unbound, src_cpu);
+	}
+}
+
+/*
+ * idle_task_exit() is an empty inline in <linux/sched/hotplug.h> now; the
+ * outgoing CPU switches back to init_mm from sched_cpu_wait_empty() instead.
+ */
+#else /* CONFIG_HOTPLUG_CPU */
+static void unbind_zero(int src_cpu) {}
+#endif /* CONFIG_HOTPLUG_CPU */
+
+void sched_set_stop_task(int cpu, struct task_struct *stop)
+{
+	struct sched_param stop_param = { .sched_priority = STOP_PRIO };
+	struct sched_param start_param = { .sched_priority = 0 };
+	struct task_struct *old_stop = cpu_rq(cpu)->stop;
+
+	if (stop) {
+		/*
+		 * Make it appear like a SCHED_FIFO task, its something
+		 * userspace knows about and won't get confused about.
+		 *
+		 * Also, it will make PI more or less work without too
+		 * much confusion -- but then, stop work should not
+		 * rely on PI working anyway.
+		 */
+		sched_setscheduler_nocheck(stop, SCHED_FIFO, &stop_param);
+	}
+
+	cpu_rq(cpu)->stop = stop;
+
+	if (old_stop) {
+		/*
+		 * Reset it back to a normal scheduling policy so that
+		 * it can die in pieces.
+		 */
+		sched_setscheduler_nocheck(old_stop, SCHED_NORMAL, &start_param);
+	}
+}
+
+#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL)
+
+static struct ctl_table sd_ctl_dir[] = {
+	{
+		.procname	= "sched_domain",
+		.mode		= 0555,
+	},
+	{}
+};
+
+static struct ctl_table sd_ctl_root[] = {
+	{
+		.procname	= "kernel",
+		.mode		= 0555,
+		.child		= sd_ctl_dir,
+	},
+	{}
+};
+
+static struct ctl_table *sd_alloc_ctl_entry(int n)
+{
+	struct ctl_table *entry =
+		kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL);
+
+	return entry;
+}
+
+static void sd_free_ctl_entry(struct ctl_table **tablep)
+{
+	struct ctl_table *entry;
+
+	/*
+	 * In the intermediate directories, both the child directory and
+	 * procname are dynamically allocated and could fail but the mode
+	 * will always be set. In the lowest directory the names are
+	 * static strings and all have proc handlers.
+	 */
+	for (entry = *tablep; entry->mode; entry++) {
+		if (entry->child)
+			sd_free_ctl_entry(&entry->child);
+		if (entry->proc_handler == NULL)
+			kfree(entry->procname);
+	}
+
+	kfree(*tablep);
+	*tablep = NULL;
+}
+
+static void
+set_table_entry(struct ctl_table *entry,
+		const char *procname, void *data, int maxlen,
+		umode_t mode, proc_handler *proc_handler)
+{
+	entry->procname = procname;
+	entry->data = data;
+	entry->maxlen = maxlen;
+	entry->mode = mode;
+	entry->proc_handler = proc_handler;
+}
+
+static struct ctl_table *
+sd_alloc_ctl_domain_table(struct sched_domain *sd)
+{
+	struct ctl_table *table = sd_alloc_ctl_entry(9);
+
+	if (table == NULL)
+		return NULL;
+
+	set_table_entry(&table[0], "min_interval",	  &sd->min_interval,	    sizeof(long), 0644, proc_doulongvec_minmax);
+	set_table_entry(&table[1], "max_interval",	  &sd->max_interval,	    sizeof(long), 0644, proc_doulongvec_minmax);
+	set_table_entry(&table[2], "busy_factor",	  &sd->busy_factor,	    sizeof(int),  0644, proc_dointvec_minmax);
+	set_table_entry(&table[3], "imbalance_pct",	  &sd->imbalance_pct,	    sizeof(int),  0644, proc_dointvec_minmax);
+	set_table_entry(&table[4], "cache_nice_tries",	  &sd->cache_nice_tries,    sizeof(int),  0644, proc_dointvec_minmax);
+	set_table_entry(&table[5], "flags",		  &sd->flags,		    sizeof(int),  0644, proc_dointvec_minmax);
+	set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax);
+	set_table_entry(&table[7], "name",		  sd->name,	       CORENAME_MAX_SIZE, 0444, proc_dostring);
+	/* &table[8] is terminator */
+
+	return table;
+}
+
+static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu)
+{
+	struct ctl_table *entry, *table;
+	struct sched_domain *sd;
+	int domain_num = 0, i;
+	char buf[32];
+
+	for_each_domain(cpu, sd)
+		domain_num++;
+	entry = table = sd_alloc_ctl_entry(domain_num + 1);
+	if (table == NULL)
+		return NULL;
+
+	i = 0;
+	for_each_domain(cpu, sd) {
+		snprintf(buf, 32, "domain%d", i);
+		entry->procname = kstrdup(buf, GFP_KERNEL);
+		entry->mode = 0555;
+		entry->child = sd_alloc_ctl_domain_table(sd);
+		entry++;
+		i++;
+	}
+	return table;
+}
+
+static cpumask_var_t sd_sysctl_cpus;
+static struct ctl_table_header *sd_sysctl_header;
+
+void register_sched_domain_sysctl(void)
+{
+	static struct ctl_table *cpu_entries;
+	static struct ctl_table **cpu_idx;
+	char buf[32];
+	int i;
+
+	if (!cpu_entries) {
+		cpu_entries = sd_alloc_ctl_entry(num_possible_cpus() + 1);
+		if (!cpu_entries)
+			return;
+
+		WARN_ON(sd_ctl_dir[0].child);
+		sd_ctl_dir[0].child = cpu_entries;
+	}
+
+	if (!cpu_idx) {
+		struct ctl_table *e = cpu_entries;
+
+		cpu_idx = kcalloc(nr_cpu_ids, sizeof(struct ctl_table*), GFP_KERNEL);
+		if (!cpu_idx)
+			return;
+
+		/* deal with sparse possible map */
+		for_each_possible_cpu(i) {
+			cpu_idx[i] = e;
+			e++;
+		}
+	}
+
+	if (!cpumask_available(sd_sysctl_cpus)) {
+		if (!alloc_cpumask_var(&sd_sysctl_cpus, GFP_KERNEL))
+			return;
+
+		/* init to possible to not have holes in @cpu_entries */
+		cpumask_copy(sd_sysctl_cpus, cpu_possible_mask);
+	}
+
+	for_each_cpu(i, sd_sysctl_cpus) {
+		struct ctl_table *e = cpu_idx[i];
+
+		if (e->child)
+			sd_free_ctl_entry(&e->child);
+
+		if (!e->procname) {
+			snprintf(buf, 32, "cpu%d", i);
+			e->procname = kstrdup(buf, GFP_KERNEL);
+		}
+		e->mode = 0555;
+		e->child = sd_alloc_ctl_cpu_table(i);
+
+		__cpumask_clear_cpu(i, sd_sysctl_cpus);
+	}
+
+	WARN_ON(sd_sysctl_header);
+	sd_sysctl_header = register_sysctl_table(sd_ctl_root);
+}
+
+void dirty_sched_domain_sysctl(int cpu)
+{
+	if (cpumask_available(sd_sysctl_cpus))
+		__cpumask_set_cpu(cpu, sd_sysctl_cpus);
+}
+
+/* may be called multiple times per register */
+void unregister_sched_domain_sysctl(void)
+{
+	unregister_sysctl_table(sd_sysctl_header);
+	sd_sysctl_header = NULL;
+}
+#endif /* CONFIG_SYSCTL */
+
+void set_rq_online(struct rq *rq)
+{
+	if (!rq->online) {
+		cpumask_set_cpu(cpu_of(rq), rq->rd->online);
+		rq->online = true;
+	}
+}
+
+void set_rq_offline(struct rq *rq)
+{
+	if (rq->online) {
+		int cpu = cpu_of(rq);
+
+		cpumask_clear_cpu(cpu, rq->rd->online);
+		rq->online = false;
+		clear_cpuidle_map(cpu);
+	}
+}
+
+/*
+ * used to mark begin/end of suspend/resume:
+ */
+static int num_cpus_frozen;
+
+/*
+ * Update cpusets according to cpu_active mask.  If cpusets are
+ * disabled, cpuset_update_active_cpus() becomes a simple wrapper
+ * around partition_sched_domains().
+ *
+ * If we come here as part of a suspend/resume, don't touch cpusets because we
+ * want to restore it back to its original state upon resume anyway.
+ */
+static void cpuset_cpu_active(void)
+{
+	if (cpuhp_tasks_frozen) {
+		/*
+		 * num_cpus_frozen tracks how many CPUs are involved in suspend
+		 * resume sequence. As long as this is not the last online
+		 * operation in the resume sequence, just build a single sched
+		 * domain, ignoring cpusets.
+		 */
+		partition_sched_domains(1, NULL, NULL);
+		if (--num_cpus_frozen)
+			return;
+		/*
+		 * This is the last CPU online operation. So fall through and
+		 * restore the original sched domains by considering the
+		 * cpuset configurations.
+		 */
+		cpuset_force_rebuild();
+	}
+
+	cpuset_update_active_cpus();
+}
+
+static int cpuset_cpu_inactive(unsigned int cpu)
+{
+	if (!cpuhp_tasks_frozen) {
+		cpuset_update_active_cpus();
+	} else {
+		num_cpus_frozen++;
+		partition_sched_domains(1, NULL, NULL);
+	}
+	return 0;
+}
+
+int sched_cpu_activate(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	/*
+	 * When going up, increment the number of cores with SMT present.
+	 * cpu_smt_mask() is cpumask_of(cpu) when !CONFIG_SCHED_SMT, so this
+	 * never fires there.
+	 */
+	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
+		static_branch_inc_cpuslocked(&sched_smt_present);
+
+	set_cpu_active(cpu, true);
+
+	if (sched_smp_initialized) {
+		sched_domains_numa_masks_set(cpu);
+		cpuset_cpu_active();
+	}
+
+	/*
+	 * Put the rq online, if not already. This happens:
+	 *
+	 * 1) In the early boot process, because we build the real domains
+	 *    after all CPUs have been brought up.
+	 *
+	 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
+	 *    domains.
+	 */
+	rq_lock_irqsave(rq, &rf);
+	if (rq->rd) {
+		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
+		set_rq_online(rq);
+	}
+	rq_unlock_irqrestore(rq, &rf);
+	/*
+	 * unbind_zero() takes each task's pi+rq lock; do not nest that
+	 * under this CPU's rq lock.
+	 */
+	unbind_zero(cpu);
+
+	return 0;
+}
+
+int sched_cpu_deactivate(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+	int ret;
+
+	set_cpu_active(cpu, false);
+	/*
+	 * We've cleared cpu_active_mask, wait for all preempt-disabled and RCU
+	 * users of this state to go away such that all new such users will
+	 * observe it.
+	 *
+	 * Do sync before park smpboot threads to take care the rcu boost case.
+	 */
+	synchronize_rcu();
+
+	sched_domains_free_llc_id(cpu);
+
+	rq_lock_irqsave(rq, &rf);
+	if (rq->rd) {
+		update_rq_clock(rq);
+		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
+		set_rq_offline(rq);
+	}
+	rq_unlock_irqrestore(rq, &rf);
+
+	/*
+	 * When going down, decrement the number of cores with SMT present.
+	 */
+	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
+		static_branch_dec_cpuslocked(&sched_smt_present);
+
+	if (!sched_smp_initialized)
+		return 0;
+
+	ret = cpuset_cpu_inactive(cpu);
+	if (ret) {
+		set_cpu_active(cpu, true);
+		return ret;
+	}
+	sched_domains_numa_masks_clear(cpu);
+	return 0;
+}
+
+int sched_cpu_starting(unsigned int cpu)
+{
+	sched_tick_start(cpu);
+	return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * Invoked on the outgoing CPU in context of the CPU hotplug thread after
+ * ensuring that there are no user space tasks left on the CPU.
+ *
+ * If there is a lazy mm in use on the hotplug thread, drop it and switch to
+ * init_mm.  finish_cpu() on the control CPU drops the init_mm reference and
+ * WARNs if we left anything else behind.  5.12 did this from idle_task_exit();
+ * mainline now does it here from sched_cpu_wait_empty().
+ */
+static void __sched_force_init_mm(void)
+{
+	struct mm_struct *mm = current->active_mm;
+
+	if (mm == &init_mm)
+		return;
+
+	mmgrab_lazy_tlb(&init_mm);
+	current->active_mm = &init_mm;
+	switch_mm_irqs_off(mm, &init_mm, current);
+	finish_arch_post_lock_switch();
+	mmdrop_lazy_tlb(mm);
+}
+
+static void sched_force_init_mm(void)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	__sched_force_init_mm();
+	local_irq_restore(flags);
+}
+
+static void dump_rq_tasks(struct rq *rq, const char *loglvl)
+{
+	struct task_struct *g, *p;
+	int cpu = cpu_of(rq);
+
+	lockdep_assert_rq_held(rq);
+
+	printk("%sCPU%d tasks (nr_running=%u nr_pinned=%u):\n",
+	       loglvl, cpu, rq->nr_running, rq->nr_pinned);
+	rcu_read_lock();
+	for_each_process_thread(g, p) {
+		if (task_cpu(p) != cpu)
+			continue;
+		printk("%s\tpid: %d, name: %s, queued: %d, on_rq: %d\n",
+		       loglvl, p->pid, p->comm, task_queued(p), p->on_rq);
+	}
+	rcu_read_unlock();
+}
+
+/*
+ * Invoked on the outgoing CPU after per-CPU kthreads have been parked.
+ * bind_zero() takes affinity off this CPU and moves anyone still queued
+ * here. Then wait until only this hotplug thread remains and every
+ * migrate_disable() pin has dropped, so finish_cpu() cannot see a user mm.
+ */
+int sched_cpu_wait_empty(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+
+	WARN_ON_ONCE(cpu != smp_processor_id());
+
+	for (;;) {
+		bind_zero(cpu);
+		if (READ_ONCE(rq->nr_running) <= 1 && !READ_ONCE(rq->nr_pinned))
+			break;
+		schedule_timeout_uninterruptible(1);
+	}
+
+	sched_force_init_mm();
+	return 0;
+}
+
+int sched_cpu_dying(unsigned int cpu)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+
+	/* Handle pending wakeups; tasks were already moved in wait_empty. */
+	sched_tick_stop(cpu);
+
+	rq_lock_irqsave(rq, &rf);
+	if (rq->rd) {
+		BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
+		set_rq_offline(rq);
+	}
+	if (rq->nr_running > 1 || rq->nr_pinned) {
+		WARN(true, "Dying CPU not properly vacated!");
+		dump_rq_tasks(rq, KERN_WARNING);
+	}
+	rq_unlock_irqrestore(rq, &rf);
+
+	sched_start_tick(rq, cpu);
+	hrexpiry_clear(rq);
+	/*
+	 * Belt-and-suspenders: wait_empty already switched to init_mm,
+	 * but a preempting migrate_disable() task could have left a
+	 * lazy user mm on current. Drop it again now that this is the
+	 * last thing to run before idle takes over.
+	 */
+	local_irq_disable();
+	__sched_force_init_mm();
+	local_irq_enable();
+
+	return 0;
+}
+#endif
+
+#if defined(CONFIG_SCHED_SMT) || defined(CONFIG_SCHED_MC)
+/*
+ * Cheaper version of the below functions in case support for SMT and MC is
+ * compiled in but CPUs have no siblings.
+ */
+static bool sole_cpu_idle(struct rq *rq)
+{
+	return rq_idle(rq);
+}
+#endif
+#ifdef CONFIG_SCHED_SMT
+static const cpumask_t *thread_cpumask(int cpu)
+{
+	return topology_sibling_cpumask(cpu);
+}
+#endif
+#ifdef CONFIG_SCHED_MC
+static const cpumask_t *core_cpumask(int cpu)
+{
+	return topology_core_cpumask(cpu);
+}
+/* All this CPU's shared cache siblings are idle */
+static bool cache_cpu_idle(struct rq *rq)
+{
+	return cpumask_subset(&rq->core_mask, &cpu_idle_map);
+}
+/* MC siblings CPU mask which share the same LLC */
+static const cpumask_t *llc_core_cpumask(int cpu)
+{
+#ifdef CONFIG_X86
+	return per_cpu(cpu_llc_shared_map, cpu);
+#else
+	return topology_core_cpumask(cpu);
+#endif
+}
+#endif
+
+enum sched_domain_level {
+	SD_LV_NONE = 0,
+	SD_LV_SIBLING,
+	SD_LV_MC,
+	SD_LV_BOOK,
+	SD_LV_CPU,
+	SD_LV_NODE,
+	SD_LV_ALLNODES,
+	SD_LV_MAX
+};
+
+#ifdef CONFIG_SMT_NICE
+/*
+ * Recorded by select_leaders() and consumed by sched_init_smp() once the
+ * runqueue locks have been dropped. See the comment where it is set.
+ */
+static bool __initdata smt_nice_needed;
+#endif
+
+/*
+ * Set up the relative cache distance of each online cpu from each
+ * other in a simple array for quick lookup. Locality is determined
+ * by the closest sched_domain that CPUs are separated by. CPUs with
+ * shared cache in SMT and MC are treated as local. Separate CPUs
+ * (within the same package or physically) within the same node are
+ * treated as not local. CPUs not even in the same domain (different
+ * nodes) are treated as very distant.
+ *
+ * Called with interrupts disabled and every runqueue lock held, so nothing
+ * here may sleep or wait on another CPU.
+ */
+static void __init select_leaders(void)
+{
+	struct rq *rq, *other_rq, *leader;
+	struct sched_domain *sd;
+	int cpu, other_cpu;
+#ifdef CONFIG_SCHED_SMT
+	bool smt_threads = false;
+#endif
+
+	for (cpu = 0; cpu < num_online_cpus(); cpu++) {
+		rq = cpu_rq(cpu);
+		leader = NULL;
+		/* First check if this cpu is in the same node */
+		for_each_domain(cpu, sd) {
+			if (sd->level > SD_LV_MC)
+				continue;
+			if (rqshare != RQSHARE_ALL)
+				leader = NULL;
+			/* Set locality to local node if not already found lower */
+			for_each_cpu(other_cpu, sched_domain_span(sd)) {
+				if (rqshare >= RQSHARE_SMP) {
+					other_rq = cpu_rq(other_cpu);
+
+					/* Set the smp_leader to the first CPU */
+					if (!leader)
+						leader = rq;
+					if (!other_rq->smp_leader)
+						other_rq->smp_leader = leader;
+				}
+				if (rq->cpu_locality[other_cpu] > LOCALITY_SMP)
+					rq->cpu_locality[other_cpu] = LOCALITY_SMP;
+			}
+		}
+
+		/*
+		 * Each runqueue has its own function in case it doesn't have
+		 * siblings of its own allowing mixed topologies.
+		 */
+#ifdef CONFIG_SCHED_MC
+		leader = NULL;
+		if (cpumask_weight(core_cpumask(cpu)) > 1) {
+			cpumask_copy(&rq->core_mask, llc_core_cpumask(cpu));
+			cpumask_clear_cpu(cpu, &rq->core_mask);
+			for_each_cpu(other_cpu, core_cpumask(cpu)) {
+				if (rqshare == RQSHARE_MC ||
+					(rqshare == RQSHARE_MC_LLC && cpumask_test_cpu(other_cpu, llc_core_cpumask(cpu)))) {
+					other_rq = cpu_rq(other_cpu);
+
+					/* Set the mc_leader to the first CPU */
+					if (!leader)
+						leader = rq;
+					if (!other_rq->mc_leader)
+						other_rq->mc_leader = leader;
+				}
+				if (rq->cpu_locality[other_cpu] > LOCALITY_MC) {
+					/* this is to get LLC into play even in case LLC sharing is not used */
+					if (cpumask_test_cpu(other_cpu, llc_core_cpumask(cpu)))
+						rq->cpu_locality[other_cpu] = LOCALITY_MC_LLC;
+					else
+						rq->cpu_locality[other_cpu] = LOCALITY_MC;
+				}
+			}
+			rq->cache_idle = cache_cpu_idle;
+		}
+#endif
+#ifdef CONFIG_SCHED_SMT
+		leader = NULL;
+		if (cpumask_weight(thread_cpumask(cpu)) > 1) {
+			cpumask_copy(&rq->thread_mask, thread_cpumask(cpu));
+			cpumask_clear_cpu(cpu, &rq->thread_mask);
+			for_each_cpu(other_cpu, thread_cpumask(cpu)) {
+				if (rqshare == RQSHARE_SMT) {
+					other_rq = cpu_rq(other_cpu);
+
+					/* Set the smt_leader to the first CPU */
+					if (!leader)
+						leader = rq;
+					if (!other_rq->smt_leader)
+						other_rq->smt_leader = leader;
+				}
+				if (rq->cpu_locality[other_cpu] > LOCALITY_SMT)
+					rq->cpu_locality[other_cpu] = LOCALITY_SMT;
+			}
+			rq->has_smt_sibling = true;
+			smt_threads = true;
+		}
+#endif
+	}
+
+#ifdef CONFIG_SMT_NICE
+	/*
+	 * Only record it here; sched_init_smp() turns SMT nice on once it has
+	 * dropped the runqueue locks and re-enabled interrupts.
+	 *
+	 * static_branch_enable() must not be called from this context. It
+	 * takes cpus_read_lock() and jump_label_mutex, either of which can
+	 * sleep - and scheduling from here would try to take a runqueue lock
+	 * this CPU already holds - and it then patches the jump site with
+	 * text_poke_bp(), which waits for every other CPU to answer a sync
+	 * IPI. Any CPU spinning on one of the runqueue locks held here has
+	 * interrupts disabled and can never answer, so the wait never ends and
+	 * the machine is gone with nothing on the console.
+	 */
+	smt_nice_needed = smt_threads;
+#endif
+
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		for_each_online_cpu(other_cpu) {
+			printk(KERN_DEBUG "MuQSS locality CPU %d to %d: %d\n", cpu, other_cpu, rq->cpu_locality[other_cpu]);
+		}
+	}
+}
+
+/*
+ * Fold @rq into @leader while all runqueue locks are already held by
+ * lock_all_rqs().  Secondary CPUs may already have tasks queued by the time
+ * sched_init_smp() runs (hotplug kthreads, RCU, early kworkers), so the
+ * skiplist must be drained onto the leader before its storage is freed.
+ *
+ * Lock handoff: rq_lock() is a plain raw_spin_lock(rq->lock) with no re-check,
+ * so a CPU that had already read the follower's lock pointer would acquire the
+ * old lock after we drop it and then walk the leader's skiplist without the
+ * leader's lock.  share_rqs() therefore runs from stop_machine(), where no
+ * other CPU can be inside or waiting on any rq lock, which is what makes
+ * repointing rq->lock and freeing the old one safe.
+ */
+static void __init share_and_free_rq(struct rq *leader, struct rq *rq)
+{
+	raw_spinlock_t *old_lock = rq->lock;
+	skiplist_node *old_node = rq->node;
+	skiplist *old_sl = rq->sl;
+
+	/*
+	 * Move every queued task onto the leader skiplist.  nr_running is
+	 * accounted against the rq the task belongs to (task_rq(p), which is
+	 * unchanged here) rather than the rq owning the skiplist, so hand the
+	 * counts straight back or the follower underflows on the next dequeue
+	 * and the leader keeps a phantom entry forever.
+	 */
+	while (rq->sl->entries > 0) {
+		struct task_struct *p = container_of(rq->node->next[0],
+						     struct task_struct, node);
+
+		dequeue_task(rq, p, DEQUEUE_SAVE);
+		enqueue_task(leader, p, ENQUEUE_RESTORE);
+		leader->nr_running--;
+		rq->nr_running++;
+		if (rt_task(p)) {
+			leader->rt_nr_running--;
+			rq->rt_nr_running++;
+		}
+	}
+
+	/*
+	 * A non-idle curr is not on the skiplist but still accounts for one
+	 * nr_running.  Leave that count on @rq; only the shared skiplist and
+	 * lock are merged.
+	 */
+	WARN_ON_ONCE(rq->sl->entries != 0);
+
+	/* Point the follower at the leader's skiplist and lock. */
+	rq->node = leader->node;
+	rq->sl = leader->sl;
+	rq->lock = leader->lock;
+	rq->is_leader = false;
+
+	/*
+	 * Drop the follower's private lock, taken by lock_all_rqs().  Only the
+	 * leader lock remains held for @rq, and nothing can be waiting on the
+	 * old one, so free it.
+	 */
+	do_raw_spin_unlock(old_lock);
+
+	kfree(old_node);
+	skiplist_free(old_sl);
+	kfree(old_lock);
+}
+
+/*
+ * Called with every runqueue lock held via lock_all_rqs() and IRQs off.
+ * Must not take locks again.  After this, only leader rqs own a unique lock;
+ * unlock_all_rqs() is replaced by unlock_leader_rqs().
+ */
+static void __init share_rqs(void)
+{
+	struct rq *rq, *leader;
+	int cpu;
+
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		leader = rq->smp_leader;
+
+		if (leader && rq != leader) {
+			printk(KERN_INFO "MuQSS sharing SMP runqueue from CPU %d to CPU %d\n",
+			       leader->cpu, rq->cpu);
+			share_and_free_rq(leader, rq);
+		}
+	}
+
+#ifdef CONFIG_SCHED_MC
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		leader = rq->mc_leader;
+
+		if (leader && rq != leader) {
+			printk(KERN_INFO "MuQSS sharing MC runqueue from CPU %d to CPU %d\n",
+			       leader->cpu, rq->cpu);
+			share_and_free_rq(leader, rq);
+		}
+	}
+#endif /* CONFIG_SCHED_MC */
+
+#ifdef CONFIG_SCHED_SMT
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		leader = rq->smt_leader;
+
+		if (leader && rq != leader) {
+			printk(KERN_INFO "MuQSS sharing SMT runqueue from CPU %d to CPU %d\n",
+			       leader->cpu, rq->cpu);
+			share_and_free_rq(leader, rq);
+		}
+	}
+#endif /* CONFIG_SCHED_SMT */
+}
+
+/* Unlock each unique runqueue lock once after share_rqs(). */
+static inline void unlock_leader_rqs(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct rq *rq = cpu_rq(cpu);
+
+		if (rq->is_leader)
+			do_raw_spin_unlock(rq->lock);
+	}
+	preempt_enable();
+}
+
+/*
+ * Fold the runqueues from stop_machine() context.  Every other CPU is parked
+ * in the stopper with interrupts disabled, so none of them is inside an rq
+ * lock or spinning on one while share_and_free_rq() repoints a follower's
+ * rq->lock at its leader and frees the old lock.  Any single CPU can run this.
+ */
+static int __init share_rqs_stopper(void *unused)
+{
+	lock_all_rqs();
+	share_rqs();
+	unlock_leader_rqs();
+
+	return 0;
+}
+
+static void __init set_rq_order(struct rq *rq, int idx, struct rq *other)
+{
+	rq->rq_order[idx] = other;
+	rq->sl_order[idx] = other->sl;
+}
+
+static void __init setup_rq_orders(void)
+{
+	int *selected_cpus, *ordered_cpus;
+	struct rq *rq, *other_rq;
+	int cpu, other_cpu, i;
+
+	selected_cpus = kmalloc(sizeof(int) * NR_CPUS, GFP_ATOMIC);
+	ordered_cpus = kmalloc(sizeof(int) * NR_CPUS, GFP_ATOMIC);
+
+	total_runqueues = 0;
+	for_each_online_cpu(cpu) {
+		int locality, total_rqs = 0, total_cpus = 0;
+
+		rq = cpu_rq(cpu);
+		if (rq->is_leader)
+			total_runqueues++;
+
+		for (locality = LOCALITY_SAME; locality <= LOCALITY_DISTANT; locality++) {
+			int selected_cpu_cnt, selected_cpu_idx, test_cpu_idx, cpu_idx, best_locality, test_cpu;
+			int ordered_cpus_idx;
+
+			ordered_cpus_idx = -1;
+			selected_cpu_cnt = 0;
+
+			for_each_online_cpu(test_cpu) {
+				if (cpu < num_online_cpus() / 2)
+					other_cpu = cpu + test_cpu;
+				else
+					other_cpu = cpu - test_cpu;
+				if (other_cpu < 0)
+					other_cpu += num_online_cpus();
+				else
+					other_cpu %= num_online_cpus();
+				/* gather CPUs of the same locality */
+				if (rq->cpu_locality[other_cpu] == locality) {
+					selected_cpus[selected_cpu_cnt] = other_cpu;
+					selected_cpu_cnt++;
+				}
+			}
+
+			/* reserve first CPU as starting point */
+			if (selected_cpu_cnt > 0) {
+				ordered_cpus_idx++;
+				ordered_cpus[ordered_cpus_idx] = selected_cpus[ordered_cpus_idx];
+				selected_cpus[ordered_cpus_idx] = -1;
+			}
+
+			/* take each CPU and sort it within the same locality based on each inter-CPU localities */
+			for (test_cpu_idx = 1; test_cpu_idx < selected_cpu_cnt; test_cpu_idx++) {
+				/* starting point with worst locality and current CPU */
+				best_locality = LOCALITY_DISTANT;
+				selected_cpu_idx = test_cpu_idx;
+
+				/* try to find the best locality within group */
+				for (cpu_idx = 1; cpu_idx < selected_cpu_cnt; cpu_idx++) {
+					/* if CPU has not been used and locality is better */
+					if (selected_cpus[cpu_idx] > -1) {
+						other_rq = cpu_rq(ordered_cpus[ordered_cpus_idx]);
+						if (best_locality > other_rq->cpu_locality[selected_cpus[cpu_idx]]) {
+							/* assign best locality and best CPU idx in array */
+							best_locality = other_rq->cpu_locality[selected_cpus[cpu_idx]];
+							selected_cpu_idx = cpu_idx;
+						}
+					}
+				}
+
+				/* add our next best CPU to ordered list */
+				ordered_cpus_idx++;
+				ordered_cpus[ordered_cpus_idx] = selected_cpus[selected_cpu_idx];
+				/* mark this CPU as used */
+				selected_cpus[selected_cpu_idx] =  -1;
+			}
+
+			/* set up RQ and CPU orders */
+			for (test_cpu = 0; test_cpu <= ordered_cpus_idx; test_cpu++) {
+				other_rq = cpu_rq(ordered_cpus[test_cpu]);
+				/* set up cpu orders */
+				rq->cpu_order[total_cpus++] = other_rq;
+				if (other_rq->is_leader) {
+					/* set up RQ orders */
+					set_rq_order(rq, total_rqs++, other_rq);
+				}
+			}
+		}
+	}
+
+	kfree(selected_cpus);
+	kfree(ordered_cpus);
+
+#ifdef CONFIG_X86
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		for (i = 0; i < total_runqueues; i++) {
+			printk(KERN_DEBUG "MuQSS CPU %d llc %d RQ order %d RQ %d llc %d\n", cpu, per_cpu_llc_id(cpu), i,
+			       rq->rq_order[i]->cpu, per_cpu_llc_id(rq->rq_order[i]->cpu));
+		}
+	}
+
+	for_each_online_cpu(cpu) {
+		rq = cpu_rq(cpu);
+		for (i = 0; i < num_online_cpus(); i++) {
+			printk(KERN_DEBUG "MuQSS CPU %d llc %d CPU order %d RQ %d llc %d\n", cpu, per_cpu_llc_id(cpu), i,
+			       rq->cpu_order[i]->cpu, per_cpu_llc_id(rq->cpu_order[i]->cpu));
+		}
+	}
+#endif
+}
+
+void __init sched_init_smp(void)
+{
+	sched_init_numa(NUMA_NO_NODE);
+
+	/*
+	 * There's no userspace yet to cause hotplug operations; hence all the
+	 * cpu masks are stable and all blatant races in the below code cannot
+	 * happen.
+	 */
+	mutex_lock(&sched_domains_mutex);
+	sched_init_domains(cpu_active_mask);
+	mutex_unlock(&sched_domains_mutex);
+
+	/* Move init over to a non-isolated CPU */
+	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0)
+		BUG();
+
+	/*
+	 * Take the sleeping lock before disabling interrupts - the 5.12 order
+	 * trips "sleeping function called from invalid context" under
+	 * CONFIG_DEBUG_ATOMIC_SLEEP.  select_leaders() walks the domain tree,
+	 * which is what the mutex is for.
+	 */
+	mutex_lock(&sched_domains_mutex);
+	local_irq_disable();
+	lock_all_rqs();
+
+	printk(KERN_INFO "MuQSS possible/present/online CPUs: %d/%d/%d\n",
+		num_possible_cpus(), num_present_cpus(), num_online_cpus());
+
+	select_leaders();
+
+	unlock_all_rqs();
+	local_irq_enable();
+	mutex_unlock(&sched_domains_mutex);
+
+#ifdef CONFIG_SMT_NICE
+	/*
+	 * Safe to patch the jump site only now that the runqueue locks are
+	 * dropped and interrupts are back on; see select_leaders(). Until
+	 * this point smt_schedule() just returns true and check_siblings()/
+	 * wake_siblings() are the no-op variants, which is correct behaviour,
+	 * merely without SMT nice.
+	 */
+	if (smt_nice_needed) {
+		check_siblings = &check_smt_siblings;
+		wake_siblings = &wake_smt_siblings;
+		static_branch_enable(&smt_nice_enabled);
+	}
+#endif
+
+	/*
+	 * Only now fold the runqueues together, and do it from stop_machine():
+	 * 7.1 has the secondary CPUs running by this point, so a follower's rq
+	 * lock can be held - or waited on - by another CPU exactly while it is
+	 * handed over to the leader and freed.  Quiescing everybody closes that
+	 * window without putting a re-check in the rq_lock() fast path.  Note
+	 * stop_machine() takes cpus_read_lock(), so it must not nest inside
+	 * sched_domains_mutex.
+	 */
+	stop_machine(share_rqs_stopper, NULL, cpumask_of(raw_smp_processor_id()));
+
+	setup_rq_orders();
+
+	switch (rqshare) {
+		case RQSHARE_ALL:
+			/* This should only ever read 1 */
+			printk(KERN_INFO "MuQSS runqueue share type ALL total runqueues: %d\n",
+			       total_runqueues);
+			break;
+		case RQSHARE_SMP:
+			printk(KERN_INFO "MuQSS runqueue share type SMP total runqueues: %d\n",
+			       total_runqueues);
+			break;
+		case RQSHARE_MC:
+			printk(KERN_INFO "MuQSS runqueue share type MC total runqueues: %d\n",
+			       total_runqueues);
+			break;
+		case RQSHARE_MC_LLC:
+			printk(KERN_INFO "MuQSS runqueue share type LLC total runqueues: %d\n",
+			       total_runqueues);
+			break;
+		case RQSHARE_SMT:
+			printk(KERN_INFO "MuQSS runqueue share type SMT total runqueues: %d\n",
+			       total_runqueues);
+			break;
+		case RQSHARE_NONE:
+			printk(KERN_INFO "MuQSS runqueue share type NONE total runqueues: %d\n",
+			       total_runqueues);
+			break;
+	}
+
+	sched_smp_initialized = true;
+}
+#else /* !CONFIG_SMP */
+void __init sched_init_smp(void)
+{
+	sched_smp_initialized = true;
+}
+
+/*
+ * 7.1 dropped the !CONFIG_SMP inline stubs these used to have in
+ * <linux/sched.h>, so the scheduler has to provide them on UP too. There is
+ * only ever cpu 0 to run on, so affinity is either trivially satisfied or
+ * impossible.
+ */
+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
+{
+	if (!cpumask_test_cpu(0, new_mask))
+		return -EINVAL;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
+
+void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask)
+{
+	cpumask_copy(&p->cpus_mask, new_mask);
+	p->nr_cpus_allowed = cpumask_weight(new_mask);
+}
+
+#ifdef CONFIG_NO_HZ_COMMON
+void nohz_balance_enter_idle(int cpu) {}
+
+/*
+ * The only CPU is the one already running this, so it is by definition not
+ * idle and needs no kick.
+ */
+void wake_up_nohz_cpu(int cpu)
+{
+}
+#endif /* CONFIG_NO_HZ_COMMON */
+
+/*
+ * topology.c is not built on UP, but <linux/sched/topology.h> declares this
+ * unconditionally and amd-pstate calls it. There are no sched domains to
+ * update.
+ */
+void sched_update_asym_prefer_cpu(int cpu, int old_prio, int new_prio)
+{
+}
+#endif /* CONFIG_SMP */
+
+int in_sched_functions(unsigned long addr)
+{
+	return in_lock_functions(addr) ||
+		(addr >= (unsigned long)__sched_text_start
+		&& addr < (unsigned long)__sched_text_end);
+}
+
+#ifdef CONFIG_CGROUP_SCHED
+/*
+ * Default CFS bandwidth period (100ms).  Used only for accept-and-ignore
+ * readback of cpu.max / cfs_period_us — MuQSS does not enforce quotas.
+ */
+#define MUQSS_CGROUP_PERIOD_DFL_US	100000ULL
+
+/* task group related information */
+struct task_group {
+	struct cgroup_subsys_state css;
+
+	struct rcu_head rcu;
+	struct list_head list;
+
+	struct task_group *parent;
+	struct list_head siblings;
+	struct list_head children;
+
+	/*
+	 * cpu controller knobs stored for ABI/readback only.  Writes are
+	 * validated then ignored for scheduling — MuQSS has no group fairness
+	 * or bandwidth enforcement.  Keeps systemd/docker/podman happy.
+	 */
+	unsigned long weight;	/* cgroup weight [CGROUP_WEIGHT_MIN, MAX] */
+	s64 idle;
+	u64 period_us;
+	s64 quota_us;		/* -1 == unlimited ("max") */
+	u64 burst_us;
+};
+
+/*
+ * Default task group.
+ * Every task in system belongs to this group at bootup.
+ */
+struct task_group root_task_group;
+LIST_HEAD(task_groups);
+
+/* Cacheline aligned slab cache for task_group */
+static struct kmem_cache *task_group_cache __read_mostly;
+
+static void init_tg_cgroup_defaults(struct task_group *tg)
+{
+	tg->weight = CGROUP_WEIGHT_DFL;
+	tg->idle = 0;
+	tg->period_us = MUQSS_CGROUP_PERIOD_DFL_US;
+	tg->quota_us = -1;
+	tg->burst_us = 0;
+}
+#endif /* CONFIG_CGROUP_SCHED */
+
+void __init sched_init(void)
+{
+#ifdef CONFIG_SMP
+	int cpu_ids;
+#endif
+	int i;
+	struct rq *rq;
+
+	wait_bit_init();
+
+	prio_ratios[0] = 128;
+	for (i = 1 ; i < NICE_WIDTH ; i++)
+		prio_ratios[i] = prio_ratios[i - 1] * 11 / 10;
+
+	skiplist_node_init(&init_task.node);
+
+#ifdef CONFIG_SMP
+	init_defrootdomain();
+	cpumask_clear(&cpu_idle_map);
+#else
+	uprq = &per_cpu(runqueues, 0);
+#endif
+
+#ifdef CONFIG_CGROUP_SCHED
+	task_group_cache = KMEM_CACHE(task_group, 0);
+
+	list_add(&root_task_group.list, &task_groups);
+	INIT_LIST_HEAD(&root_task_group.children);
+	INIT_LIST_HEAD(&root_task_group.siblings);
+	init_tg_cgroup_defaults(&root_task_group);
+#endif /* CONFIG_CGROUP_SCHED */
+	skiplist_cache_init();
+	for_each_possible_cpu(i) {
+		rq = cpu_rq(i);
+		rq->node = kmalloc(sizeof(skiplist_node), GFP_ATOMIC);
+		skiplist_init(rq->node);
+		rq->sl = new_skiplist(rq->node);
+		rq->lock = kmalloc(sizeof(raw_spinlock_t), GFP_ATOMIC);
+		raw_spin_lock_init(rq->lock);
+		rq->nr_running = 0;
+		rq->nr_uninterruptible = 0;
+		rq->nr_switches = 0;
+		rq->clock = rq->niffies = rq->jiffy_niffies = 0;
+		rq->last_jiffy = jiffies;
+		rq->user_ns = rq->nice_ns = rq->softirq_ns = rq->system_ns =
+			      rq->iowait_ns = rq->idle_ns = 0;
+		rq->dither = 0;
+		set_rq_task(rq, &init_task);
+		rq->iso_ticks = 0;
+		rq->iso_refractory = false;
+#ifdef CONFIG_SMP
+		rq->is_leader = true;
+		rq->smp_leader = NULL;
+#ifdef CONFIG_SCHED_MC
+		rq->mc_leader = NULL;
+#endif
+#ifdef CONFIG_SCHED_SMT
+		rq->smt_leader = NULL;
+#endif
+		rq->sd = NULL;
+		rq->rd = NULL;
+		rq->online = false;
+		rq->cpu = i;
+		rq_attach_root(rq, &def_root_domain);
+#endif /* CONFIG_SMP */
+		init_rq_hrexpiry(rq);
+		atomic_set(&rq->nr_iowait, 0);
+	}
+
+#ifdef CONFIG_SMP
+	cpu_ids = i;
+	/*
+	 * Set the base locality for cpu cache distance calculation to
+	 * "distant" (3). Make sure the distance from a CPU to itself is 0.
+	 */
+	for_each_possible_cpu(i) {
+		int j;
+
+		rq = cpu_rq(i);
+#ifdef CONFIG_SCHED_MC
+		rq->cache_idle = sole_cpu_idle;
+#endif
+		rq->cpu_locality = kmalloc(cpu_ids * sizeof(int *), GFP_ATOMIC);
+		for_each_possible_cpu(j) {
+			if (i == j)
+				rq->cpu_locality[j] = LOCALITY_SAME;
+			else
+				rq->cpu_locality[j] = LOCALITY_DISTANT;
+		}
+		/* sl_order is O(possible_cpus²) pointers, same as rq_order. */
+		rq->rq_order = kmalloc(cpu_ids * sizeof(struct rq *), GFP_ATOMIC);
+		rq->sl_order = kmalloc(cpu_ids * sizeof(skiplist *), GFP_ATOMIC);
+		rq->cpu_order = kmalloc(cpu_ids * sizeof(struct rq *), GFP_ATOMIC);
+		set_rq_order(rq, 0, rq);
+		rq->cpu_order[0] = rq;
+		for (j = 1; j < cpu_ids; j++) {
+			set_rq_order(rq, j, cpu_rq(j));
+			rq->cpu_order[j] = cpu_rq(j);
+		}
+	}
+#endif
+
+	/*
+	 * The boot idle thread does lazy MMU switching as well:
+	 */
+	mmgrab_lazy_tlb(&init_mm);
+	enter_lazy_tlb(&init_mm, current);
+
+	/*
+	 * Make us the idle thread. Technically, schedule() should not be
+	 * called from this thread, however somewhere below it might be,
+	 * but because we are the idle thread, we just pick up running again
+	 * when this runqueue becomes "idle".
+	 */
+	init_idle(current, smp_processor_id());
+
+#ifdef CONFIG_SMP
+	idle_thread_set_boot_cpu();
+#endif /* SMP */
+
+	init_schedstats();
+
+	psi_init();
+
+	preempt_dynamic_init();
+
+	print_scheduler_version();
+}
+
+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+static inline int preempt_count_equals(int preempt_offset)
+{
+	int nested = preempt_count() + rcu_preempt_depth();
+
+	return (nested == preempt_offset);
+}
+
+void __might_sleep(const char *file, int line)
+{
+	unsigned int state = get_current_state();
+	/*
+	 * Blocking primitives will set (and therefore destroy) current->__state,
+	 * since we will exit with TASK_RUNNING make sure we enter with it,
+	 * otherwise we will destroy state.
+	 */
+	WARN_ONCE(state != TASK_RUNNING && current->task_state_change,
+			"do not call blocking ops when !TASK_RUNNING; "
+			"state=%x set at [<%p>] %pS\n", state,
+			(void *)current->task_state_change,
+			(void *)current->task_state_change);
+
+	__might_resched(file, line, 0);
+}
+EXPORT_SYMBOL(__might_sleep);
+
+void __cant_migrate(const char *file, int line)
+{
+	static unsigned long prev_jiffy;
+
+	if (irqs_disabled())
+		return;
+
+	if (is_migration_disabled(current))
+		return;
+
+	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
+		return;
+
+	if (preempt_count() > 0)
+		return;
+
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	pr_err("BUG: assuming non migratable context at %s:%d\n", file, line);
+	pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n",
+	       in_atomic(), irqs_disabled(), is_migration_disabled(current),
+	       current->pid, current->comm);
+
+	debug_show_held_locks(current);
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL_GPL(__cant_migrate);
+
+void __cant_sleep(const char *file, int line, int preempt_offset)
+{
+	static unsigned long prev_jiffy;
+
+	if (irqs_disabled())
+		return;
+
+	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
+		return;
+
+	if (preempt_count() > preempt_offset)
+		return;
+
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
+	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
+			in_atomic(), irqs_disabled(),
+			current->pid, current->comm);
+
+	debug_show_held_locks(current);
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL_GPL(__cant_sleep);
+
+void __might_resched(const char *file, int line, unsigned int offsets)
+{
+	/* Ratelimiting timestamp: */
+	static unsigned long prev_jiffy;
+
+	unsigned long preempt_disable_ip;
+
+	/* WARN_ON_ONCE() by default, no rate limit required: */
+	rcu_sleep_check();
+
+	if ((preempt_count_equals(offsets) && !irqs_disabled() &&
+	     !is_idle_task(current) && !current->non_block_count) ||
+	    system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
+	    oops_in_progress)
+		return;
+
+	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
+		return;
+	prev_jiffy = jiffies;
+
+	/* Save this before calling printk(), since that will clobber it: */
+	preempt_disable_ip = get_preempt_disable_ip(current);
+
+	printk(KERN_ERR
+		"BUG: sleeping function called from invalid context at %s:%d\n",
+			file, line);
+	printk(KERN_ERR
+		"in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
+			in_atomic(), irqs_disabled(), current->non_block_count,
+			current->pid, current->comm);
+
+	if (task_stack_end_corrupted(current))
+		printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
+
+	debug_show_held_locks(current);
+	if (irqs_disabled())
+		print_irqtrace_events(current);
+	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
+	    && !preempt_count_equals(offsets)) {
+		pr_err("Preemption disabled at:");
+		print_ip_sym(KERN_ERR, preempt_disable_ip);
+	}
+	dump_stack();
+	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
+}
+EXPORT_SYMBOL(__might_resched);
+#endif
+
+#ifdef CONFIG_MAGIC_SYSRQ
+static inline void normalise_rt_tasks(void)
+{
+	struct sched_attr attr = {};
+	struct task_struct *g, *p;
+	struct rq_flags rf;
+	struct rq *rq;
+
+	read_lock(&tasklist_lock);
+	for_each_process_thread(g, p) {
+		/*
+		 * Only normalize user tasks:
+		 */
+		if (p->flags & PF_KTHREAD)
+			continue;
+
+		if (!rt_task(p) && !iso_task(p))
+			continue;
+
+		rq = task_rq_lock(p, &rf);
+		__setscheduler(p, rq, SCHED_NORMAL, 0, &attr, false);
+		task_rq_unlock(rq, p, &rf);
+	}
+	read_unlock(&tasklist_lock);
+}
+
+void normalize_rt_tasks(void)
+{
+	normalise_rt_tasks();
+}
+#endif /* CONFIG_MAGIC_SYSRQ */
+
+#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB)
+/*
+ * These functions are only useful for the IA64 MCA handling, or kdb.
+ *
+ * They can only be called when the whole system has been
+ * stopped - every CPU needs to be quiescent, and no scheduling
+ * activity can take place. Using them for anything else would
+ * be a serious bug, and as a result, they aren't even visible
+ * under any other configuration.
+ */
+
+/**
+ * curr_task - return the current task for a given CPU.
+ * @cpu: the processor in question.
+ *
+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
+ *
+ * Return: The current task for @cpu.
+ */
+struct task_struct *curr_task(int cpu)
+{
+	return cpu_curr(cpu);
+}
+
+#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
+
+#ifdef CONFIG_IA64
+/**
+ * ia64_set_curr_task - set the current task for a given CPU.
+ * @cpu: the processor in question.
+ * @p: the task pointer to set.
+ *
+ * Description: This function must only be used when non-maskable interrupts
+ * are serviced on a separate stack.  It allows the architecture to switch the
+ * notion of the current task on a CPU in a non-blocking manner.  This function
+ * must be called with all CPU's synchronised, and interrupts disabled, the
+ * and caller must save the original value of the current task (see
+ * curr_task() above) and restore that value before reenabling interrupts and
+ * re-starting the system.
+ *
+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
+ */
+void ia64_set_curr_task(int cpu, struct task_struct *p)
+{
+	cpu_curr(cpu) = p;
+}
+
+#endif
+
+#ifdef CONFIG_SCHED_DEBUG
+__read_mostly bool sched_debug_enabled;
+
+void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+			  struct seq_file *m)
+{
+	seq_printf(m, "%s (%d, #threads: %d)\n", p->comm, task_pid_nr_ns(p, ns),
+		   get_nr_threads(p));
+}
+
+void proc_sched_set_task(struct task_struct *p)
+{}
+#endif
+
+#ifdef CONFIG_CGROUP_SCHED
+static void sched_free_group(struct task_group *tg)
+{
+	kmem_cache_free(task_group_cache, tg);
+}
+
+/* allocate runqueue etc for a new task group */
+struct task_group *sched_create_group(struct task_group *parent)
+{
+	struct task_group *tg;
+
+	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
+	if (!tg)
+		return ERR_PTR(-ENOMEM);
+
+	init_tg_cgroup_defaults(tg);
+	return tg;
+}
+
+void sched_online_group(struct task_group *tg, struct task_group *parent)
+{
+}
+
+/* rcu callback to free various structures associated with a task group */
+static void sched_free_group_rcu(struct rcu_head *rhp)
+{
+	/* Now it should be safe to free those cfs_rqs */
+	sched_free_group(container_of(rhp, struct task_group, rcu));
+}
+
+void sched_destroy_group(struct task_group *tg)
+{
+	/* Wait for possible concurrent references to cfs_rqs complete */
+	call_rcu(&tg->rcu, sched_free_group_rcu);
+}
+
+void sched_release_group(struct task_group *tg)
+{
+}
+
+static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+{
+	return css ? container_of(css, struct task_group, css) : NULL;
+}
+
+static struct cgroup_subsys_state *
+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
+{
+	struct task_group *parent = css_tg(parent_css);
+	struct task_group *tg;
+
+	if (!parent) {
+		/* This is early initialization for the top cgroup */
+		return &root_task_group.css;
+	}
+
+	tg = sched_create_group(parent);
+	if (IS_ERR(tg))
+		return ERR_PTR(-ENOMEM);
+	return &tg->css;
+}
+
+/* Expose task group only after completing cgroup initialization */
+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+	struct task_group *parent = css_tg(css->parent);
+
+	if (parent)
+		sched_online_group(tg, parent);
+	return 0;
+}
+
+static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+
+	sched_release_group(tg);
+}
+
+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
+{
+	struct task_group *tg = css_tg(css);
+
+	/*
+	 * Relies on the RCU grace period between css_released() and this.
+	 */
+	sched_free_group(tg);
+}
+
+static void cpu_cgroup_fork(struct task_struct *task)
+{
+}
+
+static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
+{
+	return 0;
+}
+
+static void cpu_cgroup_attach(struct cgroup_taskset *tset)
+{
+}
+
+/*
+ * Accept-and-ignore cpu controller files.
+ *
+ * Mainline wires these to CFS shares/bandwidth.  MuQSS has neither, but
+ * container runtimes and systemd write CPUWeight=/CPUQuota=/--cpus and
+ * fail if the files are missing.  Validate ranges, store for readback,
+ * and leave scheduling unaffected.
+ *
+ * nice↔weight table matches CFS (sched_prio_to_weight) so weight.nice
+ * round-trips to the same values userspace expects.
+ */
+static const int muqss_prio_to_weight[40] = {
+ /* -20 */     88761,     71755,     56483,     46273,     36291,
+ /* -15 */     29154,     23254,     18705,     14949,     11916,
+ /* -10 */      9548,      7620,      6100,      4904,      3906,
+ /*  -5 */      3121,      2501,      1991,      1586,      1277,
+ /*   0 */      1024,       820,       655,       526,       423,
+ /*   5 */       335,       272,       215,       172,       137,
+ /*  10 */       110,        87,        70,        56,        45,
+ /*  15 */        36,        29,        23,        18,        15,
+};
+
+static unsigned long muqss_weight_from_cgroup(unsigned long cgrp_weight)
+{
+	return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL);
+}
+
+static unsigned long muqss_weight_to_cgroup(unsigned long weight)
+{
+	return clamp_t(unsigned long,
+		       DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024),
+		       CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX);
+}
+
+static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return css_tg(css)->weight;
+}
+
+static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft, u64 cgrp_weight)
+{
+	if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX)
+		return -ERANGE;
+
+	css_tg(css)->weight = cgrp_weight;
+	return 0;
+}
+
+static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
+				    struct cftype *cft)
+{
+	unsigned long weight = muqss_weight_from_cgroup(css_tg(css)->weight);
+	int last_delta = INT_MAX;
+	int prio, delta;
+
+	for (prio = 0; prio < ARRAY_SIZE(muqss_prio_to_weight); prio++) {
+		delta = abs(muqss_prio_to_weight[prio] - (int)weight);
+		if (delta >= last_delta)
+			break;
+		last_delta = delta;
+	}
+
+	return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO);
+}
+
+static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
+				     struct cftype *cft, s64 nice)
+{
+	int idx;
+
+	if (nice < MIN_NICE || nice > MAX_NICE)
+		return -ERANGE;
+
+	idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO;
+	idx = array_index_nospec(idx, ARRAY_SIZE(muqss_prio_to_weight));
+	css_tg(css)->weight =
+		muqss_weight_to_cgroup(muqss_prio_to_weight[idx]);
+	return 0;
+}
+
+static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css,
+			     struct cftype *cft)
+{
+	return css_tg(css)->idle;
+}
+
+static int cpu_idle_write_s64(struct cgroup_subsys_state *css,
+			      struct cftype *cft, s64 idle)
+{
+	if (idle != 0 && idle != 1)
+		return -EINVAL;
+
+	css_tg(css)->idle = idle;
+	return 0;
+}
+
+static void cpu_period_quota_print(struct seq_file *sf, long period, long quota)
+{
+	if (quota < 0)
+		seq_puts(sf, "max");
+	else
+		seq_printf(sf, "%ld", quota);
+
+	seq_printf(sf, " %ld\n", period);
+}
+
+static int cpu_period_quota_parse(char *buf, u64 *period_us_p, u64 *quota_us_p)
+{
+	char tok[21];	/* U64_MAX */
+
+	if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1)
+		return -EINVAL;
+
+	if (sscanf(tok, "%llu", quota_us_p) < 1) {
+		if (!strcmp(tok, "max"))
+			*quota_us_p = U64_MAX;
+		else
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int cpu_max_show(struct seq_file *sf, void *v)
+{
+	struct task_group *tg = css_tg(seq_css(sf));
+	s64 quota = tg->quota_us;
+
+	cpu_period_quota_print(sf, tg->period_us, quota);
+	return 0;
+}
+
+static ssize_t cpu_max_write(struct kernfs_open_file *of,
+			     char *buf, size_t nbytes, loff_t off)
+{
+	struct task_group *tg = css_tg(of_css(of));
+	u64 period_us = tg->period_us, quota_us;
+	int ret;
+
+	ret = cpu_period_quota_parse(buf, &period_us, &quota_us);
+	if (ret)
+		return ret;
+
+	if (!period_us || period_us > USEC_PER_SEC)
+		return -EINVAL;
+
+	tg->period_us = period_us;
+	tg->quota_us = (quota_us == U64_MAX) ? -1 : (s64)quota_us;
+	return nbytes;
+}
+
+static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css,
+			      struct cftype *cft)
+{
+	return css_tg(css)->burst_us;
+}
+
+static int cpu_burst_write_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft, u64 burst_us)
+{
+	struct task_group *tg = css_tg(css);
+
+	/* Burst must not exceed a finite quota when one is set. */
+	if (tg->quota_us >= 0 && burst_us > (u64)tg->quota_us)
+		return -EINVAL;
+
+	tg->burst_us = burst_us;
+	return 0;
+}
+
+/* Legacy v1 interfaces */
+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return muqss_weight_from_cgroup(css_tg(css)->weight);
+}
+
+static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft, u64 share)
+{
+	if (share < 2 || share > 262144)
+		return -ERANGE;
+
+	css_tg(css)->weight = muqss_weight_to_cgroup(share);
+	return 0;
+}
+
+static u64 cpu_period_read_u64(struct cgroup_subsys_state *css,
+			       struct cftype *cft)
+{
+	return css_tg(css)->period_us;
+}
+
+static int cpu_period_write_u64(struct cgroup_subsys_state *css,
+				struct cftype *cft, u64 period_us)
+{
+	if (!period_us || period_us > USEC_PER_SEC)
+		return -EINVAL;
+
+	css_tg(css)->period_us = period_us;
+	return 0;
+}
+
+static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css,
+			      struct cftype *cft)
+{
+	return css_tg(css)->quota_us;
+}
+
+static int cpu_quota_write_s64(struct cgroup_subsys_state *css,
+			       struct cftype *cft, s64 quota_us)
+{
+	if (quota_us < -1 || quota_us > (s64)USEC_PER_SEC * 1024)
+		return -EINVAL;
+
+	css_tg(css)->quota_us = quota_us;
+	return 0;
+}
+
+static struct cftype cpu_legacy_files[] = {
+	{
+		.name = "shares",
+		.read_u64 = cpu_shares_read_u64,
+		.write_u64 = cpu_shares_write_u64,
+	},
+	{
+		.name = "idle",
+		.read_s64 = cpu_idle_read_s64,
+		.write_s64 = cpu_idle_write_s64,
+	},
+	{
+		.name = "cfs_period_us",
+		.read_u64 = cpu_period_read_u64,
+		.write_u64 = cpu_period_write_u64,
+	},
+	{
+		.name = "cfs_quota_us",
+		.read_s64 = cpu_quota_read_s64,
+		.write_s64 = cpu_quota_write_s64,
+	},
+	{
+		.name = "cfs_burst_us",
+		.read_u64 = cpu_burst_read_u64,
+		.write_u64 = cpu_burst_write_u64,
+	},
+	{ }	/* Terminate */
+};
+
+static struct cftype cpu_files[] = {
+	{
+		.name = "weight",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_weight_read_u64,
+		.write_u64 = cpu_weight_write_u64,
+	},
+	{
+		.name = "weight.nice",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_s64 = cpu_weight_nice_read_s64,
+		.write_s64 = cpu_weight_nice_write_s64,
+	},
+	{
+		.name = "idle",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_s64 = cpu_idle_read_s64,
+		.write_s64 = cpu_idle_write_s64,
+	},
+	{
+		.name = "max",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.seq_show = cpu_max_show,
+		.write = cpu_max_write,
+	},
+	{
+		.name = "max.burst",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = cpu_burst_read_u64,
+		.write_u64 = cpu_burst_write_u64,
+	},
+	{ }	/* terminate */
+};
+
+static int cpu_extra_stat_show(struct seq_file *sf,
+			       struct cgroup_subsys_state *css)
+{
+	return 0;
+}
+
+struct cgroup_subsys cpu_cgrp_subsys = {
+	.css_alloc	= cpu_cgroup_css_alloc,
+	.css_online	= cpu_cgroup_css_online,
+	.css_released	= cpu_cgroup_css_released,
+	.css_free	= cpu_cgroup_css_free,
+	.css_extra_stat_show = cpu_extra_stat_show,
+	.fork		= cpu_cgroup_fork,
+	.can_attach	= cpu_cgroup_can_attach,
+	.attach		= cpu_cgroup_attach,
+	.legacy_cftypes	= cpu_legacy_files,
+	.dfl_cftypes	= cpu_files,
+	.early_init	= true,
+	.threaded	= true,
+};
+#endif	/* CONFIG_CGROUP_SCHED */
+
+void call_trace_sched_update_nr_running(struct rq *rq, int count)
+{
+        trace_sched_update_nr_running_tp(rq, count);
+}
+
+/* CFS Compat */
+#ifdef CONFIG_RCU_TORTURE_TEST
+int sysctl_sched_rt_runtime;
+#endif
+
+/*
+ * Compatibility layer for core.c interfaces added after 5.12.
+ *
+ * These are all consumed by code outside the scheduler. Where the feature
+ * behind them does not exist under MuQSS (deadline bandwidth, sched_ext,
+ * mm_cid, the CFS runqueue debugfs) the implementation is deliberately inert
+ * rather than absent, so mainline callers need no #ifdef.
+ */
+
+/*
+ * Tracepoint helpers behind set_current_state()/set_need_resched(). Callers
+ * MUST guard these with a tracepoint_enabled() check, which is why they use
+ * the unguarded trace_call__<tp>() form.
+ */
+void __trace_set_current_state(int state_value)
+{
+	trace_call__sched_set_state_tp(current, state_value);
+}
+EXPORT_SYMBOL(__trace_set_current_state);
+
+void __trace_set_need_resched(struct task_struct *curr, int tif)
+{
+	trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif);
+}
+EXPORT_SYMBOL_GPL(__trace_set_need_resched);
+
+unsigned long long nr_context_switches_cpu(int cpu)
+{
+	return cpu_rq(cpu)->nr_switches;
+}
+
+/*
+ * External (non-sched/) callers cannot see task_on_rq_queued(); wrap it.
+ * Used by tick-sched nohz full path among others.
+ */
+bool sched_task_on_rq(struct task_struct *p)
+{
+	return task_on_rq_queued(p);
+}
+
+unsigned long get_wchan(struct task_struct *p)
+{
+	unsigned long ip = 0;
+	unsigned int state;
+
+	if (!p || p == current)
+		return 0;
+
+	/* Only get wchan if task is blocked and we can keep it that way. */
+	raw_spin_lock_irq(&p->pi_lock);
+	state = READ_ONCE(p->__state);
+	smp_rmb(); /* see try_to_wake_up() */
+	if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq)
+		ip = __get_wchan(p);
+	raw_spin_unlock_irq(&p->pi_lock);
+
+	return ip;
+}
+
+/*
+ * Fork path. MuQSS does no cgroup bandwidth accounting and has no sched_ext
+ * to cancel, so these only need to exist.
+ */
+int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
+{
+	return 0;
+}
+
+void sched_cancel_fork(struct task_struct *p)
+{
+}
+
+/*
+ * rt_mutex helpers. MuQSS has no proxy execution; pre/post still run the
+ * worker submit/update pair so blocking on an rt_mutex flushes plugged IO.
+ * rt_mutex_schedule() must not re-enter schedule() or submit_work runs twice.
+ */
+#ifdef CONFIG_RT_MUTEXES
+#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
+#endif
+
+void rt_mutex_pre_schedule(void)
+{
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
+#endif
+	sched_submit_work(current);
+}
+
+/* Futex callers cannot have worker state or plugged IO to submit. */
+void rt_mutex_futex_pre_schedule(void)
+{
+	lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER)));
+	lockdep_assert(!current->plug);
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
+#endif
+}
+
+void rt_mutex_schedule(void)
+{
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(current->sched_rt_mutex);
+#endif
+	__schedule_loop(SM_NONE);
+}
+
+void rt_mutex_post_schedule(void)
+{
+	sched_update_worker(current);
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
+#endif
+}
+
+void rt_mutex_futex_post_schedule(void)
+{
+#ifdef CONFIG_RT_MUTEXES
+	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
+#endif
+}
+
+int dl_task_check_affinity(struct task_struct *p __always_unused,
+			   const struct cpumask *mask __always_unused)
+{
+	/* MuQSS has no deadline bandwidth admission. */
+	return 0;
+}
+
+#ifdef CONFIG_SMP
+/* Callers must hold p->pi_lock across the read and its use. */
+static const struct cpumask *task_user_cpus(struct task_struct *p)
+{
+	lockdep_assert_held(&p->pi_lock);
+
+	if (!p->user_cpus_ptr)
+		return cpu_possible_mask;
+	return p->user_cpus_ptr;
+}
+
+/*
+ * Copy out the intersection of the task's user-requested mask and @mask.
+ * pi_lock keeps user_cpus_ptr alive for the copy: everything that frees it
+ * clears the pointer under task_rq_lock(), which nests pi_lock, and only
+ * frees once that has been dropped.
+ */
+static bool user_cpus_and(struct task_struct *p, struct cpumask *dst,
+			  const struct cpumask *mask)
+{
+	unsigned long flags;
+	bool ret;
+
+	raw_spin_lock_irqsave(&p->pi_lock, flags);
+	ret = cpumask_and(dst, task_user_cpus(p), mask);
+	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
+
+	return ret;
+}
+
+/*
+ * Intersect the task's user-requested mask with @subset_mask and apply it.
+ * Empty intersection leaves affinity unchanged.
+ */
+static int restrict_cpus_allowed_ptr(struct task_struct *p,
+				     struct cpumask *new_mask,
+				     const struct cpumask *subset_mask)
+{
+	if (!user_cpus_and(p, new_mask, subset_mask))
+		return -EINVAL;
+
+	return __set_cpus_allowed_ptr(p, new_mask, 0);
+}
+#endif
+
+/*
+ * Restrict @p to CPUs it can actually run on (task_cpu_possible_mask()).
+ * ARM64 32-bit execve uses this when 64-bit-only CPUs exist.
+ */
+void force_compatible_cpus_allowed_ptr(struct task_struct *p)
+{
+	const struct cpumask *override_mask = task_cpu_possible_mask(p);
+#ifdef CONFIG_SMP
+	cpumask_var_t new_mask;
+
+	alloc_cpumask_var(&new_mask, GFP_KERNEL);
+
+	/*
+	 * __migrate_task() can fail silently if the dest CPU is offlined
+	 * concurrently, so hold the hotplug lock across the move.
+	 */
+	cpus_read_lock();
+	if (!cpumask_available(new_mask))
+		goto out_set_mask;
+
+	if (!restrict_cpus_allowed_ptr(p, new_mask, override_mask))
+		goto out_free_mask;
+
+	cpuset_cpus_allowed(p, new_mask);
+	override_mask = new_mask;
+
+out_set_mask:
+	if (printk_ratelimit()) {
+		printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n",
+				task_pid_nr(p), p->comm,
+				cpumask_pr_args(override_mask));
+	}
+
+	WARN_ON(set_cpus_allowed_ptr(p, override_mask));
+out_free_mask:
+	cpus_read_unlock();
+	free_cpumask_var(new_mask);
+#else
+	WARN_ON(set_cpus_allowed_ptr(p, override_mask));
+#endif
+}
+
+/*
+ * Restore the affinity previously restricted by
+ * force_compatible_cpus_allowed_ptr(). Caller serialises the pair.
+ */
+void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
+{
+#ifdef CONFIG_SMP
+	cpumask_var_t cpus_allowed, new_mask;
+	int ret = -ENOMEM;
+
+	if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL))
+		goto warn;
+	if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
+		goto out_free_cpus_allowed;
+
+	cpuset_cpus_allowed(p, cpus_allowed);
+	user_cpus_and(p, new_mask, cpus_allowed);
+	ret = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK);
+
+	free_cpumask_var(new_mask);
+out_free_cpus_allowed:
+	free_cpumask_var(cpus_allowed);
+warn:
+	WARN_ON_ONCE(ret);
+#else
+	WARN_ON(set_cpus_allowed_ptr(p, cpu_possible_mask));
+#endif
+}
+
+const char *preempt_model_str(void)
+{
+	if (IS_ENABLED(CONFIG_PREEMPT_RT))
+		return "PREEMPT_RT";
+	if (preempt_model_full())
+		return "PREEMPT";
+	if (preempt_model_voluntary())
+		return "VOLUNTARY";
+	return "NONE";
+}
+
+#ifdef CONFIG_SMP
+bool cpus_equal_capacity(int this_cpu, int that_cpu)
+{
+	if (!sched_asym_cpucap_active())
+		return true;
+
+	if (this_cpu == that_cpu)
+		return true;
+
+	return arch_scale_cpu_capacity(this_cpu) == arch_scale_cpu_capacity(that_cpu);
+}
+
+void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask)
+{
+	struct affinity_context ac = {
+		.new_mask  = new_mask,
+		.user_mask = NULL,
+		.flags     = SCA_USER,
+	};
+
+	do_set_cpus_allowed(p, &ac);
+}
+
+void ___migrate_enable(void)
+{
+	__set_cpus_allowed_ptr(current, &current->cpus_mask,
+			       SCA_MIGRATE_ENABLE);
+}
+#else /* !CONFIG_SMP */
+/*
+ * Nothing ever repoints cpus_ptr away from cpus_mask on UP, so
+ * __migrate_enable() never reaches here - it just has to link.
+ */
+void ___migrate_enable(void)
+{
+}
+#endif /* CONFIG_SMP */
+EXPORT_SYMBOL_GPL(___migrate_enable);
+
+/*
+ * user_cpus_ptr records the affinity a task asked for so that it can be
+ * restored after a temporary restriction: sched_setaffinity() stores it, and
+ * bind_zero() stores the mask it displaces from a task that would otherwise
+ * have been stranded by a CPU going down. Either way the child inherits both
+ * the restriction (zerobound, copied by dup_task_struct()) and the mask that
+ * has to be put back for it.
+ */
+int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
+		      int node)
+{
+	cpumask_t *user_mask;
+	unsigned long flags;
+
+	dst->user_cpus_ptr = NULL;
+
+	/*
+	 * Racing here is harmless: losing it just means the child forgoes a
+	 * restore it would have got for free, and taking the pi_lock on every
+	 * fork to close it is not worth that.
+	 */
+	if (data_race(!src->user_cpus_ptr))
+		return 0;
+
+	user_mask = kmalloc_node(cpumask_size(), GFP_KERNEL, node);
+	if (!user_mask)
+		return -ENOMEM;
+
+	raw_spin_lock_irqsave(&src->pi_lock, flags);
+	if (src->user_cpus_ptr) {
+		swap(dst->user_cpus_ptr, user_mask);
+		cpumask_copy(dst->user_cpus_ptr, src->user_cpus_ptr);
+	}
+	raw_spin_unlock_irqrestore(&src->pi_lock, flags);
+
+	kfree(user_mask);
+
+	return 0;
+}
+
+void release_user_cpus_ptr(struct task_struct *p)
+{
+	kfree(p->user_cpus_ptr);
+	p->user_cpus_ptr = NULL;
+}
+
+/*
+ * Modules cannot see the runqueues layout, so export out-of-line wrappers
+ * (INSTANTIATE_EXPORTED_MIGRATE_DISABLE makes the inlines become externs).
+ */
+void migrate_disable(void)
+{
+	__migrate_disable();
+}
+EXPORT_SYMBOL_GPL(migrate_disable);
+
+void migrate_enable(void)
+{
+	__migrate_enable();
+}
+EXPORT_SYMBOL_GPL(migrate_enable);
+
+/*
+ * MuQSS picks the CPU for a task at schedule() time rather than at exec, so
+ * there is nothing useful to do here.
+ */
+void sched_exec(void)
+{
+}
+
+/*
+ * Deadline bandwidth accounting. There is no deadline class, so no bandwidth
+ * is ever reserved and cpuset has nothing to move between root domains.
+ */
+u64 dl_cookie;
+
+int dl_bw_alloc(int cpu, u64 dl_bw)
+{
+	return 0;
+}
+
+void dl_bw_free(int cpu, u64 dl_bw)
+{
+}
+
+void sched_set_fifo_secondary(struct task_struct *p)
+{
+	struct sched_param sp = { .sched_priority = MAX_RT_PRIO / 2 - 1 };
+
+	WARN_ON_ONCE(sched_setscheduler_nocheck(p, SCHED_FIFO, &sp) != 0);
+}
+EXPORT_SYMBOL_GPL(sched_set_fifo_secondary);
+
+/*
+ * Scheduler debugfs. debug.c is CFS/EEVDF runqueue introspection and is not
+ * built under MuQSS; /proc/<pid>/sched reports nothing.
+ */
+bool sched_debug_verbose;
+
+#ifdef CONFIG_SMP
+/* Only topology.c calls these, and it is not built on UP. */
+void update_sched_domain_debugfs(void)
+{
+}
+
+void dirty_sched_domain_sysctl(int cpu)
+{
+}
+#endif /* CONFIG_SMP */
+
+void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+			  struct seq_file *m)
+{
+}
+
+void proc_sched_set_task(struct task_struct *p)
+{
+}
diff --git a/kernel/sched/MuQSS.h b/kernel/sched/MuQSS.h
new file mode 100644
--- /dev/null
+++ b/kernel/sched/MuQSS.h
@@ -0,0 +1,1254 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef MUQSS_SCHED_H
+#define MUQSS_SCHED_H
+
+#include <linux/sched/clock.h>
+#include <linux/sched/cpufreq.h>
+#include <linux/sched/cputime.h>
+#include <linux/sched/deadline.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/init.h>
+#include <linux/sched/isolation.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/signal.h>
+#include <linux/sched/smt.h>
+#include <linux/sched/stat.h>
+#include <linux/sched/task.h>
+#include <linux/sched/task_stack.h>
+#include <linux/sched/topology.h>
+#include <linux/sched/wake_q.h>
+
+#include <uapi/linux/sched/types.h>
+
+#include <linux/cgroup.h>
+#include <linux/cpufreq.h>
+#include <linux/cpuidle.h>
+#include <linux/cpuset.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <linux/energy_model.h>
+#include <linux/freezer.h>
+#include <linux/kernel_stat.h>
+#include <linux/kthread.h>
+#include <linux/membarrier.h>
+#include <linux/livepatch.h>
+#include <linux/proc_fs.h>
+#include <linux/psi.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/skip_list.h>
+#include <linux/smp.h>
+#include <linux/stop_machine.h>
+#include <linux/suspend.h>
+#include <linux/swait.h>
+#include <linux/syscalls.h>
+#include <linux/tick.h>
+#include <linux/tsacct_kern.h>
+#include <linux/u64_stats_sync.h>
+
+#ifdef CONFIG_PARAVIRT
+#include <asm/paravirt.h>
+#endif
+
+/*
+ * cpupri.h is deliberately not included: MuQSS has no RT class priority
+ * bitmap, root_domain keeps an opaque pointer instead, and pulling the header
+ * in would clash with the no-op cpupri_init()/cpupri_cleanup() below.
+ */
+
+#include <trace/events/sched.h>
+
+#ifdef CONFIG_SCHED_DEBUG
+# define SCHED_WARN_ON(x)	WARN_ONCE(x, #x)
+#else
+# define SCHED_WARN_ON(x)	((void)(x))
+#endif
+
+/* Wake flags. The first three directly map to some SD flag value */
+#define WF_EXEC     0x02 /* Wakeup after exec; maps to SD_BALANCE_EXEC */
+#define WF_FORK     0x04 /* Wakeup after fork; maps to SD_BALANCE_FORK */
+#define WF_TTWU     0x08 /* Wakeup;            maps to SD_BALANCE_WAKE */
+
+#define WF_SYNC     0x10 /* Waker goes to sleep after wakeup */
+#define WF_MIGRATED 0x20 /* Internal use, task got migrated */
+#define WF_CURRENT_CPU 0x40 /* Prefer to move the wakee to the current CPU. */
+#define WF_ON_CPU   0x80 /* Wakee is on_cpu */
+
+#ifdef CONFIG_SMP
+static_assert(WF_EXEC == SD_BALANCE_EXEC);
+static_assert(WF_FORK == SD_BALANCE_FORK);
+static_assert(WF_TTWU == SD_BALANCE_WAKE);
+#endif
+
+/* task_struct::on_rq states: */
+#define TASK_ON_RQ_QUEUED	1
+#define TASK_ON_RQ_MIGRATING	2
+
+extern void call_trace_sched_update_nr_running(struct rq *rq, int count);
+
+struct rq;
+
+#ifdef CONFIG_SMP
+
+static inline bool sched_asym_prefer(int a, int b)
+{
+	return arch_asym_cpu_priority(a) > arch_asym_cpu_priority(b);
+}
+
+struct perf_domain {
+	struct em_perf_domain *em_pd;
+	struct perf_domain *next;
+	struct rcu_head rcu;
+};
+
+/* Scheduling group status flags */
+#define SG_OVERLOAD		0x1 /* More than one runnable task on a CPU. */
+#define SG_OVERUTILIZED		0x2 /* One or more CPUs are over-utilized. */
+
+/*
+ * We add the notion of a root-domain which will be used to define per-domain
+ * variables. Each exclusive cpuset essentially defines an island domain by
+ * fully partitioning the member cpus from any other cpuset. Whenever a new
+ * exclusive cpuset is created, we also create and attach a new root-domain
+ * object.
+ *
+ */
+struct root_domain {
+	atomic_t refcount;
+	atomic_t rto_count;
+	struct rcu_head rcu;
+	cpumask_var_t span;
+	cpumask_var_t online;
+
+	/*
+	 * Indicate pullable load on at least one CPU, e.g:
+	 * - More than one runnable task
+	 * - Running task is misfit
+	 */
+	int			overload;
+
+	/* Indicate one or more cpus over-utilized (tipping point) */
+	int			overutilized;
+
+	/*
+	 * The bit corresponding to a CPU gets set here if such CPU has more
+	 * than one runnable -deadline task (as it is below for RT tasks).
+	 */
+	cpumask_var_t dlo_mask;
+	atomic_t dlo_count;
+
+	/* Replace unused CFS structures with void */
+	//struct dl_bw dl_bw;
+	//struct cpudl cpudl;
+	void *dl_bw;
+	void *cpudl;
+	u64 visit_cookie;
+
+	/*
+	 * The "RT overload" flag: it gets set if a CPU has more than
+	 * one runnable RT task.
+	 */
+	cpumask_var_t rto_mask;
+	//struct cpupri cpupri;
+	void *cpupri;
+
+	unsigned long max_cpu_capacity;
+
+	/*
+	 * NULL-terminated list of performance domains intersecting with the
+	 * CPUs of the rd. Protected by RCU.
+	 */
+	struct perf_domain	*pd;
+};
+
+extern void init_defrootdomain(void);
+extern int sched_init_domains(const struct cpumask *cpu_map);
+extern void rq_attach_root(struct rq *rq, struct root_domain *rd);
+
+static inline void cpupri_cleanup(void __maybe_unused *cpupri)
+{
+}
+
+static inline void cpudl_cleanup(void __maybe_unused *cpudl)
+{
+}
+
+static inline void init_dl_bw(void __maybe_unused *dl_bw)
+{
+}
+
+static inline int cpudl_init(void __maybe_unused *dl_bw)
+{
+	return 0;
+}
+
+static inline int cpupri_init(void __maybe_unused *cpupri)
+{
+	return 0;
+}
+#endif /* CONFIG_SMP */
+
+/*
+ * This is the main, per-CPU runqueue data structure.
+ * This data should only be modified by the local cpu.
+ */
+struct rq {
+	raw_spinlock_t *lock;
+	raw_spinlock_t *orig_lock;
+
+	struct task_struct __rcu	*curr;
+	struct task_struct	*idle;
+	struct task_struct	*stop;
+	struct mm_struct *prev_mm;
+
+	unsigned int nr_running;
+	/*
+	 * This is part of a global counter where only the total sum
+	 * over all CPUs matters. A task can increase this counter on
+	 * one CPU and if it got migrated afterwards it may decrease
+	 * it on another CPU. Always updated under the runqueue lock:
+	 */
+	unsigned long nr_uninterruptible;
+#ifdef CONFIG_SMP
+	unsigned int		ttwu_pending;
+#endif
+	/*
+	 * migrate_disable() nesting count. Mainline's migrate_disable() and
+	 * __migrate_enable() are inline in <linux/sched.h> and reach this via
+	 * the RQ_nr_pinned offset generated from rq-offsets.c, so the member
+	 * is required whether or not MuQSS itself consults it.
+	 */
+	unsigned int nr_pinned;
+	u64 nr_switches;
+
+	/*
+	 * Isolated curr snapshot for lockless peeks. Own cacheline so
+	 * last_jiffy / niffies (and remote synchronise_niffies()) do not
+	 * invalidate it. Under SMT_NICE this is four fields; without it,
+	 * deadline + prio only. Do not pull nr_running, online, or sl in.
+	 */
+	u64 rq_deadline ____cacheline_aligned;
+	int rq_prio;
+#ifdef CONFIG_SMT_NICE
+	struct mm_struct *rq_mm;
+	int rq_smt_bias; /* Policy/nice level bias across smt siblings */
+#endif
+	u8 __rq_snap_pad[0] ____cacheline_aligned;
+
+	unsigned long last_scheduler_tick; /* Last jiffy this RQ ticked */
+	unsigned long last_jiffy; /* Last jiffy this RQ updated rq clock */
+	u64 niffies; /* Last time this RQ updated rq clock */
+	u64 jiffy_niffies; /* Niffies as counted by the tick alone */
+
+	u64 load_update; /* When we last updated load */
+	unsigned long load_avg; /* Rolling load average */
+#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
+	u64 irq_load_update; /* When we last updated IRQ load */
+	unsigned long irq_load_avg; /* Rolling IRQ load average */
+#endif
+	/* Accurate timekeeping data */
+	unsigned long user_ns, nice_ns, irq_ns, softirq_ns, system_ns,
+		iowait_ns, idle_ns;
+	atomic_t nr_iowait;
+
+#ifdef CONFIG_MEMBARRIER
+	int membarrier_state;
+#endif
+
+	/*
+	 * MuQSS runs no deadline server. topology.c unconditionally tests
+	 * rq->fair_server.dl_server when attaching a root domain, so carry a
+	 * one-byte flag that is never set rather than #ifdef that code out.
+	 * __dl_server_attach_root() below is a no-op taking void *, so the
+	 * member's type does not have to match sched_dl_entity.
+	 */
+	struct {
+		bool dl_server;
+	} fair_server;
+
+	skiplist_node *node;
+	skiplist *sl;
+#ifdef CONFIG_SMP
+	struct task_struct *preempt; /* Preempt triggered on this task */
+	struct task_struct *preempting; /* Hint only, what task is preempting */
+
+	int cpu;		/* cpu of this runqueue */
+	bool online;
+
+	struct root_domain *rd;
+	struct sched_domain *sd;
+
+	unsigned long cpu_capacity_orig;
+
+	int *cpu_locality; /* CPU relative cache distance */
+	struct rq **rq_order; /* Shared RQs ordered by relative cache distance */
+	skiplist **sl_order; /* Leaders' skiplists, parallel to rq_order */
+	struct rq **cpu_order; /* RQs of discrete CPUs ordered by distance */
+
+	bool is_leader;
+	struct rq *smp_leader; /* First physical CPU per node */
+#ifdef CONFIG_SCHED_THERMAL_PRESSURE
+	struct sched_avg	avg_thermal;
+#endif /* CONFIG_SCHED_THERMAL_PRESSURE */
+#ifdef CONFIG_SCHED_SMT
+	struct rq *smt_leader; /* First logical CPU in SMT siblings */
+	cpumask_t thread_mask;
+	bool has_smt_sibling; /* This CPU has SMT siblings at all */
+#endif /* CONFIG_SCHED_SMT */
+#ifdef CONFIG_SCHED_MC
+	struct rq *mc_leader; /* First logical CPU in MC siblings */
+	cpumask_t core_mask;
+	bool (*cache_idle)(struct rq *rq);
+	/* See if all cache siblings are idle */
+#endif /* CONFIG_SCHED_MC */
+	/* last busy→idle, wakeup idle pick */
+	unsigned long idle_jiffy;
+#endif /* CONFIG_SMP */
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+	u64 prev_irq_time;
+	u64 psi_irq_time;
+#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
+#ifdef CONFIG_PARAVIRT
+	u64 prev_steal_time;
+#endif /* CONFIG_PARAVIRT */
+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
+	u64 prev_steal_time_rq;
+#endif /* CONFIG_PARAVIRT_TIME_ACCOUNTING */
+
+	u64 clock, last_tick;
+	/* Ensure that all clocks are in the same cache line */
+	u64 clock_task ____cacheline_aligned;
+	int dither;
+
+	int iso_ticks;
+	bool iso_refractory;
+
+#ifdef CONFIG_HIGH_RES_TIMERS
+	struct hrtimer hrexpiry_timer;
+	/*
+	 * Deferred start/cancel around __schedule() — same scheme as mainline
+	 * hrtick.  Reprogramming the oneshot clockevent from set_rq_task() on
+	 * every context switch under the rq lock starves TIMER_SOFTIRQ on SMP.
+	 */
+	call_single_data_t hrexpiry_csd;
+	unsigned int hrexpiry_sched;
+	s64 hrexpiry_delay;
+	ktime_t hrexpiry_time;
+#endif
+
+	int rt_nr_running; /* Number real time tasks running */
+#ifdef CONFIG_SCHEDSTATS
+
+	/* latency stats */
+	struct sched_info rq_sched_info;
+	unsigned long long rq_cpu_time;
+	/* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
+
+	/* sys_sched_yield() stats */
+	unsigned int yld_count;
+
+	/* schedule() stats */
+	unsigned int sched_switch;
+	unsigned int sched_count;
+	unsigned int sched_goidle;
+
+	/* try_to_wake_up() stats */
+	unsigned int ttwu_count;
+	unsigned int ttwu_local;
+#endif /* CONFIG_SCHEDSTATS */
+
+#ifdef CONFIG_CPU_IDLE
+	/* Must be inspected within a rcu lock section */
+	struct cpuidle_state *idle_state;
+#endif
+};
+
+static inline u64 __rq_clock_broken(struct rq *rq)
+{
+	return READ_ONCE(rq->clock);
+}
+
+static inline u64 rq_clock(struct rq *rq)
+{
+	lockdep_assert_held(rq->lock);
+
+	return rq->clock;
+}
+
+static inline u64 rq_clock_task(struct rq *rq)
+{
+	lockdep_assert_held(rq->lock);
+
+	return rq->clock_task;
+}
+
+/**
+ * By default the decay is the default pelt decay period.
+ * The decay shift can change the decay period in
+ * multiples of 32.
+ *  Decay shift		Decay period(ms)
+ *	0			32
+ *	1			64
+ *	2			128
+ *	3			256
+ *	4			512
+ */
+extern int sched_thermal_decay_shift;
+
+static inline u64 rq_clock_thermal(struct rq *rq)
+{
+	return rq_clock_task(rq) >> sched_thermal_decay_shift;
+}
+
+struct rq_flags {
+	unsigned long flags;
+};
+
+#ifdef CONFIG_SMP
+struct rq *cpu_rq(int cpu);
+#endif
+
+#ifndef CONFIG_SMP
+extern struct rq *uprq;
+#define cpu_rq(cpu)	(uprq)
+#define this_rq()	(uprq)
+#define raw_rq()	(uprq)
+#define task_rq(p)	(uprq)
+#define cpu_curr(cpu)	((uprq)->curr)
+#else /* CONFIG_SMP */
+DECLARE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
+#define cpu_rq(cpu)		(&per_cpu(runqueues, (cpu)))
+#define this_rq()		this_cpu_ptr(&runqueues)
+#define task_rq(p)		cpu_rq(task_cpu(p))
+#define cpu_curr(cpu)		(cpu_rq(cpu)->curr)
+#define raw_rq()		raw_cpu_ptr(&runqueues)
+#endif /* CONFIG_SMP */
+
+static inline int task_current(struct rq *rq, struct task_struct *p)
+{
+	return rq->curr == p;
+}
+
+static inline int task_running(struct rq *rq, struct task_struct *p)
+{
+#ifdef CONFIG_SMP
+	return p->on_cpu;
+#else
+	return task_current(rq, p);
+#endif
+}
+
+/*
+ * Mainline name used by stats.h / psi.c / RT helpers. Equivalent to
+ * task_running() under SMP; always uses p->on_cpu so the @rq argument is
+ * unused (kept for API compatibility with mainline call sites).
+ */
+static inline int task_on_cpu(struct rq *rq, struct task_struct *p)
+{
+#ifdef CONFIG_SMP
+	return p->on_cpu;
+#else
+	return task_current(rq, p);
+#endif
+}
+
+static inline int task_on_rq_queued(struct task_struct *p)
+{
+	return p->on_rq == TASK_ON_RQ_QUEUED;
+}
+
+static inline int task_on_rq_migrating(struct task_struct *p)
+{
+	return READ_ONCE(p->on_rq) == TASK_ON_RQ_MIGRATING;
+}
+
+static inline void lockdep_assert_rq_held(struct rq *rq)
+{
+	lockdep_assert_held(rq->lock);
+}
+
+static inline void rq_lock(struct rq *rq)
+	__acquires(rq->lock)
+{
+	raw_spin_lock(rq->lock);
+}
+
+static inline void rq_unlock(struct rq *rq)
+	__releases(rq->lock)
+{
+	raw_spin_unlock(rq->lock);
+}
+
+static inline void rq_lock_irq(struct rq *rq)
+	__acquires(rq->lock)
+{
+	raw_spin_lock_irq(rq->lock);
+}
+
+static inline void rq_unlock_irq(struct rq *rq, struct rq_flags __always_unused *rf)
+	__releases(rq->lock)
+{
+	raw_spin_unlock_irq(rq->lock);
+}
+
+static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf)
+	__acquires(rq->lock)
+{
+	raw_spin_lock_irqsave(rq->lock, rf->flags);
+}
+
+static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf)
+	__releases(rq->lock)
+{
+	raw_spin_unlock_irqrestore(rq->lock, rf->flags);
+}
+
+static inline struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
+	__acquires(p->pi_lock)
+	__acquires(rq->lock)
+{
+	struct rq *rq;
+
+	while (42) {
+		raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
+		rq = task_rq(p);
+		raw_spin_lock(rq->lock);
+		if (likely(rq == task_rq(p)))
+			break;
+		raw_spin_unlock(rq->lock);
+		raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+	}
+	return rq;
+}
+
+static inline void task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
+	__releases(rq->lock)
+	__releases(p->pi_lock)
+{
+	rq_unlock(rq);
+	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
+}
+
+static inline struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags __always_unused *rf)
+	__acquires(rq->lock)
+{
+	struct rq *rq;
+
+	lockdep_assert_held(&p->pi_lock);
+
+	while (42) {
+		rq = task_rq(p);
+		raw_spin_lock(rq->lock);
+		if (likely(rq == task_rq(p)))
+			break;
+		raw_spin_unlock(rq->lock);
+	}
+	return rq;
+}
+
+/*
+ * Match mainline's 3-arg signature (stats.h / psi paths pass @p). MuQSS does
+ * not pin the lock cookie, so @p and @rf are unused beyond API compatibility.
+ */
+static inline void __task_rq_unlock(struct rq *rq,
+				    struct task_struct __always_unused *p,
+				    struct rq_flags __always_unused *rf)
+{
+	rq_unlock(rq);
+}
+
+/*
+ * scoped_guard / guard(rq_lock_irq) used by psi_cgroup_restart() and similar.
+ * MuQSS rq_lock_irq takes only the rq (no pin cookie).
+ */
+DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq,
+		    rq_lock_irq(_T->lock),
+		    rq_unlock_irq(_T->lock, &_T->rf),
+		    struct rq_flags rf)
+
+DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq,
+		    rq_lock_irqsave(_T->lock, &_T->rf),
+		    rq_unlock_irqrestore(_T->lock, &_T->rf),
+		    struct rq_flags rf)
+
+static inline struct rq *
+this_rq_lock_irq(struct rq_flags *rf)
+	__acquires(rq->lock)
+{
+	struct rq *rq;
+
+	local_irq_disable();
+	rq = this_rq();
+	rq_lock(rq);
+	return rq;
+}
+
+/*
+ * {de,en}queue flags: Most not used on MuQSS.
+ *
+ * DEQUEUE_SLEEP  - task is no longer runnable
+ * ENQUEUE_WAKEUP - task just became runnable
+ *
+ * SAVE/RESTORE - an otherwise spurious dequeue/enqueue, done to ensure tasks
+ *                are in a known state which allows modification. Such pairs
+ *                should preserve as much state as possible.
+ *
+ * MOVE - paired with SAVE/RESTORE, explicitly does not preserve the location
+ *        in the runqueue.
+ *
+ * ENQUEUE_HEAD      - place at front of runqueue (tail if not specified)
+ * ENQUEUE_REPLENISH - CBS (replenish runtime and postpone deadline)
+ * ENQUEUE_MIGRATED  - the task was migrated during wakeup
+ *
+ */
+
+#define DEQUEUE_SLEEP		0x01
+#define DEQUEUE_SAVE		0x02 /* matches ENQUEUE_RESTORE */
+
+#define ENQUEUE_WAKEUP		0x01
+#define ENQUEUE_RESTORE		0x02
+
+#ifdef CONFIG_SMP
+#define ENQUEUE_MIGRATED	0x40
+#else
+#define ENQUEUE_MIGRATED	0x00
+#endif
+
+#ifdef CONFIG_NUMA
+enum numa_topology_type {
+	NUMA_DIRECT,
+	NUMA_GLUELESS_MESH,
+	NUMA_BACKPLANE,
+};
+extern enum numa_topology_type sched_numa_topology_type;
+extern int sched_max_numa_distance;
+extern bool find_numa_distance(int distance);
+extern void sched_init_numa(int offline_node);
+extern void sched_domains_numa_masks_set(unsigned int cpu);
+extern void sched_domains_numa_masks_clear(unsigned int cpu);
+extern int sched_numa_find_closest(const struct cpumask *cpus, int cpu);
+#else
+static inline void sched_init_numa(int offline_node) { }
+static inline void sched_domains_numa_masks_set(unsigned int cpu) { }
+static inline void sched_domains_numa_masks_clear(unsigned int cpu) { }
+static inline int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
+{
+	return nr_cpu_ids;
+}
+#endif
+
+extern struct mutex sched_domains_mutex;
+extern struct static_key_false sched_schedstats;
+
+#define rcu_dereference_sched_domain(p) \
+	rcu_dereference_all_check((p), lockdep_is_held(&sched_domains_mutex))
+
+struct affinity_context {
+	const struct cpumask	*new_mask;
+	struct cpumask		*user_mask;
+	unsigned int		flags;
+};
+
+#define MDF_PUSH		0x01
+
+static inline bool is_migration_disabled(struct task_struct *p)
+{
+	return p->migration_disabled;
+}
+
+#define SCA_CHECK		0x01
+#define SCA_MIGRATE_DISABLE	0x02
+#define SCA_MIGRATE_ENABLE	0x04
+#define SCA_USER		0x08
+
+#ifdef CONFIG_SMP
+
+/*
+ * The domain tree (rq->sd) is protected by RCU's quiescent state transition.
+ * See destroy_sched_domains: call_rcu for details.
+ *
+ * The domain tree of any CPU may only be accessed from within
+ * preempt-disabled sections.
+ */
+#define for_each_domain(cpu, __sd) \
+	for (__sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); \
+			__sd; __sd = __sd->parent)
+
+/**
+ * highest_flag_domain - Return highest sched_domain containing flag.
+ * @cpu:	The cpu whose highest level of sched domain is to
+ *		be returned.
+ * @flag:	The flag to check for the highest sched_domain
+ *		for the given cpu.
+ *
+ * Returns the highest sched_domain of a cpu which contains the given flag.
+ */
+static inline struct sched_domain *highest_flag_domain(int cpu, int flag)
+{
+	struct sched_domain *sd, *hsd = NULL;
+
+	for_each_domain(cpu, sd) {
+		if (!(sd->flags & flag))
+			break;
+		hsd = sd;
+	}
+
+	return hsd;
+}
+
+static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)
+{
+	struct sched_domain *sd;
+
+	for_each_domain(cpu, sd) {
+		if (sd->flags & flag)
+			break;
+	}
+
+	return sd;
+}
+
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);
+DECLARE_PER_CPU(int, sd_llc_size);
+DECLARE_PER_CPU(int, sd_llc_id);
+DECLARE_PER_CPU(int, sd_share_id);
+DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
+DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
+
+void sched_domains_free_llc_id(int cpu);
+
+extern struct static_key_false sched_asym_cpucapacity;
+extern struct static_key_false sched_cluster_active;
+
+static __always_inline bool sched_asym_cpucap_active(void)
+{
+	return static_branch_unlikely(&sched_asym_cpucapacity);
+}
+
+struct sched_group_capacity {
+	atomic_t		ref;
+	/*
+	 * CPU capacity of this group, SCHED_CAPACITY_SCALE being max capacity
+	 * for a single CPU.
+	 */
+	unsigned long		capacity;
+	unsigned long		min_capacity;		/* Min per-CPU capacity in group */
+	unsigned long		max_capacity;		/* Max per-CPU capacity in group */
+	unsigned long		next_update;
+	int			imbalance;		/* XXX unrelated to capacity but shared group state */
+
+	int			id;
+
+	unsigned long		cpumask[];		/* Balance mask */
+};
+
+struct sched_group {
+	struct sched_group	*next;			/* Must be a circular list */
+	atomic_t		ref;
+
+	unsigned int		group_weight;
+	unsigned int		cores;
+	struct sched_group_capacity *sgc;
+	int			asym_prefer_cpu;	/* CPU of highest priority in group */
+	int			flags;
+
+	/*
+	 * The CPUs this group covers.
+	 *
+	 * NOTE: this field is variable length. (Allocated dynamically
+	 * by attaching extra space to the end of the structure,
+	 * depending on how many CPUs the kernel has booted up with)
+	 */
+	unsigned long		cpumask[];
+};
+
+static inline struct cpumask *sched_group_span(struct sched_group *sg)
+{
+	return to_cpumask(sg->cpumask);
+}
+
+/*
+ * See build_balance_mask().
+ */
+static inline struct cpumask *group_balance_mask(struct sched_group *sg)
+{
+	return to_cpumask(sg->sgc->cpumask);
+}
+
+extern int group_balance_cpu(struct sched_group *sg);
+extern void update_sched_domain_debugfs(void);
+extern void dirty_sched_domain_sysctl(int cpu);
+extern void sched_update_numa(int cpu, bool online);
+extern void sched_get_rd(struct root_domain *rd);
+extern void sched_put_rd(struct root_domain *rd);
+extern bool sched_debug_verbose;
+
+/**
+ * group_first_cpu - Returns the first cpu in the cpumask of a sched_group.
+ * @group: The group whose first cpu is to be returned.
+ */
+static inline unsigned int group_first_cpu(struct sched_group *group)
+{
+	return cpumask_first(sched_group_span(group));
+}
+
+
+/*
+ * register_sched_domain_sysctl()/unregister_sched_domain_sysctl() were
+ * replaced upstream by update_sched_domain_debugfs(), declared above with
+ * dirty_sched_domain_sysctl(). flush_smp_call_function_from_idle() likewise
+ * became flush_smp_call_function_queue(), declared in smp.h.
+ */
+
+extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx);
+
+extern void set_rq_online (struct rq *rq);
+extern void set_rq_offline(struct rq *rq);
+extern bool sched_smp_initialized;
+
+static inline void update_group_capacity(struct sched_domain *sd, int cpu)
+{
+}
+
+static inline void trigger_load_balance(struct rq *rq)
+{
+}
+
+#else /* CONFIG_SMP */
+
+static inline void flush_smp_call_function_from_idle(void) { }
+
+/*
+ * UP has no sched domains and MuQSS's rq has no ->sd, but stats.c walks the
+ * domain list unconditionally. Hand it an empty list.
+ */
+#define for_each_domain(cpu, __sd) \
+	for (__sd = NULL; __sd; __sd = __sd->parent)
+
+#endif /* CONFIG_SMP */
+
+#ifdef CONFIG_CPU_IDLE
+static inline void idle_set_state(struct rq *rq,
+				  struct cpuidle_state *idle_state)
+{
+	rq->idle_state = idle_state;
+}
+
+static inline struct cpuidle_state *idle_get_state(struct rq *rq)
+{
+	SCHED_WARN_ON(!rcu_read_lock_held());
+	return rq->idle_state;
+}
+#else
+static inline void idle_set_state(struct rq *rq,
+				  struct cpuidle_state *idle_state)
+{
+}
+
+static inline struct cpuidle_state *idle_get_state(struct rq *rq)
+{
+	return NULL;
+}
+#endif
+
+#ifdef CONFIG_SCHED_DEBUG
+extern bool sched_debug_enabled;
+#endif
+
+extern void schedule_idle(void);
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+struct irqtime {
+	u64			total;
+	u64			tick_delta;
+	u64			irq_start_time;
+	struct u64_stats_sync	sync;
+};
+
+DECLARE_PER_CPU(struct irqtime, cpu_irqtime);
+
+/*
+ * Returns the irqtime minus the softirq time computed by ksoftirqd.
+ * Otherwise ksoftirqd's sum_exec_runtime is substracted its own runtime
+ * and never move forward.
+ */
+static inline u64 irq_time_read(int cpu)
+{
+	struct irqtime *irqtime = &per_cpu(cpu_irqtime, cpu);
+	unsigned int seq;
+	u64 total;
+
+	do {
+		seq = __u64_stats_fetch_begin(&irqtime->sync);
+		total = irqtime->total;
+	} while (__u64_stats_fetch_retry(&irqtime->sync, seq));
+
+	return total;
+}
+#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
+
+static inline bool sched_stop_runnable(struct rq *rq)
+{
+	return rq->stop && task_on_rq_queued(rq->stop);
+}
+
+#ifdef CONFIG_SMP
+static inline int cpu_of(struct rq *rq)
+{
+	return rq->cpu;
+}
+#else /* CONFIG_SMP */
+static inline int cpu_of(struct rq *rq)
+{
+	return 0;
+}
+#endif
+
+#ifdef CONFIG_CPU_FREQ
+DECLARE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);
+
+static inline void cpufreq_trigger(struct rq *rq, unsigned int flags)
+{
+	struct update_util_data *data;
+
+	data = rcu_dereference_sched(*per_cpu_ptr(&cpufreq_update_util_data,
+						  cpu_of(rq)));
+
+	if (data)
+		data->func(data, rq->niffies, flags);
+}
+#else
+static inline void cpufreq_trigger(struct rq *rq, unsigned int flag)
+{
+}
+#endif /* CONFIG_CPU_FREQ */
+
+static __always_inline
+unsigned int uclamp_rq_util_with(struct rq __maybe_unused *rq, unsigned int util,
+			      struct task_struct __maybe_unused *p)
+{
+	return util;
+}
+
+static inline bool uclamp_is_used(void)
+{
+	return false;
+}
+
+#ifndef arch_scale_freq_tick
+static __always_inline
+void arch_scale_freq_tick(void)
+{
+}
+#endif
+
+#ifdef arch_scale_freq_capacity
+#ifndef arch_scale_freq_invariant
+#define arch_scale_freq_invariant()	(true)
+#endif
+#else /* arch_scale_freq_capacity */
+#define arch_scale_freq_invariant()	(false)
+#endif
+
+/*
+ * read_sum_exec_runtime() lives in cputime.c and now goes through
+ * tsk_seruntime(), so MuQSS needs no copy of it here.
+ */
+
+#ifndef arch_scale_freq_capacity
+/**
+ * arch_scale_freq_capacity - get the frequency scale factor of a given CPU.
+ * @cpu: the CPU in question.
+ *
+ * Return: the frequency scale factor normalized against SCHED_CAPACITY_SCALE, i.e.
+ *
+ *     f_curr
+ *     ------ * SCHED_CAPACITY_SCALE
+ *     f_max
+ */
+static __always_inline
+unsigned long arch_scale_freq_capacity(int cpu)
+{
+	return SCHED_CAPACITY_SCALE;
+}
+#endif
+
+#ifdef CONFIG_NO_HZ_FULL
+extern bool sched_can_stop_tick(struct rq *rq);
+extern int __init sched_tick_offload_init(void);
+
+/*
+ * Tick may be needed by tasks in the runqueue depending on their policy and
+ * requirements. If tick is needed, lets send the target an IPI to kick it out of
+ * nohz mode if necessary.
+ */
+static inline void sched_update_tick_dependency(struct rq *rq)
+{
+	int cpu = cpu_of(rq);
+
+	if (!tick_nohz_full_cpu(cpu))
+		return;
+
+	if (sched_can_stop_tick(rq))
+		tick_nohz_dep_clear_cpu(cpu, TICK_DEP_BIT_SCHED);
+	else
+		tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
+}
+#else
+static inline int sched_tick_offload_init(void) { return 0; }
+static inline void sched_update_tick_dependency(struct rq *rq) { }
+#endif
+
+#define SCHED_FLAG_SUGOV	0x10000000
+
+#ifdef CONFIG_SMP
+/*
+ * enum cpu_util_type and its FREQUENCY_UTIL/ENERGY_UTIL members were removed
+ * upstream; effective_cpu_util() now reports a utilisation figure plus the
+ * usable capacity range. MuQSS supplies it from MuQSS.c.
+ */
+unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
+				 unsigned long *min, unsigned long *max);
+
+static inline unsigned long cpu_bw_dl(struct rq *rq)
+{
+	return 0;
+}
+
+static inline unsigned long cpu_util_dl(struct rq *rq)
+{
+	return 0;
+}
+
+static inline unsigned long cpu_util_cfs(struct rq *rq)
+{
+	unsigned long ret = READ_ONCE(rq->load_avg);
+
+	if (ret > SCHED_CAPACITY_SCALE)
+		ret = SCHED_CAPACITY_SCALE;
+	return ret;
+}
+
+static inline unsigned long cpu_util_rt(struct rq *rq)
+{
+	unsigned long ret = READ_ONCE(rq->rt_nr_running);
+
+	if (ret > SCHED_CAPACITY_SCALE)
+		ret = SCHED_CAPACITY_SCALE;
+	return ret;
+}
+
+#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
+static inline unsigned long cpu_util_irq(struct rq *rq)
+{
+	unsigned long ret = READ_ONCE(rq->irq_load_avg);
+
+	if (ret > SCHED_CAPACITY_SCALE)
+		ret = SCHED_CAPACITY_SCALE;
+	return ret;
+}
+
+static inline
+unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
+{
+	util *= (max - irq);
+	util /= max;
+
+	return util;
+
+}
+#else /* CONFIG_HAVE_SCHED_AVG_IRQ */
+static inline unsigned long cpu_util_irq(struct rq *rq)
+{
+	return 0;
+}
+
+static inline
+unsigned long scale_irq_capacity(unsigned long util, unsigned long irq, unsigned long max)
+{
+	return util;
+}
+#endif /* CONFIG_HAVE_SCHED_AVG_IRQ */
+#endif /* CONFIG_SMP */
+
+#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
+#define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus)))
+
+DECLARE_STATIC_KEY_FALSE(sched_energy_present);
+
+static inline bool sched_energy_enabled(void)
+{
+	return static_branch_unlikely(&sched_energy_present);
+}
+
+#else /* ! (CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL) */
+
+#define perf_domain_span(pd) NULL
+static inline bool sched_energy_enabled(void) { return false; }
+
+#endif /* CONFIG_ENERGY_MODEL && CONFIG_CPU_FREQ_GOV_SCHEDUTIL */
+
+#ifdef CONFIG_MEMBARRIER
+/*
+ * The scheduler provides memory barriers required by membarrier between:
+ * - prior user-space memory accesses and store to rq->membarrier_state,
+ * - store to rq->membarrier_state and following user-space memory accesses.
+ * In the same way it provides those guarantees around store to rq->curr.
+ */
+static inline void membarrier_switch_mm(struct rq *rq,
+					struct mm_struct *prev_mm,
+					struct mm_struct *next_mm)
+{
+	int membarrier_state;
+
+	if (prev_mm == next_mm)
+		return;
+
+	membarrier_state = atomic_read(&next_mm->membarrier_state);
+	if (READ_ONCE(rq->membarrier_state) == membarrier_state)
+		return;
+
+	WRITE_ONCE(rq->membarrier_state, membarrier_state);
+}
+#else
+static inline void membarrier_switch_mm(struct rq *rq,
+					struct mm_struct *prev_mm,
+					struct mm_struct *next_mm)
+{
+}
+#endif
+
+#ifdef CONFIG_SMP
+static inline bool is_per_cpu_kthread(struct task_struct *p)
+{
+	if (!(p->flags & PF_KTHREAD))
+		return false;
+
+	if (p->nr_cpus_allowed != 1)
+		return false;
+
+	return true;
+}
+#endif
+
+void swake_up_all_locked(struct swait_queue_head *q);
+void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
+
+/* pelt.h compat CONFIG_SCHED_THERMAL_PRESSURE impossible with MUQSS */
+static inline int
+update_thermal_load_avg(u64 now, struct rq *rq, u64 capacity)
+{
+	return 0;
+}
+
+static inline u64 thermal_load_avg(struct rq *rq)
+{
+	return 0;
+}
+
+#ifdef CONFIG_RCU_TORTURE_TEST
+extern int sysctl_sched_rt_runtime;
+#endif
+
+/*
+ * Asymmetric CPU capacity bits, used by topology.c. MuQSS does not do
+ * capacity-aware placement, but the domain build code still maintains the
+ * list.
+ */
+struct asym_cap_data {
+	struct list_head link;
+	struct rcu_head rcu;
+	unsigned long capacity;
+	unsigned long cpus[];
+};
+
+extern struct list_head asym_cap_list;
+
+#define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus)
+
+/*
+ * Stubs for mainline scheduler machinery MuQSS does not implement. These are
+ * referenced by the support files retained in build_muqss.c.
+ */
+
+/* No deadline server: topology.c attaches none. See rq->fair_server. */
+static inline void __dl_server_attach_root(void *dl_se, struct rq *rq)
+{
+}
+
+/* sched_ext is mutually exclusive with MuQSS; schedutil still asks. */
+static inline u32 scx_cpuperf_target(s32 cpu)
+{
+	return 0;
+}
+
+static inline bool scx_switched_all(void)
+{
+	return false;
+}
+
+/* No nohz idle balancing: MuQSS balances implicitly at task selection. */
+static inline void nohz_run_idle_balance(int cpu)
+{
+}
+
+/* No utilisation clamping, and no CFS utilisation to boost or clamp. */
+static inline bool uclamp_rq_is_capped(struct rq *rq)
+{
+	return false;
+}
+
+static inline unsigned long cpu_util_cfs_boost(int cpu)
+{
+	return 0;
+}
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+DECLARE_STATIC_KEY_FALSE(sched_clock_irqtime);
+
+static inline int irqtime_enabled(void)
+{
+	return static_branch_likely(&sched_clock_irqtime);
+}
+#else
+static inline int irqtime_enabled(void)
+{
+	return 0;
+}
+#endif
+
+extern unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual,
+					      unsigned long min,
+					      unsigned long max);
+
+extern int try_to_wake_up(struct task_struct *p, unsigned int state,
+			  int wake_flags);
+
+asmlinkage void schedule_user(void);
+extern void resched_cpu(int cpu);
+extern bool available_idle_cpu(int cpu);
+extern void do_set_cpus_allowed(struct task_struct *p,
+				struct affinity_context *ctx);
+extern void __do_set_cpus_allowed(struct task_struct *p,
+				  struct affinity_context *ctx);
+
+#ifdef CONFIG_PREEMPT_DYNAMIC
+extern int sched_dynamic_mode(const char *str);
+extern void sched_dynamic_update(int mode);
+#endif
+
+#ifdef CONFIG_CGROUP_SCHED
+extern struct task_group *sched_create_group(struct task_group *parent);
+extern void sched_online_group(struct task_group *tg,
+			       struct task_group *parent);
+extern void sched_destroy_group(struct task_group *tg);
+extern void sched_release_group(struct task_group *tg);
+#endif
+
+#endif /* MUQSS_SCHED_H */
diff --git a/kernel/sched/build_muqss.c b/kernel/sched/build_muqss.c
new file mode 100644
--- /dev/null
+++ b/kernel/sched/build_muqss.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * The scheduler support files MuQSS retains from mainline, built in a single
+ * compilation unit for the same build-efficiency reasons as build_policy.c
+ * and build_utility.c.
+ *
+ * MuQSS replaces core.c, fair.c, rt.c, deadline.c, syscalls.c and stop_task.c
+ * outright, so neither of the mainline aggregates can be used as-is. The
+ * files gathered here are the subset of build_utility.c and build_policy.c
+ * that survives, and they are #included rather than built individually
+ * because several of them (clock.c, completion.c, psi.c) carry no includes of
+ * their own and only compile as part of an aggregate.
+ *
+ * Deliberately absent, and why:
+ *
+ *   fair.c rt.c deadline.c ext.c stop_task.c  policy replaced by MuQSS
+ *   cpudeadline.c cpupri.c pelt.c            supporting data for those classes
+ *   syscalls.c                               MuQSS implements the syscalls
+ *   core_sched.c                             SCHED_CORE gated off
+ *   debug.c                                  CFS/EEVDF runqueue introspection
+ *   loadavg.c                                MuQSS keeps its own load average
+ *   autogroup.c cpuacct.c                    gated off in Kconfig.MuQSS
+ */
+
+/* Headers: */
+#include <linux/sched/clock.h>
+#include <linux/sched/cputime.h>
+#include <linux/sched/debug.h>
+#include <linux/sched/hotplug.h>
+#include <linux/sched/isolation.h>
+#include <linux/sched/loadavg.h>
+#include <linux/sched/nohz.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/posix-timers.h>
+#include <linux/sched/rseq_api.h>
+#include <linux/sched/rt.h>
+#include <linux/sched/task_stack.h>
+
+#include <linux/cpufreq.h>
+#include <linux/cpuidle.h>
+#include <linux/cpumask_api.h>
+#include <linux/cpuset.h>
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <linux/energy_model.h>
+#include <linux/hashtable_api.h>
+#include <linux/irq.h>
+#include <linux/jiffies.h>
+#include <linux/kobject_api.h>
+#include <linux/livepatch.h>
+#include <linux/membarrier.h>
+#include <linux/memblock.h>
+#include <linux/mempolicy.h>
+#include <linux/nmi.h>
+#include <linux/nospec.h>
+#include <linux/percpu-rwsem.h>
+#include <linux/pm.h>
+#include <linux/proc_fs.h>
+#include <linux/psi.h>
+#include <linux/ptrace_api.h>
+#include <linux/rhashtable.h>
+#include <linux/sched_clock.h>
+#include <linux/security.h>
+#include <linux/seq_buf.h>
+#include <linux/seqlock_api.h>
+#include <linux/slab.h>
+#include <linux/spinlock_api.h>
+#include <linux/suspend.h>
+#include <linux/swait_api.h>
+#include <linux/sysrq.h>
+#include <linux/timex.h>
+#include <linux/tsacct_kern.h>
+#include <linux/utsname.h>
+#include <linux/vtime.h>
+#include <linux/wait_api.h>
+#include <linux/workqueue_api.h>
+
+#include <uapi/linux/prctl.h>
+#include <uapi/linux/sched/types.h>
+
+#include <asm/switch_to.h>
+
+#include <trace/events/power.h>
+
+#include "sched.h"
+#include "stats.h"
+
+/* psi.c needs wq_worker_last_func(); mainline gets this via sched.h. */
+#include "../workqueue_internal.h"
+
+/* Source code modules: */
+
+#include "clock.c"
+
+#ifdef CONFIG_CPU_FREQ
+# include "cpufreq.c"
+#endif
+
+#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL
+# include "cpufreq_schedutil.c"
+#endif
+
+#ifdef CONFIG_SCHEDSTATS
+# include "stats.c"
+#endif
+
+#include "completion.c"
+#include "swait.c"
+#include "wait_bit.c"
+#include "wait.c"
+
+#include "cputime.c"
+#include "idle.c"
+
+#ifdef CONFIG_SMP
+# include "topology.c"
+#endif
+
+#ifdef CONFIG_PSI
+# include "psi.c"
+#endif
+
+#ifdef CONFIG_MEMBARRIER
+# include "membarrier.c"
+#endif
+
+#ifdef CONFIG_CPU_ISOLATION
+# include "isolation.c"
+#endif
+
+#include "skip_list.c"
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -305,7 +305,7 @@ static inline u64 account_other_time(u64 max)
 #ifdef CONFIG_64BIT
 static inline u64 read_sum_exec_runtime(struct task_struct *t)
 {
-	return t->se.sum_exec_runtime;
+	return tsk_seruntime(t);
 }
 #else /* !CONFIG_64BIT: */
 static u64 read_sum_exec_runtime(struct task_struct *t)
@@ -315,7 +315,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t)
 	struct rq *rq;
 
 	rq = task_rq_lock(t, &rf);
-	ns = t->se.sum_exec_runtime;
+	ns = tsk_seruntime(t);
 	task_rq_unlock(rq, t, &rf);
 
 	return ns;
@@ -689,6 +689,19 @@ void account_process_tick(struct task_struct *p, int user_tick)
 {
 	u64 cputime, steal;
 
+	/*
+	 * MuQSS accounts this tick itself, from sched_tick() ->
+	 * update_cpu_clock_tick(), which update_process_times() calls a few
+	 * lines after calling us. Its accounting is in real nanoseconds rather
+	 * than whole ticks, and it is the same p->utime/p->stime and the same
+	 * kcpustat[] fields we would be adding to here, so running both counts
+	 * every busy tick twice. That is invisible in per-task figures, since
+	 * cputime_adjust() rescales utime/stime to sched_time, but it doubles
+	 * the busy buckets of /proc/stat. Leave it all to MuQSS.
+	 */
+	if (IS_ENABLED(CONFIG_SCHED_MUQSS))
+		return;
+
 	if (vtime_accounting_enabled_this_cpu())
 		return;
 
@@ -811,7 +824,7 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
 {
 	struct task_cputime cputime = {
-		.sum_exec_runtime = p->se.sum_exec_runtime,
+		.sum_exec_runtime = tsk_seruntime(p),
 	};
 
 	if (task_cputime(p, &cputime.utime, &cputime.stime))
diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -454,6 +454,7 @@ void cpu_startup_entry(enum cpuhp_state state)
 		do_idle();
 }
 
+#ifndef CONFIG_SCHED_MUQSS
 /*
  * idle-task scheduling class.
  */
@@ -591,3 +592,4 @@ DEFINE_SCHED_CLASS(idle) = {
 	.switching_to		= switching_to_idle,
 	.update_curr		= update_curr_idle,
 };
+#endif /* CONFIG_SCHED_MUQSS */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -5,6 +5,10 @@
 #ifndef _KERNEL_SCHED_SCHED_H
 #define _KERNEL_SCHED_SCHED_H
 
+#ifdef CONFIG_SCHED_MUQSS
+#include "MuQSS.h"
+#else /* CONFIG_SCHED_MUQSS */
+
 #include <linux/prandom.h>
 #include <linux/sched/affinity.h>
 #include <linux/sched/autogroup.h>
@@ -4216,4 +4220,6 @@ DEFINE_CLASS_IS_UNCONDITIONAL(sched_change)
 
 #include "ext/ext.h"
 
+#endif /* CONFIG_SCHED_MUQSS */
+
 #endif /* _KERNEL_SCHED_SCHED_H */
diff --git a/kernel/sched/skip_list.c b/kernel/sched/skip_list.c
new file mode 100644
--- /dev/null
+++ b/kernel/sched/skip_list.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+  Copyright (C) 2011,2016 Con Kolivas.
+
+  Code based on example originally by William Pugh.
+
+Skip Lists are a probabilistic alternative to balanced trees, as
+described in the June 1990 issue of CACM and were invented by
+William Pugh in 1987.
+
+A couple of comments about this implementation:
+The routine randomLevel has been hard-coded to generate random
+levels using p=0.25. It can be easily changed.
+
+The insertion routine has been implemented so as to use the
+dirty hack described in the CACM paper: if a random level is
+generated that is more than the current maximum level, the
+current maximum level plus one is used instead.
+
+Levels start at zero and go up to MaxLevel (which is equal to
+MaxNumberOfLevels-1).
+
+The routines defined in this file are:
+
+init: defines slnode
+
+new_skiplist: returns a new, empty list
+
+randomLevel: Returns a random level based on a u64 random seed passed to it.
+In MuQSS, the "niffy" time is used for this purpose.
+
+insert(l, node, key): inserts node into l ordered by key. This operation
+occurs in O(log n) time.
+
+delnode(slnode, l, node): deletes the node from l. This operation occurs
+in O(k) time where k is the number of levels of the node in question
+(max 4). The original delete function occurred in O(log n) time and
+involved a search.
+
+MuQSS Notes: In this implementation of skiplists, there are bidirectional
+next/prev pointers. The node is embedded in task_struct; the task is
+recovered with container_of(). The key here is chosen by the scheduler
+so as to sort tasks according to the priority list requirements. The
+scheduler lookup occurs in O(1) time because it is always the first
+item in the level 0 linked list. Since the task struct embeds the node,
+it can also remove it much faster than the original implementation with
+the aid of prev<->next pointer manipulation and no searching.
+
+*/
+
+#include <linux/compiler.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/skip_list.h>
+
+static struct kmem_cache *skiplist_cache __read_mostly;
+
+#define MaxNumberOfLevels SKIPLIST_MAXLEVEL
+#define MaxLevel (MaxNumberOfLevels - 1)
+
+void skiplist_init(skiplist_node *slnode)
+{
+	int i;
+
+	slnode->key = 0xFFFFFFFFFFFFFFFF;
+	for (i = 0; i < MaxNumberOfLevels; i++)
+		slnode->next[i] = slnode->prev[i] = slnode;
+}
+
+void __init skiplist_cache_init(void)
+{
+	skiplist_cache = kmem_cache_create("skiplist", sizeof(skiplist),
+					   0, SLAB_HWCACHE_ALIGN, NULL);
+	BUG_ON(!skiplist_cache);
+}
+
+skiplist *new_skiplist(skiplist_node *slnode)
+{
+	skiplist *l = kmem_cache_zalloc(skiplist_cache, GFP_ATOMIC);
+
+	BUG_ON(!l);
+	l->header = slnode;
+	l->best_key = ~0ULL;
+	return l;
+}
+
+void skiplist_free(skiplist *l)
+{
+	kmem_cache_free(skiplist_cache, l);
+}
+
+void free_skiplist(skiplist *l)
+{
+	skiplist_node *p, *q;
+
+	p = l->header;
+	do {
+		q = p->next[0];
+		p->next[0]->prev[0] = q->prev[0];
+		skiplist_node_init(p);
+		p = q;
+	} while (p != l->header);
+	skiplist_free(l);
+}
+
+void skiplist_node_init(skiplist_node *node)
+{
+	memset(node, 0, sizeof(skiplist_node));
+}
+
+static inline unsigned int randomLevel(const long unsigned int randseed)
+{
+	/*
+	 * Two bits per extra level (p = 0.25). Search 2*MaxLevel+1 bits
+	 * so the result stays in 0..MaxLevel; this matches the old
+	 * 8-slot arrays whose random height never exceeded 3.
+	 */
+	return find_first_bit(&randseed, MaxLevel * 2 + 1) / 2;
+}
+
+void skiplist_insert(skiplist *l, skiplist_node *node, keyType key, unsigned int randseed)
+{
+	skiplist_node *update[MaxNumberOfLevels];
+	skiplist_node *p, *q;
+	int k = l->level;
+
+	p = l->header;
+	do {
+		while (q = p->next[k], q->key <= key)
+			p = q;
+		update[k] = p;
+	} while (--k >= 0);
+
+	WRITE_ONCE(l->entries, l->entries + 1);
+	k = randomLevel(randseed);
+	if (k > l->level) {
+		k = ++l->level;
+		update[k] = l->header;
+	}
+
+	node->key = key;
+	do {
+		p = update[k];
+		node->next[k] = p->next[k];
+		p->next[k] = node;
+		node->prev[k] = p;
+		node->next[k]->prev[k] = node;
+	} while (--k >= 0);
+}
+
+/*
+ * Unlink @node from @l. Returns true if it was linked and has been removed,
+ * false if it was not on any list and nothing was changed - the caller's own
+ * accounting has to be skipped along with the removal in that case.
+ */
+bool skiplist_delete(skiplist *l, skiplist_node *node)
+{
+	int k, m;
+
+	/*
+	 * Insert fills levels contiguously from 0 up and removal zeroes the
+	 * whole node, so a NULL next[] marks the top of this node and stands
+	 * in for the level count the node used to carry.
+	 */
+	for (k = 0; k < MaxNumberOfLevels && node->next[k]; k++) {
+		node->prev[k]->next[k] = node->next[k];
+		node->next[k]->prev[k] = node->prev[k];
+	}
+	m = k - 1;
+	/*
+	 * A node that was never queued has no levels linked at all. Leave
+	 * before decrementing entries, which would otherwise go negative and
+	 * make the emptiness tests in earliest_deadline_task() pass on an
+	 * empty list, handing the caller the header as if it were a task.
+	 */
+	if (WARN_ON_ONCE(m < 0))
+		return false;
+	skiplist_node_init(node);
+	if (m == l->level) {
+		while (l->header->next[m] == l->header && l->header->prev[m] == l->header && m > 0)
+			m--;
+		l->level = m;
+	}
+	WRITE_ONCE(l->entries, l->entries - 1);
+
+	return true;
+}
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -89,6 +89,8 @@ static inline void rq_sched_info_depart  (struct rq *rq, unsigned long long delt
 
 #endif /* CONFIG_SCHEDSTATS */
 
+#ifndef CONFIG_SCHED_MUQSS
+/* Only reachable from fair.c, debug.c and core.c, none of which MuQSS builds. */
 static inline struct sched_statistics *
 __schedstats_from_se(struct sched_entity *se)
 {
@@ -98,6 +100,7 @@ __schedstats_from_se(struct sched_entity *se)
 #endif
 	return &task_of(se)->stats;
 }
+#endif /* CONFIG_SCHED_MUQSS */
 
 #ifdef CONFIG_PSI
 void psi_task_change(struct task_struct *task, int clear, int set);
@@ -135,6 +138,23 @@ static inline void psi_enqueue(struct task_struct *p, int flags)
 	if (task_on_cpu(task_rq(p), p))
 		return;
 
+#ifdef CONFIG_SCHED_MUQSS
+	/*
+	 * MuQSS has no delayed-dequeue (no task_struct::se), so a migration
+	 * always moves a runnable task and a non-migration is a wakeup.
+	 */
+	if (flags & ENQUEUE_MIGRATED) {
+		set = TSK_RUNNING;
+		if (p->in_memstall)
+			set |= TSK_MEMSTALL | TSK_MEMSTALL_RUNNING;
+	} else {
+		if (p->in_iowait)
+			clear |= TSK_IOWAIT;
+		set = TSK_RUNNING;
+		if (p->in_memstall)
+			set |= TSK_MEMSTALL_RUNNING;
+	}
+#else
 	if (p->se.sched_delayed) {
 		/* CPU migration of "sleeping" task */
 		WARN_ON_ONCE(!(flags & ENQUEUE_MIGRATED));
@@ -155,6 +175,7 @@ static inline void psi_enqueue(struct task_struct *p, int flags)
 		if (p->in_memstall)
 			set |= TSK_MEMSTALL_RUNNING;
 	}
+#endif /* CONFIG_SCHED_MUQSS */
 
 	psi_task_change(p, clear, set);
 }
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -103,18 +103,36 @@ config NO_HZ_COMMON
 	bool
 	select TICK_ONESHOT
 
+#
+# Silent symbols keep CONFIG_NO_HZ_* / CONFIG_HZ_PERIODIC available to the
+# rest of the tree. The promptable *_NODEF names are what olddefconfig can
+# stick on; renaming every choice member (including idle) means a saved
+# "# CONFIG_NO_HZ_IDLE is not set" cannot pin the choice onto full dynticks.
+#
+config HZ_PERIODIC
+	bool
+
+config NO_HZ_IDLE
+	bool
+
+config NO_HZ_FULL
+	bool
+
 choice
 	prompt "Timer tick handling"
-	default NO_HZ_IDLE if NO_HZ
+	default NO_HZ_IDLE_NODEF if NO_HZ
+	default NO_HZ_IDLE_NODEF
 
-config HZ_PERIODIC
+config HZ_PERIODIC_NODEF
 	bool "Periodic timer ticks (constant rate, no dynticks)"
+	select HZ_PERIODIC
 	help
 	  This option keeps the tick running periodically at a constant
 	  rate, even when the CPU doesn't need it.
 
-config NO_HZ_IDLE
+config NO_HZ_IDLE_NODEF
 	bool "Idle dynticks system (tickless idle)"
+	select NO_HZ_IDLE
 	select NO_HZ_COMMON
 	help
 	  This option enables a tickless idle system: timer interrupts
@@ -123,8 +141,9 @@ config NO_HZ_IDLE
 
 	  Most of the time you want to say Y here.
 
-config NO_HZ_FULL
+config NO_HZ_FULL_NODEF
 	bool "Full dynticks system (tickless)"
+	select NO_HZ_FULL
 	# NO_HZ_COMMON dependency
 	# We need at least one periodic CPU for timekeeping
 	depends on SMP
@@ -150,6 +169,8 @@ config NO_HZ_FULL
 	 By default, without passing the nohz_full parameter, this behaves just
 	 like NO_HZ_IDLE.
 
+	 Not recommended for desktops, laptops, or mobile devices.
+
 	 If you're a distro say Y.
 
 endchoice
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -194,8 +194,8 @@ int clockevents_tick_resume(struct clock_event_device *dev)
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST
 
-/* Limit min_delta to a jiffy */
-#define MIN_DELTA_LIMIT		(NSEC_PER_SEC / HZ)
+/* Limit min_delta to the configurable hrtimer granularity */
+#define MIN_DELTA_LIMIT		(hrtimer_granularity_us * NSEC_PER_USEC)
 
 /**
  * clockevents_increase_min_delta - raise minimum delta of a clock event device
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -223,7 +223,7 @@ static void task_sample_cputime(struct task_struct *p, u64 *samples)
 	u64 stime, utime;
 
 	task_cputime(p, &utime, &stime);
-	store_samples(samples, stime, utime, p->se.sum_exec_runtime);
+	store_samples(samples, stime, utime, tsk_seruntime(p));
 }
 
 static void proc_sample_cputime_atomic(struct task_cputime_atomic *at,
@@ -905,10 +905,12 @@ static void collect_posix_cputimers(struct posix_cputimers *pct, u64 *samples,
 
 static inline void check_dl_overrun(struct task_struct *tsk)
 {
+#ifndef CONFIG_SCHED_MUQSS
 	if (tsk->dl.dl_overrun) {
 		tsk->dl.dl_overrun = 0;
 		send_signal_locked(SIGXCPU, SEND_SIG_PRIV, tsk, PIDTYPE_TGID);
 	}
+#endif
 }
 
 static bool check_rlimit(u64 time, u64 limit, int signo, bool rt, bool hard)
@@ -952,7 +954,7 @@ static void check_thread_timers(struct task_struct *tsk,
 	soft = task_rlimit(tsk, RLIMIT_RTTIME);
 	if (soft != RLIM_INFINITY) {
 		/* Task RT timeout is accounted in jiffies. RTTIME is usec */
-		unsigned long rttime = tsk->rt.timeout * (USEC_PER_SEC / HZ);
+		unsigned long rttime = tsk_rttimeout(tsk) * (USEC_PER_SEC / HZ);
 		unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME);
 
 		/* At the hard limit, send SIGKILL. No further action. */
@@ -1183,7 +1185,7 @@ static inline bool fastpath_timer_check(struct task_struct *tsk)
 			return true;
 	}
 
-	if (dl_task(tsk) && tsk->dl.dl_overrun)
+	if (dl_task(tsk) && tsk_dl_overrun(tsk))
 		return true;
 
 	return false;
diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c
--- a/kernel/time/sleep_timeout.c
+++ b/kernel/time/sleep_timeout.c
@@ -4,13 +4,57 @@
  */
 
 #include <linux/delay.h>
+#include <linux/freezer.h>
 #include <linux/jiffies.h>
 #include <linux/timer.h>
 #include <linux/sched/signal.h>
 #include <linux/sched/debug.h>
+#include <linux/sysctl.h>
 
 #include "tick-internal.h"
 
+/*
+ * Granularity, in microseconds, below which a high resolution timeout is not
+ * worth arming. Also caps how far clockevents_increase_min_delta() is allowed
+ * to raise a clock event device's minimum delta.
+ */
+int __read_mostly hrtimer_granularity_us = 100;
+
+/* The timeout, in microseconds, used by schedule_min_hrtimeout(). */
+int __read_mostly hrtimeout_min_us = 500;
+
+#ifdef CONFIG_SYSCTL
+static const int hrtimeout_us_max = 10000;
+
+static const struct ctl_table hrtimeout_sysctl[] = {
+	{
+		.procname	= "hrtimer_granularity_us",
+		.data		= &hrtimer_granularity_us,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ONE,
+		.extra2		= (void *)&hrtimeout_us_max,
+	},
+	{
+		.procname	= "hrtimeout_min_us",
+		.data		= &hrtimeout_min_us,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ONE,
+		.extra2		= (void *)&hrtimeout_us_max,
+	},
+};
+
+static int __init hrtimeout_sysctl_init(void)
+{
+	register_sysctl("kernel", hrtimeout_sysctl);
+	return 0;
+}
+device_initcall(hrtimeout_sysctl_init);
+#endif /* CONFIG_SYSCTL */
+
 /*
  * Since schedule_timeout()'s timer is defined on the stack, it must store
  * the target task on the stack as well.
@@ -279,10 +323,164 @@ int __sched schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode)
 }
 EXPORT_SYMBOL_GPL(schedule_hrtimeout);
 
+#ifdef CONFIG_HIGH_RES_TIMERS
+/*
+ * As per schedule_hrtimeout but takes a millisecond value and returns how
+ * many milliseconds are left.
+ */
+long __sched schedule_msec_hrtimeout(long timeout)
+{
+	struct hrtimer_sleeper t;
+	int delta, jiffs;
+	ktime_t expires;
+
+	if (!timeout) {
+		__set_current_state(TASK_RUNNING);
+		return 0;
+	}
+
+	jiffs = msecs_to_jiffies(timeout);
+	/*
+	 * If regular timer resolution is adequate or hrtimer resolution is not
+	 * (yet) better than Hz, as would occur during startup, use regular
+	 * timers. Freezing gets the same treatment as some drivers still do not
+	 * correctly use freezable timeouts.
+	 */
+	if (jiffs > 4 || hrtimer_resolution >= NSEC_PER_SEC / HZ || pm_freezing)
+		return schedule_timeout(jiffs);
+
+	/* The jiffs > 4 test above bounds timeout well under one second. */
+	delta = (timeout % 1000) * NSEC_PER_MSEC;
+	expires = ktime_set(0, delta);
+
+	hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	/*
+	 * No slack is given here. The whole point of taking the high resolution
+	 * path is that the timeout is too short for the tick to resolve, and a
+	 * slack of delta would push the expiry out to twice the request.
+	 */
+	hrtimer_set_expires(&t.timer, expires);
+
+	hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL);
+
+	if (likely(t.task))
+		schedule();
+
+	hrtimer_cancel(&t.timer);
+	destroy_hrtimer_on_stack(&t.timer);
+
+	__set_current_state(TASK_RUNNING);
+
+	expires = hrtimer_expires_remaining(&t.timer);
+	timeout = ktime_to_ms(expires);
+	return timeout < 0 ? 0 : timeout;
+}
+
+EXPORT_SYMBOL(schedule_msec_hrtimeout);
+
+/*
+ * As per schedule_msec_hrtimeout but takes a microsecond value and returns
+ * how many microseconds are left.
+ */
+static long __sched schedule_usec_hrtimeout(long timeout)
+{
+	struct hrtimer_sleeper t;
+	ktime_t expires;
+	int delta;
+
+	if (!timeout) {
+		__set_current_state(TASK_RUNNING);
+		return 0;
+	}
+
+	if (hrtimer_resolution >= NSEC_PER_SEC / HZ)
+		return schedule_timeout(usecs_to_jiffies(timeout));
+
+	if (timeout < hrtimer_granularity_us)
+		timeout = hrtimer_granularity_us;
+	delta = (timeout % USEC_PER_SEC) * NSEC_PER_USEC;
+	expires = ktime_set(0, delta);
+
+	hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+	/* No slack is given here, as per schedule_msec_hrtimeout(). */
+	hrtimer_set_expires(&t.timer, expires);
+
+	hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL);
+
+	if (likely(t.task))
+		schedule();
+
+	hrtimer_cancel(&t.timer);
+	destroy_hrtimer_on_stack(&t.timer);
+
+	__set_current_state(TASK_RUNNING);
+
+	expires = hrtimer_expires_remaining(&t.timer);
+	timeout = ktime_to_us(expires);
+	return timeout < 0 ? 0 : timeout;
+}
+
+long __sched schedule_min_hrtimeout(void)
+{
+	return usecs_to_jiffies(schedule_usec_hrtimeout(hrtimeout_min_us));
+}
+
+EXPORT_SYMBOL(schedule_min_hrtimeout);
+
+long __sched schedule_msec_hrtimeout_interruptible(long timeout)
+{
+	__set_current_state(TASK_INTERRUPTIBLE);
+	return schedule_msec_hrtimeout(timeout);
+}
+EXPORT_SYMBOL(schedule_msec_hrtimeout_interruptible);
+
+long __sched schedule_msec_hrtimeout_uninterruptible(long timeout)
+{
+	__set_current_state(TASK_UNINTERRUPTIBLE);
+	return schedule_msec_hrtimeout(timeout);
+}
+EXPORT_SYMBOL(schedule_msec_hrtimeout_uninterruptible);
+
+#else /* !CONFIG_HIGH_RES_TIMERS */
+
+/*
+ * Without high resolution timers there is nothing better than the regular
+ * jiffy based timeouts to fall back on.
+ */
+long __sched schedule_msec_hrtimeout(long timeout)
+{
+	return schedule_timeout(msecs_to_jiffies(timeout));
+}
+EXPORT_SYMBOL(schedule_msec_hrtimeout);
+
+long __sched schedule_min_hrtimeout(void)
+{
+	return schedule_timeout(1);
+}
+EXPORT_SYMBOL(schedule_min_hrtimeout);
+
+long __sched schedule_msec_hrtimeout_interruptible(long timeout)
+{
+	return schedule_timeout_interruptible(msecs_to_jiffies(timeout));
+}
+EXPORT_SYMBOL(schedule_msec_hrtimeout_interruptible);
+
+long __sched schedule_msec_hrtimeout_uninterruptible(long timeout)
+{
+	return schedule_timeout_uninterruptible(msecs_to_jiffies(timeout));
+}
+EXPORT_SYMBOL(schedule_msec_hrtimeout_uninterruptible);
+
+#endif /* CONFIG_HIGH_RES_TIMERS */
+
 /**
  * msleep - sleep safely even with waitqueue interruptions
  * @msecs:	Requested sleep duration in milliseconds
  *
+ * Sleeps shorter than five ticks are handed to a high resolution timer instead,
+ * where that has better resolution than the tick, so the slack described below
+ * only applies to the jiffy based path.
+ *
  * msleep() uses jiffy based timeouts for the sleep duration. Because of the
  * design of the timer wheel, the maximum additional percentage delay (slack) is
  * 12.5%. This is only valid for timers which will end up in level 1 or a higher
@@ -312,7 +510,19 @@ EXPORT_SYMBOL_GPL(schedule_hrtimeout);
  */
 void msleep(unsigned int msecs)
 {
-	unsigned long timeout = msecs_to_jiffies(msecs);
+	int jiffs = msecs_to_jiffies(msecs);
+	unsigned long timeout;
+
+	/*
+	 * Use high resolution timers where the resolution of tick based
+	 * timers is inadequate.
+	 */
+	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) {
+		while (msecs)
+			msecs = schedule_msec_hrtimeout_uninterruptible(msecs);
+		return;
+	}
+	timeout = jiffs;
 
 	while (timeout)
 		timeout = schedule_timeout_uninterruptible(timeout);
@@ -333,7 +543,15 @@ EXPORT_SYMBOL(msleep);
  */
 unsigned long msleep_interruptible(unsigned int msecs)
 {
-	unsigned long timeout = msecs_to_jiffies(msecs);
+	int jiffs = msecs_to_jiffies(msecs);
+	unsigned long timeout;
+
+	if (jiffs < 5 && hrtimer_resolution < NSEC_PER_SEC / HZ && !pm_freezing) {
+		while (msecs && !signal_pending(current))
+			msecs = schedule_msec_hrtimeout_interruptible(msecs);
+		return msecs;
+	}
+	timeout = jiffs;
 
 	while (timeout && !signal_pending(current))
 		timeout = schedule_timeout_interruptible(timeout);
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -13,6 +13,9 @@ struct timer_events {
 	u64	global;
 };
 
+/* Defined in sleep_timeout.c, tunable via sysctl */
+extern int hrtimer_granularity_us;
+
 #ifdef CONFIG_GENERIC_CLOCKEVENTS
 
 # define TICK_DO_TIMER_NONE	-1
diff --git a/kernel/trace/rv/monitors/deadline/Kconfig b/kernel/trace/rv/monitors/deadline/Kconfig
--- a/kernel/trace/rv/monitors/deadline/Kconfig
+++ b/kernel/trace/rv/monitors/deadline/Kconfig
@@ -1,5 +1,5 @@
 config RV_MON_DEADLINE
-	depends on RV
+	depends on RV && !SCHED_MUQSS
 	bool "deadline monitor"
 	help
 	  Collection of monitors to check the deadline scheduler and server
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -56,6 +56,7 @@
 #include <linux/kvm_para.h>
 #include <linux/delay.h>
 #include <linux/irq_work.h>
+#include <linux/muqss_iotime.h>
 
 #include "workqueue_internal.h"
 
@@ -1438,7 +1439,7 @@ void wq_worker_running(struct task_struct *task)
 	 * CPU intensive auto-detection cares about how long a work item hogged
 	 * CPU without sleeping. Reset the starting timestamp on wakeup.
 	 */
-	worker->current_at = worker->task->se.sum_exec_runtime;
+	worker->current_at = tsk_seruntime(worker->task);
 
 	WRITE_ONCE(worker->sleeping, 0);
 }
@@ -1523,7 +1524,7 @@ void wq_worker_tick(struct task_struct *task)
 	 * We probably want to make this prettier in the future.
 	 */
 	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||
-	    worker->task->se.sum_exec_runtime - worker->current_at <
+	    tsk_seruntime(worker->task) - worker->current_at <
 	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)
 		return;
 
@@ -2449,6 +2450,7 @@ bool queue_work_on(int cpu, struct workqueue_struct *wq,
 
 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
 	    !clear_pending_if_disabled(work)) {
+		muqss_work_set_owner(work);
 		__queue_work(cpu, wq, work);
 		ret = true;
 	}
@@ -2530,6 +2532,7 @@ bool queue_work_node(int node, struct workqueue_struct *wq,
 	    !clear_pending_if_disabled(work)) {
 		int cpu = select_numa_node_cpu(node);
 
+		muqss_work_set_owner(work);
 		__queue_work(cpu, wq, work);
 		ret = true;
 	}
@@ -2617,6 +2620,7 @@ bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
 
 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
 	    !clear_pending_if_disabled(work)) {
+		muqss_work_set_owner(work);
 		__queue_delayed_work(cpu, wq, dwork, delay);
 		ret = true;
 	}
@@ -2652,8 +2656,10 @@ bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
 
 	ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags);
 
-	if (!clear_pending_if_disabled(&dwork->work))
+	if (!clear_pending_if_disabled(&dwork->work)) {
+		muqss_work_set_owner(&dwork->work);
 		__queue_delayed_work(cpu, wq, dwork, delay);
+	}
 
 	local_irq_restore(irq_flags);
 	return ret;
@@ -2690,6 +2696,7 @@ bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
 	 */
 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&
 	    !WARN_ON_ONCE(clear_pending_if_disabled(work))) {
+		muqss_work_set_owner(work);
 		rwork->wq = wq;
 		call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
 		return true;
@@ -3223,7 +3230,9 @@ __acquires(&pool->lock)
 {
 	struct pool_workqueue *pwq = get_work_pwq(work);
 	struct worker_pool *pool = worker->pool;
+	struct muqss_kern_window kern_window;
 	unsigned long work_data;
+	unsigned int muqss_owner;
 	int lockdep_start_depth, rcu_start_depth;
 	bool bh_draining = pool->flags & POOL_BH_DRAINING;
 #ifdef CONFIG_LOCKDEP
@@ -3249,11 +3258,23 @@ __acquires(&pool->lock)
 	worker->current_func = work->func;
 	worker->current_pwq = pwq;
 	if (worker->task)
-		worker->current_at = worker->task->se.sum_exec_runtime;
+		worker->current_at = tsk_seruntime(worker->task);
 	worker->current_start = jiffies;
 	work_data = *work_data_bits(work);
 	worker->current_color = get_work_color(work_data);
 
+	/*
+	 * Read while PENDING still holds this work item for us. Once it is
+	 * cleared below another CPU may requeue the same item and restamp it
+	 * with a different owner, and the one we are about to run belongs to
+	 * whoever queued it, not to them.
+	 *
+	 * A BH pool runs work in softirq context on top of whichever task was
+	 * interrupted, so worker->task is NULL and there is no worker runtime
+	 * to measure. Charge nobody there.
+	 */
+	muqss_owner = worker->task ? muqss_work_owner(work) : 0;
+
 	/*
 	 * Record wq name for cmdline and debug reporting, may get
 	 * overridden through set_worker_desc().
@@ -3318,6 +3339,13 @@ __acquires(&pool->lock)
 	 * workqueues), so hiding them isn't a problem.
 	 */
 	lockdep_invariant_state(true);
+	/*
+	 * Everything from here to muqss_kerntime_end() is done on the queueing
+	 * task's account: the CPU time it costs this worker is charged back to
+	 * it, and so is any I/O the work item submits or any page it dirties,
+	 * which would otherwise be lost to a kernel thread.
+	 */
+	muqss_kerntime_begin(&kern_window, muqss_owner);
 	trace_workqueue_execute_start(work);
 	worker->current_func(work);
 	/*
@@ -3325,6 +3353,7 @@ __acquires(&pool->lock)
 	 * point will only record its address.
 	 */
 	trace_workqueue_execute_end(work, worker->current_func);
+	muqss_kerntime_end(&kern_window);
 
 	lock_map_release(&lockdep_map);
 	if (!bh_draining)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -969,7 +969,7 @@ static void khugepaged_alloc_sleep(void)
 
 	add_wait_queue(&khugepaged_wait, &wait);
 	__set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
-	schedule_timeout(msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
+	schedule_msec_hrtimeout(khugepaged_alloc_sleep_millisecs);
 	remove_wait_queue(&khugepaged_wait, &wait);
 }
 
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -38,6 +38,7 @@
 #include <linux/sched/rt.h>
 #include <linux/sched/signal.h>
 #include <linux/mm_inline.h>
+#include <linux/muqss_iotime.h>
 #include <linux/shmem_fs.h>
 #include <trace/events/writeback.h>
 
@@ -2636,6 +2637,12 @@ static void folio_account_dirtied(struct folio *folio,
 		current->nr_dirtied += nr;
 		__this_cpu_add(bdp_ratelimits, nr);
 
+		/*
+		 * Record who dirtied this so the writeback issued for it later
+		 * by a flusher thread can still be charged to them.
+		 */
+		muqss_iotime_dirty_folio(folio);
+
 		mem_cgroup_track_foreign_dirty(folio, wb);
 	}
 }
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1401,6 +1401,7 @@ static __always_inline bool __free_pages_prepare(struct page *page,
 	}
 
 	page_cpupid_reset_last(page);
+	page_io_owner_reset(page);
 	page->flags.f &= ~PAGE_FLAGS_CHECK_AT_PREP;
 	page->private = 0;
 	reset_page_owner(page, order);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -1992,7 +1992,7 @@ static ssize_t pktgen_thread_write(struct file *file,
 		mutex_lock(&pktgen_thread_lock);
 		t->control |= T_REMDEVALL;
 		mutex_unlock(&pktgen_thread_lock);
-		schedule_timeout_interruptible(msecs_to_jiffies(125));	/* Propagate thread->control  */
+		schedule_msec_hrtimeout_interruptible(125);	/* Propagate thread->control  */
 		ret = count;
 		sprintf(pg_result, "OK: rem_device_all");
 		goto out;
@@ -2064,7 +2064,7 @@ static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
 		mutex_unlock(&pktgen_thread_lock);
 		pr_debug("%s: waiting for %s to disappear....\n",
 			 __func__, ifname);
-		schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
+		schedule_msec_hrtimeout_interruptible(msec_per_try);
 		mutex_lock(&pktgen_thread_lock);
 
 		if (++i >= max_tries) {
@@ -3321,7 +3321,7 @@ static void pktgen_run_all_threads(struct pktgen_net *pn)
 	pktgen_handle_all_threads(pn, T_RUN);
 
 	/* Propagate thread->control  */
-	schedule_timeout_interruptible(msecs_to_jiffies(125));
+	schedule_msec_hrtimeout_interruptible(125);
 
 	pktgen_wait_all_threads_run(pn);
 }
@@ -3333,7 +3333,7 @@ static void pktgen_reset_all_threads(struct pktgen_net *pn)
 	pktgen_handle_all_threads(pn, T_REMDEVALL);
 
 	/* Propagate thread->control  */
-	schedule_timeout_interruptible(msecs_to_jiffies(125));
+	schedule_msec_hrtimeout_interruptible(125);
 
 	pktgen_wait_all_threads_run(pn);
 }
diff --git a/sound/hda/core/controller.c b/sound/hda/core/controller.c
--- a/sound/hda/core/controller.c
+++ b/sound/hda/core/controller.c
@@ -340,7 +340,7 @@ static int snd_hdac_bus_get_response_rirb(struct hdac_bus *bus,
 			break;
 #define LOOP_COUNT_MAX	3000
 		if (!bus->polling_mode) {
-			schedule_timeout(msecs_to_jiffies(2));
+			schedule_msec_hrtimeout(2);
 		} else if (bus->needs_damn_long_delay ||
 			   loopcounter > LOOP_COUNT_MAX) {
 			if (loopcounter > LOOP_COUNT_MAX && !warned) {
diff --git a/sound/isa/msnd/msnd_pinnacle.c b/sound/isa/msnd/msnd_pinnacle.c
--- a/sound/isa/msnd/msnd_pinnacle.c
+++ b/sound/isa/msnd/msnd_pinnacle.c
@@ -490,7 +490,7 @@ static int snd_msnd_calibrate_adc(struct snd_msnd *chip, u16 srate)
 		       & ~0x0001, chip->SMA + SMA_wCurrHostStatusFlags);
 	if (snd_msnd_send_word(chip, 0, 0, HDEXAR_CAL_A_TO_D) == 0 &&
 	    snd_msnd_send_dsp_cmd_chk(chip, HDEX_AUX_REQ) == 0) {
-		schedule_timeout_interruptible(msecs_to_jiffies(333));
+		schedule_msec_hrtimeout_interruptible(333);
 		return 0;
 	}
 	dev_warn(chip->card->dev, LOGNAME ": ADC calibration failed\n");
diff --git a/sound/pci/maestro3.c b/sound/pci/maestro3.c
--- a/sound/pci/maestro3.c
+++ b/sound/pci/maestro3.c
@@ -1999,7 +1999,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip)
 		outw(0, io + GPIO_DATA);
 		outw(dir | GPO_PRIMARY_AC97, io + GPIO_DIRECTION);
 
-		schedule_timeout_uninterruptible(msecs_to_jiffies(delay1));
+		schedule_msec_hrtimeout_uninterruptible(delay1);
 
 		outw(GPO_PRIMARY_AC97, io + GPIO_DATA);
 		udelay(5);
@@ -2007,7 +2007,7 @@ static void snd_m3_ac97_reset(struct snd_m3 *chip)
 		outw(IO_SRAM_ENABLE | SERIAL_AC_LINK_ENABLE, io + RING_BUS_CTRL_A);
 		outw(~0, io + GPIO_MASK);
 
-		schedule_timeout_uninterruptible(msecs_to_jiffies(delay2));
+		schedule_msec_hrtimeout_uninterruptible(delay2);
 
 		if (! snd_m3_try_read_vendor(chip))
 			break;
@@ -2054,7 +2054,7 @@ static int snd_m3_mixer(struct snd_m3 *chip)
 
 	/* seems ac97 PCM needs initialization.. hack hack.. */
 	snd_ac97_write(chip->ac97, AC97_PCM, 0x8000 | (15 << 8) | 15);
-	schedule_timeout_uninterruptible(msecs_to_jiffies(100));
+	schedule_msec_hrtimeout_uninterruptible(100);
 	snd_ac97_write(chip->ac97, AC97_PCM, 0);
 
 #ifndef CONFIG_SND_MAESTRO3_INPUT
diff --git a/sound/pci/ymfpci/ymfpci_main.c b/sound/pci/ymfpci/ymfpci_main.c
--- a/sound/pci/ymfpci/ymfpci_main.c
+++ b/sound/pci/ymfpci/ymfpci_main.c
@@ -742,7 +742,7 @@ static void snd_ymfpci_irq_wait(struct snd_ymfpci *chip)
 		init_waitqueue_entry(&wait, current);
 		add_wait_queue(&chip->interrupt_sleep, &wait);
 		atomic_inc(&chip->interrupt_sleep_count);
-		schedule_timeout_uninterruptible(msecs_to_jiffies(50));
+		schedule_msec_hrtimeout_uninterruptible(50);
 		remove_wait_queue(&chip->interrupt_sleep, &wait);
 	}
 }
diff --git a/sound/soc/codecs/rt5631.c b/sound/soc/codecs/rt5631.c
--- a/sound/soc/codecs/rt5631.c
+++ b/sound/soc/codecs/rt5631.c
@@ -417,7 +417,7 @@ static void onebit_depop_mute_stage(struct snd_soc_component *component, int ena
 	hp_zc = snd_soc_component_read(component, RT5631_INT_ST_IRQ_CTRL_2);
 	snd_soc_component_write(component, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff);
 	if (enable) {
-		schedule_timeout_uninterruptible(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout_uninterruptible(10);
 		/* config one-bit depop parameter */
 		rt5631_write_index(component, RT5631_SPK_INTL_CTRL, 0x307f);
 		snd_soc_component_update_bits(component, RT5631_HP_OUT_VOL,
@@ -529,7 +529,7 @@ static void depop_seq_mute_stage(struct snd_soc_component *component, int enable
 	hp_zc = snd_soc_component_read(component, RT5631_INT_ST_IRQ_CTRL_2);
 	snd_soc_component_write(component, RT5631_INT_ST_IRQ_CTRL_2, hp_zc & 0xf7ff);
 	if (enable) {
-		schedule_timeout_uninterruptible(msecs_to_jiffies(10));
+		schedule_msec_hrtimeout_uninterruptible(10);
 
 		/* config depop sequence parameter */
 		rt5631_write_index(component, RT5631_SPK_INTL_CTRL, 0x302f);
@@ -1496,8 +1496,7 @@ static int rt5631_codec_set_dai_pll(struct snd_soc_dai *codec_dai, int pll_id,
 					"change PLL in master mode\n");
 				snd_soc_component_write(component, RT5631_PLL_CTRL,
 					codec_master_pll_div[i].reg_val);
-				schedule_timeout_uninterruptible(
-					msecs_to_jiffies(20));
+				schedule_msec_hrtimeout_uninterruptible(20);
 				snd_soc_component_update_bits(component,
 					RT5631_GLOBAL_CLK_CTRL,
 					RT5631_SYSCLK_SOUR_SEL_MASK |
@@ -1515,8 +1514,7 @@ static int rt5631_codec_set_dai_pll(struct snd_soc_dai *codec_dai, int pll_id,
 					"change PLL in slave mode\n");
 				snd_soc_component_write(component, RT5631_PLL_CTRL,
 					codec_slave_pll_div[i].reg_val);
-				schedule_timeout_uninterruptible(
-					msecs_to_jiffies(20));
+				schedule_msec_hrtimeout_uninterruptible(20);
 				snd_soc_component_update_bits(component,
 					RT5631_GLOBAL_CLK_CTRL,
 					RT5631_SYSCLK_SOUR_SEL_MASK |
diff --git a/sound/soc/codecs/wm8350.c b/sound/soc/codecs/wm8350.c
--- a/sound/soc/codecs/wm8350.c
+++ b/sound/soc/codecs/wm8350.c
@@ -234,11 +234,9 @@ static void wm8350_pga_work(struct work_struct *work)
 		    out2->ramp == WM8350_RAMP_UP) {
 			/* delay is longer over 0dB as increases are larger */
 			if (i >= WM8350_OUTn_0dB)
-				schedule_timeout_interruptible(msecs_to_jiffies
-							       (2));
+				schedule_msec_hrtimeout_interruptible(2);
 			else
-				schedule_timeout_interruptible(msecs_to_jiffies
-							       (1));
+				schedule_msec_hrtimeout_interruptible(1);
 		} else
 			udelay(50);	/* doesn't matter if we delay longer */
 	}
@@ -1122,9 +1120,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component,
 					 (platform->dis_out4 << 6));
 
 			/* wait for discharge */
-			schedule_timeout_interruptible(msecs_to_jiffies
-						       (platform->
-							cap_discharge_msecs));
+			schedule_msec_hrtimeout_interruptible(platform->cap_discharge_msecs);
 
 			/* enable antipop */
 			wm8350_reg_write(wm8350, WM8350_ANTI_POP_CONTROL,
@@ -1138,9 +1134,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component,
 					 WM8350_VBUFEN);
 
 			/* wait for vmid */
-			schedule_timeout_interruptible(msecs_to_jiffies
-						       (platform->
-							vmid_charge_msecs));
+			schedule_msec_hrtimeout_interruptible(platform->vmid_charge_msecs);
 
 			/* turn on vmid 300k  */
 			pm1 = wm8350_reg_read(wm8350, WM8350_POWER_MGMT_1) &
@@ -1189,9 +1183,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component,
 		wm8350_reg_write(wm8350, WM8350_POWER_MGMT_1, pm1);
 
 		/* wait */
-		schedule_timeout_interruptible(msecs_to_jiffies
-					       (platform->
-						vmid_discharge_msecs));
+		schedule_msec_hrtimeout_interruptible(platform->vmid_discharge_msecs);
 
 		wm8350_reg_write(wm8350, WM8350_ANTI_POP_CONTROL,
 				 (platform->vmid_s_curve << 8) |
@@ -1207,8 +1199,7 @@ static int wm8350_set_bias_level(struct snd_soc_component *component,
 				 pm1 | WM8350_OUTPUT_DRAIN_EN);
 
 		/* wait */
-		schedule_timeout_interruptible(msecs_to_jiffies
-					       (platform->drain_msecs));
+		schedule_msec_hrtimeout_interruptible(platform->drain_msecs);
 
 		pm1 &= ~WM8350_BIASEN;
 		wm8350_reg_write(wm8350, WM8350_POWER_MGMT_1, pm1);
diff --git a/sound/soc/codecs/wm8900.c b/sound/soc/codecs/wm8900.c
--- a/sound/soc/codecs/wm8900.c
+++ b/sound/soc/codecs/wm8900.c
@@ -1057,7 +1057,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component,
 				     WM8900_REG_POWER1_STARTUP_BIAS_ENA | 0x1);
 
 			/* Allow capacitors to charge */
-			schedule_timeout_interruptible(msecs_to_jiffies(400));
+			schedule_msec_hrtimeout_interruptible(400);
 
 			/* Enable bias */
 			snd_soc_component_write(component, WM8900_REG_POWER1,
@@ -1091,7 +1091,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component,
 		/* Discharge caps */
 		snd_soc_component_write(component, WM8900_REG_POWER1,
 			     WM8900_REG_POWER1_STARTUP_BIAS_ENA);
-		schedule_timeout_interruptible(msecs_to_jiffies(500));
+		schedule_msec_hrtimeout_interruptible(500);
 
 		/* Remove clamp */
 		snd_soc_component_write(component, WM8900_REG_HPCTL1, 0);
@@ -1105,7 +1105,7 @@ static int wm8900_set_bias_level(struct snd_soc_component *component,
 		/* Need to let things settle before stopping the clock
 		 * to ensure that restart works, see "Stopping the
 		 * master clock" in the datasheet. */
-		schedule_timeout_interruptible(msecs_to_jiffies(1));
+		schedule_msec_hrtimeout_interruptible(1);
 		snd_soc_component_write(component, WM8900_REG_POWER2,
 			     WM8900_REG_POWER2_SYSCLK_ENA);
 		break;
diff --git a/sound/soc/codecs/wm9713.c b/sound/soc/codecs/wm9713.c
--- a/sound/soc/codecs/wm9713.c
+++ b/sound/soc/codecs/wm9713.c
@@ -199,7 +199,7 @@ static int wm9713_voice_shutdown(struct snd_soc_dapm_widget *w,
 
 	/* Gracefully shut down the voice interface. */
 	snd_soc_component_update_bits(component, AC97_HANDSET_RATE, 0x0f00, 0x0200);
-	schedule_timeout_interruptible(msecs_to_jiffies(1));
+	schedule_msec_hrtimeout_interruptible(1);
 	snd_soc_component_update_bits(component, AC97_HANDSET_RATE, 0x0f00, 0x0f00);
 	snd_soc_component_update_bits(component, AC97_EXTENDED_MID, 0x1000, 0x1000);
 
@@ -864,7 +864,7 @@ static int wm9713_set_pll(struct snd_soc_component *component,
 	wm9713->pll_in = freq_in;
 
 	/* wait 10ms AC97 link frames for the link to stabilise */
-	schedule_timeout_interruptible(msecs_to_jiffies(10));
+	schedule_msec_hrtimeout_interruptible(10);
 	return 0;
 }
 
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -166,7 +166,7 @@ static void dapm_assert_locked(struct snd_soc_dapm_context *dapm)
 static void dapm_pop_wait(void)
 {
 	if (pop_time)
-		schedule_timeout_uninterruptible(msecs_to_jiffies(pop_time));
+		schedule_msec_hrtimeout_uninterruptible(pop_time);
 }
 
 __printf(2, 3)
diff --git a/sound/usb/line6/pcm.c b/sound/usb/line6/pcm.c
--- a/sound/usb/line6/pcm.c
+++ b/sound/usb/line6/pcm.c
@@ -127,7 +127,7 @@ static void line6_wait_clear_audio_urbs(struct snd_line6_pcm *line6pcm,
 		if (!alive)
 			break;
 		set_current_state(TASK_UNINTERRUPTIBLE);
-		schedule_timeout(1);
+		schedule_min_hrtimeout();
 	} while (--timeout > 0);
 	if (alive)
 		dev_err(line6pcm->line6->ifcdev,
diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile
--- a/tools/objtool/Makefile
+++ b/tools/objtool/Makefile
@@ -101,9 +101,19 @@ LIBOPCODES_LIBS := $(shell \
 HAVE_DISASM_STYLED := $(shell echo '$(pound)include <dis-asm.h>' | \
 			$(HOSTCC) -E -xc - 2>/dev/null | grep -q disassembler_style && echo y)
 
+#
+# Linking -lopcodes can succeed when the shared library is present without
+# development headers. That is common on Gentoo without sys-libs/binutils-libs
+# (bfd.h lives under /usr/lib64/binutils/<tuple>/<ver>/include, not
+# /usr/include). Do not enable DISAS unless the headers actually compile.
+#
+HAVE_DISAS_HEADERS := $(shell printf "$(pound)include <bfd.h>\n$(pound)include <dis-asm.h>\nint main(void) { return 0; }\n" | \
+			$(HOSTCC) $(HOSTCFLAGS) -DPACKAGE='"objtool"' -xc - -c -o /dev/null 2>/dev/null && echo y)
+
 BUILD_DISAS := n
 
 ifneq ($(LIBOPCODES_LIBS),)
+ifeq ($(HAVE_DISAS_HEADERS),y)
 	BUILD_DISAS := y
 	OBJTOOL_CFLAGS += -DDISAS -DPACKAGE='"objtool"'
 	OBJTOOL_LDFLAGS += $(LIBOPCODES_LIBS)
@@ -111,6 +121,7 @@ ifeq ($(HAVE_DISASM_STYLED),y)
 	OBJTOOL_CFLAGS += -DDISASM_INIT_STYLED
 endif
 endif
+endif
 
 export BUILD_DISAS