ZacharyZhang-NY/omarchy-pkgs
1
0 Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

Add new base and BORE-enabled Linux kernel packages for Omarchy

Browse Source 
Signed-off-by: Krzysztof Wilczyński <kwilczynski@omarchy.org>
This commit is contained in:
Krzysztof Wilczyński committed 2026-09-15 00:53:12 +09:00
1 parent 61c8f08aea
commit 770244c0e7
195 files changed
+190517
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkgbuilds/linux-omarchy/0121-smp-preempt.patch
+514
View File
@@ -0,0 +1,514 @@
diff --git a/include/linux/sched.h b/include/linux/sched.h
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -823,6 +823,17 @@ struct kmap_ctrl {
#endif
};
+#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPTION)
+struct task_ipi_mask {
+ union {
+ cpumask_t *ipi_mask_ptr;
+ unsigned long ipi_mask_val;
+ };
+};
+#else
+struct task_ipi_mask { };
+#endif
+
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
@@ -1359,6 +1370,7 @@ struct task_struct {
struct list_head perf_event_list;
struct perf_ctx_data __rcu *perf_ctx_data;
#endif
+ struct task_ipi_mask __private ipi_mask;
#ifdef CONFIG_DEBUG_PREEMPT
unsigned long preempt_disable_ip;
#endif
diff --git a/include/linux/smp.h b/include/linux/smp.h
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -47,8 +47,7 @@ extern void __smp_call_single_queue(int cpu, struct llist_node *node);
/* total number of cpus in this system (may exceed NR_CPUS) */
extern unsigned int total_cpus;
-int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,
- int wait);
+int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, bool wait);
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
void *info, bool wait, const struct cpumask *mask);
@@ -239,6 +238,18 @@ static inline int get_boot_cpu_id(void)
#endif /* !SMP */
+#if defined(CONFIG_PREEMPTION) && defined(CONFIG_SMP)
+int smp_task_ipi_mask_alloc(struct task_struct *task);
+void smp_task_ipi_mask_free(struct task_struct *task);
+#else
+static inline int smp_task_ipi_mask_alloc(struct task_struct *task)
+{
+ return 0;
+}
+
+static inline void smp_task_ipi_mask_free(struct task_struct *task) { }
+#endif
+
/*
* raw_smp_processor_id() - get the current (unstable) CPU id
*
diff --git a/kernel/fork.c b/kernel/fork.c
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -543,6 +543,7 @@ void free_task(struct task_struct *tsk)
#endif
release_user_cpus_ptr(tsk);
scs_release(tsk);
+ smp_task_ipi_mask_free(tsk);
#ifndef CONFIG_THREAD_INFO_IN_TASK
/*
@@ -941,10 +942,14 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
account_kernel_stack(tsk, 1);
- err = scs_prepare(tsk, node);
+ err = smp_task_ipi_mask_alloc(tsk);
if (err)
goto free_stack;
+ err = scs_prepare(tsk, node);
+ if (err)
+ goto free_ipi_mask;
+
#ifdef CONFIG_SECCOMP
/*
* We must handle setting up seccomp filters once we're under
@@ -1022,6 +1027,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
return tsk;
+free_ipi_mask:
+ smp_task_ipi_mask_free(tsk);
free_stack:
exit_task_stack_account(tsk);
free_thread_stack(tsk);
diff --git a/kernel/scftorture.c b/kernel/scftorture.c
--- a/kernel/scftorture.c
+++ b/kernel/scftorture.c
@@ -348,6 +348,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
int ret = 0;
struct scf_check *scfcp = NULL;
struct scf_selector *scfsp = scf_sel_rand(trsp);
+ bool is_single = (scfsp->scfs_prim == SCF_PRIM_SINGLE ||
+ scfsp->scfs_prim == SCF_PRIM_SINGLE_RPC);
if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) {
scfcp = kmalloc_obj(*scfcp, GFP_ATOMIC);
@@ -364,8 +366,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
}
if (use_cpus_read_lock)
cpus_read_lock();
- else
- preempt_disable();
switch (scfsp->scfs_prim) {
case SCF_PRIM_RESCHED:
if (IS_BUILTIN(CONFIG_SCF_TORTURE_TEST)) {
@@ -411,13 +411,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
if (!ret) {
if (use_cpus_read_lock)
cpus_read_unlock();
- else
- preempt_enable();
+
wait_for_completion(&scfcp->scfc_completion);
if (use_cpus_read_lock)
cpus_read_lock();
- else
- preempt_disable();
} else {
scfp->n_single_rpc_ofl++;
scf_add_to_free_list(scfcp);
@@ -452,7 +449,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
scfcp->scfc_out = true;
}
if (scfcp && scfsp->scfs_wait) {
- if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) &&
+ if (WARN_ON_ONCE(((use_cpus_read_lock && num_online_cpus() > 1) || is_single) &&
!scfcp->scfc_out)) {
pr_warn("%s: Memory-ordering failure, scfs_prim: %d.\n", __func__, scfsp->scfs_prim);
atomic_inc(&n_mb_out_errs); // Leak rather than trash!
@@ -463,8 +460,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra
}
if (use_cpus_read_lock)
cpus_read_unlock();
- else
- preempt_enable();
if (allocfail)
schedule_timeout_idle((1 + longwait) * HZ); // Let no-wait handlers complete.
else if (!(torture_random(trsp) & 0xfff))
diff --git a/kernel/smp.c b/kernel/smp.c
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -16,6 +16,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/gfp.h>
+#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/sched.h>
@@ -63,7 +64,14 @@ int smpcfd_prepare_cpu(unsigned int cpu)
free_cpumask_var(cfd->cpumask);
return -ENOMEM;
}
- cfd->csd = alloc_percpu(call_single_data_t);
+
+ /*
+ * Allocate the per-CPU CSD the first time a CPU comes up. It is
+ * not freed when the CPU is offlined, so csd_lock_wait() can access
+ * it even when the CPU was offlined after preemption was re-enabled.
+ */
+ if (!cfd->csd)
+ cfd->csd = alloc_percpu(call_single_data_t);
if (!cfd->csd) {
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
@@ -79,7 +87,6 @@ int smpcfd_dead_cpu(unsigned int cpu)
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
- free_percpu(cfd->csd);
return 0;
}
@@ -323,6 +330,8 @@ static void __csd_lock_wait(call_single_data_t *csd)
int bug_id = 0;
u64 ts0, ts1;
+ guard(preempt)();
+
ts1 = ts0 = ktime_get_mono_fast_ns();
for (;;) {
if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
@@ -659,17 +668,9 @@ void flush_smp_call_function_queue(void)
local_irq_restore(flags);
}
-/**
- * smp_call_function_single - Run a function on a specific CPU
- * @cpu: Specific target CPU for this function.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Returns: %0 on success, else a negative status code.
- */
-int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
- int wait)
+static int __smp_call_function_single(int cpu, smp_call_func_t func,
+ void *info, const struct cpumask *mask,
+ bool wait)
{
call_single_data_t *csd;
call_single_data_t csd_stack = {
@@ -686,6 +687,14 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
*/
this_cpu = get_cpu();
+ if (mask) {
+ /* Try for same CPU (cheapest) */
+ if (!cpumask_test_cpu(this_cpu, mask))
+ cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu));
+ else
+ cpu = this_cpu;
+ }
+
/*
* Can deadlock when called with interrupts disabled.
* We allow cpu's that are not yet online though, as no one else can
@@ -718,13 +727,32 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
err = generic_exec_single(cpu, csd);
+ /*
+ * @csd is stack-allocated when @wait is true. No concurrent access
+ * except from the IPI completion path, so we can re-enable preemption
+ * early to reduce latency.
+ */
+ put_cpu();
+
if (wait)
csd_lock_wait(csd);
- put_cpu();
-
return err;
}
+
+/**
+ * smp_call_function_single - Run a function on a specific CPU
+ * @cpu: Specific target CPU for this function.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait until function has completed on other CPUs.
+ *
+ * Returns: %0 on success, else a negative status code.
+ */
+int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait)
+{
+ return __smp_call_function_single(cpu, func, info, NULL, wait);
+}
EXPORT_SYMBOL(smp_call_function_single);
/**
@@ -775,10 +803,10 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async);
/**
* smp_call_function_any - Run a function on any of the given cpus
- * @mask: The mask of cpus it can run on.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait until function has completed.
+ * @mask: The mask of cpus it can run on.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait until function has completed.
*
* Selection preference:
* 1) current cpu if in @mask
@@ -789,20 +817,54 @@ EXPORT_SYMBOL_GPL(smp_call_function_single_async);
int smp_call_function_any(const struct cpumask *mask,
smp_call_func_t func, void *info, int wait)
{
- unsigned int cpu;
- int ret;
-
- /* Try for same CPU (cheapest) */
- cpu = get_cpu();
- if (!cpumask_test_cpu(cpu, mask))
- cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu));
-
- ret = smp_call_function_single(cpu, func, info, wait);
- put_cpu();
- return ret;
+ return __smp_call_function_single(-1, func, info, mask, wait);
}
EXPORT_SYMBOL_GPL(smp_call_function_any);
+static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined);
+
+#ifdef CONFIG_PREEMPTION
+
+int smp_task_ipi_mask_alloc(struct task_struct *task)
+{
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return 0;
+
+ ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr =
+ kmalloc(cpumask_size(), GFP_KERNEL);
+ if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void smp_task_ipi_mask_free(struct task_struct *task)
+{
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return;
+
+ kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr);
+}
+
+static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
+{
+ /*
+ * If cpumask_size() is smaller than or equal to the pointer
+ * size, it stashes the cpumask in the pointer itself to
+ * avoid extra memory allocations.
+ */
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val;
+
+ return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr;
+}
+#else
+static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
+{
+ return NULL;
+}
+#endif
+
/*
* Flags to be used as scf_flags argument of smp_call_function_many_cond().
*
@@ -817,13 +879,20 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
unsigned int scf_flags,
smp_cond_func_t cond_func)
{
- int cpu, last_cpu, this_cpu = smp_processor_id();
- struct call_function_data *cfd;
+ struct cpumask *cpumask, *task_mask;
bool wait = scf_flags & SCF_WAIT;
- int nr_cpus = 0;
+ struct call_function_data *cfd;
+ int cpu, last_cpu, this_cpu;
bool run_remote = false;
+ int nr_cpus = 0;
- lockdep_assert_preemption_disabled();
+ this_cpu = get_cpu();
+ cfd = this_cpu_ptr(&cfd_data);
+ task_mask = smp_task_ipi_mask(current);
+ if (task_mask)
+ cpumask = task_mask;
+ else
+ cpumask = cfd->cpumask;
/*
* Can deadlock when called with interrupts disabled.
@@ -845,16 +914,15 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
/* Check if we need remote execution, i.e., any CPU excluding this one. */
if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) {
- cfd = this_cpu_ptr(&cfd_data);
- cpumask_and(cfd->cpumask, mask, cpu_online_mask);
- __cpumask_clear_cpu(this_cpu, cfd->cpumask);
+ cpumask_and(cpumask, mask, cpu_online_mask);
+ __cpumask_clear_cpu(this_cpu, cpumask);
cpumask_clear(cfd->cpumask_ipi);
- for_each_cpu(cpu, cfd->cpumask) {
+ for_each_cpu(cpu, cpumask) {
call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
if (cond_func && !cond_func(cpu, info)) {
- __cpumask_clear_cpu(cpu, cfd->cpumask);
+ __cpumask_clear_cpu(cpu, cpumask);
continue;
}
@@ -904,8 +972,18 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
local_irq_restore(flags);
}
+ /*
+ * The IPI work has been queued and dispatched. On PREEMPT kernels,
+ * tasks created through dup_task_struct() have task-local wait masks.
+ * The boot init_task can fall back to cfd->cpumask when the mask is
+ * not inlined, but other tasks still use task-local masks and cannot
+ * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule
+ * another task, so the per-CPU mask remains protected.
+ */
+ put_cpu();
+
if (run_remote && wait) {
- for_each_cpu(cpu, cfd->cpumask) {
+ for_each_cpu(cpu, cpumask) {
call_single_data_t *csd;
csd = per_cpu_ptr(cfd->csd, cpu);
@@ -916,15 +994,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
/**
* smp_call_function_many() - Run a function on a set of CPUs.
- * @mask: The set of cpus to run on (only runs on online subset).
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed
- * on other CPUs.
+ * @mask: The set of cpus to run on (only runs on online subset).
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed
+ * on other CPUs.
*
* You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler. Preemption
- * must be disabled when calling this function.
+ * hardware interrupt handler or from a bottom half handler.
*
* @func is not called on the local CPU even if @mask contains it. Consider
* using on_each_cpu_cond_mask() instead if this is not desirable.
@@ -938,10 +1015,10 @@ EXPORT_SYMBOL(smp_call_function_many);
/**
* smp_call_function() - Run a function on all other CPUs.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed
- * on other CPUs.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed
+ * on other CPUs.
*
* If @wait is true, then returns once @func has returned; otherwise
* it returns just before the target cpu calls @func.
@@ -951,9 +1028,8 @@ EXPORT_SYMBOL(smp_call_function_many);
*/
void smp_call_function(smp_call_func_t func, void *info, int wait)
{
- preempt_disable();
- smp_call_function_many(cpu_online_mask, func, info, wait);
- preempt_enable();
+ smp_call_function_many_cond(cpu_online_mask, func, info,
+ wait ? SCF_WAIT : 0, NULL);
}
EXPORT_SYMBOL(smp_call_function);
@@ -1019,6 +1095,9 @@ EXPORT_SYMBOL(nr_cpu_ids);
void __init setup_nr_cpu_ids(void)
{
set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
+
+ if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long))
+ static_branch_enable(&ipi_mask_inlined);
}
/* Called by boot processor to activate the rest. */
@@ -1055,12 +1134,14 @@ void __init smp_init(void)
* @func: The function to run on all applicable CPUs.
* This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to both functions.
- * @wait: If true, wait (atomically) until function has
- * completed on other CPUs.
+ * @wait: If true, wait until function has completed on other CPUs.
* @mask: The set of cpus to run on (only runs on online subset).
*
- * Preemption is disabled to protect against CPUs going offline but not online.
- * CPUs going online during the call will not be seen or sent an IPI.
+ * Target CPU selection and work queueing are done with preemption
+ * disabled. This protects against CPUs going offline, but not against
+ * CPUs coming online concurrently; newly online CPUs are not guaranteed
+ * to be seen or sent an IPI. If @wait is true, the final wait for remote
+ * completion happens after that preemption-disabled section.
*
* You must not call this function with disabled interrupts or
* from a hardware interrupt handler or from a bottom half handler.
@@ -1073,9 +1154,7 @@ void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
if (wait)
scf_flags |= SCF_WAIT;
- preempt_disable();
smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
- preempt_enable();
}
EXPORT_SYMBOL(on_each_cpu_cond_mask);
diff --git a/kernel/up.c b/kernel/up.c
--- a/kernel/up.c
+++ b/kernel/up.c
@@ -9,8 +9,7 @@
#include <linux/smp.h>
#include <linux/hypervisor.h>
-int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
- int wait)
+int smp_call_function_single(int cpu, void (*func)(void *info), void *info, bool wait)
{
unsigned long flags;