diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -50,7 +50,7 @@ void ptdump_walk_user_pgd_level_checkwx(void); extern spinlock_t pgd_lock; extern struct list_head pgd_list; -extern struct mm_struct *pgd_page_get_mm(struct page *page); +struct mm_struct *pgd_page_get_mm(struct ptdesc *pt); extern pmdval_t early_pmd_flags; diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h --- a/arch/x86/include/asm/pgtable_types.h +++ b/arch/x86/include/asm/pgtable_types.h @@ -512,7 +512,7 @@ static inline pgprot_t pgprot_large_2_4k(pgprot_t pgprot) return __pgprot(protval_large_2_4k(pgprot_val(pgprot))); } - +struct ptdesc; typedef struct page *pgtable_t; extern pteval_t __supported_pte_mask; diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -211,6 +212,12 @@ extern u16 invlpgb_count_max; extern void initialize_tlbstate_and_flush(void); +/* + * Keep stack-allocated flush_tlb_info cacheline aligned, but cap the + * alignment to avoid excessive stack usage on large-cacheline systems. + */ +#define FLUSH_TLB_INFO_ALIGN MIN(SMP_CACHE_BYTES, 64) + /* * TLB flushing: * @@ -249,7 +256,7 @@ struct flush_tlb_info { u8 stride_shift; u8 freed_tables; u8 trim_cpumask; -}; +} __aligned(FLUSH_TLB_INFO_ALIGN); void flush_tlb_local(void); void flush_tlb_one_user(unsigned long addr); diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -663,8 +663,10 @@ static void kvm_flush_tlb_multi(const struct cpumask *cpumask, u8 state; int cpu; struct kvm_steal_time *src; - struct cpumask *flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); + struct cpumask *flushmask; + guard(preempt)(); + flushmask = this_cpu_cpumask_var_ptr(__pv_cpu_mask); cpumask_copy(flushmask, cpumask); /* * We have to call flush only on online vCPUs. And diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -275,17 +275,17 @@ void arch_sync_kernel_mappings(unsigned long start, unsigned long end) for (addr = start & PMD_MASK; addr >= TASK_SIZE_MAX && addr < VMALLOC_END; addr += PMD_SIZE) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { spinlock_t *pgt_lock; /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); - vmalloc_sync_one(page_address(page), addr); + vmalloc_sync_one(ptdesc_address(ptdesc), addr); spin_unlock(pgt_lock); } spin_unlock(&pgd_lock); diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c @@ -136,7 +136,7 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { const pgd_t *pgd_ref = pgd_offset_k(addr); - struct page *page; + struct ptdesc *ptdesc; /* Check for overflow */ if (addr < start) @@ -146,13 +146,13 @@ static void sync_global_pgds_l5(unsigned long start, unsigned long end) continue; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; spinlock_t *pgt_lock; - pgd = (pgd_t *)page_address(page) + pgd_index(addr); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); if (!pgd_none(*pgd_ref) && !pgd_none(*pgd)) @@ -174,7 +174,7 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) { pgd_t *pgd_ref = pgd_offset_k(addr); const p4d_t *p4d_ref; - struct page *page; + struct ptdesc *ptdesc; /* * With folded p4d, pgd_none() is always false, we need to @@ -187,15 +187,15 @@ static void sync_global_pgds_l4(unsigned long start, unsigned long end) continue; spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; p4d_t *p4d; spinlock_t *pgt_lock; - pgd = (pgd_t *)page_address(page) + pgd_index(addr); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); p4d = p4d_offset(pgd, addr); /* the pgt_lock only for Xen */ - pgt_lock = &pgd_page_get_mm(page)->page_table_lock; + pgt_lock = &pgd_page_get_mm(ptdesc)->page_table_lock; spin_lock(pgt_lock); if (!p4d_none(*p4d_ref) && !p4d_none(*p4d)) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -88,9 +88,8 @@ static unsigned long direct_pages_count[PG_LEVEL_NUM]; void update_page_count(int level, unsigned long pages) { /* Protect against CPA */ - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); direct_pages_count[level] += pages; - spin_unlock(&pgd_lock); } static void split_page_count(int level) @@ -420,7 +419,7 @@ static void __cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; - spin_lock(&cpa_lock); + guard(spinlock)(&cpa_lock); if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) @@ -435,10 +434,8 @@ static void __cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) { - spin_unlock(&cpa_lock); + if (!collapsed) return; - } flush_tlb_all(); @@ -454,8 +451,6 @@ static void __cpa_collapse_large_pages(struct cpa_data *cpa) else pagetable_free(ptdesc); } - - spin_unlock(&cpa_lock); } static void cpa_collapse_large_pages(struct cpa_data *cpa) @@ -916,24 +911,23 @@ static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) { /* change init_mm */ set_pte_atomic(kpte, pte); -#ifdef CONFIG_X86_32 - { - struct page *page; - list_for_each_entry(page, &pgd_list, lru) { + if (IS_ENABLED(CONFIG_X86_32)) { + struct ptdesc *ptdesc; + + list_for_each_entry(ptdesc, &pgd_list, pt_list) { pgd_t *pgd; p4d_t *p4d; pud_t *pud; pmd_t *pmd; - pgd = (pgd_t *)page_address(page) + pgd_index(address); + pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(address); p4d = p4d_offset(pgd, address); pud = pud_offset(p4d, address); pmd = pmd_offset(pud, address); set_pte_atomic((pte_t *)pmd, pte); } } -#endif } static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot) @@ -1103,16 +1097,11 @@ static int __should_split_large_page(pte_t *kpte, unsigned long address, static int should_split_large_page(pte_t *kpte, unsigned long address, struct cpa_data *cpa) { - int do_split; - if (cpa->force_split) return 1; - spin_lock(&pgd_lock); - do_split = __should_split_large_page(kpte, address, cpa); - spin_unlock(&pgd_lock); - - return do_split; + guard(spinlock)(&pgd_lock); + return __should_split_large_page(kpte, address, cpa); } static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, @@ -1162,16 +1151,14 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, bool nx, rw; pte_t *tmp; - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); /* * Check for races, another CPU might have split this page * up for us already: */ tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); - if (tmp != kpte) { - spin_unlock(&pgd_lock); + if (tmp != kpte) return 1; - } paravirt_alloc_pte(&init_mm, page_to_pfn(base)); @@ -1204,7 +1191,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, break; default: - spin_unlock(&pgd_lock); return 1; } @@ -1252,7 +1238,6 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, * just split large page entry. */ flush_tlb_all(); - spin_unlock(&pgd_lock); return 0; } @@ -1327,11 +1312,11 @@ static int collapse_pmd_page(pmd_t *pmd, unsigned long addr, list_add(&page_ptdesc(pmd_page(old_pmd))->pt_list, pgtables); if (IS_ENABLED(CONFIG_X86_32)) { - struct page *page; + struct ptdesc *ptdesc; /* Update all PGD tables to use the same large page */ - list_for_each_entry(page, &pgd_list, lru) { - pgd_t *pgd = (pgd_t *)page_address(page) + pgd_index(addr); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + pgd_t *pgd = (pgd_t *)ptdesc_address(ptdesc) + pgd_index(addr); p4d_t *p4d = p4d_offset(pgd, addr); pud_t *pud = pud_offset(p4d, addr); pmd_t *pmd = pmd_offset(pud, addr); @@ -1405,7 +1390,7 @@ static int collapse_pud_page(pud_t *pud, unsigned long addr, */ static int collapse_large_pages(unsigned long addr, struct list_head *pgtables) { - int collapsed = 0; + int collapsed; pgd_t *pgd; p4d_t *p4d; pud_t *pud; @@ -1413,26 +1398,24 @@ static int collapse_large_pages(unsigned long addr, struct list_head *pgtables) addr &= PMD_MASK; - spin_lock(&pgd_lock); + guard(spinlock)(&pgd_lock); pgd = pgd_offset_k(addr); if (pgd_none(*pgd)) - goto out; + return 0; p4d = p4d_offset(pgd, addr); if (p4d_none(*p4d)) - goto out; + return 0; pud = pud_offset(p4d, addr); if (!pud_present(*pud) || pud_leaf(*pud)) - goto out; + return 0; pmd = pmd_offset(pud, addr); if (!pmd_present(*pmd) || pmd_leaf(*pmd)) - goto out; + return 0; collapsed = collapse_pmd_page(pmd, addr, pgtables); if (collapsed) collapsed += collapse_pud_page(pud, addr, pgtables); -out: - spin_unlock(&pgd_lock); return collapsed; } diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c @@ -74,9 +74,9 @@ static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) virt_to_ptdesc(pgd)->pt_mm = mm; } -struct mm_struct *pgd_page_get_mm(struct page *page) +struct mm_struct *pgd_page_get_mm(struct ptdesc *pt) { - return page_ptdesc(page)->pt_mm; + return pt->pt_mm; } static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -1373,35 +1373,19 @@ void flush_tlb_multi(const struct cpumask *cpumask, */ unsigned long tlb_single_page_flush_ceiling __read_mostly = 33; -static DEFINE_PER_CPU_SHARED_ALIGNED(struct flush_tlb_info, flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM -static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx); -#endif - -static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, - unsigned long start, unsigned long end, - unsigned int stride_shift, bool freed_tables, - u64 new_tlb_gen) +static void init_flush_tlb_info(struct flush_tlb_info *info, + struct mm_struct *mm, + unsigned long start, unsigned long end, + unsigned int stride_shift, bool freed_tables, + u64 new_tlb_gen) { - struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info); - -#ifdef CONFIG_DEBUG_VM - /* - * Ensure that the following code is non-reentrant and flush_tlb_info - * is not overwritten. This means no TLB flushing is initiated by - * interrupt handlers and machine-check exception handlers. - */ - BUG_ON(this_cpu_inc_return(flush_tlb_info_idx) != 1); -#endif - /* * If the number of flushes is so large that a full flush * would be faster, do a full flush. */ if ((end - start) >> stride_shift > tlb_single_page_flush_ceiling) { - start = 0; - end = TLB_FLUSH_ALL; + start = 0; + end = TLB_FLUSH_ALL; } info->start = start; @@ -1412,32 +1396,21 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, info->new_tlb_gen = new_tlb_gen; info->initiating_cpu = smp_processor_id(); info->trim_cpumask = 0; - - return info; -} - -static void put_flush_tlb_info(void) -{ -#ifdef CONFIG_DEBUG_VM - /* Complete reentrancy prevention checks */ - barrier(); - this_cpu_dec(flush_tlb_info_idx); -#endif } void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int stride_shift, bool freed_tables) { - struct flush_tlb_info *info; + struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); u64 new_tlb_gen; /* This is also a barrier that synchronizes with switch_mm(). */ new_tlb_gen = inc_mm_tlb_gen(mm); - info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables, - new_tlb_gen); + init_flush_tlb_info(&info, mm, start, end, stride_shift, freed_tables, new_tlb_gen); /* * flush_tlb_multi() is not optimized for the common case in which only @@ -1445,20 +1418,24 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, * flush_tlb_func_local() directly in this case. */ if (mm_global_asid(mm)) { - broadcast_tlb_flush(info); + broadcast_tlb_flush(&info); } else if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) { - info->trim_cpumask = should_trim_cpumask(mm); - flush_tlb_multi(mm_cpumask(mm), info); - consider_global_asid(mm); + remote_flush = true; } else if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } - put_flush_tlb_info(); put_cpu(); + + if (remote_flush) { + info.trim_cpumask = should_trim_cpumask(mm); + flush_tlb_multi(mm_cpumask(mm), &info); + consider_global_asid(mm); + } + mmu_notifier_arch_invalidate_secondary_tlbs(mm, start, end); } @@ -1527,19 +1504,16 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info) void flush_tlb_kernel_range(unsigned long start, unsigned long end) { - struct flush_tlb_info *info; + struct flush_tlb_info info; guard(preempt)(); + init_flush_tlb_info(&info, NULL, start, end, PAGE_SHIFT, false, + TLB_GENERATION_INVALID); - info = get_flush_tlb_info(NULL, start, end, PAGE_SHIFT, false, - TLB_GENERATION_INVALID); - - if (info->end == TLB_FLUSH_ALL) - kernel_tlb_flush_all(info); + if (info.end == TLB_FLUSH_ALL) + kernel_tlb_flush_all(&info); else - kernel_tlb_flush_range(info); - - put_flush_tlb_info(); + kernel_tlb_flush_range(&info); } /* @@ -1707,12 +1681,12 @@ EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - struct flush_tlb_info *info; - + struct flush_tlb_info info; + bool remote_flush = false; int cpu = get_cpu(); - info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false, - TLB_GENERATION_INVALID); + init_flush_tlb_info(&info, NULL, 0, TLB_FLUSH_ALL, 0, false, + TLB_GENERATION_INVALID); /* * flush_tlb_multi() is not optimized for the common case in which only * a local TLB flush is needed. Optimize this use-case by calling @@ -1722,18 +1696,20 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) invlpgb_flush_all_nonglobals(); batch->unmapped_pages = false; } else if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) { - flush_tlb_multi(&batch->cpumask, info); + remote_flush = true; } else if (cpumask_test_cpu(cpu, &batch->cpumask)) { lockdep_assert_irqs_enabled(); local_irq_disable(); - flush_tlb_func(info); + flush_tlb_func(&info); local_irq_enable(); } - cpumask_clear(&batch->cpumask); - - put_flush_tlb_info(); put_cpu(); + + if (remote_flush) + flush_tlb_multi(&batch->cpumask, &info); + + cpumask_clear(&batch->cpumask); } /* diff --git a/arch/x86/xen/mmu_pv.c b/arch/x86/xen/mmu_pv.c --- a/arch/x86/xen/mmu_pv.c +++ b/arch/x86/xen/mmu_pv.c @@ -836,15 +836,15 @@ static void xen_pgd_pin(struct mm_struct *mm) */ void xen_mm_pin_all(void) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&init_mm.page_table_lock); spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { - if (!PagePinned(page)) { - __xen_pgd_pin(&init_mm, (pgd_t *)page_address(page)); - SetPageSavePinned(page); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + if (!PagePinned(ptdesc_page(ptdesc))) { + __xen_pgd_pin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); + SetPageSavePinned(ptdesc_page(ptdesc)); } } @@ -947,16 +947,16 @@ static void xen_pgd_unpin(struct mm_struct *mm) */ void xen_mm_unpin_all(void) { - struct page *page; + struct ptdesc *ptdesc; spin_lock(&init_mm.page_table_lock); spin_lock(&pgd_lock); - list_for_each_entry(page, &pgd_list, lru) { - if (PageSavePinned(page)) { - BUG_ON(!PagePinned(page)); - __xen_pgd_unpin(&init_mm, (pgd_t *)page_address(page)); - ClearPageSavePinned(page); + list_for_each_entry(ptdesc, &pgd_list, pt_list) { + if (PageSavePinned(ptdesc_page(ptdesc))) { + BUG_ON(!PagePinned(ptdesc_page(ptdesc))); + __xen_pgd_unpin(&init_mm, (pgd_t *)ptdesc_address(ptdesc)); + ClearPageSavePinned(ptdesc_page(ptdesc)); } }