diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -1498,6 +1498,32 @@ enum kswapd_opportunistic_compaction_type { KSWAPD_OPPORTUNISTIC_COMPACTION, }; +/* + * pgdat->kswapd_request holds the order and the hint above in one word. A + * waker folds both in with a single cmpxchg and kswapd takes both with a + * single exchange, so a run cannot serve one waker's order under another + * waker's hint. + */ +#define KSWAPD_REQUEST_HINT_MASK 3 +#define KSWAPD_REQUEST_ORDER_SHIFT 2 + +static inline int kswapd_request(int order, + enum kswapd_opportunistic_compaction_type hint) +{ + return (order << KSWAPD_REQUEST_ORDER_SHIFT) | hint; +} + +static inline int kswapd_request_order(int request) +{ + return request >> KSWAPD_REQUEST_ORDER_SHIFT; +} + +static inline enum kswapd_opportunistic_compaction_type +kswapd_request_hint(int request) +{ + return request & KSWAPD_REQUEST_HINT_MASK; +} + /* * On NUMA machines, each NUMA node would have a pg_data_t to describe * it's memory layout. On UMA machines there is a single pglist_data which @@ -1561,14 +1587,15 @@ typedef struct pglist_data { struct mutex kswapd_lock; #endif struct task_struct *kswapd; /* Protected by kswapd_lock */ - int kswapd_order; /* - * Aggregated opportunistic-compaction hint for the next kswapd run. - * Updated by wakeup_kswapd() based on the gfp flags / order of each - * waker, and consumed (and reset) by kswapd before balance_pgdat(). - * See enum kswapd_opportunistic_compaction_type for the state machine. + * The next kswapd run's request: the highest order asked for since + * kswapd last consumed one, and the aggregated opportunistic- + * compaction hint, in one word. Updated by wakeup_kswapd() from the + * gfp flags and order of each waker, and consumed by kswapd before + * balance_pgdat(). See kswapd_request() and + * enum kswapd_opportunistic_compaction_type. */ - atomic_t kswapd_opportunistic_compaction; + atomic_t kswapd_request; enum zone_type kswapd_highest_zoneidx; atomic_t kswapd_failures; /* Number of 'reclaimed == 0' runs */ diff --git a/mm/mm_init.c b/mm/mm_init.c --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1558,7 +1558,8 @@ int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) * when it starts in the near future. */ pgdat->nr_zones = 0; - pgdat->kswapd_order = 0; + atomic_set(&pgdat->kswapd_request, + kswapd_request(0, KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION)); pgdat->kswapd_highest_zoneidx = 0; pgdat->node_start_pfn = 0; pgdat->node_present_pages = 0; diff --git a/mm/vmscan.c b/mm/vmscan.c --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -7413,6 +7413,49 @@ static enum zone_type kswapd_highest_zoneidx(pg_data_t *pgdat, return curr_idx == MAX_NR_ZONES ? prev_highest_zoneidx : curr_idx; } +/* + * How a caller votes on the opportunistic-compaction hint when it folds a + * request into pgdat->kswapd_request. + */ +enum kswapd_vote { + KSWAPD_VOTE_NONE, + KSWAPD_VOTE_NO_OPPORTUNISTIC, + KSWAPD_VOTE_OPPORTUNISTIC, +}; + +/* + * Raise the requested order to @order and fold in @vote, in one step. + * + * The hint is sticky in the "NO" direction: once any waker votes against + * opportunistic treatment, the hint stays cleared until kswapd consumes the + * request. KSWAPD_VOTE_NONE raises the order and leaves the hint alone. + * + * Order and hint live in the same word, so a waker whose order kswapd + * observes always contributed its vote to the hint kswapd observes with it. + */ +static void kswapd_request_fold(pg_data_t *pgdat, int order, + enum kswapd_vote vote) +{ + int old = atomic_read(&pgdat->kswapd_request); + int new; + + do { + enum kswapd_opportunistic_compaction_type hint = + kswapd_request_hint(old); + int req_order = max(kswapd_request_order(old), order); + + if (vote == KSWAPD_VOTE_NO_OPPORTUNISTIC) + hint = KSWAPD_NO_OPPORTUNISTIC_COMPACTION; + else if (vote == KSWAPD_VOTE_OPPORTUNISTIC && + hint == KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION) + hint = KSWAPD_OPPORTUNISTIC_COMPACTION; + + new = kswapd_request(req_order, hint); + if (new == old) + return; + } while (!atomic_try_cmpxchg(&pgdat->kswapd_request, &old, new)); +} + static void kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, unsigned int highest_zoneidx, @@ -7460,13 +7503,11 @@ kswapd_try_to_sleep(pg_data_t *pgdat, int alloc_order, int reclaim_order, kswapd_highest_zoneidx(pgdat, highest_zoneidx)); - if (READ_ONCE(pgdat->kswapd_order) < reclaim_order) - WRITE_ONCE(pgdat->kswapd_order, reclaim_order); - - if (kswapd_opportunistic_compaction == - KSWAPD_NO_OPPORTUNISTIC_COMPACTION) - atomic_set(&pgdat->kswapd_opportunistic_compaction, - KSWAPD_NO_OPPORTUNISTIC_COMPACTION); + kswapd_request_fold(pgdat, reclaim_order, + kswapd_opportunistic_compaction == + KSWAPD_NO_OPPORTUNISTIC_COMPACTION ? + KSWAPD_VOTE_NO_OPPORTUNISTIC : + KSWAPD_VOTE_NONE); } finish_wait(&pgdat->kswapd_wait, &wait); @@ -7524,6 +7565,7 @@ static int kswapd(void *p) pg_data_t *pgdat = (pg_data_t *)p; struct task_struct *tsk = current; enum kswapd_opportunistic_compaction_type kswapd_opportunistic_compaction; + int request; /* * Tell the memory management that we're a "memory allocator", @@ -7540,32 +7582,30 @@ static int kswapd(void *p) tsk->flags |= PF_MEMALLOC | PF_KSWAPD; set_freezable(); - WRITE_ONCE(pgdat->kswapd_order, 0); - atomic_set(&pgdat->kswapd_opportunistic_compaction, - KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION); + atomic_set(&pgdat->kswapd_request, + kswapd_request(0, KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION)); WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); atomic_set(&pgdat->nr_writeback_throttled, 0); for ( ; ; ) { bool was_frozen; - alloc_order = reclaim_order = READ_ONCE(pgdat->kswapd_order); + request = atomic_read(&pgdat->kswapd_request); + alloc_order = reclaim_order = kswapd_request_order(request); highest_zoneidx = kswapd_highest_zoneidx(pgdat, highest_zoneidx); - kswapd_opportunistic_compaction = - atomic_read(&pgdat->kswapd_opportunistic_compaction); + kswapd_opportunistic_compaction = kswapd_request_hint(request); kswapd_try_sleep: kswapd_try_to_sleep(pgdat, alloc_order, reclaim_order, highest_zoneidx, kswapd_opportunistic_compaction); - /* Read the new order and highest_zoneidx */ - alloc_order = READ_ONCE(pgdat->kswapd_order); - kswapd_opportunistic_compaction = - atomic_xchg(&pgdat->kswapd_opportunistic_compaction, - KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION); + /* Take the new request and highest_zoneidx */ + request = atomic_xchg(&pgdat->kswapd_request, + kswapd_request(0, KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION)); + alloc_order = kswapd_request_order(request); + kswapd_opportunistic_compaction = kswapd_request_hint(request); highest_zoneidx = kswapd_highest_zoneidx(pgdat, highest_zoneidx); - WRITE_ONCE(pgdat->kswapd_order, 0); WRITE_ONCE(pgdat->kswapd_highest_zoneidx, MAX_NR_ZONES); if (kthread_freezable_should_stop(&was_frozen)) @@ -7625,30 +7665,15 @@ void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order, if (curr_idx == MAX_NR_ZONES || curr_idx < highest_zoneidx) WRITE_ONCE(pgdat->kswapd_highest_zoneidx, highest_zoneidx); - if (READ_ONCE(pgdat->kswapd_order) < order) - WRITE_ONCE(pgdat->kswapd_order, order); - /* - * Fold this waker into the per-pgdat opportunistic-compaction hint - * that kswapd will pick up at the start of its next run. - * - * The state is sticky in the "NO" direction: once any waker in this - * batch is order-0 or a non-failable high-order allocation, the hint - * stays cleared until kswapd consumes it. Only when every waker so - * far is a failable high-order allocation do we set - * KSWAPD_OPPORTUNISTIC_COMPACTION, asking shrinkers to skip work - * that won't realistically help compaction. + * Fold this waker's order and its vote on the opportunistic-compaction + * hint into the request kswapd will pick up at the start of its next + * run. Both change together, so the run that serves this order also + * carries this waker's vote. */ - if (atomic_read(&pgdat->kswapd_opportunistic_compaction) != - KSWAPD_NO_OPPORTUNISTIC_COMPACTION) { - if (!opportunistic) - atomic_set(&pgdat->kswapd_opportunistic_compaction, - KSWAPD_NO_OPPORTUNISTIC_COMPACTION); - else - atomic_cmpxchg(&pgdat->kswapd_opportunistic_compaction, - KSWAPD_UNSET_OPPORTUNISTIC_COMPACTION, - KSWAPD_OPPORTUNISTIC_COMPACTION); - } + kswapd_request_fold(pgdat, order, + opportunistic ? KSWAPD_VOTE_OPPORTUNISTIC : + KSWAPD_VOTE_NO_OPPORTUNISTIC); if (!waitqueue_active(&pgdat->kswapd_wait)) return;