162 lines
5.9 KiB
Diff
162 lines
5.9 KiB
Diff
diff --git a/mm/ksm.c b/mm/ksm.c
|
|
--- a/mm/ksm.c
|
|
+++ b/mm/ksm.c
|
|
@@ -195,22 +195,28 @@ struct ksm_stable_node {
|
|
* @node: rb node of this rmap_item in the unstable tree
|
|
* @head: pointer to stable_node heading this list in the stable tree
|
|
* @hlist: link into hlist of rmap_items hanging off that stable_node
|
|
- * @age: number of scan iterations since creation
|
|
- * @remaining_skips: how many scans to skip
|
|
+ * @age: number of scan iterations since creation (unstable node)
|
|
+ * @remaining_skips: how many scans to skip (unstable node)
|
|
+ * @linear_page_index: the original page's index before merged by KSM (stable node)
|
|
*/
|
|
struct ksm_rmap_item {
|
|
struct ksm_rmap_item *rmap_list;
|
|
union {
|
|
- struct anon_vma *anon_vma; /* when stable */
|
|
+ struct anon_vma *anon_vma; /* for reverse mapping, when stable */
|
|
#ifdef CONFIG_NUMA
|
|
int nid; /* when node of unstable tree */
|
|
#endif
|
|
};
|
|
struct mm_struct *mm;
|
|
unsigned long address; /* + low bits used for flags below */
|
|
- unsigned int oldchecksum; /* when unstable */
|
|
- rmap_age_t age;
|
|
- rmap_age_t remaining_skips;
|
|
+ union {
|
|
+ struct {
|
|
+ unsigned int oldchecksum;
|
|
+ rmap_age_t age;
|
|
+ rmap_age_t remaining_skips;
|
|
+ }; /* when unstable */
|
|
+ unsigned long linear_page_index; /* for reverse mapping, when stable */
|
|
+ };
|
|
union {
|
|
struct rb_node node; /* when node of unstable tree */
|
|
struct { /* when listed from stable tree */
|
|
@@ -776,6 +782,11 @@ static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm,
|
|
return vma;
|
|
}
|
|
|
|
+/*
|
|
+ * break_cow: actively break COW, replacing the KSM page by a fresh anonymous
|
|
+ * page. This is called when rmap_item has not yet become stable, but page
|
|
+ * has been merged.
|
|
+ */
|
|
static void break_cow(struct ksm_rmap_item *rmap_item)
|
|
{
|
|
struct mm_struct *mm = rmap_item->mm;
|
|
@@ -787,6 +798,11 @@ static void break_cow(struct ksm_rmap_item *rmap_item)
|
|
* to undo, we also need to drop a reference to the anon_vma.
|
|
*/
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
+ /*
|
|
+ * Reset linear_page_index that might overlay age-related
|
|
+ * information. (it's still unstable node)
|
|
+ */
|
|
+ rmap_item->linear_page_index = 0;
|
|
|
|
mmap_read_lock(mm);
|
|
vma = find_mergeable_vma(mm, addr);
|
|
@@ -899,6 +915,8 @@ static void remove_node_from_stable_tree(struct ksm_stable_node *stable_node)
|
|
VM_BUG_ON(stable_node->rmap_hlist_len <= 0);
|
|
stable_node->rmap_hlist_len--;
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
+ /* Reset linear_page_index that might overlay age-related information. */
|
|
+ rmap_item->linear_page_index = 0;
|
|
rmap_item->address &= PAGE_MASK;
|
|
cond_resched();
|
|
}
|
|
@@ -1052,6 +1070,8 @@ static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item)
|
|
stable_node->rmap_hlist_len--;
|
|
|
|
put_anon_vma(rmap_item->anon_vma);
|
|
+ /* Reset linear_page_index that might overlay age-related information. */
|
|
+ rmap_item->linear_page_index = 0;
|
|
rmap_item->head = NULL;
|
|
rmap_item->address &= PAGE_MASK;
|
|
|
|
@@ -1598,8 +1618,15 @@ static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
|
|
/* Unstable nid is in union with stable anon_vma: remove first */
|
|
remove_rmap_item_from_tree(rmap_item);
|
|
|
|
- /* Must get reference to anon_vma while still holding mmap_lock */
|
|
+ /*
|
|
+ * We can consider the VMA only while still holding the mmap lock,
|
|
+ * so lock, so reference the anon_vma and calculate the linear
|
|
+ * page index early, before stable_tree_append(). If anything goes
|
|
+ * wrong that prevents the rmap_item from being added to the
|
|
+ * stable_tree, break_cow() will clean it up.
|
|
+ */
|
|
rmap_item->anon_vma = vma->anon_vma;
|
|
+ rmap_item->linear_page_index = linear_page_index(vma, rmap_item->address);
|
|
get_anon_vma(vma->anon_vma);
|
|
out:
|
|
mmap_read_unlock(mm);
|
|
@@ -2458,6 +2485,13 @@ static bool should_skip_rmap_item(struct folio *folio,
|
|
if (folio_test_ksm(folio))
|
|
return false;
|
|
|
|
+ /*
|
|
+ * There is no age information in stable-tree nodes. We might end up
|
|
+ * here without a KSM page for example after COW.
|
|
+ */
|
|
+ if (rmap_item->address & STABLE_FLAG)
|
|
+ return false;
|
|
+
|
|
age = rmap_item->age;
|
|
if (age != U8_MAX)
|
|
rmap_item->age++;
|
|
@@ -3173,6 +3207,7 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
|
|
hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
|
|
/* Ignore the stable/unstable/sqnr flags */
|
|
const unsigned long addr = rmap_item->address & PAGE_MASK;
|
|
+ const unsigned long index = rmap_item->linear_page_index;
|
|
struct anon_vma *anon_vma = rmap_item->anon_vma;
|
|
struct anon_vma_chain *vmac;
|
|
struct vm_area_struct *vma;
|
|
@@ -3186,8 +3221,18 @@ void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc)
|
|
anon_vma_lock_read(anon_vma);
|
|
}
|
|
|
|
+ /*
|
|
+ * Currently, KSM folios are always small folios, so it's
|
|
+ * sufficient to search for a single page. We can simply use
|
|
+ * the linear_page_index of the original de-duplicate
|
|
+ * anonymous page that we remembered in the rmap_item while
|
|
+ * de-duplicating. Note that mremap() always de-duplicates KSM
|
|
+ * folios: so if there was mremap() in our parent or our child,
|
|
+ * we wouldn't have the KSM folio mapped in these processes
|
|
+ * anymore.
|
|
+ */
|
|
anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
|
|
- 0, ULONG_MAX) {
|
|
+ index, index) {
|
|
|
|
cond_resched();
|
|
vma = vmac->vma;
|
|
@@ -3243,17 +3288,17 @@ void collect_procs_ksm(const struct folio *folio, const struct page *page,
|
|
rcu_read_lock();
|
|
for_each_process(tsk) {
|
|
struct anon_vma_chain *vmac;
|
|
- unsigned long addr;
|
|
+ const unsigned long addr = rmap_item->address & PAGE_MASK;
|
|
+ const unsigned long index = rmap_item->linear_page_index;
|
|
struct task_struct *t =
|
|
task_early_kill(tsk, force_early);
|
|
if (!t)
|
|
continue;
|
|
- anon_vma_interval_tree_foreach(vmac, &av->rb_root, 0,
|
|
- ULONG_MAX)
|
|
+ anon_vma_interval_tree_foreach(vmac, &av->rb_root, index,
|
|
+ index)
|
|
{
|
|
vma = vmac->vma;
|
|
if (vma->vm_mm == t->mm) {
|
|
- addr = rmap_item->address & PAGE_MASK;
|
|
add_to_kill_ksm(t, page, vma, to_kill,
|
|
addr);
|
|
}
|