diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 9a98bc985a71..2e8ba4ca0908 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1668,6 +1668,74 @@ const struct file_operations proc_pagemap_operations = { #endif /* CONFIG_PROC_PAGE_MONITOR */ #ifdef CONFIG_PROCESS_RECLAIM +static BLOCKING_NOTIFIER_HEAD(proc_reclaim_notifier); + +int proc_reclaim_notifier_register(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&proc_reclaim_notifier, nb); +} + +int proc_reclaim_notifier_unregister(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&proc_reclaim_notifier, nb); +} + +static void proc_reclaim_notify(unsigned long pid, void *rp) +{ + blocking_notifier_call_chain(&proc_reclaim_notifier, pid, rp); +} + +int reclaim_address_space(struct address_space *mapping, + struct reclaim_param *rp, struct vm_area_struct *vma) +{ + struct radix_tree_iter iter; + void __rcu **slot; + pgoff_t start; + struct page *page; + LIST_HEAD(page_list); + int reclaimed; + int ret = NOTIFY_OK; + + lru_add_drain(); + start = 0; + rcu_read_lock(); + + radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { + + page = radix_tree_deref_slot(slot); + + if (radix_tree_deref_retry(page)) { + slot = radix_tree_iter_retry(&iter); + continue; + } + + if (radix_tree_exceptional_entry(page)) + continue; + + if (isolate_lru_page(page)) + continue; + + rp->nr_scanned++; + + list_add(&page->lru, &page_list); + inc_node_page_state(page, NR_ISOLATED_ANON + + page_is_file_cache(page)); + + if (need_resched()) { + slot = radix_tree_iter_resume(slot, &iter); + cond_resched_rcu(); + } + } + rcu_read_unlock(); + reclaimed = reclaim_pages_from_list(&page_list, vma); + rp->nr_reclaimed += reclaimed; + + if (rp->nr_scanned >= rp->nr_to_reclaim) + ret = NOTIFY_DONE; + + return ret; +} + static int reclaim_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -1739,6 +1807,29 @@ enum reclaim_type { RECLAIM_RANGE, }; +struct reclaim_param reclaim_task_nomap(struct task_struct *task, + int nr_to_reclaim) +{ + struct mm_struct *mm; + struct reclaim_param rp = { + .nr_to_reclaim = nr_to_reclaim, + }; + + get_task_struct(task); + mm = get_task_mm(task); + if (!mm) + goto out; + down_read(&mm->mmap_sem); + + proc_reclaim_notify(task_tgid_nr(task), (void *)&rp); + + up_read(&mm->mmap_sem); + mmput(mm); +out: + put_task_struct(task); + return rp; +} + struct reclaim_param reclaim_task_anon(struct task_struct *task, int nr_to_reclaim) { diff --git a/include/linux/mm.h b/include/linux/mm.h index 2b39bdabf7db..719fa2f841ed 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3028,6 +3028,12 @@ struct reclaim_param { }; extern struct reclaim_param reclaim_task_anon(struct task_struct *task, int nr_to_reclaim); +extern struct reclaim_param reclaim_task_nomap(struct task_struct *task, + int nr_to_reclaim); +extern int reclaim_address_space(struct address_space *mapping, + struct reclaim_param *rp, struct vm_area_struct *vma); +extern int proc_reclaim_notifier_register(struct notifier_block *nb); +extern int proc_reclaim_notifier_unregister(struct notifier_block *nb); #endif #endif /* __KERNEL__ */ diff --git a/mm/process_reclaim.c b/mm/process_reclaim.c index 08c826b95ec0..084131312d1f 100644 --- a/mm/process_reclaim.c +++ b/mm/process_reclaim.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved. + * Copyright (c) 2015-2020, The Linux Foundation. All rights reserved. */ #include #include @@ -32,6 +32,12 @@ module_param_named(per_swap_size, per_swap_size, int, 0644); int reclaim_avg_efficiency; module_param_named(reclaim_avg_efficiency, reclaim_avg_efficiency, int, 0444); +static unsigned long reclaimed_anon; +module_param_named(reclaimed_anon, reclaimed_anon, ulong, 0444); + +static unsigned long reclaimed_nomap; +module_param_named(reclaimed_nomap, reclaimed_nomap, ulong, 0444); + /* The vmpressure region where process reclaim operates */ static unsigned long pressure_min = 50; static unsigned long pressure_max = 90; @@ -183,6 +189,13 @@ static void swap_fn(struct work_struct *work) nr_to_reclaim); total_scan += rp.nr_scanned; total_reclaimed += rp.nr_reclaimed; + reclaimed_anon += rp.nr_reclaimed; + + rp = reclaim_task_nomap(selected[si].p, nr_to_reclaim); + total_scan += rp.nr_scanned; + total_reclaimed += rp.nr_reclaimed; + reclaimed_nomap += rp.nr_reclaimed; + put_task_struct(selected[si].p); }