mm: process_reclaim: support reclaim of driver owned pages

GPU drivers can allocate memory via shmem and these pages
may or may not be mapped to userspace at any moment. But
these pages are allocated on behalf of userspace tasks.
It is ideal to swapout the unmapped pages of a task during
process reclaim. But as process reclaims works on mappings
to derive pages, it misses the unmapped gpu pages. Add an
option to reclaim these pages. Drivers can register to the
reclaim notifier and call into the reclaim function with the
mapping that contains these mapped/unmapped pages.

Change-Id: Ia5de99d9830c0bdf0e9e56d860987f79f95df435
Signed-off-by: Vinayak Menon <vinmenon@codeaurora.org>

fs/proc/task_mmu.c

@@ -1668,6 +1668,74 @@ const struct file_operations proc_pagemap_operations = {
 #endif /* CONFIG_PROC_PAGE_MONITOR */
 
 #ifdef CONFIG_PROCESS_RECLAIM
+static BLOCKING_NOTIFIER_HEAD(proc_reclaim_notifier);
+
+int proc_reclaim_notifier_register(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_register(&proc_reclaim_notifier, nb);
+}
+
+int proc_reclaim_notifier_unregister(struct notifier_block *nb)
+{
+	return blocking_notifier_chain_unregister(&proc_reclaim_notifier, nb);
+}
+
+static void proc_reclaim_notify(unsigned long pid, void *rp)
+{
+	blocking_notifier_call_chain(&proc_reclaim_notifier, pid, rp);
+}
+
+int reclaim_address_space(struct address_space *mapping,
+			struct reclaim_param *rp, struct vm_area_struct *vma)
+{
+	struct radix_tree_iter iter;
+	void __rcu **slot;
+	pgoff_t start;
+	struct page *page;
+	LIST_HEAD(page_list);
+	int reclaimed;
+	int ret = NOTIFY_OK;
+
+	lru_add_drain();
+	start = 0;
+	rcu_read_lock();
+
+	radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
+
+		page = radix_tree_deref_slot(slot);
+
+		if (radix_tree_deref_retry(page)) {
+			slot = radix_tree_iter_retry(&iter);
+			continue;
+		}
+
+		if (radix_tree_exceptional_entry(page))
+			continue;
+
+		if (isolate_lru_page(page))
+			continue;
+
+		rp->nr_scanned++;
+
+		list_add(&page->lru, &page_list);
+		inc_node_page_state(page, NR_ISOLATED_ANON +
+				page_is_file_cache(page));
+
+		if (need_resched()) {
+			slot = radix_tree_iter_resume(slot, &iter);
+			cond_resched_rcu();
+		}
+	}
+	rcu_read_unlock();
+	reclaimed = reclaim_pages_from_list(&page_list, vma);
+	rp->nr_reclaimed += reclaimed;
+
+	if (rp->nr_scanned >= rp->nr_to_reclaim)
+		ret = NOTIFY_DONE;
+
+	return ret;
+}
+
 static int reclaim_pte_range(pmd_t *pmd, unsigned long addr,
 				unsigned long end, struct mm_walk *walk)
 {
@@ -1739,6 +1807,29 @@ enum reclaim_type {
 	RECLAIM_RANGE,
 };
 
+struct reclaim_param reclaim_task_nomap(struct task_struct *task,
+		int nr_to_reclaim)
+{
+	struct mm_struct *mm;
+	struct reclaim_param rp = {
+		.nr_to_reclaim = nr_to_reclaim,
+	};
+
+	get_task_struct(task);
+	mm = get_task_mm(task);
+	if (!mm)
+		goto out;
+	down_read(&mm->mmap_sem);
+
+	proc_reclaim_notify(task_tgid_nr(task), (void *)&rp);
+
+	up_read(&mm->mmap_sem);
+	mmput(mm);
+out:
+	put_task_struct(task);
+	return rp;
+}
+
 struct reclaim_param reclaim_task_anon(struct task_struct *task,
 		int nr_to_reclaim)
 {

include/linux/mm.h

@@ -3028,6 +3028,12 @@ struct reclaim_param {
 };
 extern struct reclaim_param reclaim_task_anon(struct task_struct *task,
 		int nr_to_reclaim);
+extern struct reclaim_param reclaim_task_nomap(struct task_struct *task,
+		int nr_to_reclaim);
+extern int reclaim_address_space(struct address_space *mapping,
+		struct reclaim_param *rp, struct vm_area_struct *vma);
+extern int proc_reclaim_notifier_register(struct notifier_block *nb);
+extern int proc_reclaim_notifier_unregister(struct notifier_block *nb);
 #endif
 
 #endif /* __KERNEL__ */

mm/process_reclaim.c

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2015-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) 2015-2020, The Linux Foundation. All rights reserved.
  */
 #include <linux/module.h>
 #include <linux/kernel.h>
@@ -32,6 +32,12 @@ module_param_named(per_swap_size, per_swap_size, int, 0644);
 int reclaim_avg_efficiency;
 module_param_named(reclaim_avg_efficiency, reclaim_avg_efficiency, int, 0444);
 
+static unsigned long reclaimed_anon;
+module_param_named(reclaimed_anon, reclaimed_anon, ulong, 0444);
+
+static unsigned long reclaimed_nomap;
+module_param_named(reclaimed_nomap, reclaimed_nomap, ulong, 0444);
+
 /* The vmpressure region where process reclaim operates */
 static unsigned long pressure_min = 50;
 static unsigned long pressure_max = 90;
@@ -183,6 +189,13 @@ static void swap_fn(struct work_struct *work)
 				nr_to_reclaim);
 		total_scan += rp.nr_scanned;
 		total_reclaimed += rp.nr_reclaimed;
+		reclaimed_anon += rp.nr_reclaimed;
+
+		rp = reclaim_task_nomap(selected[si].p, nr_to_reclaim);
+		total_scan += rp.nr_scanned;
+		total_reclaimed += rp.nr_reclaimed;
+		reclaimed_nomap += rp.nr_reclaimed;
+
 		put_task_struct(selected[si].p);
 	}