path: root/features/cgroups/vm_cgroup-private-writable-mappings-accounting.patch

From ae6487ef6152ee9ba05fe9a69b94d3b0aff637a6 Mon Sep 17 00:00:00 2001
From: Vladimir Davydov <vdavydov@parallels.com>
Date: Fri, 8 Aug 2014 09:20:35 +0800
Subject: [PATCH] vm_cgroup: private writable mappings accounting

Taken from https://lkml.org/lkml/2014/7/3/408

Address space that contributes to memory overcommit consists of two
parts - private writable mappings and shared memory. This patch adds
private writable mappings accounting.

The implementation is quite simple. Each mm holds a reference to the vm
cgroup it is accounted to. The reference is initialized with the current
cgroup on mm creation and released only on mm destruction. For
simplicity, task migrations as well as mm owner changes are not handled
yet, so an offline cgroup will be pinned in memory until all mm's
accounted to it die.

Signed-off-by: Vladimir Davydov <vdavydov@parallels.com>
Signed-off-by: He Zhe <zhe.he@windriver.com>
Signed-off-by: Bruce Ashfield <bruce.ashfield@windriver.com>
---
 include/linux/mm_types.h  |  3 +++
 include/linux/vm_cgroup.h | 29 ++++++++++++++++++++
 kernel/fork.c             | 12 ++++++++-
 mm/mmap.c                 | 43 ++++++++++++++++++++++++------
 mm/mprotect.c             |  8 +++++-
 mm/mremap.c               | 15 ++++++++---
 mm/vm_cgroup.c            | 67 +++++++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 164 insertions(+), 13 deletions(-)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 8f374ede64e0..8f4ffa4c9ea6 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -421,6 +421,9 @@ struct mm_struct {
 	 */
 	struct task_struct __rcu *owner;
 #endif
+#ifdef CONFIG_CGROUP_VM
+	struct vm_cgroup *vmcg;		/* vm_cgroup this mm is accounted to */
+#endif
 
 	/* store ref to file /proc/<pid>/exe symlink points to */
 	struct file *exe_file;
diff --git a/include/linux/vm_cgroup.h b/include/linux/vm_cgroup.h
index 3694bc447231..c591d5c477ba 100644
--- a/include/linux/vm_cgroup.h
+++ b/include/linux/vm_cgroup.h
@@ -1,6 +1,8 @@
 #ifndef _LINUX_VM_CGROUP_H
 #define _LINUX_VM_CGROUP_H
 
+struct mm_struct;
+
 #ifdef CONFIG_CGROUP_VM
 static inline bool vm_cgroup_disabled(void)
 {
@@ -8,11 +10,38 @@ static inline bool vm_cgroup_disabled(void)
 		return true;
 	return false;
 }
+
+extern void mm_init_vm_cgroup(struct mm_struct *mm, struct task_struct *p);
+extern void mm_release_vm_cgroup(struct mm_struct *mm);
+extern int vm_cgroup_charge_memory_mm(struct mm_struct *mm,
+				      unsigned long nr_pages);
+extern void vm_cgroup_uncharge_memory_mm(struct mm_struct *mm,
+					 unsigned long nr_pages);
 #else /* !CONFIG_CGROUP_VM */
 static inline bool vm_cgroup_disabled(void)
 {
 	return true;
 }
+
+static inline void mm_init_vm_cgroup(struct mm_struct *mm,
+				     struct task_struct *p)
+{
+}
+
+static inline void mm_release_vm_cgroup(struct mm_struct *mm)
+{
+}
+
+static inline int vm_cgroup_charge_memory_mm(struct mm_struct *mm,
+					     unsigned long nr_pages)
+{
+	return 0;
+}
+
+static inline void vm_cgroup_uncharge_memory_mm(struct mm_struct *mm,
+						unsigned long nr_pages)
+{
+}
 #endif /* CONFIG_CGROUP_VM */
 
 #endif /* _LINUX_VM_CGROUP_H */
diff --git a/kernel/fork.c b/kernel/fork.c
index 87ef21e5c0ee..0e8a98005740 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -74,6 +74,7 @@
 #include <linux/uprobes.h>
 #include <linux/aio.h>
 #include <linux/compiler.h>
+#include <linux/vm_cgroup.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -402,8 +403,13 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
 		if (mpnt->vm_flags & VM_ACCOUNT) {
 			unsigned long len = vma_pages(mpnt);
 
-			if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
+			if (vm_cgroup_charge_memory_mm(mm, len))
 				goto fail_nomem;
+
+			if (security_vm_enough_memory_mm(oldmm, len)) {
+				vm_cgroup_uncharge_memory_mm(mm, len);
+				goto fail_nomem;
+			}
 			charge = len;
 		}
 		tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
@@ -487,6 +493,7 @@ fail_nomem_policy:
 fail_nomem:
 	retval = -ENOMEM;
 	vm_unacct_memory(charge);
+	vm_cgroup_uncharge_memory_mm(mm, charge);
 	goto out;
 }
 
@@ -576,6 +583,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
 	}
 
 	if (mm_alloc_pgd(mm))
+		mm_init_vm_cgroup(mm, current);
 		goto fail_nopgd;
 
 	if (init_new_context(p, mm))
@@ -584,6 +592,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p)
 	return mm;
 
 fail_nocontext:
+	mm_release_vm_cgroup(mm);
 	mm_free_pgd(mm);
 fail_nopgd:
 	free_mm(mm);
@@ -630,6 +639,7 @@ struct mm_struct *mm_alloc(void)
 void __mmdrop(struct mm_struct *mm)
 {
 	BUG_ON(mm == &init_mm);
+	mm_release_vm_cgroup(mm);
 	mm_free_pgd(mm);
 	destroy_context(mm);
 	mmu_notifier_mm_destroy(mm);
diff --git a/mm/mmap.c b/mm/mmap.c
index 8c434e0b46b3..ea684c74c93c 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -41,6 +41,7 @@
 #include <linux/notifier.h>
 #include <linux/memory.h>
 #include <linux/printk.h>
+#include <linux/vm_cgroup.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -1540,8 +1541,12 @@ munmap_back:
 	 */
 	if (accountable_mapping(file, vm_flags)) {
 		charged = len >> PAGE_SHIFT;
-		if (security_vm_enough_memory_mm(mm, charged))
+		if (vm_cgroup_charge_memory_mm(mm, charged))
 			return -ENOMEM;
+		if (security_vm_enough_memory_mm(mm, charged)) {
+			vm_cgroup_uncharge_memory_mm(mm, charged);
+			return -ENOMEM;
+		}
 		vm_flags |= VM_ACCOUNT;
 	}
 
@@ -1675,8 +1680,10 @@ allow_write_and_free_vma:
 free_vma:
 	kmem_cache_free(vm_area_cachep, vma);
 unacct_error:
-	if (charged)
+	if (charged) {
 		vm_unacct_memory(charged);
+		vm_cgroup_uncharge_memory_mm(mm, charged);
+	}
 	return error;
 }
 
@@ -2107,12 +2114,16 @@ static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, uns
 	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
 		return -EFAULT;
 
+	if (vm_cgroup_charge_memory_mm(mm, grow))
+		return -ENOMEM;
 	/*
 	 * Overcommit..  This must be the final test, as it will
 	 * update security statistics.
 	 */
-	if (security_vm_enough_memory_mm(mm, grow))
+	if (security_vm_enough_memory_mm(mm, grow)) {
+		vm_cgroup_uncharge_memory_mm(mm, grow);
 		return -ENOMEM;
+	}
 
 	/* Ok, everything looks good - let it rip */
 	if (vma->vm_flags & VM_LOCKED)
@@ -2364,6 +2375,7 @@ static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
 		vma = remove_vma(vma);
 	} while (vma);
 	vm_unacct_memory(nr_accounted);
+	vm_cgroup_uncharge_memory_mm(mm, nr_accounted);
 	validate_mm(mm);
 }
 
@@ -2626,6 +2638,7 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
 	unsigned long flags;
 	struct rb_node ** rb_link, * rb_parent;
 	pgoff_t pgoff = addr >> PAGE_SHIFT;
+	unsigned long charged;
 	int error;
 
 	len = PAGE_ALIGN(len);
@@ -2665,8 +2678,13 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
 	if (mm->map_count > sysctl_max_map_count)
 		return -ENOMEM;
 
-	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
+	charged = len >> PAGE_SHIFT;
+	if (vm_cgroup_charge_memory_mm(mm, charged))
+		return -ENOMEM;
+	if (security_vm_enough_memory_mm(mm, charged)) {
+		vm_cgroup_uncharge_memory_mm(mm, charged);
 		return -ENOMEM;
+	}
 
 	/* Can we just expand an old private anonymous mapping? */
 	vma = vma_merge(mm, prev, addr, addr + len, flags,
@@ -2679,7 +2697,8 @@ static unsigned long do_brk(unsigned long addr, unsigned long len)
 	 */
 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
 	if (!vma) {
-		vm_unacct_memory(len >> PAGE_SHIFT);
+		vm_unacct_memory(charged);
+		vm_cgroup_uncharge_memory_mm(mm, charged);
 		return -ENOMEM;
 	}
 
@@ -2761,6 +2780,7 @@ void exit_mmap(struct mm_struct *mm)
 		vma = remove_vma(vma);
 	}
 	vm_unacct_memory(nr_accounted);
+	vm_cgroup_uncharge_memory_mm(mm, nr_accounted);
 
 	WARN_ON(atomic_long_read(&mm->nr_ptes) >
 			(FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
@@ -2794,9 +2814,16 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
 	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
 			   &prev, &rb_link, &rb_parent))
 		return -ENOMEM;
-	if ((vma->vm_flags & VM_ACCOUNT) &&
-	     security_vm_enough_memory_mm(mm, vma_pages(vma)))
-		return -ENOMEM;
+	if ((vma->vm_flags & VM_ACCOUNT)) {
+		unsigned long charged = vma_pages(vma);
+
+		if (vm_cgroup_charge_memory_mm(mm, charged))
+			return -ENOMEM;
+		if (security_vm_enough_memory_mm(mm, charged)) {
+			vm_cgroup_uncharge_memory_mm(mm, charged);
+			return -ENOMEM;
+		}
+	}
 
 	vma_link(mm, vma, prev, rb_link, rb_parent);
 	return 0;
diff --git a/mm/mprotect.c b/mm/mprotect.c
index c43d557941f8..f76d1cadb3c1 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -24,6 +24,7 @@
 #include <linux/migrate.h>
 #include <linux/perf_event.h>
 #include <linux/ksm.h>
+#include <linux/vm_cgroup.h>
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
 #include <asm/cacheflush.h>
@@ -283,8 +284,12 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
 		if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
 						VM_SHARED|VM_NORESERVE))) {
 			charged = nrpages;
-			if (security_vm_enough_memory_mm(mm, charged))
+			if (vm_cgroup_charge_memory_mm(mm, charged))
 				return -ENOMEM;
+			if (security_vm_enough_memory_mm(mm, charged)) {
+				vm_cgroup_uncharge_memory_mm(mm, charged);
+				return -ENOMEM;
+			}
 			newflags |= VM_ACCOUNT;
 		}
 	}
@@ -338,6 +343,7 @@ success:
 
 fail:
 	vm_unacct_memory(charged);
+	vm_cgroup_uncharge_memory_mm(mm, charged);
 	return error;
 }
 
diff --git a/mm/mremap.c b/mm/mremap.c
index 05f1180e9f21..1cf5709acce5 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -21,6 +21,7 @@
 #include <linux/syscalls.h>
 #include <linux/mmu_notifier.h>
 #include <linux/sched/sysctl.h>
+#include <linux/vm_cgroup.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -313,6 +314,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
 	if (do_munmap(mm, old_addr, old_len) < 0) {
 		/* OOM: unable to split vma, just get accounts right */
 		vm_unacct_memory(excess >> PAGE_SHIFT);
+		vm_cgroup_uncharge_memory_mm(mm, excess >> PAGE_SHIFT);
 		excess = 0;
 	}
 	mm->hiwater_vm = hiwater_vm;
@@ -374,8 +376,13 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
 
 	if (vma->vm_flags & VM_ACCOUNT) {
 		unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-		if (security_vm_enough_memory_mm(mm, charged))
+
+		if (vm_cgroup_charge_memory_mm(mm, charged))
+			goto Efault;
+		if (security_vm_enough_memory_mm(mm, charged)) {
+			vm_cgroup_uncharge_memory_mm(mm, charged);
 			goto Efault;
+		}
 		*p = charged;
 	}
 
@@ -447,7 +454,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
 		goto out;
 out1:
 	vm_unacct_memory(charged);
-
+	vm_cgroup_uncharge_memory_mm(mm, charged);
 out:
 	return ret;
 }
@@ -578,8 +585,10 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
 		ret = move_vma(vma, addr, old_len, new_len, new_addr, &locked);
 	}
 out:
-	if (ret & ~PAGE_MASK)
+	if (ret & ~PAGE_MASK) {
 		vm_unacct_memory(charged);
+		vm_cgroup_uncharge_memory_mm(mm, charged);
+	}
 	up_write(&current->mm->mmap_sem);
 	if (locked && new_len > old_len)
 		mm_populate(new_addr + old_len, new_len - old_len);
diff --git a/mm/vm_cgroup.c b/mm/vm_cgroup.c
index 9f3178e506ba..0d34ffcaf92e 100644
--- a/mm/vm_cgroup.c
+++ b/mm/vm_cgroup.c
@@ -2,6 +2,7 @@
 #include <linux/res_counter.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
+#include <linux/rcupdate.h>
 #include <linux/vm_cgroup.h>
 
 struct vm_cgroup {
@@ -25,6 +26,72 @@ static struct vm_cgroup *vm_cgroup_from_css(struct cgroup_subsys_state *s)
 	return s ? container_of(s, struct vm_cgroup, css) : NULL;
 }
 
+static struct vm_cgroup *vm_cgroup_from_task(struct task_struct *p)
+{
+	return vm_cgroup_from_css(task_css(p, vm_cgrp_id));
+}
+
+static struct vm_cgroup *get_vm_cgroup_from_task(struct task_struct *p)
+{
+	struct vm_cgroup *vmcg;
+
+	rcu_read_lock();
+	do {
+		vmcg = vm_cgroup_from_task(p);
+	} while (!css_tryget_online(&vmcg->css));
+	rcu_read_unlock();
+
+	return vmcg;
+}
+
+void mm_init_vm_cgroup(struct mm_struct *mm, struct task_struct *p)
+{
+	if (!vm_cgroup_disabled())
+		mm->vmcg = get_vm_cgroup_from_task(p);
+}
+
+void mm_release_vm_cgroup(struct mm_struct *mm)
+{
+	struct vm_cgroup *vmcg = mm->vmcg;
+
+	if (vmcg)
+		css_put(&vmcg->css);
+}
+
+static int vm_cgroup_do_charge(struct vm_cgroup *vmcg,
+			       unsigned long nr_pages)
+{
+	unsigned long val = nr_pages << PAGE_SHIFT;
+	struct res_counter *fail_res;
+
+	return res_counter_charge(&vmcg->res, val, &fail_res);
+}
+
+static void vm_cgroup_do_uncharge(struct vm_cgroup *vmcg,
+				  unsigned long nr_pages)
+{
+	unsigned long val = nr_pages << PAGE_SHIFT;
+
+	res_counter_uncharge(&vmcg->res, val);
+}
+
+int vm_cgroup_charge_memory_mm(struct mm_struct *mm, unsigned long nr_pages)
+{
+	struct vm_cgroup *vmcg = mm->vmcg;
+
+	if (vmcg)
+		return vm_cgroup_do_charge(vmcg, nr_pages);
+	return 0;
+}
+
+void vm_cgroup_uncharge_memory_mm(struct mm_struct *mm, unsigned long nr_pages)
+{
+	struct vm_cgroup *vmcg = mm->vmcg;
+
+	if (vmcg)
+		vm_cgroup_do_uncharge(vmcg, nr_pages);
+}
+
 static struct cgroup_subsys_state *
 vm_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
-- 
1.8.1.2