1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
From d4e7b31f217ee00026ee090133e88a9bcc33826b Mon Sep 17 00:00:00 2001
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Wed, 28 Jan 2015 17:14:16 +0100
Subject: [PATCH 115/191] mm/memcontrol: Replace local_irq_disable with local
locks
There are a few local_irq_disable() which then take sleeping locks. This
patch converts them local locks.
[bigeasy: Move unlock after memcg_check_events() in mem_cgroup_swapout(),
pointed out by Matt Fleming <matt@codeblueprint.co.uk>]
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
mm/memcontrol.c | 29 +++++++++++++++++++++--------
1 file changed, 21 insertions(+), 8 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0d741fcf9f53..ae52cebfecfc 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -66,6 +66,7 @@
#include <net/sock.h>
#include <net/ip.h>
#include "slab.h"
+#include <linux/local_lock.h>
#include <linux/uaccess.h>
@@ -96,6 +97,13 @@ bool cgroup_memory_noswap __read_mostly;
static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
#endif
+struct event_lock {
+ local_lock_t l;
+};
+static DEFINE_PER_CPU(struct event_lock, event_lock) = {
+ .l = INIT_LOCAL_LOCK(l),
+};
+
/* Whether legacy memory+swap accounting is active */
static bool do_memsw_account(void)
{
@@ -5677,12 +5685,12 @@ static int mem_cgroup_move_account(struct page *page,
ret = 0;
- local_irq_disable();
+ local_lock_irq(&event_lock.l);
mem_cgroup_charge_statistics(to, page, nr_pages);
memcg_check_events(to, page);
mem_cgroup_charge_statistics(from, page, -nr_pages);
memcg_check_events(from, page);
- local_irq_enable();
+ local_unlock_irq(&event_lock.l);
out_unlock:
unlock_page(page);
out:
@@ -6739,10 +6747,10 @@ int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
css_get(&memcg->css);
commit_charge(page, memcg);
- local_irq_disable();
+ local_lock_irq(&event_lock.l);
mem_cgroup_charge_statistics(memcg, page, nr_pages);
memcg_check_events(memcg, page);
- local_irq_enable();
+ local_unlock_irq(&event_lock.l);
/*
* Cgroup1's unified memory+swap counter has been charged with the
@@ -6798,11 +6806,11 @@ static void uncharge_batch(const struct uncharge_gather *ug)
memcg_oom_recover(ug->memcg);
}
- local_irq_save(flags);
+ local_lock_irqsave(&event_lock.l, flags);
__count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
__this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
memcg_check_events(ug->memcg, ug->dummy_page);
- local_irq_restore(flags);
+ local_unlock_irqrestore(&event_lock.l, flags);
/* drop reference from uncharge_page */
css_put(&ug->memcg->css);
@@ -6935,10 +6943,10 @@ void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
css_get(&memcg->css);
commit_charge(newpage, memcg);
- local_irq_save(flags);
+ local_lock_irqsave(&event_lock.l, flags);
mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
memcg_check_events(memcg, newpage);
- local_irq_restore(flags);
+ local_unlock_irqrestore(&event_lock.l, flags);
}
DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
@@ -7121,6 +7129,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
struct mem_cgroup *memcg, *swap_memcg;
unsigned int nr_entries;
unsigned short oldid;
+ unsigned long flags;
VM_BUG_ON_PAGE(PageLRU(page), page);
VM_BUG_ON_PAGE(page_count(page), page);
@@ -7169,9 +7178,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
* important here to have the interrupts disabled because it is the
* only synchronisation we have for updating the per-CPU variables.
*/
+ local_lock_irqsave(&event_lock.l, flags);
+#ifndef CONFIG_PREEMPT_RT
VM_BUG_ON(!irqs_disabled());
+#endif
mem_cgroup_charge_statistics(memcg, page, -nr_entries);
memcg_check_events(memcg, page);
+ local_unlock_irqrestore(&event_lock.l, flags);
css_put(&memcg->css);
}
--
2.19.1
|
