[PATCH v3 2/6] mm: prepare to premature release of per-node lruvec_stat_cpu

From: Roman Gushchin
Date: Wed Mar 13 2019 - 14:40:47 EST


Similar to the memcg's vmstats_percpu, per-memcg per-node stats
consists of percpu- and atomic counterparts, and we do expect
that both coexist during the whole life-cycle of the memcg.

To prepare for a premature release of percpu per-node data,
let's pretend that lruvec_stat_cpu is a rcu-protected pointer,
which can be NULL. This patch adds corresponding checks whenever
required.

Signed-off-by: Roman Gushchin <guro@xxxxxx>
Acked-by: Johannes Weiner <hannes@xxxxxxxxxxx>
---
include/linux/memcontrol.h | 21 +++++++++++++++------
mm/memcontrol.c | 14 +++++++++++---
2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 05ca77767c6a..8ac04632002a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -126,7 +126,7 @@ struct memcg_shrinker_map {
struct mem_cgroup_per_node {
struct lruvec lruvec;

- struct lruvec_stat __percpu *lruvec_stat_cpu;
+ struct lruvec_stat __rcu /* __percpu */ *lruvec_stat_cpu;
atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS];

unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
@@ -682,6 +682,7 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
static inline void __mod_lruvec_state(struct lruvec *lruvec,
enum node_stat_item idx, int val)
{
+ struct lruvec_stat __percpu *lruvec_stat_cpu;
struct mem_cgroup_per_node *pn;
long x;

@@ -697,12 +698,20 @@ static inline void __mod_lruvec_state(struct lruvec *lruvec,
__mod_memcg_state(pn->memcg, idx, val);

/* Update lruvec */
- x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
- if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
- atomic_long_add(x, &pn->lruvec_stat[idx]);
- x = 0;
+ rcu_read_lock();
+ lruvec_stat_cpu = (struct lruvec_stat __percpu *)
+ rcu_dereference(pn->lruvec_stat_cpu);
+ if (likely(lruvec_stat_cpu)) {
+ x = val + __this_cpu_read(lruvec_stat_cpu->count[idx]);
+ if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
+ atomic_long_add(x, &pn->lruvec_stat[idx]);
+ x = 0;
+ }
+ __this_cpu_write(lruvec_stat_cpu->count[idx], x);
+ } else {
+ atomic_long_add(val, &pn->lruvec_stat[idx]);
}
- __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
+ rcu_read_unlock();
}

static inline void mod_lruvec_state(struct lruvec *lruvec,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 803c772f354b..5ef4098f3f8d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2122,6 +2122,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
static int memcg_hotplug_cpu_dead(unsigned int cpu)
{
struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+ struct lruvec_stat __percpu *lruvec_stat_cpu;
struct memcg_stock_pcp *stock;
struct mem_cgroup *memcg;

@@ -2152,7 +2153,12 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu)
struct mem_cgroup_per_node *pn;

pn = mem_cgroup_nodeinfo(memcg, nid);
- x = this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
+
+ lruvec_stat_cpu = (struct lruvec_stat __percpu*)
+ rcu_dereference(pn->lruvec_stat_cpu);
+ if (!lruvec_stat_cpu)
+ continue;
+ x = this_cpu_xchg(lruvec_stat_cpu->count[i], 0);
if (x)
atomic_long_add(x, &pn->lruvec_stat[i]);
}
@@ -4414,6 +4420,7 @@ struct mem_cgroup *mem_cgroup_from_id(unsigned short id)

static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
{
+ struct lruvec_stat __percpu *lruvec_stat_cpu;
struct mem_cgroup_per_node *pn;
int tmp = node;
/*
@@ -4430,11 +4437,12 @@ static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
if (!pn)
return 1;

- pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
- if (!pn->lruvec_stat_cpu) {
+ lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
+ if (!lruvec_stat_cpu) {
kfree(pn);
return 1;
}
+ rcu_assign_pointer(pn->lruvec_stat_cpu, lruvec_stat_cpu);

lruvec_init(&pn->lruvec);
pn->usage_in_excess = 0;
--
2.20.1