mm: fix race between kmem_cache destroy, create and deactivate
[muen/linux.git] / mm / slab_common.c
index 98dcdc3520623bf776164982224b7b258e9a2291..42aca26d61d01c67d0ba381fd08f2c11fce6af4a 100644 (file)
@@ -136,6 +136,7 @@ void slab_init_memcg_params(struct kmem_cache *s)
        s->memcg_params.root_cache = NULL;
        RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL);
        INIT_LIST_HEAD(&s->memcg_params.children);
+       s->memcg_params.dying = false;
 }
 
 static int init_memcg_params(struct kmem_cache *s,
@@ -608,7 +609,7 @@ void memcg_create_kmem_cache(struct mem_cgroup *memcg,
         * The memory cgroup could have been offlined while the cache
         * creation work was pending.
         */
-       if (memcg->kmem_state != KMEM_ONLINE)
+       if (memcg->kmem_state != KMEM_ONLINE || root_cache->memcg_params.dying)
                goto out_unlock;
 
        idx = memcg_cache_id(memcg);
@@ -712,6 +713,9 @@ void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,
            WARN_ON_ONCE(s->memcg_params.deact_fn))
                return;
 
+       if (s->memcg_params.root_cache->memcg_params.dying)
+               return;
+
        /* pin memcg so that @s doesn't get destroyed in the middle */
        css_get(&s->memcg_params.memcg->css);
 
@@ -823,11 +827,36 @@ static int shutdown_memcg_caches(struct kmem_cache *s)
                return -EBUSY;
        return 0;
 }
+
+static void flush_memcg_workqueue(struct kmem_cache *s)
+{
+       mutex_lock(&slab_mutex);
+       s->memcg_params.dying = true;
+       mutex_unlock(&slab_mutex);
+
+       /*
+        * SLUB deactivates the kmem_caches through call_rcu_sched. Make
+        * sure all registered rcu callbacks have been invoked.
+        */
+       if (IS_ENABLED(CONFIG_SLUB))
+               rcu_barrier_sched();
+
+       /*
+        * SLAB and SLUB create memcg kmem_caches through workqueue and SLUB
+        * deactivates the memcg kmem_caches through workqueue. Make sure all
+        * previous workitems on workqueue are processed.
+        */
+       flush_workqueue(memcg_kmem_cache_wq);
+}
 #else
 static inline int shutdown_memcg_caches(struct kmem_cache *s)
 {
        return 0;
 }
+
+static inline void flush_memcg_workqueue(struct kmem_cache *s)
+{
+}
 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */
 
 void slab_kmem_cache_release(struct kmem_cache *s)
@@ -845,6 +874,8 @@ void kmem_cache_destroy(struct kmem_cache *s)
        if (unlikely(!s))
                return;
 
+       flush_memcg_workqueue(s);
+
        get_online_cpus();
        get_online_mems();