slab, slub: skip unnecessary kasan_cache_shutdown()
authorShakeel Butt <shakeelb@google.com>
Thu, 5 Apr 2018 23:21:57 +0000 (16:21 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Fri, 6 Apr 2018 04:36:24 +0000 (21:36 -0700)
The kasan quarantine is designed to delay freeing slab objects to catch
use-after-free.  The quarantine can be large (several percent of machine
memory size).  When kmem_caches are deleted related objects are flushed
from the quarantine but this requires scanning the entire quarantine
which can be very slow.  We have seen the kernel busily working on this
while holding slab_mutex and badly affecting cache_reaper, slabinfo
readers and memcg kmem cache creations.

It can easily reproduced by following script:

yes . | head -1000000 | xargs stat > /dev/null
for i in `seq 1 10`; do
seq 500 | (cd /cg/memory && xargs mkdir)
seq 500 | xargs -I{} sh -c 'echo $BASHPID > \
/cg/memory/{}/tasks && exec stat .' > /dev/null
seq 500 | (cd /cg/memory && xargs rmdir)
done

The busy stack:
    kasan_cache_shutdown
    shutdown_cache
    memcg_destroy_kmem_caches
    mem_cgroup_css_free
    css_free_rwork_fn
    process_one_work
    worker_thread
    kthread
    ret_from_fork

This patch is based on the observation that if the kmem_cache to be
destroyed is empty then there should not be any objects of this cache in
the quarantine.

Without the patch the script got stuck for couple of hours.  With the
patch the script completed within a second.

Link: http://lkml.kernel.org/r/20180327230603.54721-1-shakeelb@google.com
Signed-off-by: Shakeel Butt <shakeelb@google.com>
Reviewed-by: Andrew Morton <akpm@linux-foundation.org>
Acked-by: Andrey Ryabinin <aryabinin@virtuozzo.com>
Acked-by: Christoph Lameter <cl@linux.com>
Cc: Vladimir Davydov <vdavydov.dev@gmail.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: David Rientjes <rientjes@google.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
mm/kasan/kasan.c
mm/slab.c
mm/slab.h
mm/slub.c

index f7a5e1d1ba87e5a1cc8d575bd34fb50cacde3e31..bc0e68f7dc756104ff47005a4f9403af7e08fe4e 100644 (file)
@@ -382,7 +382,8 @@ void kasan_cache_shrink(struct kmem_cache *cache)
 
 void kasan_cache_shutdown(struct kmem_cache *cache)
 {
 
 void kasan_cache_shutdown(struct kmem_cache *cache)
 {
-       quarantine_remove_cache(cache);
+       if (!__kmem_cache_empty(cache))
+               quarantine_remove_cache(cache);
 }
 
 size_t kasan_metadata_size(struct kmem_cache *cache)
 }
 
 size_t kasan_metadata_size(struct kmem_cache *cache)
index fb106e8277b77fae33b25524f9a8ae768ed95c2b..e3a9b8e2330642e98755ad12b6b722957fbea78c 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2291,6 +2291,18 @@ out:
        return nr_freed;
 }
 
        return nr_freed;
 }
 
+bool __kmem_cache_empty(struct kmem_cache *s)
+{
+       int node;
+       struct kmem_cache_node *n;
+
+       for_each_kmem_cache_node(s, node, n)
+               if (!list_empty(&n->slabs_full) ||
+                   !list_empty(&n->slabs_partial))
+                       return false;
+       return true;
+}
+
 int __kmem_cache_shrink(struct kmem_cache *cachep)
 {
        int ret = 0;
 int __kmem_cache_shrink(struct kmem_cache *cachep)
 {
        int ret = 0;
index e8981e811c4572aefa6587ba2df4afba07cabc55..68bdf498da3b30d6a204553ead374a1a5b009566 100644 (file)
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -166,6 +166,7 @@ static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
                              SLAB_TEMPORARY | \
                              SLAB_ACCOUNT)
 
                              SLAB_TEMPORARY | \
                              SLAB_ACCOUNT)
 
+bool __kmem_cache_empty(struct kmem_cache *);
 int __kmem_cache_shutdown(struct kmem_cache *);
 void __kmem_cache_release(struct kmem_cache *);
 int __kmem_cache_shrink(struct kmem_cache *);
 int __kmem_cache_shutdown(struct kmem_cache *);
 void __kmem_cache_release(struct kmem_cache *);
 int __kmem_cache_shrink(struct kmem_cache *);
index f48d942a487e3d7b74aae1016393d9728afe345f..4fb037c9878251fb88395e35a96acf1193b3b1b7 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3696,6 +3696,17 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
                discard_slab(s, page);
 }
 
                discard_slab(s, page);
 }
 
+bool __kmem_cache_empty(struct kmem_cache *s)
+{
+       int node;
+       struct kmem_cache_node *n;
+
+       for_each_kmem_cache_node(s, node, n)
+               if (n->nr_partial || slabs_node(s, node))
+                       return false;
+       return true;
+}
+
 /*
  * Release all resources used by a slab cache.
  */
 /*
  * Release all resources used by a slab cache.
  */