From e33660165c901d18e7d3df2290db070d3e4b46df Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:18 +0000 Subject: slab: Use common kmalloc_index/kmalloc_size functions Make slab use the common functions. We can get rid of a lot of old ugly stuff as a results. Among them the sizes array and the weird include/linux/kmalloc_sizes file and some pretty bad #include statements in slab_def.h. The one thing that is different in slab is that the 32 byte cache will also be created for arches that have page sizes larger than 4K. There are numerous smaller allocations that SLOB and SLUB can handle better because of their support for smaller allocation sizes so lets keep the 32 byte slab also for arches with > 4K pages. Reviewed-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 169 ++++++++++++++++++++++++++++---------------------------------- 1 file changed, 76 insertions(+), 93 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index e7667a3584bc..2a7132ec4ff6 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -318,34 +318,18 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int len, static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); static void cache_reap(struct work_struct *unused); -/* - * This function must be completely optimized away if a constant is passed to - * it. Mostly the same as what is in linux/slab.h except it returns an index. - */ -static __always_inline int index_of(const size_t size) -{ - extern void __bad_size(void); - - if (__builtin_constant_p(size)) { - int i = 0; +struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(kmalloc_caches); -#define CACHE(x) \ - if (size <=x) \ - return i; \ - else \ - i++; -#include -#undef CACHE - __bad_size(); - } else - __bad_size(); - return 0; -} +#ifdef CONFIG_ZONE_DMA +struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; +EXPORT_SYMBOL(kmalloc_dma_caches); +#endif static int slab_early_init = 1; -#define INDEX_AC index_of(sizeof(struct arraycache_init)) -#define INDEX_L3 index_of(sizeof(struct kmem_list3)) +#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) +#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3)) static void kmem_list3_init(struct kmem_list3 *parent) { @@ -524,30 +508,6 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return reciprocal_divide(offset, cache->reciprocal_buffer_size); } -/* - * These are the default caches for kmalloc. Custom caches can have other sizes. - */ -struct cache_sizes malloc_sizes[] = { -#define CACHE(x) { .cs_size = (x) }, -#include - CACHE(ULONG_MAX) -#undef CACHE -}; -EXPORT_SYMBOL(malloc_sizes); - -/* Must match cache_sizes above. Out of line to keep cache footprint low. */ -struct cache_names { - char *name; - char *name_dma; -}; - -static struct cache_names __initdata cache_names[] = { -#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, -#include - {NULL,} -#undef CACHE -}; - static struct arraycache_init initarray_generic = { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; @@ -625,19 +585,23 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) static void init_node_lock_keys(int q) { - struct cache_sizes *s = malloc_sizes; + int i; if (slab_state < UP) return; - for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { struct kmem_list3 *l3; + struct kmem_cache *cache = kmalloc_caches[i]; + + if (!cache) + continue; - l3 = s->cs_cachep->nodelists[q]; - if (!l3 || OFF_SLAB(s->cs_cachep)) + l3 = cache->nodelists[q]; + if (!l3 || OFF_SLAB(cache)) continue; - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, + slab_set_lock_classes(cache, &on_slab_l3_key, &on_slab_alc_key, q); } } @@ -705,20 +669,19 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) static inline struct kmem_cache *__find_general_cachep(size_t size, gfp_t gfpflags) { - struct cache_sizes *csizep = malloc_sizes; + int i; #if DEBUG /* This happens if someone tries to call * kmem_cache_create(), or __kmalloc(), before * the generic caches are initialized. */ - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); + BUG_ON(kmalloc_caches[INDEX_AC] == NULL); #endif if (!size) return ZERO_SIZE_PTR; - while (size > csizep->cs_size) - csizep++; + i = kmalloc_index(size); /* * Really subtle: The last entry with cs->cs_size==ULONG_MAX @@ -727,9 +690,9 @@ static inline struct kmem_cache *__find_general_cachep(size_t size, */ #ifdef CONFIG_ZONE_DMA if (unlikely(gfpflags & GFP_DMA)) - return csizep->cs_dmacachep; + return kmalloc_dma_caches[i]; #endif - return csizep->cs_cachep; + return kmalloc_caches[i]; } static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) @@ -1602,8 +1565,6 @@ static void setup_nodelists_pointer(struct kmem_cache *cachep) */ void __init kmem_cache_init(void) { - struct cache_sizes *sizes; - struct cache_names *names; int i; kmem_cache = &kmem_cache_boot; @@ -1657,8 +1618,6 @@ void __init kmem_cache_init(void) list_add(&kmem_cache->list, &slab_caches); /* 2+3) create the kmalloc caches */ - sizes = malloc_sizes; - names = cache_names; /* * Initialize the caches that provide memory for the array cache and the @@ -1666,35 +1625,39 @@ void __init kmem_cache_init(void) * bug. */ - sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name, - sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS); + kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac", + kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS); if (INDEX_AC != INDEX_L3) - sizes[INDEX_L3].cs_cachep = - create_kmalloc_cache(names[INDEX_L3].name, - sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS); + kmalloc_caches[INDEX_L3] = + create_kmalloc_cache("kmalloc-l3", + kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS); slab_early_init = 0; - while (sizes->cs_size != ULONG_MAX) { - /* - * For performance, all the general caches are L1 aligned. - * This should be particularly beneficial on SMP boxes, as it - * eliminates "false sharing". - * Note for systems short on memory removing the alignment will - * allow tighter packing of the smaller caches. - */ - if (!sizes->cs_cachep) - sizes->cs_cachep = create_kmalloc_cache(names->name, - sizes->cs_size, ARCH_KMALLOC_FLAGS); + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + size_t cs_size = kmalloc_size(i); + + if (cs_size < KMALLOC_MIN_SIZE) + continue; + + if (!kmalloc_caches[i]) { + /* + * For performance, all the general caches are L1 aligned. + * This should be particularly beneficial on SMP boxes, as it + * eliminates "false sharing". + * Note for systems short on memory removing the alignment will + * allow tighter packing of the smaller caches. + */ + kmalloc_caches[i] = create_kmalloc_cache("kmalloc", + cs_size, ARCH_KMALLOC_FLAGS); + } #ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = create_kmalloc_cache( - names->name_dma, sizes->cs_size, + kmalloc_dma_caches[i] = create_kmalloc_cache( + "kmalloc-dma", cs_size, SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS); #endif - sizes++; - names++; } /* 4) Replace the bootstrap head arrays */ { @@ -1713,17 +1676,16 @@ void __init kmem_cache_init(void) ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); - BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) + BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC]) != &initarray_generic.cache); - memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), + memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), sizeof(struct arraycache_init)); /* * Do not assume that spinlocks can be initialized via memcpy: */ spin_lock_init(&ptr->lock); - malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = - ptr; + kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; } /* 5) Replace the bootstrap kmem_list3's */ { @@ -1732,17 +1694,39 @@ void __init kmem_cache_init(void) for_each_online_node(nid) { init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid); - init_list(malloc_sizes[INDEX_AC].cs_cachep, + init_list(kmalloc_caches[INDEX_AC], &initkmem_list3[SIZE_AC + nid], nid); if (INDEX_AC != INDEX_L3) { - init_list(malloc_sizes[INDEX_L3].cs_cachep, + init_list(kmalloc_caches[INDEX_L3], &initkmem_list3[SIZE_L3 + nid], nid); } } } slab_state = UP; + + /* Create the proper names */ + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + char *s; + struct kmem_cache *c = kmalloc_caches[i]; + + if (!c) + continue; + + s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i)); + + BUG_ON(!s); + c->name = s; + +#ifdef CONFIG_ZONE_DMA + c = kmalloc_dma_caches[i]; + BUG_ON(!c); + s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i)); + BUG_ON(!s); + c->name = s; +#endif + } } void __init kmem_cache_init_late(void) @@ -2428,10 +2412,9 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) size += BYTES_PER_WORD; } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) - if (size >= malloc_sizes[INDEX_L3 + 1].cs_size - && cachep->object_size > cache_line_size() - && ALIGN(size, cachep->align) < PAGE_SIZE) { - cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); + if (size >= kmalloc_size(INDEX_L3 + 1) + && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { + cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); size = PAGE_SIZE; } #endif -- cgit v1.2.3 From 6744f087ba2a49f6d6935d9daa0b20a0f03567b5 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:12:17 +0000 Subject: slab: Common name for the per node structures Rename the structure used for the per node structures in slab to have a name that expresses that fact. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 87 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 43 insertions(+), 44 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 2a7132ec4ff6..7c0da4c86973 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -288,7 +288,7 @@ struct arraycache_init { /* * The slab lists for all objects. */ -struct kmem_list3 { +struct kmem_cache_node { struct list_head slabs_partial; /* partial list first, better asm code */ struct list_head slabs_full; struct list_head slabs_free; @@ -306,13 +306,13 @@ struct kmem_list3 { * Need this for bootstrapping a per node allocator. */ #define NUM_INIT_LISTS (3 * MAX_NUMNODES) -static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS]; +static struct kmem_cache_node __initdata initkmem_list3[NUM_INIT_LISTS]; #define CACHE_CACHE 0 #define SIZE_AC MAX_NUMNODES #define SIZE_L3 (2 * MAX_NUMNODES) static int drain_freelist(struct kmem_cache *cache, - struct kmem_list3 *l3, int tofree); + struct kmem_cache_node *l3, int tofree); static void free_block(struct kmem_cache *cachep, void **objpp, int len, int node); static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); @@ -329,9 +329,9 @@ EXPORT_SYMBOL(kmalloc_dma_caches); static int slab_early_init = 1; #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) -#define INDEX_L3 kmalloc_index(sizeof(struct kmem_list3)) +#define INDEX_L3 kmalloc_index(sizeof(struct kmem_cache_node)) -static void kmem_list3_init(struct kmem_list3 *parent) +static void kmem_list3_init(struct kmem_cache_node *parent) { INIT_LIST_HEAD(&parent->slabs_full); INIT_LIST_HEAD(&parent->slabs_partial); @@ -546,7 +546,7 @@ static void slab_set_lock_classes(struct kmem_cache *cachep, int q) { struct array_cache **alc; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; int r; l3 = cachep->nodelists[q]; @@ -591,7 +591,7 @@ static void init_node_lock_keys(int q) return; for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { - struct kmem_list3 *l3; + struct kmem_cache_node *l3; struct kmem_cache *cache = kmalloc_caches[i]; if (!cache) @@ -608,9 +608,8 @@ static void init_node_lock_keys(int q) static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) { - struct kmem_list3 *l3; - l3 = cachep->nodelists[q]; - if (!l3) + + if (!cachep->nodelists[q]) return; slab_set_lock_classes(cachep, &on_slab_l3_key, @@ -901,7 +900,7 @@ static inline bool is_slab_pfmemalloc(struct slab *slabp) static void recheck_pfmemalloc_active(struct kmem_cache *cachep, struct array_cache *ac) { - struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()]; + struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()]; struct slab *slabp; unsigned long flags; @@ -934,7 +933,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */ if (unlikely(is_obj_pfmemalloc(objp))) { - struct kmem_list3 *l3; + struct kmem_cache_node *l3; if (gfp_pfmemalloc_allowed(flags)) { clear_obj_pfmemalloc(&objp); @@ -1106,7 +1105,7 @@ static void free_alien_cache(struct array_cache **ac_ptr) static void __drain_alien_cache(struct kmem_cache *cachep, struct array_cache *ac, int node) { - struct kmem_list3 *rl3 = cachep->nodelists[node]; + struct kmem_cache_node *rl3 = cachep->nodelists[node]; if (ac->avail) { spin_lock(&rl3->list_lock); @@ -1127,7 +1126,7 @@ static void __drain_alien_cache(struct kmem_cache *cachep, /* * Called from cache_reap() to regularly drain alien caches round robin. */ -static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3) +static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *l3) { int node = __this_cpu_read(slab_reap_node); @@ -1162,7 +1161,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) { struct slab *slabp = virt_to_slab(objp); int nodeid = slabp->nodeid; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; struct array_cache *alien = NULL; int node; @@ -1207,8 +1206,8 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) static int init_cache_nodelists_node(int node) { struct kmem_cache *cachep; - struct kmem_list3 *l3; - const int memsize = sizeof(struct kmem_list3); + struct kmem_cache_node *l3; + const int memsize = sizeof(struct kmem_cache_node); list_for_each_entry(cachep, &slab_caches, list) { /* @@ -1244,7 +1243,7 @@ static int init_cache_nodelists_node(int node) static void __cpuinit cpuup_canceled(long cpu) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; + struct kmem_cache_node *l3 = NULL; int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); @@ -1309,7 +1308,7 @@ free_array_cache: static int __cpuinit cpuup_prepare(long cpu) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; + struct kmem_cache_node *l3 = NULL; int node = cpu_to_mem(cpu); int err; @@ -1463,7 +1462,7 @@ static int __meminit drain_cache_nodelists_node(int node) int ret = 0; list_for_each_entry(cachep, &slab_caches, list) { - struct kmem_list3 *l3; + struct kmem_cache_node *l3; l3 = cachep->nodelists[node]; if (!l3) @@ -1516,15 +1515,15 @@ out: /* * swap the static kmem_list3 with kmalloced memory */ -static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list, +static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, int nodeid) { - struct kmem_list3 *ptr; + struct kmem_cache_node *ptr; - ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid); + ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid); BUG_ON(!ptr); - memcpy(ptr, list, sizeof(struct kmem_list3)); + memcpy(ptr, list, sizeof(struct kmem_cache_node)); /* * Do not assume that spinlocks can be initialized via memcpy: */ @@ -1556,7 +1555,7 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index) */ static void setup_nodelists_pointer(struct kmem_cache *cachep) { - cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; + cachep->nodelists = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids]; } /* @@ -1613,7 +1612,7 @@ void __init kmem_cache_init(void) */ create_boot_cache(kmem_cache, "kmem_cache", offsetof(struct kmem_cache, array[nr_cpu_ids]) + - nr_node_ids * sizeof(struct kmem_list3 *), + nr_node_ids * sizeof(struct kmem_cache_node *), SLAB_HWCACHE_ALIGN); list_add(&kmem_cache->list, &slab_caches); @@ -1787,7 +1786,7 @@ __initcall(cpucache_init); static noinline void slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) { - struct kmem_list3 *l3; + struct kmem_cache_node *l3; struct slab *slabp; unsigned long flags; int node; @@ -2279,7 +2278,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) int node; for_each_online_node(node) { cachep->nodelists[node] = - kmalloc_node(sizeof(struct kmem_list3), + kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); BUG_ON(!cachep->nodelists[node]); kmem_list3_init(cachep->nodelists[node]); @@ -2547,7 +2546,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) #define check_spinlock_acquired_node(x, y) do { } while(0) #endif -static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, struct array_cache *ac, int force, int node); @@ -2567,7 +2566,7 @@ static void do_drain(void *arg) static void drain_cpu_caches(struct kmem_cache *cachep) { - struct kmem_list3 *l3; + struct kmem_cache_node *l3; int node; on_each_cpu(do_drain, cachep, 1); @@ -2592,7 +2591,7 @@ static void drain_cpu_caches(struct kmem_cache *cachep) * Returns the actual number of slabs released. */ static int drain_freelist(struct kmem_cache *cache, - struct kmem_list3 *l3, int tofree) + struct kmem_cache_node *l3, int tofree) { struct list_head *p; int nr_freed; @@ -2630,7 +2629,7 @@ out: static int __cache_shrink(struct kmem_cache *cachep) { int ret = 0, i = 0; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; drain_cpu_caches(cachep); @@ -2672,7 +2671,7 @@ EXPORT_SYMBOL(kmem_cache_shrink); int __kmem_cache_shutdown(struct kmem_cache *cachep) { int i; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; int rc = __cache_shrink(cachep); if (rc) @@ -2869,7 +2868,7 @@ static int cache_grow(struct kmem_cache *cachep, struct slab *slabp; size_t offset; gfp_t local_flags; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; /* * Be lazy and only check for valid flags here, keeping it out of the @@ -3059,7 +3058,7 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags, bool force_refill) { int batchcount; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; struct array_cache *ac; int node; @@ -3391,7 +3390,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, { struct list_head *entry; struct slab *slabp; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; void *obj; int x; @@ -3586,7 +3585,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, int node) { int i; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; for (i = 0; i < nr_objects; i++) { void *objp; @@ -3632,7 +3631,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) { int batchcount; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; int node = numa_mem_id(); batchcount = ac->batchcount; @@ -3924,7 +3923,7 @@ EXPORT_SYMBOL(kfree); static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) { int node; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; struct array_cache *new_shared; struct array_cache **new_alien = NULL; @@ -3969,7 +3968,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) free_alien_cache(new_alien); continue; } - l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node); + l3 = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); if (!l3) { free_alien_cache(new_alien); kfree(new_shared); @@ -4165,7 +4164,7 @@ skip_setup: * necessary. Note that the l3 listlock also protects the array_cache * if drain_array() is used on the shared array. */ -static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, struct array_cache *ac, int force, int node) { int tofree; @@ -4204,7 +4203,7 @@ static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, static void cache_reap(struct work_struct *w) { struct kmem_cache *searchp; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; int node = numa_mem_id(); struct delayed_work *work = to_delayed_work(w); @@ -4268,7 +4267,7 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) const char *name; char *error = NULL; int node; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; active_objs = 0; num_slabs = 0; @@ -4482,7 +4481,7 @@ static int leaks_show(struct seq_file *m, void *p) { struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; - struct kmem_list3 *l3; + struct kmem_cache_node *l3; const char *name; unsigned long *n = m->private; int node; -- cgit v1.2.3 From 6a67368c36e2c0c2578ba62f6264ab739af08cce Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Rename nodelists to node Have a common naming between both slab caches for future changes. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 135 +++++++++++++++++++++++++++++++------------------------------- 1 file changed, 67 insertions(+), 68 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 7c0da4c86973..3416f4c544b3 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -347,7 +347,7 @@ static void kmem_list3_init(struct kmem_cache_node *parent) #define MAKE_LIST(cachep, listp, slab, nodeid) \ do { \ INIT_LIST_HEAD(listp); \ - list_splice(&(cachep->nodelists[nodeid]->slab), listp); \ + list_splice(&(cachep->node[nodeid]->slab), listp); \ } while (0) #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ @@ -549,7 +549,7 @@ static void slab_set_lock_classes(struct kmem_cache *cachep, struct kmem_cache_node *l3; int r; - l3 = cachep->nodelists[q]; + l3 = cachep->node[q]; if (!l3) return; @@ -597,7 +597,7 @@ static void init_node_lock_keys(int q) if (!cache) continue; - l3 = cache->nodelists[q]; + l3 = cache->node[q]; if (!l3 || OFF_SLAB(cache)) continue; @@ -608,8 +608,7 @@ static void init_node_lock_keys(int q) static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) { - - if (!cachep->nodelists[q]) + if (!cachep->node[q]) return; slab_set_lock_classes(cachep, &on_slab_l3_key, @@ -900,7 +899,7 @@ static inline bool is_slab_pfmemalloc(struct slab *slabp) static void recheck_pfmemalloc_active(struct kmem_cache *cachep, struct array_cache *ac) { - struct kmem_cache_node *l3 = cachep->nodelists[numa_mem_id()]; + struct kmem_cache_node *l3 = cachep->node[numa_mem_id()]; struct slab *slabp; unsigned long flags; @@ -955,7 +954,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, * If there are empty slabs on the slabs_free list and we are * being forced to refill the cache, mark this one !pfmemalloc. */ - l3 = cachep->nodelists[numa_mem_id()]; + l3 = cachep->node[numa_mem_id()]; if (!list_empty(&l3->slabs_free) && force_refill) { struct slab *slabp = virt_to_slab(objp); ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem)); @@ -1105,7 +1104,7 @@ static void free_alien_cache(struct array_cache **ac_ptr) static void __drain_alien_cache(struct kmem_cache *cachep, struct array_cache *ac, int node) { - struct kmem_cache_node *rl3 = cachep->nodelists[node]; + struct kmem_cache_node *rl3 = cachep->node[node]; if (ac->avail) { spin_lock(&rl3->list_lock); @@ -1174,7 +1173,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) if (likely(slabp->nodeid == node)) return 0; - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; STATS_INC_NODEFREES(cachep); if (l3->alien && l3->alien[nodeid]) { alien = l3->alien[nodeid]; @@ -1186,24 +1185,24 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) ac_put_obj(cachep, alien, objp); spin_unlock(&alien->lock); } else { - spin_lock(&(cachep->nodelists[nodeid])->list_lock); + spin_lock(&(cachep->node[nodeid])->list_lock); free_block(cachep, &objp, 1, nodeid); - spin_unlock(&(cachep->nodelists[nodeid])->list_lock); + spin_unlock(&(cachep->node[nodeid])->list_lock); } return 1; } #endif /* - * Allocates and initializes nodelists for a node on each slab cache, used for + * Allocates and initializes node for a node on each slab cache, used for * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3 * will be allocated off-node since memory is not yet online for the new node. - * When hotplugging memory or a cpu, existing nodelists are not replaced if + * When hotplugging memory or a cpu, existing node are not replaced if * already in use. * * Must hold slab_mutex. */ -static int init_cache_nodelists_node(int node) +static int init_cache_node_node(int node) { struct kmem_cache *cachep; struct kmem_cache_node *l3; @@ -1215,7 +1214,7 @@ static int init_cache_nodelists_node(int node) * begin anything. Make sure some other cpu on this * node has not already allocated this */ - if (!cachep->nodelists[node]) { + if (!cachep->node[node]) { l3 = kmalloc_node(memsize, GFP_KERNEL, node); if (!l3) return -ENOMEM; @@ -1228,14 +1227,14 @@ static int init_cache_nodelists_node(int node) * go. slab_mutex is sufficient * protection here. */ - cachep->nodelists[node] = l3; + cachep->node[node] = l3; } - spin_lock_irq(&cachep->nodelists[node]->list_lock); - cachep->nodelists[node]->free_limit = + spin_lock_irq(&cachep->node[node]->list_lock); + cachep->node[node]->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - spin_unlock_irq(&cachep->nodelists[node]->list_lock); + spin_unlock_irq(&cachep->node[node]->list_lock); } return 0; } @@ -1255,7 +1254,7 @@ static void __cpuinit cpuup_canceled(long cpu) /* cpu is dead; no one can alloc from it. */ nc = cachep->array[cpu]; cachep->array[cpu] = NULL; - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) goto free_array_cache; @@ -1298,7 +1297,7 @@ free_array_cache: * shrink each nodelist to its limit. */ list_for_each_entry(cachep, &slab_caches, list) { - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) continue; drain_freelist(cachep, l3, l3->free_objects); @@ -1318,7 +1317,7 @@ static int __cpuinit cpuup_prepare(long cpu) * kmalloc_node allows us to add the slab to the right * kmem_list3 and not this cpu's kmem_list3 */ - err = init_cache_nodelists_node(node); + err = init_cache_node_node(node); if (err < 0) goto bad; @@ -1353,7 +1352,7 @@ static int __cpuinit cpuup_prepare(long cpu) } } cachep->array[cpu] = nc; - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; BUG_ON(!l3); spin_lock_irq(&l3->list_lock); @@ -1456,7 +1455,7 @@ static struct notifier_block __cpuinitdata cpucache_notifier = { * * Must hold slab_mutex. */ -static int __meminit drain_cache_nodelists_node(int node) +static int __meminit drain_cache_node_node(int node) { struct kmem_cache *cachep; int ret = 0; @@ -1464,7 +1463,7 @@ static int __meminit drain_cache_nodelists_node(int node) list_for_each_entry(cachep, &slab_caches, list) { struct kmem_cache_node *l3; - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) continue; @@ -1493,12 +1492,12 @@ static int __meminit slab_memory_callback(struct notifier_block *self, switch (action) { case MEM_GOING_ONLINE: mutex_lock(&slab_mutex); - ret = init_cache_nodelists_node(nid); + ret = init_cache_node_node(nid); mutex_unlock(&slab_mutex); break; case MEM_GOING_OFFLINE: mutex_lock(&slab_mutex); - ret = drain_cache_nodelists_node(nid); + ret = drain_cache_node_node(nid); mutex_unlock(&slab_mutex); break; case MEM_ONLINE: @@ -1530,7 +1529,7 @@ static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node * spin_lock_init(&ptr->list_lock); MAKE_ALL_LISTS(cachep, ptr, nodeid); - cachep->nodelists[nodeid] = ptr; + cachep->node[nodeid] = ptr; } /* @@ -1542,8 +1541,8 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index) int node; for_each_online_node(node) { - cachep->nodelists[node] = &initkmem_list3[index + node]; - cachep->nodelists[node]->next_reap = jiffies + + cachep->node[node] = &initkmem_list3[index + node]; + cachep->node[node]->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; } @@ -1551,11 +1550,11 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index) /* * The memory after the last cpu cache pointer is used for the - * the nodelists pointer. + * the node pointer. */ -static void setup_nodelists_pointer(struct kmem_cache *cachep) +static void setup_node_pointer(struct kmem_cache *cachep) { - cachep->nodelists = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids]; + cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids]; } /* @@ -1567,7 +1566,7 @@ void __init kmem_cache_init(void) int i; kmem_cache = &kmem_cache_boot; - setup_nodelists_pointer(kmem_cache); + setup_node_pointer(kmem_cache); if (num_possible_nodes() == 1) use_alien_caches = 0; @@ -1756,7 +1755,7 @@ void __init kmem_cache_init_late(void) #ifdef CONFIG_NUMA /* * Register a memory hotplug callback that initializes and frees - * nodelists. + * node. */ hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); #endif @@ -1801,7 +1800,7 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) unsigned long active_objs = 0, num_objs = 0, free_objects = 0; unsigned long active_slabs = 0, num_slabs = 0; - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) continue; @@ -2277,15 +2276,15 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) } else { int node; for_each_online_node(node) { - cachep->nodelists[node] = + cachep->node[node] = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); - BUG_ON(!cachep->nodelists[node]); - kmem_list3_init(cachep->nodelists[node]); + BUG_ON(!cachep->node[node]); + kmem_list3_init(cachep->node[node]); } } } - cachep->nodelists[numa_mem_id()]->next_reap = + cachep->node[numa_mem_id()]->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; @@ -2388,7 +2387,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) else gfp = GFP_NOWAIT; - setup_nodelists_pointer(cachep); + setup_node_pointer(cachep); #if DEBUG /* @@ -2527,7 +2526,7 @@ static void check_spinlock_acquired(struct kmem_cache *cachep) { #ifdef CONFIG_SMP check_irq_off(); - assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock); + assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock); #endif } @@ -2535,7 +2534,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) { #ifdef CONFIG_SMP check_irq_off(); - assert_spin_locked(&cachep->nodelists[node]->list_lock); + assert_spin_locked(&cachep->node[node]->list_lock); #endif } @@ -2558,9 +2557,9 @@ static void do_drain(void *arg) check_irq_off(); ac = cpu_cache_get(cachep); - spin_lock(&cachep->nodelists[node]->list_lock); + spin_lock(&cachep->node[node]->list_lock); free_block(cachep, ac->entry, ac->avail, node); - spin_unlock(&cachep->nodelists[node]->list_lock); + spin_unlock(&cachep->node[node]->list_lock); ac->avail = 0; } @@ -2572,13 +2571,13 @@ static void drain_cpu_caches(struct kmem_cache *cachep) on_each_cpu(do_drain, cachep, 1); check_irq_on(); for_each_online_node(node) { - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (l3 && l3->alien) drain_alien_cache(cachep, l3->alien); } for_each_online_node(node) { - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (l3) drain_array(cachep, l3, l3->shared, 1, node); } @@ -2635,7 +2634,7 @@ static int __cache_shrink(struct kmem_cache *cachep) check_irq_on(); for_each_online_node(i) { - l3 = cachep->nodelists[i]; + l3 = cachep->node[i]; if (!l3) continue; @@ -2682,7 +2681,7 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep) /* NUMA: free the list3 structures */ for_each_online_node(i) { - l3 = cachep->nodelists[i]; + l3 = cachep->node[i]; if (l3) { kfree(l3->shared); free_alien_cache(l3->alien); @@ -2879,7 +2878,7 @@ static int cache_grow(struct kmem_cache *cachep, /* Take the l3 list lock to change the colour_next on this node */ check_irq_off(); - l3 = cachep->nodelists[nodeid]; + l3 = cachep->node[nodeid]; spin_lock(&l3->list_lock); /* Get colour for the slab, and cal the next value. */ @@ -3077,7 +3076,7 @@ retry: */ batchcount = BATCHREFILL_LIMIT; } - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; BUG_ON(ac->avail > 0 || !l3); spin_lock(&l3->list_lock); @@ -3299,7 +3298,7 @@ static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) /* * Fallback function if there was no memory available and no objects on a * certain node and fall back is permitted. First we scan all the - * available nodelists for available objects. If that fails then we + * available node for available objects. If that fails then we * perform an allocation without specifying a node. This allows the page * allocator to do its reclaim / fallback magic. We then insert the * slab into the proper nodelist and then allocate from it. @@ -3333,8 +3332,8 @@ retry: nid = zone_to_nid(zone); if (cpuset_zone_allowed_hardwall(zone, flags) && - cache->nodelists[nid] && - cache->nodelists[nid]->free_objects) { + cache->node[nid] && + cache->node[nid]->free_objects) { obj = ____cache_alloc_node(cache, flags | GFP_THISNODE, nid); if (obj) @@ -3394,7 +3393,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, void *obj; int x; - l3 = cachep->nodelists[nodeid]; + l3 = cachep->node[nodeid]; BUG_ON(!l3); retry: @@ -3479,7 +3478,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, if (nodeid == NUMA_NO_NODE) nodeid = slab_node; - if (unlikely(!cachep->nodelists[nodeid])) { + if (unlikely(!cachep->node[nodeid])) { /* Node not bootstrapped yet */ ptr = fallback_alloc(cachep, flags); goto out; @@ -3595,7 +3594,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, objp = objpp[i]; slabp = virt_to_slab(objp); - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; list_del(&slabp->list); check_spinlock_acquired_node(cachep, node); check_slabp(cachep, slabp); @@ -3639,7 +3638,7 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) BUG_ON(!batchcount || batchcount > ac->avail); #endif check_irq_off(); - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; spin_lock(&l3->list_lock); if (l3->shared) { struct array_cache *shared_array = l3->shared; @@ -3946,7 +3945,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) } } - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (l3) { struct array_cache *shared = l3->shared; @@ -3982,7 +3981,7 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) l3->alien = new_alien; l3->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - cachep->nodelists[node] = l3; + cachep->node[node] = l3; } return 0; @@ -3991,13 +3990,13 @@ fail: /* Cache is not active yet. Roll back what we did */ node--; while (node >= 0) { - if (cachep->nodelists[node]) { - l3 = cachep->nodelists[node]; + if (cachep->node[node]) { + l3 = cachep->node[node]; kfree(l3->shared); free_alien_cache(l3->alien); kfree(l3); - cachep->nodelists[node] = NULL; + cachep->node[node] = NULL; } node--; } @@ -4057,9 +4056,9 @@ static int __do_tune_cpucache(struct kmem_cache *cachep, int limit, struct array_cache *ccold = new->new[i]; if (!ccold) continue; - spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock); + spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i)); - spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock); + spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); kfree(ccold); } kfree(new); @@ -4219,7 +4218,7 @@ static void cache_reap(struct work_struct *w) * have established with reasonable certainty that * we can do some work if the lock was obtained. */ - l3 = searchp->nodelists[node]; + l3 = searchp->node[node]; reap_alien(searchp, l3); @@ -4272,7 +4271,7 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) active_objs = 0; num_slabs = 0; for_each_online_node(node) { - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) continue; @@ -4497,7 +4496,7 @@ static int leaks_show(struct seq_file *m, void *p) n[1] = 0; for_each_online_node(node) { - l3 = cachep->nodelists[node]; + l3 = cachep->node[node]; if (!l3) continue; -- cgit v1.2.3 From 9425c58e5445277699ff3c2a87bac1cfebc1b48d Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:12:17 +0000 Subject: slab: Common definition for the array of kmalloc caches Have a common definition fo the kmalloc cache arrays in SLAB and SLUB Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 3416f4c544b3..357f0bdc5e43 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -318,14 +318,6 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int len, static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); static void cache_reap(struct work_struct *unused); -struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; -EXPORT_SYMBOL(kmalloc_caches); - -#ifdef CONFIG_ZONE_DMA -struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; -EXPORT_SYMBOL(kmalloc_dma_caches); -#endif - static int slab_early_init = 1; #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) -- cgit v1.2.3 From f97d5f634d3b5133951424fae751db1f339548bd Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:12:17 +0000 Subject: slab: Common function to create the kmalloc array The kmalloc array is created in similar ways in both SLAB and SLUB. Create a common function and have both allocators call that function. V1->V2: Whitespace cleanup Reviewed-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 48 +----------------------------------------------- 1 file changed, 1 insertion(+), 47 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 357f0bdc5e43..08ba44f81a28 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1625,30 +1625,6 @@ void __init kmem_cache_init(void) slab_early_init = 0; - for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { - size_t cs_size = kmalloc_size(i); - - if (cs_size < KMALLOC_MIN_SIZE) - continue; - - if (!kmalloc_caches[i]) { - /* - * For performance, all the general caches are L1 aligned. - * This should be particularly beneficial on SMP boxes, as it - * eliminates "false sharing". - * Note for systems short on memory removing the alignment will - * allow tighter packing of the smaller caches. - */ - kmalloc_caches[i] = create_kmalloc_cache("kmalloc", - cs_size, ARCH_KMALLOC_FLAGS); - } - -#ifdef CONFIG_ZONE_DMA - kmalloc_dma_caches[i] = create_kmalloc_cache( - "kmalloc-dma", cs_size, - SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS); -#endif - } /* 4) Replace the bootstrap head arrays */ { struct array_cache *ptr; @@ -1694,29 +1670,7 @@ void __init kmem_cache_init(void) } } - slab_state = UP; - - /* Create the proper names */ - for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { - char *s; - struct kmem_cache *c = kmalloc_caches[i]; - - if (!c) - continue; - - s = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i)); - - BUG_ON(!s); - c->name = s; - -#ifdef CONFIG_ZONE_DMA - c = kmalloc_dma_caches[i]; - BUG_ON(!c); - s = kasprintf(GFP_NOWAIT, "dma-kmalloc-%d", kmalloc_size(i)); - BUG_ON(!s); - c->name = s; -#endif - } + create_kmalloc_caches(ARCH_KMALLOC_FLAGS); } void __init kmem_cache_init_late(void) -- cgit v1.2.3 From 2c59dd6544212faa5ce761920d2251f4152f408d Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common Kmalloc cache determination Extract the optimized lookup functions from slub and put them into slab_common.c. Then make slab use these functions as well. Joonsoo notes that this fixes some issues with constant folding which also reduces the code size for slub. https://lkml.org/lkml/2012/10/20/82 Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 40 +++------------------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 08ba44f81a28..62629b11df38 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -656,40 +656,6 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) return cachep->array[smp_processor_id()]; } -static inline struct kmem_cache *__find_general_cachep(size_t size, - gfp_t gfpflags) -{ - int i; - -#if DEBUG - /* This happens if someone tries to call - * kmem_cache_create(), or __kmalloc(), before - * the generic caches are initialized. - */ - BUG_ON(kmalloc_caches[INDEX_AC] == NULL); -#endif - if (!size) - return ZERO_SIZE_PTR; - - i = kmalloc_index(size); - - /* - * Really subtle: The last entry with cs->cs_size==ULONG_MAX - * has cs_{dma,}cachep==NULL. Thus no special case - * for large kmalloc calls required. - */ -#ifdef CONFIG_ZONE_DMA - if (unlikely(gfpflags & GFP_DMA)) - return kmalloc_dma_caches[i]; -#endif - return kmalloc_caches[i]; -} - -static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) -{ - return __find_general_cachep(size, gfpflags); -} - static size_t slab_mgmt_size(size_t nr_objs, size_t align) { return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align); @@ -2426,7 +2392,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) cachep->reciprocal_buffer_size = reciprocal_value(size); if (flags & CFLGS_OFF_SLAB) { - cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u); + cachep->slabp_cache = kmalloc_slab(slab_size, 0u); /* * This is a possibility for one of the malloc_sizes caches. * But since we go off slab only for object size greater than @@ -3729,7 +3695,7 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) { struct kmem_cache *cachep; - cachep = kmem_find_general_cachep(size, flags); + cachep = kmalloc_slab(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; return kmem_cache_alloc_node_trace(cachep, flags, node, size); @@ -3774,7 +3740,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, * Then kmalloc uses the uninlined functions instead of the inline * functions. */ - cachep = __find_general_cachep(size, flags); + cachep = kmalloc_slab(size, flags); if (unlikely(ZERO_OR_NULL_PTR(cachep))) return cachep; ret = slab_alloc(cachep, flags, caller); -- cgit v1.2.3 From ce8eb6c424c794d7fb4d1a6667d267990ca28072 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Rename list3/l3 to node The list3 or l3 pointers are pointing to per node structures. Reflect that in the names of variables used. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 516 +++++++++++++++++++++++++++++++------------------------------- 1 file changed, 258 insertions(+), 258 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 62629b11df38..c162b2eb493a 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -306,13 +306,13 @@ struct kmem_cache_node { * Need this for bootstrapping a per node allocator. */ #define NUM_INIT_LISTS (3 * MAX_NUMNODES) -static struct kmem_cache_node __initdata initkmem_list3[NUM_INIT_LISTS]; +static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS]; #define CACHE_CACHE 0 #define SIZE_AC MAX_NUMNODES -#define SIZE_L3 (2 * MAX_NUMNODES) +#define SIZE_NODE (2 * MAX_NUMNODES) static int drain_freelist(struct kmem_cache *cache, - struct kmem_cache_node *l3, int tofree); + struct kmem_cache_node *n, int tofree); static void free_block(struct kmem_cache *cachep, void **objpp, int len, int node); static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); @@ -321,9 +321,9 @@ static void cache_reap(struct work_struct *unused); static int slab_early_init = 1; #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) -#define INDEX_L3 kmalloc_index(sizeof(struct kmem_cache_node)) +#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node)) -static void kmem_list3_init(struct kmem_cache_node *parent) +static void kmem_cache_node_init(struct kmem_cache_node *parent) { INIT_LIST_HEAD(&parent->slabs_full); INIT_LIST_HEAD(&parent->slabs_partial); @@ -538,15 +538,15 @@ static void slab_set_lock_classes(struct kmem_cache *cachep, int q) { struct array_cache **alc; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; int r; - l3 = cachep->node[q]; - if (!l3) + n = cachep->node[q]; + if (!n) return; - lockdep_set_class(&l3->list_lock, l3_key); - alc = l3->alien; + lockdep_set_class(&n->list_lock, l3_key); + alc = n->alien; /* * FIXME: This check for BAD_ALIEN_MAGIC * should go away when common slab code is taught to @@ -583,14 +583,14 @@ static void init_node_lock_keys(int q) return; for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { - struct kmem_cache_node *l3; + struct kmem_cache_node *n; struct kmem_cache *cache = kmalloc_caches[i]; if (!cache) continue; - l3 = cache->node[q]; - if (!l3 || OFF_SLAB(cache)) + n = cache->node[q]; + if (!n || OFF_SLAB(cache)) continue; slab_set_lock_classes(cache, &on_slab_l3_key, @@ -857,29 +857,29 @@ static inline bool is_slab_pfmemalloc(struct slab *slabp) static void recheck_pfmemalloc_active(struct kmem_cache *cachep, struct array_cache *ac) { - struct kmem_cache_node *l3 = cachep->node[numa_mem_id()]; + struct kmem_cache_node *n = cachep->node[numa_mem_id()]; struct slab *slabp; unsigned long flags; if (!pfmemalloc_active) return; - spin_lock_irqsave(&l3->list_lock, flags); - list_for_each_entry(slabp, &l3->slabs_full, list) + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(slabp, &n->slabs_full, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_partial, list) + list_for_each_entry(slabp, &n->slabs_partial, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_free, list) + list_for_each_entry(slabp, &n->slabs_free, list) if (is_slab_pfmemalloc(slabp)) goto out; pfmemalloc_active = false; out: - spin_unlock_irqrestore(&l3->list_lock, flags); + spin_unlock_irqrestore(&n->list_lock, flags); } static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, @@ -890,7 +890,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */ if (unlikely(is_obj_pfmemalloc(objp))) { - struct kmem_cache_node *l3; + struct kmem_cache_node *n; if (gfp_pfmemalloc_allowed(flags)) { clear_obj_pfmemalloc(&objp); @@ -912,8 +912,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, * If there are empty slabs on the slabs_free list and we are * being forced to refill the cache, mark this one !pfmemalloc. */ - l3 = cachep->node[numa_mem_id()]; - if (!list_empty(&l3->slabs_free) && force_refill) { + n = cachep->node[numa_mem_id()]; + if (!list_empty(&n->slabs_free) && force_refill) { struct slab *slabp = virt_to_slab(objp); ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem)); clear_obj_pfmemalloc(&objp); @@ -990,7 +990,7 @@ static int transfer_objects(struct array_cache *to, #ifndef CONFIG_NUMA #define drain_alien_cache(cachep, alien) do { } while (0) -#define reap_alien(cachep, l3) do { } while (0) +#define reap_alien(cachep, n) do { } while (0) static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) { @@ -1062,33 +1062,33 @@ static void free_alien_cache(struct array_cache **ac_ptr) static void __drain_alien_cache(struct kmem_cache *cachep, struct array_cache *ac, int node) { - struct kmem_cache_node *rl3 = cachep->node[node]; + struct kmem_cache_node *n = cachep->node[node]; if (ac->avail) { - spin_lock(&rl3->list_lock); + spin_lock(&n->list_lock); /* * Stuff objects into the remote nodes shared array first. * That way we could avoid the overhead of putting the objects * into the free lists and getting them back later. */ - if (rl3->shared) - transfer_objects(rl3->shared, ac, ac->limit); + if (n->shared) + transfer_objects(n->shared, ac, ac->limit); free_block(cachep, ac->entry, ac->avail, node); ac->avail = 0; - spin_unlock(&rl3->list_lock); + spin_unlock(&n->list_lock); } } /* * Called from cache_reap() to regularly drain alien caches round robin. */ -static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *l3) +static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) { int node = __this_cpu_read(slab_reap_node); - if (l3->alien) { - struct array_cache *ac = l3->alien[node]; + if (n->alien) { + struct array_cache *ac = n->alien[node]; if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { __drain_alien_cache(cachep, ac, node); @@ -1118,7 +1118,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) { struct slab *slabp = virt_to_slab(objp); int nodeid = slabp->nodeid; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; struct array_cache *alien = NULL; int node; @@ -1131,10 +1131,10 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) if (likely(slabp->nodeid == node)) return 0; - l3 = cachep->node[node]; + n = cachep->node[node]; STATS_INC_NODEFREES(cachep); - if (l3->alien && l3->alien[nodeid]) { - alien = l3->alien[nodeid]; + if (n->alien && n->alien[nodeid]) { + alien = n->alien[nodeid]; spin_lock(&alien->lock); if (unlikely(alien->avail == alien->limit)) { STATS_INC_ACOVERFLOW(cachep); @@ -1153,7 +1153,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) /* * Allocates and initializes node for a node on each slab cache, used for - * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3 + * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node * will be allocated off-node since memory is not yet online for the new node. * When hotplugging memory or a cpu, existing node are not replaced if * already in use. @@ -1163,7 +1163,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) static int init_cache_node_node(int node) { struct kmem_cache *cachep; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; const int memsize = sizeof(struct kmem_cache_node); list_for_each_entry(cachep, &slab_caches, list) { @@ -1173,11 +1173,11 @@ static int init_cache_node_node(int node) * node has not already allocated this */ if (!cachep->node[node]) { - l3 = kmalloc_node(memsize, GFP_KERNEL, node); - if (!l3) + n = kmalloc_node(memsize, GFP_KERNEL, node); + if (!n) return -ENOMEM; - kmem_list3_init(l3); - l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + + kmem_cache_node_init(n); + n->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; /* @@ -1185,7 +1185,7 @@ static int init_cache_node_node(int node) * go. slab_mutex is sufficient * protection here. */ - cachep->node[node] = l3; + cachep->node[node] = n; } spin_lock_irq(&cachep->node[node]->list_lock); @@ -1200,7 +1200,7 @@ static int init_cache_node_node(int node) static void __cpuinit cpuup_canceled(long cpu) { struct kmem_cache *cachep; - struct kmem_cache_node *l3 = NULL; + struct kmem_cache_node *n = NULL; int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); @@ -1212,34 +1212,34 @@ static void __cpuinit cpuup_canceled(long cpu) /* cpu is dead; no one can alloc from it. */ nc = cachep->array[cpu]; cachep->array[cpu] = NULL; - l3 = cachep->node[node]; + n = cachep->node[node]; - if (!l3) + if (!n) goto free_array_cache; - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); - /* Free limit for this kmem_list3 */ - l3->free_limit -= cachep->batchcount; + /* Free limit for this kmem_cache_node */ + n->free_limit -= cachep->batchcount; if (nc) free_block(cachep, nc->entry, nc->avail, node); if (!cpumask_empty(mask)) { - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); goto free_array_cache; } - shared = l3->shared; + shared = n->shared; if (shared) { free_block(cachep, shared->entry, shared->avail, node); - l3->shared = NULL; + n->shared = NULL; } - alien = l3->alien; - l3->alien = NULL; + alien = n->alien; + n->alien = NULL; - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); kfree(shared); if (alien) { @@ -1255,17 +1255,17 @@ free_array_cache: * shrink each nodelist to its limit. */ list_for_each_entry(cachep, &slab_caches, list) { - l3 = cachep->node[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - drain_freelist(cachep, l3, l3->free_objects); + drain_freelist(cachep, n, n->free_objects); } } static int __cpuinit cpuup_prepare(long cpu) { struct kmem_cache *cachep; - struct kmem_cache_node *l3 = NULL; + struct kmem_cache_node *n = NULL; int node = cpu_to_mem(cpu); int err; @@ -1273,7 +1273,7 @@ static int __cpuinit cpuup_prepare(long cpu) * We need to do this right in the beginning since * alloc_arraycache's are going to use this list. * kmalloc_node allows us to add the slab to the right - * kmem_list3 and not this cpu's kmem_list3 + * kmem_cache_node and not this cpu's kmem_cache_node */ err = init_cache_node_node(node); if (err < 0) @@ -1310,25 +1310,25 @@ static int __cpuinit cpuup_prepare(long cpu) } } cachep->array[cpu] = nc; - l3 = cachep->node[node]; - BUG_ON(!l3); + n = cachep->node[node]; + BUG_ON(!n); - spin_lock_irq(&l3->list_lock); - if (!l3->shared) { + spin_lock_irq(&n->list_lock); + if (!n->shared) { /* * We are serialised from CPU_DEAD or * CPU_UP_CANCELLED by the cpucontrol lock */ - l3->shared = shared; + n->shared = shared; shared = NULL; } #ifdef CONFIG_NUMA - if (!l3->alien) { - l3->alien = alien; + if (!n->alien) { + n->alien = alien; alien = NULL; } #endif - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); kfree(shared); free_alien_cache(alien); if (cachep->flags & SLAB_DEBUG_OBJECTS) @@ -1383,9 +1383,9 @@ static int __cpuinit cpuup_callback(struct notifier_block *nfb, case CPU_DEAD_FROZEN: /* * Even if all the cpus of a node are down, we don't free the - * kmem_list3 of any cache. This to avoid a race between + * kmem_cache_node of any cache. This to avoid a race between * cpu_down, and a kmalloc allocation from another cpu for - * memory from the node of the cpu going down. The list3 + * memory from the node of the cpu going down. The node * structure is usually allocated from kmem_cache_create() and * gets destroyed at kmem_cache_destroy(). */ @@ -1419,16 +1419,16 @@ static int __meminit drain_cache_node_node(int node) int ret = 0; list_for_each_entry(cachep, &slab_caches, list) { - struct kmem_cache_node *l3; + struct kmem_cache_node *n; - l3 = cachep->node[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - drain_freelist(cachep, l3, l3->free_objects); + drain_freelist(cachep, n, n->free_objects); - if (!list_empty(&l3->slabs_full) || - !list_empty(&l3->slabs_partial)) { + if (!list_empty(&n->slabs_full) || + !list_empty(&n->slabs_partial)) { ret = -EBUSY; break; } @@ -1470,7 +1470,7 @@ out: #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */ /* - * swap the static kmem_list3 with kmalloced memory + * swap the static kmem_cache_node with kmalloced memory */ static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, int nodeid) @@ -1491,15 +1491,15 @@ static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node * } /* - * For setting up all the kmem_list3s for cache whose buffer_size is same as - * size of kmem_list3. + * For setting up all the kmem_cache_node for cache whose buffer_size is same as + * size of kmem_cache_node. */ -static void __init set_up_list3s(struct kmem_cache *cachep, int index) +static void __init set_up_node(struct kmem_cache *cachep, int index) { int node; for_each_online_node(node) { - cachep->node[node] = &initkmem_list3[index + node]; + cachep->node[node] = &init_kmem_cache_node[index + node]; cachep->node[node]->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; @@ -1530,9 +1530,9 @@ void __init kmem_cache_init(void) use_alien_caches = 0; for (i = 0; i < NUM_INIT_LISTS; i++) - kmem_list3_init(&initkmem_list3[i]); + kmem_cache_node_init(&init_kmem_cache_node[i]); - set_up_list3s(kmem_cache, CACHE_CACHE); + set_up_node(kmem_cache, CACHE_CACHE); /* * Fragmentation resistance on low memory - only use bigger @@ -1548,7 +1548,7 @@ void __init kmem_cache_init(void) * kmem_cache structures of all caches, except kmem_cache itself: * kmem_cache is statically allocated. * Initially an __init data area is used for the head array and the - * kmem_list3 structures, it's replaced with a kmalloc allocated + * kmem_cache_node structures, it's replaced with a kmalloc allocated * array at the end of the bootstrap. * 2) Create the first kmalloc cache. * The struct kmem_cache for the new cache is allocated normally. @@ -1557,7 +1557,7 @@ void __init kmem_cache_init(void) * head arrays. * 4) Replace the __init data head arrays for kmem_cache and the first * kmalloc cache with kmalloc allocated arrays. - * 5) Replace the __init data for kmem_list3 for kmem_cache and + * 5) Replace the __init data for kmem_cache_node for kmem_cache and * the other cache's with kmalloc allocated memory. * 6) Resize the head arrays of the kmalloc caches to their final sizes. */ @@ -1577,17 +1577,17 @@ void __init kmem_cache_init(void) /* * Initialize the caches that provide memory for the array cache and the - * kmem_list3 structures first. Without this, further allocations will + * kmem_cache_node structures first. Without this, further allocations will * bug. */ kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac", kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS); - if (INDEX_AC != INDEX_L3) - kmalloc_caches[INDEX_L3] = - create_kmalloc_cache("kmalloc-l3", - kmalloc_size(INDEX_L3), ARCH_KMALLOC_FLAGS); + if (INDEX_AC != INDEX_NODE) + kmalloc_caches[INDEX_NODE] = + create_kmalloc_cache("kmalloc-node", + kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS); slab_early_init = 0; @@ -1619,19 +1619,19 @@ void __init kmem_cache_init(void) kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; } - /* 5) Replace the bootstrap kmem_list3's */ + /* 5) Replace the bootstrap kmem_cache_node */ { int nid; for_each_online_node(nid) { - init_list(kmem_cache, &initkmem_list3[CACHE_CACHE + nid], nid); + init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid); init_list(kmalloc_caches[INDEX_AC], - &initkmem_list3[SIZE_AC + nid], nid); + &init_kmem_cache_node[SIZE_AC + nid], nid); - if (INDEX_AC != INDEX_L3) { - init_list(kmalloc_caches[INDEX_L3], - &initkmem_list3[SIZE_L3 + nid], nid); + if (INDEX_AC != INDEX_NODE) { + init_list(kmalloc_caches[INDEX_NODE], + &init_kmem_cache_node[SIZE_NODE + nid], nid); } } } @@ -1697,7 +1697,7 @@ __initcall(cpucache_init); static noinline void slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) { - struct kmem_cache_node *l3; + struct kmem_cache_node *n; struct slab *slabp; unsigned long flags; int node; @@ -1712,24 +1712,24 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) unsigned long active_objs = 0, num_objs = 0, free_objects = 0; unsigned long active_slabs = 0, num_slabs = 0; - l3 = cachep->node[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; - spin_lock_irqsave(&l3->list_lock, flags); - list_for_each_entry(slabp, &l3->slabs_full, list) { + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(slabp, &n->slabs_full, list) { active_objs += cachep->num; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_partial, list) { + list_for_each_entry(slabp, &n->slabs_partial, list) { active_objs += slabp->inuse; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_free, list) + list_for_each_entry(slabp, &n->slabs_free, list) num_slabs++; - free_objects += l3->free_objects; - spin_unlock_irqrestore(&l3->list_lock, flags); + free_objects += n->free_objects; + spin_unlock_irqrestore(&n->list_lock, flags); num_slabs += active_slabs; num_objs = num_slabs * cachep->num; @@ -2154,7 +2154,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) if (slab_state == DOWN) { /* * Note: Creation of first cache (kmem_cache). - * The setup_list3s is taken care + * The setup_node is taken care * of by the caller of __kmem_cache_create */ cachep->array[smp_processor_id()] = &initarray_generic.cache; @@ -2168,13 +2168,13 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) cachep->array[smp_processor_id()] = &initarray_generic.cache; /* - * If the cache that's used by kmalloc(sizeof(kmem_list3)) is - * the second cache, then we need to set up all its list3s, + * If the cache that's used by kmalloc(sizeof(kmem_cache_node)) is + * the second cache, then we need to set up all its node/, * otherwise the creation of further caches will BUG(). */ - set_up_list3s(cachep, SIZE_AC); - if (INDEX_AC == INDEX_L3) - slab_state = PARTIAL_L3; + set_up_node(cachep, SIZE_AC); + if (INDEX_AC == INDEX_NODE) + slab_state = PARTIAL_NODE; else slab_state = PARTIAL_ARRAYCACHE; } else { @@ -2183,8 +2183,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) kmalloc(sizeof(struct arraycache_init), gfp); if (slab_state == PARTIAL_ARRAYCACHE) { - set_up_list3s(cachep, SIZE_L3); - slab_state = PARTIAL_L3; + set_up_node(cachep, SIZE_NODE); + slab_state = PARTIAL_NODE; } else { int node; for_each_online_node(node) { @@ -2192,7 +2192,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); BUG_ON(!cachep->node[node]); - kmem_list3_init(cachep->node[node]); + kmem_cache_node_init(cachep->node[node]); } } } @@ -2322,7 +2322,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) size += BYTES_PER_WORD; } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) - if (size >= kmalloc_size(INDEX_L3 + 1) + if (size >= kmalloc_size(INDEX_NODE + 1) && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); size = PAGE_SIZE; @@ -2457,7 +2457,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) #define check_spinlock_acquired_node(x, y) do { } while(0) #endif -static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, struct array_cache *ac, int force, int node); @@ -2477,21 +2477,21 @@ static void do_drain(void *arg) static void drain_cpu_caches(struct kmem_cache *cachep) { - struct kmem_cache_node *l3; + struct kmem_cache_node *n; int node; on_each_cpu(do_drain, cachep, 1); check_irq_on(); for_each_online_node(node) { - l3 = cachep->node[node]; - if (l3 && l3->alien) - drain_alien_cache(cachep, l3->alien); + n = cachep->node[node]; + if (n && n->alien) + drain_alien_cache(cachep, n->alien); } for_each_online_node(node) { - l3 = cachep->node[node]; - if (l3) - drain_array(cachep, l3, l3->shared, 1, node); + n = cachep->node[node]; + if (n) + drain_array(cachep, n, n->shared, 1, node); } } @@ -2502,19 +2502,19 @@ static void drain_cpu_caches(struct kmem_cache *cachep) * Returns the actual number of slabs released. */ static int drain_freelist(struct kmem_cache *cache, - struct kmem_cache_node *l3, int tofree) + struct kmem_cache_node *n, int tofree) { struct list_head *p; int nr_freed; struct slab *slabp; nr_freed = 0; - while (nr_freed < tofree && !list_empty(&l3->slabs_free)) { + while (nr_freed < tofree && !list_empty(&n->slabs_free)) { - spin_lock_irq(&l3->list_lock); - p = l3->slabs_free.prev; - if (p == &l3->slabs_free) { - spin_unlock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); + p = n->slabs_free.prev; + if (p == &n->slabs_free) { + spin_unlock_irq(&n->list_lock); goto out; } @@ -2527,8 +2527,8 @@ static int drain_freelist(struct kmem_cache *cache, * Safe to drop the lock. The slab is no longer linked * to the cache. */ - l3->free_objects -= cache->num; - spin_unlock_irq(&l3->list_lock); + n->free_objects -= cache->num; + spin_unlock_irq(&n->list_lock); slab_destroy(cache, slabp); nr_freed++; } @@ -2540,20 +2540,20 @@ out: static int __cache_shrink(struct kmem_cache *cachep) { int ret = 0, i = 0; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; drain_cpu_caches(cachep); check_irq_on(); for_each_online_node(i) { - l3 = cachep->node[i]; - if (!l3) + n = cachep->node[i]; + if (!n) continue; - drain_freelist(cachep, l3, l3->free_objects); + drain_freelist(cachep, n, n->free_objects); - ret += !list_empty(&l3->slabs_full) || - !list_empty(&l3->slabs_partial); + ret += !list_empty(&n->slabs_full) || + !list_empty(&n->slabs_partial); } return (ret ? 1 : 0); } @@ -2582,7 +2582,7 @@ EXPORT_SYMBOL(kmem_cache_shrink); int __kmem_cache_shutdown(struct kmem_cache *cachep) { int i; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; int rc = __cache_shrink(cachep); if (rc) @@ -2591,13 +2591,13 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep) for_each_online_cpu(i) kfree(cachep->array[i]); - /* NUMA: free the list3 structures */ + /* NUMA: free the node structures */ for_each_online_node(i) { - l3 = cachep->node[i]; - if (l3) { - kfree(l3->shared); - free_alien_cache(l3->alien); - kfree(l3); + n = cachep->node[i]; + if (n) { + kfree(n->shared); + free_alien_cache(n->alien); + kfree(n); } } return 0; @@ -2779,7 +2779,7 @@ static int cache_grow(struct kmem_cache *cachep, struct slab *slabp; size_t offset; gfp_t local_flags; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; /* * Be lazy and only check for valid flags here, keeping it out of the @@ -2788,17 +2788,17 @@ static int cache_grow(struct kmem_cache *cachep, BUG_ON(flags & GFP_SLAB_BUG_MASK); local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); - /* Take the l3 list lock to change the colour_next on this node */ + /* Take the node list lock to change the colour_next on this node */ check_irq_off(); - l3 = cachep->node[nodeid]; - spin_lock(&l3->list_lock); + n = cachep->node[nodeid]; + spin_lock(&n->list_lock); /* Get colour for the slab, and cal the next value. */ - offset = l3->colour_next; - l3->colour_next++; - if (l3->colour_next >= cachep->colour) - l3->colour_next = 0; - spin_unlock(&l3->list_lock); + offset = n->colour_next; + n->colour_next++; + if (n->colour_next >= cachep->colour) + n->colour_next = 0; + spin_unlock(&n->list_lock); offset *= cachep->colour_off; @@ -2835,13 +2835,13 @@ static int cache_grow(struct kmem_cache *cachep, if (local_flags & __GFP_WAIT) local_irq_disable(); check_irq_off(); - spin_lock(&l3->list_lock); + spin_lock(&n->list_lock); /* Make slab active. */ - list_add_tail(&slabp->list, &(l3->slabs_free)); + list_add_tail(&slabp->list, &(n->slabs_free)); STATS_INC_GROWN(cachep); - l3->free_objects += cachep->num; - spin_unlock(&l3->list_lock); + n->free_objects += cachep->num; + spin_unlock(&n->list_lock); return 1; opps1: kmem_freepages(cachep, objp); @@ -2969,7 +2969,7 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags, bool force_refill) { int batchcount; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; struct array_cache *ac; int node; @@ -2988,14 +2988,14 @@ retry: */ batchcount = BATCHREFILL_LIMIT; } - l3 = cachep->node[node]; + n = cachep->node[node]; - BUG_ON(ac->avail > 0 || !l3); - spin_lock(&l3->list_lock); + BUG_ON(ac->avail > 0 || !n); + spin_lock(&n->list_lock); /* See if we can refill from the shared array */ - if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) { - l3->shared->touched = 1; + if (n->shared && transfer_objects(ac, n->shared, batchcount)) { + n->shared->touched = 1; goto alloc_done; } @@ -3003,11 +3003,11 @@ retry: struct list_head *entry; struct slab *slabp; /* Get slab alloc is to come from. */ - entry = l3->slabs_partial.next; - if (entry == &l3->slabs_partial) { - l3->free_touched = 1; - entry = l3->slabs_free.next; - if (entry == &l3->slabs_free) + entry = n->slabs_partial.next; + if (entry == &n->slabs_partial) { + n->free_touched = 1; + entry = n->slabs_free.next; + if (entry == &n->slabs_free) goto must_grow; } @@ -3035,15 +3035,15 @@ retry: /* move slabp to correct slabp list: */ list_del(&slabp->list); if (slabp->free == BUFCTL_END) - list_add(&slabp->list, &l3->slabs_full); + list_add(&slabp->list, &n->slabs_full); else - list_add(&slabp->list, &l3->slabs_partial); + list_add(&slabp->list, &n->slabs_partial); } must_grow: - l3->free_objects -= ac->avail; + n->free_objects -= ac->avail; alloc_done: - spin_unlock(&l3->list_lock); + spin_unlock(&n->list_lock); if (unlikely(!ac->avail)) { int x; @@ -3301,21 +3301,21 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, { struct list_head *entry; struct slab *slabp; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; void *obj; int x; - l3 = cachep->node[nodeid]; - BUG_ON(!l3); + n = cachep->node[nodeid]; + BUG_ON(!n); retry: check_irq_off(); - spin_lock(&l3->list_lock); - entry = l3->slabs_partial.next; - if (entry == &l3->slabs_partial) { - l3->free_touched = 1; - entry = l3->slabs_free.next; - if (entry == &l3->slabs_free) + spin_lock(&n->list_lock); + entry = n->slabs_partial.next; + if (entry == &n->slabs_partial) { + n->free_touched = 1; + entry = n->slabs_free.next; + if (entry == &n->slabs_free) goto must_grow; } @@ -3331,20 +3331,20 @@ retry: obj = slab_get_obj(cachep, slabp, nodeid); check_slabp(cachep, slabp); - l3->free_objects--; + n->free_objects--; /* move slabp to correct slabp list: */ list_del(&slabp->list); if (slabp->free == BUFCTL_END) - list_add(&slabp->list, &l3->slabs_full); + list_add(&slabp->list, &n->slabs_full); else - list_add(&slabp->list, &l3->slabs_partial); + list_add(&slabp->list, &n->slabs_partial); - spin_unlock(&l3->list_lock); + spin_unlock(&n->list_lock); goto done; must_grow: - spin_unlock(&l3->list_lock); + spin_unlock(&n->list_lock); x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL); if (x) goto retry; @@ -3496,7 +3496,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, int node) { int i; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; for (i = 0; i < nr_objects; i++) { void *objp; @@ -3506,19 +3506,19 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, objp = objpp[i]; slabp = virt_to_slab(objp); - l3 = cachep->node[node]; + n = cachep->node[node]; list_del(&slabp->list); check_spinlock_acquired_node(cachep, node); check_slabp(cachep, slabp); slab_put_obj(cachep, slabp, objp, node); STATS_DEC_ACTIVE(cachep); - l3->free_objects++; + n->free_objects++; check_slabp(cachep, slabp); /* fixup slab chains */ if (slabp->inuse == 0) { - if (l3->free_objects > l3->free_limit) { - l3->free_objects -= cachep->num; + if (n->free_objects > n->free_limit) { + n->free_objects -= cachep->num; /* No need to drop any previously held * lock here, even if we have a off-slab slab * descriptor it is guaranteed to come from @@ -3527,14 +3527,14 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, */ slab_destroy(cachep, slabp); } else { - list_add(&slabp->list, &l3->slabs_free); + list_add(&slabp->list, &n->slabs_free); } } else { /* Unconditionally move a slab to the end of the * partial list on free - maximum time for the * other objects to be freed, too. */ - list_add_tail(&slabp->list, &l3->slabs_partial); + list_add_tail(&slabp->list, &n->slabs_partial); } } } @@ -3542,7 +3542,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) { int batchcount; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; int node = numa_mem_id(); batchcount = ac->batchcount; @@ -3550,10 +3550,10 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) BUG_ON(!batchcount || batchcount > ac->avail); #endif check_irq_off(); - l3 = cachep->node[node]; - spin_lock(&l3->list_lock); - if (l3->shared) { - struct array_cache *shared_array = l3->shared; + n = cachep->node[node]; + spin_lock(&n->list_lock); + if (n->shared) { + struct array_cache *shared_array = n->shared; int max = shared_array->limit - shared_array->avail; if (max) { if (batchcount > max) @@ -3572,8 +3572,8 @@ free_done: int i = 0; struct list_head *p; - p = l3->slabs_free.next; - while (p != &(l3->slabs_free)) { + p = n->slabs_free.next; + while (p != &(n->slabs_free)) { struct slab *slabp; slabp = list_entry(p, struct slab, list); @@ -3585,7 +3585,7 @@ free_done: STATS_SET_FREEABLE(cachep, i); } #endif - spin_unlock(&l3->list_lock); + spin_unlock(&n->list_lock); ac->avail -= batchcount; memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); } @@ -3829,12 +3829,12 @@ void kfree(const void *objp) EXPORT_SYMBOL(kfree); /* - * This initializes kmem_list3 or resizes various caches for all nodes. + * This initializes kmem_cache_node or resizes various caches for all nodes. */ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) { int node; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; struct array_cache *new_shared; struct array_cache **new_alien = NULL; @@ -3857,43 +3857,43 @@ static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) } } - l3 = cachep->node[node]; - if (l3) { - struct array_cache *shared = l3->shared; + n = cachep->node[node]; + if (n) { + struct array_cache *shared = n->shared; - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); if (shared) free_block(cachep, shared->entry, shared->avail, node); - l3->shared = new_shared; - if (!l3->alien) { - l3->alien = new_alien; + n->shared = new_shared; + if (!n->alien) { + n->alien = new_alien; new_alien = NULL; } - l3->free_limit = (1 + nr_cpus_node(node)) * + n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); kfree(shared); free_alien_cache(new_alien); continue; } - l3 = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); - if (!l3) { + n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); + if (!n) { free_alien_cache(new_alien); kfree(new_shared); goto fail; } - kmem_list3_init(l3); - l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + + kmem_cache_node_init(n); + n->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; - l3->shared = new_shared; - l3->alien = new_alien; - l3->free_limit = (1 + nr_cpus_node(node)) * + n->shared = new_shared; + n->alien = new_alien; + n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - cachep->node[node] = l3; + cachep->node[node] = n; } return 0; @@ -3903,11 +3903,11 @@ fail: node--; while (node >= 0) { if (cachep->node[node]) { - l3 = cachep->node[node]; + n = cachep->node[node]; - kfree(l3->shared); - free_alien_cache(l3->alien); - kfree(l3); + kfree(n->shared); + free_alien_cache(n->alien); + kfree(n); cachep->node[node] = NULL; } node--; @@ -4071,11 +4071,11 @@ skip_setup: } /* - * Drain an array if it contains any elements taking the l3 lock only if - * necessary. Note that the l3 listlock also protects the array_cache + * Drain an array if it contains any elements taking the node lock only if + * necessary. Note that the node listlock also protects the array_cache * if drain_array() is used on the shared array. */ -static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, +static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, struct array_cache *ac, int force, int node) { int tofree; @@ -4085,7 +4085,7 @@ static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, if (ac->touched && !force) { ac->touched = 0; } else { - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); if (ac->avail) { tofree = force ? ac->avail : (ac->limit + 4) / 5; if (tofree > ac->avail) @@ -4095,7 +4095,7 @@ static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, memmove(ac->entry, &(ac->entry[tofree]), sizeof(void *) * ac->avail); } - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); } } @@ -4114,7 +4114,7 @@ static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *l3, static void cache_reap(struct work_struct *w) { struct kmem_cache *searchp; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; int node = numa_mem_id(); struct delayed_work *work = to_delayed_work(w); @@ -4126,33 +4126,33 @@ static void cache_reap(struct work_struct *w) check_irq_on(); /* - * We only take the l3 lock if absolutely necessary and we + * We only take the node lock if absolutely necessary and we * have established with reasonable certainty that * we can do some work if the lock was obtained. */ - l3 = searchp->node[node]; + n = searchp->node[node]; - reap_alien(searchp, l3); + reap_alien(searchp, n); - drain_array(searchp, l3, cpu_cache_get(searchp), 0, node); + drain_array(searchp, n, cpu_cache_get(searchp), 0, node); /* * These are racy checks but it does not matter * if we skip one check or scan twice. */ - if (time_after(l3->next_reap, jiffies)) + if (time_after(n->next_reap, jiffies)) goto next; - l3->next_reap = jiffies + REAPTIMEOUT_LIST3; + n->next_reap = jiffies + REAPTIMEOUT_LIST3; - drain_array(searchp, l3, l3->shared, 0, node); + drain_array(searchp, n, n->shared, 0, node); - if (l3->free_touched) - l3->free_touched = 0; + if (n->free_touched) + n->free_touched = 0; else { int freed; - freed = drain_freelist(searchp, l3, (l3->free_limit + + freed = drain_freelist(searchp, n, (n->free_limit + 5 * searchp->num - 1) / (5 * searchp->num)); STATS_ADD_REAPED(searchp, freed); } @@ -4178,25 +4178,25 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) const char *name; char *error = NULL; int node; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; active_objs = 0; num_slabs = 0; for_each_online_node(node) { - l3 = cachep->node[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; check_irq_on(); - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); - list_for_each_entry(slabp, &l3->slabs_full, list) { + list_for_each_entry(slabp, &n->slabs_full, list) { if (slabp->inuse != cachep->num && !error) error = "slabs_full accounting error"; active_objs += cachep->num; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_partial, list) { + list_for_each_entry(slabp, &n->slabs_partial, list) { if (slabp->inuse == cachep->num && !error) error = "slabs_partial inuse accounting error"; if (!slabp->inuse && !error) @@ -4204,16 +4204,16 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) active_objs += slabp->inuse; active_slabs++; } - list_for_each_entry(slabp, &l3->slabs_free, list) { + list_for_each_entry(slabp, &n->slabs_free, list) { if (slabp->inuse && !error) error = "slabs_free/inuse accounting error"; num_slabs++; } - free_objects += l3->free_objects; - if (l3->shared) - shared_avail += l3->shared->avail; + free_objects += n->free_objects; + if (n->shared) + shared_avail += n->shared->avail; - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); } num_slabs += active_slabs; num_objs = num_slabs * cachep->num; @@ -4239,7 +4239,7 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) { #if STATS - { /* list3 stats */ + { /* node stats */ unsigned long high = cachep->high_mark; unsigned long allocs = cachep->num_allocations; unsigned long grown = cachep->grown; @@ -4392,7 +4392,7 @@ static int leaks_show(struct seq_file *m, void *p) { struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); struct slab *slabp; - struct kmem_cache_node *l3; + struct kmem_cache_node *n; const char *name; unsigned long *n = m->private; int node; @@ -4408,18 +4408,18 @@ static int leaks_show(struct seq_file *m, void *p) n[1] = 0; for_each_online_node(node) { - l3 = cachep->node[node]; - if (!l3) + n = cachep->node[node]; + if (!n) continue; check_irq_on(); - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); - list_for_each_entry(slabp, &l3->slabs_full, list) + list_for_each_entry(slabp, &n->slabs_full, list) handle_slab(n, cachep, slabp); - list_for_each_entry(slabp, &l3->slabs_partial, list) + list_for_each_entry(slabp, &n->slabs_partial, list) handle_slab(n, cachep, slabp); - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_lock); } name = cachep->name; if (n[0] == n[1]) { -- cgit v1.2.3 From ca34956b804b7554fc4e88826773380d9d5122a8 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common definition for kmem_cache_node Put the definitions for the kmem_cache_node structures together so that we have one structure. That will allow us to create more common fields in the future which could yield more opportunities to share code. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index c162b2eb493a..17f859614546 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -285,23 +285,6 @@ struct arraycache_init { void *entries[BOOT_CPUCACHE_ENTRIES]; }; -/* - * The slab lists for all objects. - */ -struct kmem_cache_node { - struct list_head slabs_partial; /* partial list first, better asm code */ - struct list_head slabs_full; - struct list_head slabs_free; - unsigned long free_objects; - unsigned int free_limit; - unsigned int colour_next; /* Per-node cache coloring */ - spinlock_t list_lock; - struct array_cache *shared; /* shared per node */ - struct array_cache **alien; /* on other nodes */ - unsigned long next_reap; /* updated without locking */ - int free_touched; /* updated without locking */ -}; - /* * Need this for bootstrapping a per node allocator. */ -- cgit v1.2.3 From db8450673458e724229e91050ac9a92253b01234 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 5 Feb 2013 18:45:23 +0000 Subject: slab: Fixup CONFIG_PAGE_ALLOC/DEBUG_SLAB_LEAK sections Variables were not properly converted and the conversion caused a naming conflict. Signed-off-by: Tetsuo Handa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- mm/slab.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 17f859614546..910df6326831 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -2306,8 +2306,9 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) } #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) if (size >= kmalloc_size(INDEX_NODE + 1) - && cachep->object_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { - cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); + && cachep->object_size > cache_line_size() + && ALIGN(size, cachep->align) < PAGE_SIZE) { + cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); size = PAGE_SIZE; } #endif @@ -4377,7 +4378,7 @@ static int leaks_show(struct seq_file *m, void *p) struct slab *slabp; struct kmem_cache_node *n; const char *name; - unsigned long *n = m->private; + unsigned long *x = m->private; int node; int i; @@ -4388,7 +4389,7 @@ static int leaks_show(struct seq_file *m, void *p) /* OK, we can do it */ - n[1] = 0; + x[1] = 0; for_each_online_node(node) { n = cachep->node[node]; @@ -4399,32 +4400,32 @@ static int leaks_show(struct seq_file *m, void *p) spin_lock_irq(&n->list_lock); list_for_each_entry(slabp, &n->slabs_full, list) - handle_slab(n, cachep, slabp); + handle_slab(x, cachep, slabp); list_for_each_entry(slabp, &n->slabs_partial, list) - handle_slab(n, cachep, slabp); + handle_slab(x, cachep, slabp); spin_unlock_irq(&n->list_lock); } name = cachep->name; - if (n[0] == n[1]) { + if (x[0] == x[1]) { /* Increase the buffer size */ mutex_unlock(&slab_mutex); - m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL); + m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL); if (!m->private) { /* Too bad, we are really out */ - m->private = n; + m->private = x; mutex_lock(&slab_mutex); return -ENOMEM; } - *(unsigned long *)m->private = n[0] * 2; - kfree(n); + *(unsigned long *)m->private = x[0] * 2; + kfree(x); mutex_lock(&slab_mutex); /* Now make sure this entry will be retried */ m->count = m->size; return 0; } - for (i = 0; i < n[1]; i++) { - seq_printf(m, "%s: %lu ", name, n[2*i+3]); - show_symbol(m, n[2*i+2]); + for (i = 0; i < x[1]; i++) { + seq_printf(m, "%s: %lu ", name, x[2*i+3]); + show_symbol(m, x[2*i+2]); seq_putc(m, '\n'); } -- cgit v1.2.3 From 14e50c6a9bc2b283bb4021026226268312ceefdd Mon Sep 17 00:00:00 2001 From: Aaron Tomlin Date: Fri, 26 Apr 2013 16:15:34 +0100 Subject: mm: slab: Verify the nodeid passed to ____cache_alloc_node If the nodeid is > num_online_nodes() this can cause an Oops and a panic(). The purpose of this patch is to assert if this condition is true to aid debugging efforts rather than some random NULL pointer dereference or page fault. This patch is in response to BZ#42967 [1]. Using VM_BUG_ON so it's used only when CONFIG_DEBUG_VM is set, given that ____cache_alloc_node() is a hot code path. [1]: https://bugzilla.kernel.org/show_bug.cgi?id=42967 Signed-off-by: Aaron Tomlin Reviewed-by: Rik van Riel Acked-by: Christoph Lameter Acked-by: Rafael Aquini Acked-by: David Rientjes Signed-off-by: Pekka Enberg --- mm/slab.c | 1 + 1 file changed, 1 insertion(+) (limited to 'mm/slab.c') diff --git a/mm/slab.c b/mm/slab.c index 910df6326831..a98f8db93670 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3289,6 +3289,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, void *obj; int x; + VM_BUG_ON(nodeid > num_online_nodes()); n = cachep->node[nodeid]; BUG_ON(!n); -- cgit v1.2.3