diff options
| -rw-r--r-- | fs/proc/stat.c | 2 | ||||
| -rw-r--r-- | include/linux/kmalloc_sizes.h | 45 | ||||
| -rw-r--r-- | include/linux/slab.h | 231 | ||||
| -rw-r--r-- | include/linux/slab_def.h | 54 | ||||
| -rw-r--r-- | include/linux/slub_def.h | 136 | ||||
| -rw-r--r-- | mm/slab.c | 790 | ||||
| -rw-r--r-- | mm/slab.h | 43 | ||||
| -rw-r--r-- | mm/slab_common.c | 174 | ||||
| -rw-r--r-- | mm/slub.c | 221 |
9 files changed, 781 insertions, 915 deletions
diff --git a/fs/proc/stat.c b/fs/proc/stat.c index e296572c73e..1cf86c0e868 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c @@ -184,7 +184,7 @@ static int show_stat(struct seq_file *p, void *v) static int stat_open(struct inode *inode, struct file *file) { - unsigned size = 1024 + 128 * num_possible_cpus(); + size_t size = 1024 + 128 * num_possible_cpus(); char *buf; struct seq_file *m; int res; diff --git a/include/linux/kmalloc_sizes.h b/include/linux/kmalloc_sizes.h deleted file mode 100644 index e576b848ce1..00000000000 --- a/include/linux/kmalloc_sizes.h +++ /dev/null @@ -1,45 +0,0 @@ -#if (PAGE_SIZE == 4096) - CACHE(32) -#endif - CACHE(64) -#if L1_CACHE_BYTES < 64 - CACHE(96) -#endif - CACHE(128) -#if L1_CACHE_BYTES < 128 - CACHE(192) -#endif - CACHE(256) - CACHE(512) - CACHE(1024) - CACHE(2048) - CACHE(4096) - CACHE(8192) - CACHE(16384) - CACHE(32768) - CACHE(65536) - CACHE(131072) -#if KMALLOC_MAX_SIZE >= 262144 - CACHE(262144) -#endif -#if KMALLOC_MAX_SIZE >= 524288 - CACHE(524288) -#endif -#if KMALLOC_MAX_SIZE >= 1048576 - CACHE(1048576) -#endif -#if KMALLOC_MAX_SIZE >= 2097152 - CACHE(2097152) -#endif -#if KMALLOC_MAX_SIZE >= 4194304 - CACHE(4194304) -#endif -#if KMALLOC_MAX_SIZE >= 8388608 - CACHE(8388608) -#endif -#if KMALLOC_MAX_SIZE >= 16777216 - CACHE(16777216) -#endif -#if KMALLOC_MAX_SIZE >= 33554432 - CACHE(33554432) -#endif diff --git a/include/linux/slab.h b/include/linux/slab.h index 5d168d7e0a2..0c621752caa 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -94,29 +94,6 @@ #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ (unsigned long)ZERO_SIZE_PTR) -/* - * Common fields provided in kmem_cache by all slab allocators - * This struct is either used directly by the allocator (SLOB) - * or the allocator must include definitions for all fields - * provided in kmem_cache_common in their definition of kmem_cache. - * - * Once we can do anonymous structs (C11 standard) we could put a - * anonymous struct definition in these allocators so that the - * separate allocations in the kmem_cache structure of SLAB and - * SLUB is no longer needed. - */ -#ifdef CONFIG_SLOB -struct kmem_cache { - unsigned int object_size;/* The original size of the object */ - unsigned int size; /* The aligned/padded/added on size */ - unsigned int align; /* Alignment as calculated */ - unsigned long flags; /* Active flags on the slab */ - const char *name; /* Slab name for sysfs */ - int refcount; /* Use counter */ - void (*ctor)(void *); /* Called on object slot creation */ - struct list_head list; /* List of all slab caches on the system */ -}; -#endif struct mem_cgroup; /* @@ -148,7 +125,63 @@ void kmem_cache_free(struct kmem_cache *, void *); (__flags), NULL) /* - * The largest kmalloc size supported by the slab allocators is + * Common kmalloc functions provided by all allocators + */ +void * __must_check __krealloc(const void *, size_t, gfp_t); +void * __must_check krealloc(const void *, size_t, gfp_t); +void kfree(const void *); +void kzfree(const void *); +size_t ksize(const void *); + +/* + * Some archs want to perform DMA into kmalloc caches and need a guaranteed + * alignment larger than the alignment of a 64-bit integer. + * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. + */ +#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 +#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN +#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN +#define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) +#else +#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) +#endif + +#ifdef CONFIG_SLOB +/* + * Common fields provided in kmem_cache by all slab allocators + * This struct is either used directly by the allocator (SLOB) + * or the allocator must include definitions for all fields + * provided in kmem_cache_common in their definition of kmem_cache. + * + * Once we can do anonymous structs (C11 standard) we could put a + * anonymous struct definition in these allocators so that the + * separate allocations in the kmem_cache structure of SLAB and + * SLUB is no longer needed. + */ +struct kmem_cache { + unsigned int object_size;/* The original size of the object */ + unsigned int size; /* The aligned/padded/added on size */ + unsigned int align; /* Alignment as calculated */ + unsigned long flags; /* Active flags on the slab */ + const char *name; /* Slab name for sysfs */ + int refcount; /* Use counter */ + void (*ctor)(void *); /* Called on object slot creation */ + struct list_head list; /* List of all slab caches on the system */ +}; + +#define KMALLOC_MAX_SIZE (1UL << 30) + +#include <linux/slob_def.h> + +#else /* CONFIG_SLOB */ + +/* + * Kmalloc array related definitions + */ + +#ifdef CONFIG_SLAB +/* + * The largest kmalloc size supported by the SLAB allocators is * 32 megabyte (2^25) or the maximum allocatable page order if that is * less than 32 MB. * @@ -158,22 +191,120 @@ void kmem_cache_free(struct kmem_cache *, void *); */ #define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ (MAX_ORDER + PAGE_SHIFT - 1) : 25) +#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifndef KMALLOC_SHIFT_LOW +#define KMALLOC_SHIFT_LOW 5 +#endif +#else +/* + * SLUB allocates up to order 2 pages directly and otherwise + * passes the request to the page allocator. + */ +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) +#ifndef KMALLOC_SHIFT_LOW +#define KMALLOC_SHIFT_LOW 3 +#endif +#endif -#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_HIGH) -#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_HIGH - PAGE_SHIFT) +/* Maximum allocatable size */ +#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) +/* Maximum size for which we actually use a slab cache */ +#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) +/* Maximum order allocatable via the slab allocagtor */ +#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) /* - * Some archs want to perform DMA into kmalloc caches and need a guaranteed - * alignment larger than the alignment of a 64-bit integer. - * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. + * Kmalloc subsystem. */ -#ifdef ARCH_DMA_MINALIGN -#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN +#ifndef KMALLOC_MIN_SIZE +#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) +#endif + +extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +#ifdef CONFIG_ZONE_DMA +extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; +#endif + +/* + * Figure out which kmalloc slab an allocation of a certain size + * belongs to. + * 0 = zero alloc + * 1 = 65 .. 96 bytes + * 2 = 120 .. 192 bytes + * n = 2^(n-1) .. 2^n -1 + */ +static __always_inline int kmalloc_index(size_t size) +{ + if (!size) + return 0; + + if (size <= KMALLOC_MIN_SIZE) + return KMALLOC_SHIFT_LOW; + + if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) + return 1; + if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) + return 2; + if (size <= 8) return 3; + if (size <= 16) return 4; + if (size <= 32) return 5; + if (size <= 64) return 6; + if (size <= 128) return 7; + if (size <= 256) return 8; + if (size <= 512) return 9; + if (size <= 1024) return 10; + if (size <= 2 * 1024) return 11; + if (size <= 4 * 1024) return 12; + if (size <= 8 * 1024) return 13; + if (size <= 16 * 1024) return 14; + if (size <= 32 * 1024) return 15; + if (size <= 64 * 1024) return 16; + if (size <= 128 * 1024) return 17; + if (size <= 256 * 1024) return 18; + if (size <= 512 * 1024) return 19; + if (size <= 1024 * 1024) return 20; + if (size <= 2 * 1024 * 1024) return 21; + if (size <= 4 * 1024 * 1024) return 22; + if (size <= 8 * 1024 * 1024) return 23; + if (size <= 16 * 1024 * 1024) return 24; + if (size <= 32 * 1024 * 1024) return 25; + if (size <= 64 * 1024 * 1024) return 26; + BUG(); + + /* Will never be reached. Needed because the compiler may complain */ + return -1; +} + +#ifdef CONFIG_SLAB +#include <linux/slab_def.h> +#elif defined(CONFIG_SLUB) +#include <linux/slub_def.h> #else -#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) +#error "Unknown slab allocator" #endif /* + * Determine size used for the nth kmalloc cache. + * return size or 0 if a kmalloc cache for that + * size does not exist + */ +static __always_inline int kmalloc_size(int n) +{ + if (n > 2) + return 1 << n; + + if (n == 1 && KMALLOC_MIN_SIZE <= 32) + return 96; + + if (n == 2 && KMALLOC_MIN_SIZE <= 64) + return 192; + + return 0; +} +#endif /* !CONFIG_SLOB */ + +/* * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. * Intended for arches that get misalignment faults even for 64 bit integer * aligned buffers. @@ -224,42 +355,6 @@ struct seq_file; int cache_show(struct kmem_cache *s, struct seq_file *m); void print_slabinfo_header(struct seq_file *m); -/* - * Common kmalloc functions provided by all allocators - */ -void * __must_check __krealloc(const void *, size_t, gfp_t); -void * __must_check krealloc(const void *, size_t, gfp_t); -void kfree(const void *); -void kzfree(const void *); -size_t ksize(const void *); - -/* - * Allocator specific definitions. These are mainly used to establish optimized - * ways to convert kmalloc() calls to kmem_cache_alloc() invocations by - * selecting the appropriate general cache at compile time. - * - * Allocators must define at least: - * - * kmem_cache_alloc() - * __kmalloc() - * kmalloc() - * - * Those wishing to support NUMA must also define: - * - * kmem_cache_alloc_node() - * kmalloc_node() - * - * See each allocator definition file for additional comments and - * implementation notes. - */ -#ifdef CONFIG_SLUB -#include <linux/slub_def.h> -#elif defined(CONFIG_SLOB) -#include <linux/slob_def.h> -#else -#include <linux/slab_def.h> -#endif - /** * kmalloc_array - allocate memory for an array. * @n: number of elements. diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 8bb6e0eaf3c..cd401580bdd 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -11,8 +11,6 @@ */ #include <linux/init.h> -#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */ -#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */ #include <linux/compiler.h> /* @@ -97,23 +95,13 @@ struct kmem_cache { * pointer for each node since "nodelists" uses the remainder of * available pointers. */ - struct kmem_list3 **nodelists; + struct kmem_cache_node **node; struct array_cache *array[NR_CPUS + MAX_NUMNODES]; /* * Do not add fields after array[] */ }; -/* Size description struct for general caches. */ -struct cache_sizes { - size_t cs_size; - struct kmem_cache *cs_cachep; -#ifdef CONFIG_ZONE_DMA - struct kmem_cache *cs_dmacachep; -#endif -}; -extern struct cache_sizes malloc_sizes[]; - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); @@ -133,26 +121,22 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) void *ret; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - return NULL; -found: + if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) + return NULL; + + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; ret = kmem_cache_alloc_trace(cachep, flags, size); @@ -186,26 +170,22 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) struct kmem_cache *cachep; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - return NULL; -found: + if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) + return NULL; + + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; return kmem_cache_alloc_node_trace(cachep, flags, node, size); } diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 9db4825cd39..027276fa871 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -53,17 +53,6 @@ struct kmem_cache_cpu { #endif }; -struct kmem_cache_node { - spinlock_t list_lock; /* Protect partial list and nr_partial */ - unsigned long nr_partial; - struct list_head partial; -#ifdef CONFIG_SLUB_DEBUG - atomic_long_t nr_slabs; - atomic_long_t total_objects; - struct list_head full; -#endif -}; - /* * Word size structure that can be atomically updated or read and that * contains both the order and the number of objects that a slab of the @@ -115,111 +104,6 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -/* - * Kmalloc subsystem. - */ -#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 -#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN -#else -#define KMALLOC_MIN_SIZE 8 -#endif - -#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) - -/* - * Maximum kmalloc object size handled by SLUB. Larger object allocations - * are passed through to the page allocator. The page allocator "fastpath" - * is relatively slow so we need this value sufficiently high so that - * performance critical objects are allocated through the SLUB fastpath. - * - * This should be dropped to PAGE_SIZE / 2 once the page allocator - * "fastpath" becomes competitive with the slab allocator fastpaths. - */ -#define SLUB_MAX_SIZE (2 * PAGE_SIZE) - -#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2) - -#ifdef CONFIG_ZONE_DMA -#define SLUB_DMA __GFP_DMA -#else -/* Disable DMA functionality */ -#define SLUB_DMA (__force gfp_t)0 -#endif - -/* - * We keep the general caches in an array of slab caches that are used for - * 2^x bytes of allocations. - */ -extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT]; - -/* - * Sorry that the following has to be that ugly but some versions of GCC - * have trouble with constant propagation and loops. - */ -static __always_inline int kmalloc_index(size_t size) -{ - if (!size) - return 0; - - if (size <= KMALLOC_MIN_SIZE) - return KMALLOC_SHIFT_LOW; - - if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) - return 1; - if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) - return 2; - if (size <= 8) return 3; - if (size <= 16) return 4; - if (size <= 32) return 5; - if (size <= 64) return 6; - if (size <= 128) return 7; - if (size <= 256) return 8; - if (size <= 512) return 9; - if (size <= 1024) return 10; - if (size <= 2 * 1024) return 11; - if (size <= 4 * 1024) return 12; -/* - * The following is only needed to support architectures with a larger page - * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page - * size we would have to go up to 128k. - */ - if (size <= 8 * 1024) return 13; - if (size <= 16 * 1024) return 14; - if (size <= 32 * 1024) return 15; - if (size <= 64 * 1024) return 16; - if (size <= 128 * 1024) return 17; - if (size <= 256 * 1024) return 18; - if (size <= 512 * 1024) return 19; - if (size <= 1024 * 1024) return 20; - if (size <= 2 * 1024 * 1024) return 21; - BUG(); - return -1; /* Will never be reached */ - -/* - * What we really wanted to do and cannot do because of compiler issues is: - * int i; - * for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) - * if (size <= (1 << i)) - * return i; - */ -} - -/* - * Find the slab cache for a given combination of allocation flags and size. - * - * This ought to end up with a global pointer to the right cache - * in kmalloc_caches. - */ -static __always_inline struct kmem_cache *kmalloc_slab(size_t size) -{ - int index = kmalloc_index(size); - - if (index == 0) - return NULL; - - return kmalloc_caches[index]; -} - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); @@ -274,16 +158,17 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags) static __always_inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { - if (size > SLUB_MAX_SIZE) + if (size > KMALLOC_MAX_CACHE_SIZE) return kmalloc_large(size, flags); - if (!(flags & SLUB_DMA)) { - struct kmem_cache *s = kmalloc_slab(size); + if (!(flags & GFP_DMA)) { + int index = kmalloc_index(size); - if (!s) + if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_trace(s, flags, size); + return kmem_cache_alloc_trace(kmalloc_caches[index], + flags, size); } } return __kmalloc(size, flags); @@ -310,13 +195,14 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { if (__builtin_constant_p(size) && - size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) { - struct kmem_cache *s = kmalloc_slab(size); + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { + int index = kmalloc_index(size); - if (!s) + if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_node_trace(s, flags, node, size); + return kmem_cache_alloc_node_trace(kmalloc_caches[index], + flags, node, size); } return __kmalloc_node(size, flags, node); } diff --git a/mm/slab.c b/mm/slab.c index 96079244c86..8ccd296c6d9 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -286,68 +286,27 @@ struct arraycache_init { }; /* - * The slab lists for all objects. - */ -struct kmem_list3 { - struct list_head slabs_partial; /* partial list first, better asm code */ - struct list_head slabs_full; - struct list_head slabs_free; - unsigned long free_objects; - unsigned int free_limit; - unsigned int colour_next; /* Per-node cache coloring */ - spinlock_t list_lock; - struct array_cache *shared; /* shared per node */ - struct array_cache **alien; /* on other nodes */ - unsigned long next_reap; /* updated without locking */ - int free_touched; /* updated without locking */ -}; - -/* * Need this for bootstrapping a per node allocator. */ #define NUM_INIT_LISTS (3 * MAX_NUMNODES) -static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS]; +static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS]; #define CACHE_CACHE 0 #define SIZE_AC MAX_NUMNODES -#define SIZE_L3 (2 * MAX_NUMNODES) +#define SIZE_NODE (2 * MAX_NUMNODES) static int drain_freelist(struct kmem_cache *cache, - struct kmem_list3 *l3, int tofree); + struct kmem_cache_node *n, int tofree); static void free_block(struct kmem_cache *cachep, void **objpp, int len, int node); static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); static void cache_reap(struct work_struct *unused); -/* - * This function must be completely optimized away if a constant is passed to - * it. Mostly the same as what is in linux/slab.h except it returns an index. - */ -static __always_inline int index_of(const size_t size) -{ - extern void __bad_size(void); - - if (__builtin_constant_p(size)) { - int i = 0; - -#define CACHE(x) \ - if (size <=x) \ - return i; \ - else \ - i++; -#include <linux/kmalloc_sizes.h> -#undef CACHE - __bad_size(); - } else - __bad_size(); - return 0; -} - static int slab_early_init = 1; -#define INDEX_AC index_of(sizeof(struct arraycache_init)) -#define INDEX_L3 index_of(sizeof(struct kmem_list3)) +#define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) +#define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node)) -static void kmem_list3_init(struct kmem_list3 *parent) +static void kmem_cache_node_init(struct kmem_cache_node *parent) { INIT_LIST_HEAD(&parent->slabs_full); INIT_LIST_HEAD(&parent->slabs_partial); @@ -363,7 +322,7 @@ static void kmem_list3_init(struct kmem_list3 *parent) #define MAKE_LIST(cachep, listp, slab, nodeid) \ do { \ INIT_LIST_HEAD(listp); \ - list_splice(&(cachep->nodelists[nodeid]->slab), listp); \ + list_splice(&(cachep->node[nodeid]->slab), listp); \ } while (0) #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ @@ -524,30 +483,6 @@ static inline unsigned int obj_to_index(const struct kmem_cache *cache, return reciprocal_divide(offset, cache->reciprocal_buffer_size); } -/* - * These are the default caches for kmalloc. Custom caches can have other sizes. - */ -struct cache_sizes malloc_sizes[] = { -#define CACHE(x) { .cs_size = (x) }, -#include <linux/kmalloc_sizes.h> - CACHE(ULONG_MAX) -#undef CACHE -}; -EXPORT_SYMBOL(malloc_sizes); - -/* Must match cache_sizes above. Out of line to keep cache footprint low. */ -struct cache_names { - char *name; - char *name_dma; -}; - -static struct cache_names __initdata cache_names[] = { -#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, -#include <linux/kmalloc_sizes.h> - {NULL,} -#undef CACHE -}; - static struct arraycache_init initarray_generic = { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; @@ -586,15 +521,15 @@ static void slab_set_lock_classes(struct kmem_cache *cachep, int q) { struct array_cache **alc; - struct kmem_list3 *l3; + struct kmem_cache_node *n; int r; - l3 = cachep->nodelists[q]; - if (!l3) + n = cachep->node[q]; + if (!n) return; - lockdep_set_class(&l3->list_lock, l3_key); - alc = l3->alien; + lockdep_set_class(&n->list_lock, l3_key); + alc = n->alien; /* * FIXME: This check for BAD_ALIEN_MAGIC * should go away when common slab code is taught to @@ -625,28 +560,30 @@ static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) static void init_node_lock_keys(int q) { - struct cache_sizes *s = malloc_sizes; + int i; if (slab_state < UP) return; - for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { - struct kmem_list3 *l3; + for (i = 1; i < PAGE_SHIFT + MAX_ORDER; i++) { + struct kmem_cache_node *n; + struct kmem_cache *cache = kmalloc_caches[i]; + + if (!cache) + continue; - l3 = s->cs_cachep->nodelists[q]; - if (!l3 || OFF_SLAB(s->cs_cachep)) + n = cache->node[q]; + if (!n || OFF_SLAB(cache)) continue; - slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, + slab_set_lock_classes(cache, &on_slab_l3_key, &on_slab_alc_key, q); } } static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) { - struct kmem_list3 *l3; - l3 = cachep->nodelists[q]; - if (!l3) + if (!cachep->node[q]) return; slab_set_lock_classes(cachep, &on_slab_l3_key, @@ -702,41 +639,6 @@ static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) return cachep->array[smp_processor_id()]; } -static inline struct kmem_cache *__find_general_cachep(size_t size, - gfp_t gfpflags) -{ - struct cache_sizes *csizep = malloc_sizes; - -#if DEBUG - /* This happens if someone tries to call - * kmem_cache_create(), or __kmalloc(), before - * the generic caches are initialized. - */ - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); -#endif - if (!size) - return ZERO_SIZE_PTR; - - while (size > csizep->cs_size) - csizep++; - - /* - * Really subtle: The last entry with cs->cs_size==ULONG_MAX - * has cs_{dma,}cachep==NULL. Thus no special case - * for large kmalloc calls required. - */ -#ifdef CONFIG_ZONE_DMA - if (unlikely(gfpflags & GFP_DMA)) - return csizep->cs_dmacachep; -#endif - return csizep->cs_cachep; -} - -static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) -{ - return __find_general_cachep(size, gfpflags); -} - static size_t slab_mgmt_size(size_t nr_objs, size_t align) { return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align); @@ -938,29 +840,29 @@ static inline bool is_slab_pfmemalloc(struct slab *slabp) static void recheck_pfmemalloc_active(struct kmem_cache *cachep, struct array_cache *ac) { - struct kmem_list3 *l3 = cachep->nodelists[numa_mem_id()]; + struct kmem_cache_node *n = cachep->node[numa_mem_id()]; struct slab *slabp; unsigned long flags; if (!pfmemalloc_active) return; - spin_lock_irqsave(&l3->list_lock, flags); - list_for_each_entry(slabp, &l3->slabs_full, list) + spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(slabp, &n->slabs_full, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_partial, list) + list_for_each_entry(slabp, &n->slabs_partial, list) if (is_slab_pfmemalloc(slabp)) goto out; - list_for_each_entry(slabp, &l3->slabs_free, list) + list_for_each_entry(slabp, &n->slabs_free, list) if (is_slab_pfmemalloc(slabp)) goto out; pfmemalloc_active = false; out: - spin_unlock_irqrestore(&l3->list_lock, flags); + spin_unlock_irqrestore(&n->list_lock, flags); } static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, @@ -971,7 +873,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */ if (unlikely(is_obj_pfmemalloc(objp))) { - struct kmem_list3 *l3; + struct kmem_cache_node *n; if (gfp_pfmemalloc_allowed(flags)) { clear_obj_pfmemalloc(&objp); @@ -993,8 +895,8 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, * If there are empty slabs on the slabs_free list and we are * being forced to refill the cache, mark this one !pfmemalloc. */ - l3 = cachep->nodelists[numa_mem_id()]; - if (!list_empty(&l3->slabs_free) && force_refill) { + n = cachep->node[numa_mem_id()]; + if (!list_empty(&n->slabs_free) && force_refill) { struct slab *slabp = virt_to_slab(objp); ClearPageSlabPfmemalloc(virt_to_head_page(slabp->s_mem)); clear_obj_pfmemalloc(&objp); @@ -1071,7 +973,7 @@ static int transfer_objects(struct array_cache *to, #ifndef CONFIG_NUMA #define drain_alien_cache(cachep, alien) do { } while (0) -#define reap_alien(cachep, l3) do { } while (0) +#define reap_alien(cachep, n) do { } while (0) static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) { @@ -1143,33 +1045,33 @@ static void free_alien_cache(struct array_cache **ac_ptr) static void __drain_alien_cache(struct kmem_cache *cachep, struct array_cache *ac, int node) { - struct kmem_list3 *rl3 = cachep->nodelists[node]; + struct kmem_cache_node *n = cachep->node[node]; if (ac->avail) { - spin_lock(&rl3->list_lock); + spin_lock(&n->list_lock); /* * Stuff objects into the remote nodes shared array first. * That way we could avoid the overhead of putting the objects * into the free lists and getting them back later. */ - if (rl3->shared) - transfer_objects(rl3->shared, ac, ac->limit); + if (n->shared) + transfer_objects(n->shared, ac, ac->limit); free_block(cachep, ac->entry, ac->avail, node); ac->avail = 0; - spin_unlock(&rl3->list_lock); + spin_unlock(&n->list_lock); } } /* * Called from cache_reap() to regularly drain alien caches round robin. */ -static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3) +static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) { int node = __this_cpu_read(slab_reap_node); - if (l3->alien) { - struct array_cache *ac = l3->alien[node]; + if (n->alien) { + struct array_cache *ac = n->alien[node]; if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { __drain_alien_cache(cachep, ac, node); @@ -1199,7 +1101,7 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) { struct slab *slabp = virt_to_slab(objp); int nodeid = slabp->nodeid; - struct kmem_list3 *l3; + struct kmem_cache_node *n; struct array_cache *alien = NULL; int node; @@ -1212,10 +1114,10 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) if (likely(slabp->nodeid == node)) return 0; - l3 = cachep->nodelists[node]; + n = cachep->node[node]; STATS_INC_NODEFREES(cachep); - if (l3->alien && l3->alien[nodeid]) { - alien = l3->alien[nodeid]; + if (n->alien && n->alien[nodeid]) { + alien = n->alien[nodeid]; spin_lock(&alien->lock); if (unlikely(alien->avail == alien->limit)) { STATS_INC_ACOVERFLOW(cachep); @@ -1224,28 +1126,28 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) ac_put_obj(cachep, alien, objp); spin_unlock(&alien->lock); } else { - spin_lock(&(cachep->nodelists[nodeid])->list_lock); + spin_lock(&(cachep->node[nodeid])->list_lock); free_block(cachep, &objp, 1, nodeid); - spin_unlock(&(cachep->nodelists[nodeid])->list_lock); + spin_unlock(&(cachep->node[nodeid])->list_lock); } return 1; } #endif /* - * Allocates and initializes nodelists for a node on each slab cache, used for - * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3 + * Allocates and initializes node for a node on each slab cache, used for + * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node * will be allocated off-node since memory is not yet online for the new node. - * When hotplugging memory or a cpu, existing nodelists are not replaced if + * When hotplugging memory or a cpu, existing node are not replaced if * already in use. * * Must hold slab_mutex. */ -static int init_cache_nodelists_node(int node) +static int init_cache_node_node(int node) { struct kmem_cache *cachep; - struct kmem_list3 *l3; - const int memsize = sizeof(struct kmem_list3); + struct kmem_cache_node *n; + const int memsize = sizeof(struct kmem_cache_node); list_for_each_entry(cachep, &slab_caches, list) { /* @@ -1253,12 +1155,12 @@ static int init_cache_nodelists_node(int node) * begin anything. Make sure some other cpu on this * node has not already allocated this */ - if (!cachep->nodelists[node]) { - l3 = kmalloc_node(memsize, GFP_KERNEL, node); - if (!l3) + if (!cachep->node[node]) { + n = kmalloc_node(memsize, GFP_KERNEL, node); + if (!n) return -ENOMEM; - kmem_list3_init(l3); - l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + + kmem_cache_node_init(n); + n->next_reap = jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; /* @@ -1266,14 +1168,14 @@ static int init_cache_nodelists_node(int node) * go. slab_mutex is sufficient * protection here. */ - cachep->nodelists[node] = l3; + cachep->node[node] = n; } - spin_lock_irq(&cachep->nodelists[node]->list_lock); - cachep->nodelists[node]->free_limit = + spin_lock_irq(&cachep->node[node]->list_lock); + cachep->node[node]->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; - spin_unlock_irq(&cachep->nodelists[node]->list_lock); + spin_unlock_irq(&cachep->node[node]->list_lock); } return 0; } @@ -1281,7 +1183,7 @@ static int init_cache_nodelists_node(int node) static void __cpuinit cpuup_canceled(long cpu) { struct kmem_cache *cachep; - struct kmem_list3 *l3 = NULL; + struct kmem_cache_node *n = NULL; int node = cpu_to_mem(cpu); const struct cpumask *mask = cpumask_of_node(node); @@ -1293,34 +1195,34 @@ static void __cpuinit cpuup_canceled(long cpu) /* cpu is dead; no one can alloc from it. */ nc = cachep->array[cpu]; cachep->array[cpu] = NULL; - l3 = cachep->nodelists[node]; + n = cachep->node[node]; - if (!l3) + if (!n) goto free_array_cache; - spin_lock_irq(&l3->list_lock); + spin_lock_irq(&n->list_lock); - /* Free limit for this kmem_list3 */ - l3->free_limit -= cachep->batchcount; + /* Free limit for this kmem_cache_node */ + n->free_limit -= cachep->batchcount; if (nc) free_block(cachep, nc->entry, nc->avail, node); if (!cpumask_empty(mask)) { - spin_unlock_irq(&l3->list_lock); + spin_unlock_irq(&n->list_l |