From 345046673449b5c35840e5cc34a60059cbec9305 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:00:53 +0000 Subject: slab: Move kmalloc related function defs Move these functions higher up in slab.h so that they are grouped with other generic kmalloc related definitions. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index 5d168d7e0a2..ccbb37685c6 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -147,6 +147,15 @@ void kmem_cache_free(struct kmem_cache *, void *); sizeof(struct __struct), __alignof__(struct __struct),\ (__flags), NULL) +/* + * Common kmalloc functions provided by all allocators + */ +void * __must_check __krealloc(const void *, size_t, gfp_t); +void * __must_check krealloc(const void *, size_t, gfp_t); +void kfree(const void *); +void kzfree(const void *); +size_t ksize(const void *); + /* * The largest kmalloc size supported by the slab allocators is * 32 megabyte (2^25) or the maximum allocatable page order if that is @@ -224,15 +233,6 @@ struct seq_file; int cache_show(struct kmem_cache *s, struct seq_file *m); void print_slabinfo_header(struct seq_file *m); -/* - * Common kmalloc functions provided by all allocators - */ -void * __must_check __krealloc(const void *, size_t, gfp_t); -void * __must_check krealloc(const void *, size_t, gfp_t); -void kfree(const void *); -void kzfree(const void *); -size_t ksize(const void *); - /* * Allocator specific definitions. These are mainly used to establish optimized * ways to convert kmalloc() calls to kmem_cache_alloc() invocations by -- cgit v1.2.3-18-g5258 From ce6a50263d4ddeba1f0d08f16716a82770c03690 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common kmalloc slab index determination Extract the function to determine the index of the slab within the array of kmalloc caches as well as a function to determine maximum object size from the nr of the kmalloc slab. This is used here only to simplify slub bootstrap but will be used later also for SLAB. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 172 +++++++++++++++++++++++++++++++++-------------- include/linux/slub_def.h | 63 ----------------- 2 files changed, 122 insertions(+), 113 deletions(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index ccbb37685c6..c97fe92532d 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -94,29 +94,6 @@ #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ (unsigned long)ZERO_SIZE_PTR) -/* - * Common fields provided in kmem_cache by all slab allocators - * This struct is either used directly by the allocator (SLOB) - * or the allocator must include definitions for all fields - * provided in kmem_cache_common in their definition of kmem_cache. - * - * Once we can do anonymous structs (C11 standard) we could put a - * anonymous struct definition in these allocators so that the - * separate allocations in the kmem_cache structure of SLAB and - * SLUB is no longer needed. - */ -#ifdef CONFIG_SLOB -struct kmem_cache { - unsigned int object_size;/* The original size of the object */ - unsigned int size; /* The aligned/padded/added on size */ - unsigned int align; /* Alignment as calculated */ - unsigned long flags; /* Active flags on the slab */ - const char *name; /* Slab name for sysfs */ - int refcount; /* Use counter */ - void (*ctor)(void *); /* Called on object slot creation */ - struct list_head list; /* List of all slab caches on the system */ -}; -#endif struct mem_cgroup; /* @@ -156,6 +133,35 @@ void kfree(const void *); void kzfree(const void *); size_t ksize(const void *); +#ifdef CONFIG_SLOB +/* + * Common fields provided in kmem_cache by all slab allocators + * This struct is either used directly by the allocator (SLOB) + * or the allocator must include definitions for all fields + * provided in kmem_cache_common in their definition of kmem_cache. + * + * Once we can do anonymous structs (C11 standard) we could put a + * anonymous struct definition in these allocators so that the + * separate allocations in the kmem_cache structure of SLAB and + * SLUB is no longer needed. + */ +struct kmem_cache { + unsigned int object_size;/* The original size of the object */ + unsigned int size; /* The aligned/padded/added on size */ + unsigned int align; /* Alignment as calculated */ + unsigned long flags; /* Active flags on the slab */ + const char *name; /* Slab name for sysfs */ + int refcount; /* Use counter */ + void (*ctor)(void *); /* Called on object slot creation */ + struct list_head list; /* List of all slab caches on the system */ +}; + +#define KMALLOC_MAX_SIZE (1UL << 30) + +#include + +#else /* CONFIG_SLOB */ + /* * The largest kmalloc size supported by the slab allocators is * 32 megabyte (2^25) or the maximum allocatable page order if that is @@ -171,6 +177,99 @@ size_t ksize(const void *); #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_HIGH) #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_HIGH - PAGE_SHIFT) +/* + * Kmalloc subsystem. + */ +#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 +#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN +#else +#ifdef CONFIG_SLAB +#define KMALLOC_MIN_SIZE 32 +#else +#define KMALLOC_MIN_SIZE 8 +#endif +#endif + +#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) + +/* + * Figure out which kmalloc slab an allocation of a certain size + * belongs to. + * 0 = zero alloc + * 1 = 65 .. 96 bytes + * 2 = 120 .. 192 bytes + * n = 2^(n-1) .. 2^n -1 + */ +static __always_inline int kmalloc_index(size_t size) +{ + if (!size) + return 0; + + if (size <= KMALLOC_MIN_SIZE) + return KMALLOC_SHIFT_LOW; + + if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) + return 1; + if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) + return 2; + if (size <= 8) return 3; + if (size <= 16) return 4; + if (size <= 32) return 5; + if (size <= 64) return 6; + if (size <= 128) return 7; + if (size <= 256) return 8; + if (size <= 512) return 9; + if (size <= 1024) return 10; + if (size <= 2 * 1024) return 11; + if (size <= 4 * 1024) return 12; + if (size <= 8 * 1024) return 13; + if (size <= 16 * 1024) return 14; + if (size <= 32 * 1024) return 15; + if (size <= 64 * 1024) return 16; + if (size <= 128 * 1024) return 17; + if (size <= 256 * 1024) return 18; + if (size <= 512 * 1024) return 19; + if (size <= 1024 * 1024) return 20; + if (size <= 2 * 1024 * 1024) return 21; + if (size <= 4 * 1024 * 1024) return 22; + if (size <= 8 * 1024 * 1024) return 23; + if (size <= 16 * 1024 * 1024) return 24; + if (size <= 32 * 1024 * 1024) return 25; + if (size <= 64 * 1024 * 1024) return 26; + BUG(); + + /* Will never be reached. Needed because the compiler may complain */ + return -1; +} + +#ifdef CONFIG_SLAB +#include +#elif defined(CONFIG_SLUB) +#include +#else +#error "Unknown slab allocator" +#endif + +/* + * Determine size used for the nth kmalloc cache. + * return size or 0 if a kmalloc cache for that + * size does not exist + */ +static __always_inline int kmalloc_size(int n) +{ + if (n > 2) + return 1 << n; + + if (n == 1 && KMALLOC_MIN_SIZE <= 32) + return 96; + + if (n == 2 && KMALLOC_MIN_SIZE <= 64) + return 192; + + return 0; +} +#endif /* !CONFIG_SLOB */ + /* * Some archs want to perform DMA into kmalloc caches and need a guaranteed * alignment larger than the alignment of a 64-bit integer. @@ -233,33 +332,6 @@ struct seq_file; int cache_show(struct kmem_cache *s, struct seq_file *m); void print_slabinfo_header(struct seq_file *m); -/* - * Allocator specific definitions. These are mainly used to establish optimized - * ways to convert kmalloc() calls to kmem_cache_alloc() invocations by - * selecting the appropriate general cache at compile time. - * - * Allocators must define at least: - * - * kmem_cache_alloc() - * __kmalloc() - * kmalloc() - * - * Those wishing to support NUMA must also define: - * - * kmem_cache_alloc_node() - * kmalloc_node() - * - * See each allocator definition file for additional comments and - * implementation notes. - */ -#ifdef CONFIG_SLUB -#include -#elif defined(CONFIG_SLOB) -#include -#else -#include -#endif - /** * kmalloc_array - allocate memory for an array. * @n: number of elements. diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 9db4825cd39..99c3e05ff1f 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -115,17 +115,6 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -/* - * Kmalloc subsystem. - */ -#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 -#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN -#else -#define KMALLOC_MIN_SIZE 8 -#endif - -#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) - /* * Maximum kmalloc object size handled by SLUB. Larger object allocations * are passed through to the page allocator. The page allocator "fastpath" @@ -152,58 +141,6 @@ struct kmem_cache { */ extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT]; -/* - * Sorry that the following has to be that ugly but some versions of GCC - * have trouble with constant propagation and loops. - */ -static __always_inline int kmalloc_index(size_t size) -{ - if (!size) - return 0; - - if (size <= KMALLOC_MIN_SIZE) - return KMALLOC_SHIFT_LOW; - - if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) - return 1; - if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) - return 2; - if (size <= 8) return 3; - if (size <= 16) return 4; - if (size <= 32) return 5; - if (size <= 64) return 6; - if (size <= 128) return 7; - if (size <= 256) return 8; - if (size <= 512) return 9; - if (size <= 1024) return 10; - if (size <= 2 * 1024) return 11; - if (size <= 4 * 1024) return 12; -/* - * The following is only needed to support architectures with a larger page - * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page - * size we would have to go up to 128k. - */ - if (size <= 8 * 1024) return 13; - if (size <= 16 * 1024) return 14; - if (size <= 32 * 1024) return 15; - if (size <= 64 * 1024) return 16; - if (size <= 128 * 1024) return 17; - if (size <= 256 * 1024) return 18; - if (size <= 512 * 1024) return 19; - if (size <= 1024 * 1024) return 20; - if (size <= 2 * 1024 * 1024) return 21; - BUG(); - return -1; /* Will never be reached */ - -/* - * What we really wanted to do and cannot do because of compiler issues is: - * int i; - * for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) - * if (size <= (1 << i)) - * return i; - */ -} - /* * Find the slab cache for a given combination of allocation flags and size. * -- cgit v1.2.3-18-g5258 From e33660165c901d18e7d3df2290db070d3e4b46df Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:18 +0000 Subject: slab: Use common kmalloc_index/kmalloc_size functions Make slab use the common functions. We can get rid of a lot of old ugly stuff as a results. Among them the sizes array and the weird include/linux/kmalloc_sizes file and some pretty bad #include statements in slab_def.h. The one thing that is different in slab is that the 32 byte cache will also be created for arches that have page sizes larger than 4K. There are numerous smaller allocations that SLOB and SLUB can handle better because of their support for smaller allocation sizes so lets keep the 32 byte slab also for arches with > 4K pages. Reviewed-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/kmalloc_sizes.h | 45 ----------------------------------------- include/linux/slab_def.h | 47 +++++++++++-------------------------------- 2 files changed, 12 insertions(+), 80 deletions(-) delete mode 100644 include/linux/kmalloc_sizes.h (limited to 'include') diff --git a/include/linux/kmalloc_sizes.h b/include/linux/kmalloc_sizes.h deleted file mode 100644 index e576b848ce1..00000000000 --- a/include/linux/kmalloc_sizes.h +++ /dev/null @@ -1,45 +0,0 @@ -#if (PAGE_SIZE == 4096) - CACHE(32) -#endif - CACHE(64) -#if L1_CACHE_BYTES < 64 - CACHE(96) -#endif - CACHE(128) -#if L1_CACHE_BYTES < 128 - CACHE(192) -#endif - CACHE(256) - CACHE(512) - CACHE(1024) - CACHE(2048) - CACHE(4096) - CACHE(8192) - CACHE(16384) - CACHE(32768) - CACHE(65536) - CACHE(131072) -#if KMALLOC_MAX_SIZE >= 262144 - CACHE(262144) -#endif -#if KMALLOC_MAX_SIZE >= 524288 - CACHE(524288) -#endif -#if KMALLOC_MAX_SIZE >= 1048576 - CACHE(1048576) -#endif -#if KMALLOC_MAX_SIZE >= 2097152 - CACHE(2097152) -#endif -#if KMALLOC_MAX_SIZE >= 4194304 - CACHE(4194304) -#endif -#if KMALLOC_MAX_SIZE >= 8388608 - CACHE(8388608) -#endif -#if KMALLOC_MAX_SIZE >= 16777216 - CACHE(16777216) -#endif -#if KMALLOC_MAX_SIZE >= 33554432 - CACHE(33554432) -#endif diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 8bb6e0eaf3c..e0f30ef9525 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -11,8 +11,6 @@ */ #include -#include /* kmalloc_sizes.h needs PAGE_SIZE */ -#include /* kmalloc_sizes.h needs L1_CACHE_BYTES */ #include /* @@ -104,15 +102,8 @@ struct kmem_cache { */ }; -/* Size description struct for general caches. */ -struct cache_sizes { - size_t cs_size; - struct kmem_cache *cs_cachep; -#ifdef CONFIG_ZONE_DMA - struct kmem_cache *cs_dmacachep; -#endif -}; -extern struct cache_sizes malloc_sizes[]; +extern struct kmem_cache *kmalloc_caches[PAGE_SHIFT + MAX_ORDER]; +extern struct kmem_cache *kmalloc_dma_caches[PAGE_SHIFT + MAX_ORDER]; void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); @@ -133,26 +124,19 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) void *ret; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include -#undef CACHE - return NULL; -found: + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; ret = kmem_cache_alloc_trace(cachep, flags, size); @@ -186,26 +170,19 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) struct kmem_cache *cachep; if (__builtin_constant_p(size)) { - int i = 0; + int i; if (!size) return ZERO_SIZE_PTR; -#define CACHE(x) \ - if (size <= x) \ - goto found; \ - else \ - i++; -#include -#undef CACHE - return NULL; -found: + i = kmalloc_index(size); + #ifdef CONFIG_ZONE_DMA if (flags & GFP_DMA) - cachep = malloc_sizes[i].cs_dmacachep; + cachep = kmalloc_dma_caches[i]; else #endif - cachep = malloc_sizes[i].cs_cachep; + cachep = kmalloc_caches[i]; return kmem_cache_alloc_node_trace(cachep, flags, node, size); } -- cgit v1.2.3-18-g5258 From 6744f087ba2a49f6d6935d9daa0b20a0f03567b5 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:12:17 +0000 Subject: slab: Common name for the per node structures Rename the structure used for the per node structures in slab to have a name that expresses that fact. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index e0f30ef9525..8b5b2f6b36d 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -95,7 +95,7 @@ struct kmem_cache { * pointer for each node since "nodelists" uses the remainder of * available pointers. */ - struct kmem_list3 **nodelists; + struct kmem_cache_node **nodelists; struct array_cache *array[NR_CPUS + MAX_NUMNODES]; /* * Do not add fields after array[] -- cgit v1.2.3-18-g5258 From 6a67368c36e2c0c2578ba62f6264ab739af08cce Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Rename nodelists to node Have a common naming between both slab caches for future changes. Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 8b5b2f6b36d..4ff50e8d1a2 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -95,7 +95,7 @@ struct kmem_cache { * pointer for each node since "nodelists" uses the remainder of * available pointers. */ - struct kmem_cache_node **nodelists; + struct kmem_cache_node **node; struct array_cache *array[NR_CPUS + MAX_NUMNODES]; /* * Do not add fields after array[] -- cgit v1.2.3-18-g5258 From 95a05b428cc675694321c8f762591984f3fd2b1e Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common constants for kmalloc boundaries Standardize the constants that describe the smallest and largest object kept in the kmalloc arrays for SLAB and SLUB. Differentiate between the maximum size for which a slab cache is used (KMALLOC_MAX_CACHE_SIZE) and the maximum allocatable size (KMALLOC_MAX_SIZE, KMALLOC_MAX_ORDER). Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 34 ++++++++++++++++++++++++---------- include/linux/slub_def.h | 19 +++---------------- 2 files changed, 27 insertions(+), 26 deletions(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index c97fe92532d..c0178054005 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -163,7 +163,12 @@ struct kmem_cache { #else /* CONFIG_SLOB */ /* - * The largest kmalloc size supported by the slab allocators is + * Kmalloc array related definitions + */ + +#ifdef CONFIG_SLAB +/* + * The largest kmalloc size supported by the SLAB allocators is * 32 megabyte (2^25) or the maximum allocatable page order if that is * less than 32 MB. * @@ -173,9 +178,24 @@ struct kmem_cache { */ #define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ (MAX_ORDER + PAGE_SHIFT - 1) : 25) +#define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#define KMALLOC_SHIFT_LOW 5 +#else +/* + * SLUB allocates up to order 2 pages directly and otherwise + * passes the request to the page allocator. + */ +#define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) +#define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) +#define KMALLOC_SHIFT_LOW 3 +#endif -#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_HIGH) -#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_HIGH - PAGE_SHIFT) +/* Maximum allocatable size */ +#define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) +/* Maximum size for which we actually use a slab cache */ +#define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) +/* Maximum order allocatable via the slab allocagtor */ +#define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) /* * Kmalloc subsystem. @@ -183,15 +203,9 @@ struct kmem_cache { #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN #else -#ifdef CONFIG_SLAB -#define KMALLOC_MIN_SIZE 32 -#else -#define KMALLOC_MIN_SIZE 8 -#endif +#define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) #endif -#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE) - /* * Figure out which kmalloc slab an allocation of a certain size * belongs to. diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 99c3e05ff1f..032028ef9a3 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -115,19 +115,6 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -/* - * Maximum kmalloc object size handled by SLUB. Larger object allocations - * are passed through to the page allocator. The page allocator "fastpath" - * is relatively slow so we need this value sufficiently high so that - * performance critical objects are allocated through the SLUB fastpath. - * - * This should be dropped to PAGE_SIZE / 2 once the page allocator - * "fastpath" becomes competitive with the slab allocator fastpaths. - */ -#define SLUB_MAX_SIZE (2 * PAGE_SIZE) - -#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2) - #ifdef CONFIG_ZONE_DMA #define SLUB_DMA __GFP_DMA #else @@ -139,7 +126,7 @@ struct kmem_cache { * We keep the general caches in an array of slab caches that are used for * 2^x bytes of allocations. */ -extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT]; +extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; /* * Find the slab cache for a given combination of allocation flags and size. @@ -211,7 +198,7 @@ static __always_inline void *kmalloc_large(size_t size, gfp_t flags) static __always_inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { - if (size > SLUB_MAX_SIZE) + if (size > KMALLOC_MAX_CACHE_SIZE) return kmalloc_large(size, flags); if (!(flags & SLUB_DMA)) { @@ -247,7 +234,7 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { if (__builtin_constant_p(size) && - size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) { + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) { struct kmem_cache *s = kmalloc_slab(size); if (!s) -- cgit v1.2.3-18-g5258 From 9425c58e5445277699ff3c2a87bac1cfebc1b48d Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:12:17 +0000 Subject: slab: Common definition for the array of kmalloc caches Have a common definition fo the kmalloc cache arrays in SLAB and SLUB Acked-by: Glauber Costa Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 5 +++++ include/linux/slab_def.h | 3 --- include/linux/slub_def.h | 6 ------ 3 files changed, 5 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index c0178054005..f2327a898a8 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -206,6 +206,11 @@ struct kmem_cache { #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) #endif +extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; +#ifdef CONFIG_ZONE_DMA +extern struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; +#endif + /* * Figure out which kmalloc slab an allocation of a certain size * belongs to. diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 4ff50e8d1a2..113ec080313 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -102,9 +102,6 @@ struct kmem_cache { */ }; -extern struct kmem_cache *kmalloc_caches[PAGE_SHIFT + MAX_ORDER]; -extern struct kmem_cache *kmalloc_dma_caches[PAGE_SHIFT + MAX_ORDER]; - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 032028ef9a3..3701896f7f8 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -122,12 +122,6 @@ struct kmem_cache { #define SLUB_DMA (__force gfp_t)0 #endif -/* - * We keep the general caches in an array of slab caches that are used for - * 2^x bytes of allocations. - */ -extern struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; - /* * Find the slab cache for a given combination of allocation flags and size. * -- cgit v1.2.3-18-g5258 From 2c59dd6544212faa5ce761920d2251f4152f408d Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common Kmalloc cache determination Extract the optimized lookup functions from slub and put them into slab_common.c. Then make slab use these functions as well. Joonsoo notes that this fixes some issues with constant folding which also reduces the code size for slub. https://lkml.org/lkml/2012/10/20/82 Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 41 ++++++++++------------------------------- 1 file changed, 10 insertions(+), 31 deletions(-) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 3701896f7f8..16341e5316d 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -115,29 +115,6 @@ struct kmem_cache { struct kmem_cache_node *node[MAX_NUMNODES]; }; -#ifdef CONFIG_ZONE_DMA -#define SLUB_DMA __GFP_DMA -#else -/* Disable DMA functionality */ -#define SLUB_DMA (__force gfp_t)0 -#endif - -/* - * Find the slab cache for a given combination of allocation flags and size. - * - * This ought to end up with a global pointer to the right cache - * in kmalloc_caches. - */ -static __always_inline struct kmem_cache *kmalloc_slab(size_t size) -{ - int index = kmalloc_index(size); - - if (index == 0) - return NULL; - - return kmalloc_caches[index]; -} - void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); @@ -195,13 +172,14 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) if (size > KMALLOC_MAX_CACHE_SIZE) return kmalloc_large(size, flags); - if (!(flags & SLUB_DMA)) { - struct kmem_cache *s = kmalloc_slab(size); + if (!(flags & GFP_DMA)) { + int index = kmalloc_index(size); - if (!s) + if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_trace(s, flags, size); + return kmem_cache_alloc_trace(kmalloc_caches[index], + flags, size); } } return __kmalloc(size, flags); @@ -228,13 +206,14 @@ kmem_cache_alloc_node_trace(struct kmem_cache *s, static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) { if (__builtin_constant_p(size) && - size <= KMALLOC_MAX_CACHE_SIZE && !(flags & SLUB_DMA)) { - struct kmem_cache *s = kmalloc_slab(size); + size <= KMALLOC_MAX_CACHE_SIZE && !(flags & GFP_DMA)) { + int index = kmalloc_index(size); - if (!s) + if (!index) return ZERO_SIZE_PTR; - return kmem_cache_alloc_node_trace(s, flags, node, size); + return kmem_cache_alloc_node_trace(kmalloc_caches[index], + flags, node, size); } return __kmalloc_node(size, flags, node); } -- cgit v1.2.3-18-g5258 From ca34956b804b7554fc4e88826773380d9d5122a8 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 10 Jan 2013 19:14:19 +0000 Subject: slab: Common definition for kmem_cache_node Put the definitions for the kmem_cache_node structures together so that we have one structure. That will allow us to create more common fields in the future which could yield more opportunities to share code. Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slub_def.h | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'include') diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 16341e5316d..027276fa871 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -53,17 +53,6 @@ struct kmem_cache_cpu { #endif }; -struct kmem_cache_node { - spinlock_t list_lock; /* Protect partial list and nr_partial */ - unsigned long nr_partial; - struct list_head partial; -#ifdef CONFIG_SLUB_DEBUG - atomic_long_t nr_slabs; - atomic_long_t total_objects; - struct list_head full; -#endif -}; - /* * Word size structure that can be atomically updated or read and that * contains both the order and the number of objects that a slab of the -- cgit v1.2.3-18-g5258 From c601fd6956e92b0eb268d4af754073c76155b99d Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 5 Feb 2013 16:36:47 +0000 Subject: slab: Handle ARCH_DMA_MINALIGN correctly James Hogan hit boot problems in next-20130204 on Meta: META213-Thread0 DSP [LogF] kobject (4fc03980): tried to init an initialized object, something is seriously wrong. META213-Thread0 DSP [LogF] META213-Thread0 DSP [LogF] Call trace: META213-Thread0 DSP [LogF] [<4000888c>] _show_stack+0x68/0x7c META213-Thread0 DSP [LogF] [<400088b4>] _dump_stack+0x14/0x28 META213-Thread0 DSP [LogF] [<40103794>] _kobject_init+0x58/0x9c META213-Thread0 DSP [LogF] [<40103810>] _kobject_create+0x38/0x64 META213-Thread0 DSP [LogF] [<40103eac>] _kobject_create_and_add+0x14/0x8c META213-Thread0 DSP [LogF] [<40190ac4>] _mnt_init+0xd8/0x220 META213-Thread0 DSP [LogF] [<40190508>] _vfs_caches_init+0xb0/0x160 META213-Thread0 DSP [LogF] [<401851f4>] _start_kernel+0x274/0x340 META213-Thread0 DSP [LogF] [<40188424>] _metag_start_kernel+0x58/0x6c META213-Thread0 DSP [LogF] [<40000044>] __start+0x44/0x48 META213-Thread0 DSP [LogF] META213-Thread0 DSP [LogF] devtmpfs: initialized META213-Thread0 DSP [LogF] L2 Cache: Not present META213-Thread0 DSP [LogF] BUG: failure at fs/sysfs/dir.c:736/sysfs_read_ns_type()! META213-Thread0 DSP [LogF] Kernel panic - not syncing: BUG! META213-Thread0 DSP [Thread Exit] Thread has exited - return code = 4294967295 And bisected the problem to commit 95a05b4 ("slab: Common constants for kmalloc boundaries"). As it turns out, a fixed KMALLOC_SHIFT_LOW does not work for arches with higher alignment requirements. Determine KMALLOC_SHIFT_LOW from ARCH_DMA_MINALIGN instead. Reported-and-tested-by: James Hogan Signed-off-by: Christoph Lameter Signed-off-by: Pekka Enberg --- include/linux/slab.h | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) (limited to 'include') diff --git a/include/linux/slab.h b/include/linux/slab.h index f2327a898a8..0c621752caa 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -133,6 +133,19 @@ void kfree(const void *); void kzfree(const void *); size_t ksize(const void *); +/* + * Some archs want to perform DMA into kmalloc caches and need a guaranteed + * alignment larger than the alignment of a 64-bit integer. + * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. + */ +#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 +#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN +#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN +#define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) +#else +#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) +#endif + #ifdef CONFIG_SLOB /* * Common fields provided in kmem_cache by all slab allocators @@ -179,7 +192,9 @@ struct kmem_cache { #define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ (MAX_ORDER + PAGE_SHIFT - 1) : 25) #define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH +#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 5 +#endif #else /* * SLUB allocates up to order 2 pages directly and otherwise @@ -187,8 +202,10 @@ struct kmem_cache { */ #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT) +#ifndef KMALLOC_SHIFT_LOW #define KMALLOC_SHIFT_LOW 3 #endif +#endif /* Maximum allocatable size */ #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) @@ -200,9 +217,7 @@ struct kmem_cache { /* * Kmalloc subsystem. */ -#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 -#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN -#else +#ifndef KMALLOC_MIN_SIZE #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) #endif @@ -289,17 +304,6 @@ static __always_inline int kmalloc_size(int n) } #endif /* !CONFIG_SLOB */ -/* - * Some archs want to perform DMA into kmalloc caches and need a guaranteed - * alignment larger than the alignment of a 64-bit integer. - * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. - */ -#ifdef ARCH_DMA_MINALIGN -#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN -#else -#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) -#endif - /* * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. * Intended for arches that get misalignment faults even for 64 bit integer -- cgit v1.2.3-18-g5258 From 6286ae97d10ea2b5cd90532163797ab217bfdbdf Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Fri, 3 May 2013 15:43:18 +0000 Subject: slab: Return NULL for oversized allocations The inline path seems to have changed the SLAB behavior for very large kmalloc allocations with commit e3366016 ("slab: Use common kmalloc_index/kmalloc_size functions"). This patch restores the old behavior but also adds diagnostics so that we can figure where in the code these large allocations occur. Reported-and-tested-by: Tetsuo Handa Signed-off-by: Christoph Lameter Link: http://lkml.kernel.org/r/201305040348.CIF81716.OStQOHFJMFLOVF@I-love.SAKURA.ne.jp [ penberg@kernel.org: use WARN_ON_ONCE ] Signed-off-by: Pekka Enberg --- include/linux/slab_def.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include') diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h index 113ec080313..cd401580bdd 100644 --- a/include/linux/slab_def.h +++ b/include/linux/slab_def.h @@ -126,6 +126,9 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags) if (!size) return ZERO_SIZE_PTR; + if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) + return NULL; + i = kmalloc_index(size); #ifdef CONFIG_ZONE_DMA @@ -172,6 +175,9 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) if (!size) return ZERO_SIZE_PTR; + if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE)) + return NULL; + i = kmalloc_index(size); #ifdef CONFIG_ZONE_DMA -- cgit v1.2.3-18-g5258