diff options
| -rw-r--r-- | arch/ia64/mm/init.c | 2 | ||||
| -rw-r--r-- | fs/hugetlbfs/inode.c | 2 | ||||
| -rw-r--r-- | include/linux/mm.h | 33 | ||||
| -rw-r--r-- | include/linux/page-flags.h | 14 | ||||
| -rw-r--r-- | mm/internal.h | 2 | ||||
| -rw-r--r-- | mm/page_alloc.c | 29 | ||||
| -rw-r--r-- | mm/slab.c | 6 | ||||
| -rw-r--r-- | mm/slub.c | 19 | ||||
| -rw-r--r-- | mm/swap.c | 2 |
9 files changed, 72 insertions, 37 deletions
diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 4f36987eea7..2da84111072 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -121,7 +121,7 @@ lazy_mmu_prot_update (pte_t pte) return; /* i-cache is already coherent with d-cache */ if (PageCompound(page)) { - order = (unsigned long) (page[1].lru.prev); + order = compound_order(page); flush_icache_range(addr, addr + (1UL << order << PAGE_SHIFT)); } else diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9ba71b252f3..8e1b7825e2f 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -450,7 +450,7 @@ static int hugetlbfs_symlink(struct inode *dir, */ static int hugetlbfs_set_page_dirty(struct page *page) { - struct page *head = (struct page *)page_private(page); + struct page *head = compound_head(page); SetPageDirty(head); return 0; diff --git a/include/linux/mm.h b/include/linux/mm.h index c95d96ebd5a..8c149fa4491 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -267,17 +267,28 @@ static inline int get_page_unless_zero(struct page *page) return atomic_inc_not_zero(&page->_count); } +static inline struct page *compound_head(struct page *page) +{ + /* + * We could avoid the PageCompound(page) check if + * we would not overload PageTail(). + * + * This check has to be done in several performance critical + * paths of the slab etc. IMHO PageTail deserves its own flag. + */ + if (unlikely(PageCompound(page) && PageTail(page))) + return page->first_page; + return page; +} + static inline int page_count(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); - return atomic_read(&page->_count); + return atomic_read(&compound_head(page)->_count); } static inline void get_page(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); VM_BUG_ON(atomic_read(&page->_count) == 0); atomic_inc(&page->_count); } @@ -314,6 +325,18 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page) return (compound_page_dtor *)page[1].lru.next; } +static inline int compound_order(struct page *page) +{ + if (!PageCompound(page) || PageTail(page)) + return 0; + return (unsigned long)page[1].lru.prev; +} + +static inline void set_compound_order(struct page *page, unsigned long order) +{ + page[1].lru.prev = (void *)order; +} + /* * Multiple processes may "see" the same page. E.g. for untouched * mappings of /dev/null, all processes see the same page full of diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 96326594e55..a1e14363494 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -94,6 +94,12 @@ /* PG_owner_priv_1 users should have descriptive aliases */ #define PG_checked PG_owner_priv_1 /* Used by some filesystems */ +/* + * Marks tail portion of a compound page. We currently do not reclaim + * compound pages so we can reuse a flag only used for reclaim here. + */ +#define PG_tail PG_reclaim + #if (BITS_PER_LONG > 32) /* * 64-bit-only flags build down from bit 31 @@ -241,6 +247,14 @@ static inline void SetPageUptodate(struct page *page) #define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags) #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) +/* + * Note: PG_tail is an alias of another page flag. The result of PageTail() + * is only valid if PageCompound(page) is true. + */ +#define PageTail(page) test_bit(PG_tail, &(page)->flags) +#define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags) +#define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags) + #ifdef CONFIG_SWAP #define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) #define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags) diff --git a/mm/internal.h b/mm/internal.h index d527b80b292..a3110c02aea 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -24,7 +24,7 @@ static inline void set_page_count(struct page *page, int v) */ static inline void set_page_refcounted(struct page *page) { - VM_BUG_ON(PageCompound(page) && page_private(page) != (unsigned long)page); + VM_BUG_ON(PageCompound(page) && PageTail(page)); VM_BUG_ON(atomic_read(&page->_count)); set_page_count(page, 1); } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 542fc088ff5..fc241fe295a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -225,7 +225,7 @@ static void bad_page(struct page *page) static void free_compound_page(struct page *page) { - __free_pages_ok(page, (unsigned long)page[1].lru.prev); + __free_pages_ok(page, compound_order(page)); } static void prep_compound_page(struct page *page, unsigned long order) @@ -234,12 +234,14 @@ static void prep_compound_page(struct page *page, unsigned long order) int nr_pages = 1 << order; set_compound_page_dtor(page, free_compound_page); - page[1].lru.prev = (void *)order; - for (i = 0; i < nr_pages; i++) { + set_compound_order(page, order); + __SetPageCompound(page); + for (i = 1; i < nr_pages; i++) { struct page *p = page + i; + __SetPageTail(p); __SetPageCompound(p); - set_page_private(p, (unsigned long)page); + p->first_page = page; } } @@ -248,15 +250,19 @@ static void destroy_compound_page(struct page *page, unsigned long order) int i; int nr_pages = 1 << order; - if (unlikely((unsigned long)page[1].lru.prev != order)) + if (unlikely(compound_order(page) != order)) bad_page(page); - for (i = 0; i < nr_pages; i++) { + if (unlikely(!PageCompound(page))) + bad_page(page); + __ClearPageCompound(page); + for (i = 1; i < nr_pages; i++) { struct page *p = page + i; - if (unlikely(!PageCompound(p) | - (page_private(p) != (unsigned long)page))) + if (unlikely(!PageCompound(p) | !PageTail(p) | + (p->first_page != page))) bad_page(page); + __ClearPageTail(p); __ClearPageCompound(p); } } @@ -429,13 +435,18 @@ static inline int free_pages_check(struct page *page) 1 << PG_private | 1 << PG_locked | 1 << PG_active | - 1 << PG_reclaim | 1 << PG_slab | 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | 1 << PG_buddy )))) bad_page(page); + /* + * PageReclaim == PageTail. It is only an error + * for PageReclaim to be set if PageCompound is clear. + */ + if (unlikely(!PageCompound(page) && PageReclaim(page))) + bad_page(page); if (PageDirty(page)) __ClearPageDirty(page); /* diff --git a/mm/slab.c b/mm/slab.c index 9cd01fa6000..f4b2e22b5c6 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -602,8 +602,7 @@ static inline void page_set_cache(struct page *page, struct kmem_cache *cache) static inline struct kmem_cache *page_get_cache(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); BUG_ON(!PageSlab(page)); return (struct kmem_cache *)page->lru.next; } @@ -615,8 +614,7 @@ static inline void page_set_slab(struct page *page, struct slab *slab) static inline struct slab *page_get_slab(struct page *page) { - if (unlikely(PageCompound(page))) - page = (struct page *)page_private(page); + page = compound_head(page); BUG_ON(!PageSlab(page)); return (struct slab *)page->lru.prev; } diff --git a/mm/slub.c b/mm/slub.c index 9d52cce7c99..8fa1c6e937f 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1325,9 +1325,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x) page = virt_to_page(x); - if (unlikely(PageCompound(page))) - page = page->first_page; - + page = compound_head(page); if (unlikely(PageError(page) && (s->flags & SLAB_STORE_USER))) set_tracking(s, x, TRACK_FREE); @@ -1338,10 +1336,7 @@ EXPORT_SYMBOL(kmem_cache_free); /* Figure out on which slab object the object resides */ static struct page *get_object_page(const void *x) { - struct page *page = virt_to_page(x); - - if (unlikely(PageCompound(page))) - page = page->first_page; + struct page *page = compound_head(virt_to_page(x)); if (!PageSlab(page)) return NULL; @@ -2081,10 +2076,7 @@ void kfree(const void *x) if (!x) return; - page = virt_to_page(x); - - if (unlikely(PageCompound(page))) - page = page->first_page; + page = compound_head(virt_to_page(x)); s = page->slab; @@ -2120,10 +2112,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags) return NULL; } - page = virt_to_page(p); - - if (unlikely(PageCompound(page))) - page = page->first_page; + page = compound_head(virt_to_page(p)); new_cache = get_slab(new_size, flags); diff --git a/mm/swap.c b/mm/swap.c index 2ed7be39795..218c52a24a2 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -55,7 +55,7 @@ static void fastcall __page_cache_release(struct page *page) static void put_compound_page(struct page *page) { - page = (struct page *)page_private(page); + page = compound_head(page); if (put_page_testzero(page)) { compound_page_dtor *dtor; |