diff options
| -rw-r--r-- | include/linux/hugetlb.h | 6 | ||||
| -rw-r--r-- | mm/hugetlb.c | 17 | ||||
| -rw-r--r-- | mm/swap.c | 107 |
3 files changed, 101 insertions, 29 deletions
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 32697c186c7..4bc9445222f 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -24,6 +24,7 @@ struct hugepage_subpool *hugepage_new_subpool(long nr_blocks); void hugepage_put_subpool(struct hugepage_subpool *spool); int PageHuge(struct page *page); +int PageHeadHuge(struct page *page_head); void reset_vma_resv_huge_pages(struct vm_area_struct *vma); int hugetlb_sysctl_handler(struct ctl_table *, int, void __user *, size_t *, loff_t *); @@ -88,6 +89,11 @@ static inline int PageHuge(struct page *page) return 0; } +static inline int PageHeadHuge(struct page *page_head) +{ + return 0; +} + static inline void reset_vma_resv_huge_pages(struct vm_area_struct *vma) { } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index ddf2128f93f..3a5aae2d6ee 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -679,6 +679,23 @@ int PageHuge(struct page *page) } EXPORT_SYMBOL_GPL(PageHuge); +/* + * PageHeadHuge() only returns true for hugetlbfs head page, but not for + * normal or transparent huge pages. + */ +int PageHeadHuge(struct page *page_head) +{ + compound_page_dtor *dtor; + + if (!PageHead(page_head)) + return 0; + + dtor = get_compound_page_dtor(page_head); + + return dtor == free_huge_page; +} +EXPORT_SYMBOL_GPL(PageHeadHuge); + pgoff_t __basepage_index(struct page *page) { struct page *page_head = compound_head(page); diff --git a/mm/swap.c b/mm/swap.c index a5c7567b22b..a4b90161be6 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -78,18 +78,6 @@ static void __put_compound_page(struct page *page) static void put_compound_page(struct page *page) { - /* - * hugetlbfs pages cannot be split from under us. If this is a - * hugetlbfs page, check refcount on head page and release the page if - * the refcount becomes zero. - */ - if (PageHuge(page)) { - page = compound_head(page); - if (put_page_testzero(page)) - __put_compound_page(page); - return; - } - if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_trans_head(page); @@ -97,6 +85,35 @@ static void put_compound_page(struct page *page) if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; + + if (PageHeadHuge(page_head)) { + if (likely(PageTail(page))) { + /* + * __split_huge_page_refcount + * cannot race here. + */ + VM_BUG_ON(!PageHead(page_head)); + atomic_dec(&page->_mapcount); + if (put_page_testzero(page_head)) + VM_BUG_ON(1); + if (put_page_testzero(page_head)) + __put_compound_page(page_head); + return; + } else { + /* + * __split_huge_page_refcount + * run before us, "page" was a + * THP tail. The split + * page_head has been freed + * and reallocated as slab or + * hugetlbfs page of smaller + * order (only possible if + * reallocated as slab on + * x86). + */ + goto skip_lock; + } + } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time @@ -108,9 +125,29 @@ static void put_compound_page(struct page *page) /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); VM_BUG_ON(PageHead(page_head)); - if (put_page_testzero(page_head)) - __put_single_page(page_head); - out_put_single: +skip_lock: + if (put_page_testzero(page_head)) { + /* + * The head page may have been + * freed and reallocated as a + * compound page of smaller + * order and then freed again. + * All we know is that it + * cannot have become: a THP + * page, a compound page of + * higher order, a tail page. + * That is because we still + * hold the refcount of the + * split THP tail and + * page_head was the THP head + * before the split. + */ + if (PageHead(page_head)) + __put_compound_page(page_head); + else + __put_single_page(page_head); + } +out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; @@ -174,21 +211,34 @@ bool __get_page_tail(struct page *page) */ unsigned long flags; bool got = false; - struct page *page_head; + struct page *page_head = compound_trans_head(page); - /* - * If this is a hugetlbfs page it cannot be split under us. Simply - * increment refcount for the head page. - */ - if (PageHuge(page)) { - page_head = compound_head(page); - atomic_inc(&page_head->_count); - got = true; - goto out; - } - - page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { + /* Ref to put_compound_page() comment. */ + if (PageHeadHuge(page_head)) { + if (likely(PageTail(page))) { + /* + * This is a hugetlbfs + * page. __split_huge_page_refcount + * cannot race here. + */ + VM_BUG_ON(!PageHead(page_head)); + __get_page_tail_foll(page, false); + return true; + } else { + /* + * __split_huge_page_refcount run + * before us, "page" was a THP + * tail. The split page_head has been + * freed and reallocated as slab or + * hugetlbfs page of smaller order + * (only possible if reallocated as + * slab on x86). + */ + put_page(page_head); + return false; + } + } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time @@ -205,7 +255,6 @@ bool __get_page_tail(struct page *page) if (unlikely(!got)) put_page(page_head); } -out: return got; } EXPORT_SYMBOL(__get_page_tail); |