source: src/linux/universal/linux-4.9/mm/huge_memory.c @ 31885

Last change on this file since 31885 was 31885, checked in by brainslayer, 3 months ago

update

File size: 63.7 KB
Line 
1/*
2 *  Copyright (C) 2009  Red Hat, Inc.
3 *
4 *  This work is licensed under the terms of the GNU GPL, version 2. See
5 *  the COPYING file in the top-level directory.
6 */
7
8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
10#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/highmem.h>
13#include <linux/hugetlb.h>
14#include <linux/mmu_notifier.h>
15#include <linux/rmap.h>
16#include <linux/swap.h>
17#include <linux/shrinker.h>
18#include <linux/mm_inline.h>
19#include <linux/swapops.h>
20#include <linux/dax.h>
21#include <linux/khugepaged.h>
22#include <linux/freezer.h>
23#include <linux/pfn_t.h>
24#include <linux/mman.h>
25#include <linux/memremap.h>
26#include <linux/pagemap.h>
27#include <linux/debugfs.h>
28#include <linux/migrate.h>
29#include <linux/hashtable.h>
30#include <linux/userfaultfd_k.h>
31#include <linux/page_idle.h>
32#include <linux/shmem_fs.h>
33
34#include <asm/tlb.h>
35#include <asm/pgalloc.h>
36#include "internal.h"
37
38/*
39 * By default transparent hugepage support is disabled in order that avoid
40 * to risk increase the memory footprint of applications without a guaranteed
41 * benefit. When transparent hugepage support is enabled, is for all mappings,
42 * and khugepaged scans all mappings.
43 * Defrag is invoked by khugepaged hugepage allocations and by page faults
44 * for all hugepage allocations.
45 */
46unsigned long transparent_hugepage_flags __read_mostly =
47#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
48        (1<<TRANSPARENT_HUGEPAGE_FLAG)|
49#endif
50#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
51        (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
52#endif
53        (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
54        (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
55        (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
56
57static struct shrinker deferred_split_shrinker;
58
59static atomic_t huge_zero_refcount;
60struct page *huge_zero_page __read_mostly;
61
62static struct page *get_huge_zero_page(void)
63{
64        struct page *zero_page;
65retry:
66        if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
67                return READ_ONCE(huge_zero_page);
68
69        zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
70                        HPAGE_PMD_ORDER);
71        if (!zero_page) {
72                count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
73                return NULL;
74        }
75        count_vm_event(THP_ZERO_PAGE_ALLOC);
76        preempt_disable();
77        if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
78                preempt_enable();
79                __free_pages(zero_page, compound_order(zero_page));
80                goto retry;
81        }
82
83        /* We take additional reference here. It will be put back by shrinker */
84        atomic_set(&huge_zero_refcount, 2);
85        preempt_enable();
86        return READ_ONCE(huge_zero_page);
87}
88
89static void put_huge_zero_page(void)
90{
91        /*
92         * Counter should never go to zero here. Only shrinker can put
93         * last reference.
94         */
95        BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
96}
97
98struct page *mm_get_huge_zero_page(struct mm_struct *mm)
99{
100        if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
101                return READ_ONCE(huge_zero_page);
102
103        if (!get_huge_zero_page())
104                return NULL;
105
106        if (test_and_set_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
107                put_huge_zero_page();
108
109        return READ_ONCE(huge_zero_page);
110}
111
112void mm_put_huge_zero_page(struct mm_struct *mm)
113{
114        if (test_bit(MMF_HUGE_ZERO_PAGE, &mm->flags))
115                put_huge_zero_page();
116}
117
118static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
119                                        struct shrink_control *sc)
120{
121        /* we can free zero page only if last reference remains */
122        return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
123}
124
125static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
126                                       struct shrink_control *sc)
127{
128        if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
129                struct page *zero_page = xchg(&huge_zero_page, NULL);
130                BUG_ON(zero_page == NULL);
131                __free_pages(zero_page, compound_order(zero_page));
132                return HPAGE_PMD_NR;
133        }
134
135        return 0;
136}
137
138static struct shrinker huge_zero_page_shrinker = {
139        .count_objects = shrink_huge_zero_page_count,
140        .scan_objects = shrink_huge_zero_page_scan,
141        .seeks = DEFAULT_SEEKS,
142};
143
144#ifdef CONFIG_SYSFS
145
146static ssize_t triple_flag_store(struct kobject *kobj,
147                                 struct kobj_attribute *attr,
148                                 const char *buf, size_t count,
149                                 enum transparent_hugepage_flag enabled,
150                                 enum transparent_hugepage_flag deferred,
151                                 enum transparent_hugepage_flag req_madv)
152{
153        if (!memcmp("defer", buf,
154                    min(sizeof("defer")-1, count))) {
155                if (enabled == deferred)
156                        return -EINVAL;
157                clear_bit(enabled, &transparent_hugepage_flags);
158                clear_bit(req_madv, &transparent_hugepage_flags);
159                set_bit(deferred, &transparent_hugepage_flags);
160        } else if (!memcmp("always", buf,
161                    min(sizeof("always")-1, count))) {
162                clear_bit(deferred, &transparent_hugepage_flags);
163                clear_bit(req_madv, &transparent_hugepage_flags);
164                set_bit(enabled, &transparent_hugepage_flags);
165        } else if (!memcmp("madvise", buf,
166                           min(sizeof("madvise")-1, count))) {
167                clear_bit(enabled, &transparent_hugepage_flags);
168                clear_bit(deferred, &transparent_hugepage_flags);
169                set_bit(req_madv, &transparent_hugepage_flags);
170        } else if (!memcmp("never", buf,
171                           min(sizeof("never")-1, count))) {
172                clear_bit(enabled, &transparent_hugepage_flags);
173                clear_bit(req_madv, &transparent_hugepage_flags);
174                clear_bit(deferred, &transparent_hugepage_flags);
175        } else
176                return -EINVAL;
177
178        return count;
179}
180
181static ssize_t enabled_show(struct kobject *kobj,
182                            struct kobj_attribute *attr, char *buf)
183{
184        if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
185                return sprintf(buf, "[always] madvise never\n");
186        else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
187                return sprintf(buf, "always [madvise] never\n");
188        else
189                return sprintf(buf, "always madvise [never]\n");
190}
191
192static ssize_t enabled_store(struct kobject *kobj,
193                             struct kobj_attribute *attr,
194                             const char *buf, size_t count)
195{
196        ssize_t ret;
197
198        ret = triple_flag_store(kobj, attr, buf, count,
199                                TRANSPARENT_HUGEPAGE_FLAG,
200                                TRANSPARENT_HUGEPAGE_FLAG,
201                                TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
202
203        if (ret > 0) {
204                int err = start_stop_khugepaged();
205                if (err)
206                        ret = err;
207        }
208
209        return ret;
210}
211static struct kobj_attribute enabled_attr =
212        __ATTR(enabled, 0644, enabled_show, enabled_store);
213
214ssize_t single_hugepage_flag_show(struct kobject *kobj,
215                                struct kobj_attribute *attr, char *buf,
216                                enum transparent_hugepage_flag flag)
217{
218        return sprintf(buf, "%d\n",
219                       !!test_bit(flag, &transparent_hugepage_flags));
220}
221
222ssize_t single_hugepage_flag_store(struct kobject *kobj,
223                                 struct kobj_attribute *attr,
224                                 const char *buf, size_t count,
225                                 enum transparent_hugepage_flag flag)
226{
227        unsigned long value;
228        int ret;
229
230        ret = kstrtoul(buf, 10, &value);
231        if (ret < 0)
232                return ret;
233        if (value > 1)
234                return -EINVAL;
235
236        if (value)
237                set_bit(flag, &transparent_hugepage_flags);
238        else
239                clear_bit(flag, &transparent_hugepage_flags);
240
241        return count;
242}
243
244/*
245 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
246 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
247 * memory just to allocate one more hugepage.
248 */
249static ssize_t defrag_show(struct kobject *kobj,
250                           struct kobj_attribute *attr, char *buf)
251{
252        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
253                return sprintf(buf, "[always] defer madvise never\n");
254        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
255                return sprintf(buf, "always [defer] madvise never\n");
256        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
257                return sprintf(buf, "always defer [madvise] never\n");
258        else
259                return sprintf(buf, "always defer madvise [never]\n");
260
261}
262static ssize_t defrag_store(struct kobject *kobj,
263                            struct kobj_attribute *attr,
264                            const char *buf, size_t count)
265{
266        return triple_flag_store(kobj, attr, buf, count,
267                                 TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
268                                 TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
269                                 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
270}
271static struct kobj_attribute defrag_attr =
272        __ATTR(defrag, 0644, defrag_show, defrag_store);
273
274static ssize_t use_zero_page_show(struct kobject *kobj,
275                struct kobj_attribute *attr, char *buf)
276{
277        return single_hugepage_flag_show(kobj, attr, buf,
278                                TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
279}
280static ssize_t use_zero_page_store(struct kobject *kobj,
281                struct kobj_attribute *attr, const char *buf, size_t count)
282{
283        return single_hugepage_flag_store(kobj, attr, buf, count,
284                                 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
285}
286static struct kobj_attribute use_zero_page_attr =
287        __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
288#ifdef CONFIG_DEBUG_VM
289static ssize_t debug_cow_show(struct kobject *kobj,
290                                struct kobj_attribute *attr, char *buf)
291{
292        return single_hugepage_flag_show(kobj, attr, buf,
293                                TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
294}
295static ssize_t debug_cow_store(struct kobject *kobj,
296                               struct kobj_attribute *attr,
297                               const char *buf, size_t count)
298{
299        return single_hugepage_flag_store(kobj, attr, buf, count,
300                                 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
301}
302static struct kobj_attribute debug_cow_attr =
303        __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
304#endif /* CONFIG_DEBUG_VM */
305
306static struct attribute *hugepage_attr[] = {
307        &enabled_attr.attr,
308        &defrag_attr.attr,
309        &use_zero_page_attr.attr,
310#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
311        &shmem_enabled_attr.attr,
312#endif
313#ifdef CONFIG_DEBUG_VM
314        &debug_cow_attr.attr,
315#endif
316        NULL,
317};
318
319static struct attribute_group hugepage_attr_group = {
320        .attrs = hugepage_attr,
321};
322
323static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
324{
325        int err;
326
327        *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
328        if (unlikely(!*hugepage_kobj)) {
329                pr_err("failed to create transparent hugepage kobject\n");
330                return -ENOMEM;
331        }
332
333        err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
334        if (err) {
335                pr_err("failed to register transparent hugepage group\n");
336                goto delete_obj;
337        }
338
339        err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
340        if (err) {
341                pr_err("failed to register transparent hugepage group\n");
342                goto remove_hp_group;
343        }
344
345        return 0;
346
347remove_hp_group:
348        sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
349delete_obj:
350        kobject_put(*hugepage_kobj);
351        return err;
352}
353
354static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
355{
356        sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
357        sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
358        kobject_put(hugepage_kobj);
359}
360#else
361static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
362{
363        return 0;
364}
365
366static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
367{
368}
369#endif /* CONFIG_SYSFS */
370
371static int __init hugepage_init(void)
372{
373        int err;
374        struct kobject *hugepage_kobj;
375
376        if (!has_transparent_hugepage()) {
377                transparent_hugepage_flags = 0;
378                return -EINVAL;
379        }
380
381        /*
382         * hugepages can't be allocated by the buddy allocator
383         */
384        MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
385        /*
386         * we use page->mapping and page->index in second tail page
387         * as list_head: assuming THP order >= 2
388         */
389        MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
390
391        err = hugepage_init_sysfs(&hugepage_kobj);
392        if (err)
393                goto err_sysfs;
394
395        err = khugepaged_init();
396        if (err)
397                goto err_slab;
398
399        err = register_shrinker(&huge_zero_page_shrinker);
400        if (err)
401                goto err_hzp_shrinker;
402        err = register_shrinker(&deferred_split_shrinker);
403        if (err)
404                goto err_split_shrinker;
405
406        /*
407         * By default disable transparent hugepages on smaller systems,
408         * where the extra memory used could hurt more than TLB overhead
409         * is likely to save.  The admin can still enable it through /sys.
410         */
411        if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
412                transparent_hugepage_flags = 0;
413                return 0;
414        }
415
416        err = start_stop_khugepaged();
417        if (err)
418                goto err_khugepaged;
419
420        return 0;
421err_khugepaged:
422        unregister_shrinker(&deferred_split_shrinker);
423err_split_shrinker:
424        unregister_shrinker(&huge_zero_page_shrinker);
425err_hzp_shrinker:
426        khugepaged_destroy();
427err_slab:
428        hugepage_exit_sysfs(hugepage_kobj);
429err_sysfs:
430        return err;
431}
432subsys_initcall(hugepage_init);
433
434static int __init setup_transparent_hugepage(char *str)
435{
436        int ret = 0;
437        if (!str)
438                goto out;
439        if (!strcmp(str, "always")) {
440                set_bit(TRANSPARENT_HUGEPAGE_FLAG,
441                        &transparent_hugepage_flags);
442                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
443                          &transparent_hugepage_flags);
444                ret = 1;
445        } else if (!strcmp(str, "madvise")) {
446                clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
447                          &transparent_hugepage_flags);
448                set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
449                        &transparent_hugepage_flags);
450                ret = 1;
451        } else if (!strcmp(str, "never")) {
452                clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
453                          &transparent_hugepage_flags);
454                clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
455                          &transparent_hugepage_flags);
456                ret = 1;
457        }
458out:
459        if (!ret)
460                pr_warn("transparent_hugepage= cannot parse, ignored\n");
461        return ret;
462}
463__setup("transparent_hugepage=", setup_transparent_hugepage);
464
465pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
466{
467        if (likely(vma->vm_flags & VM_WRITE))
468                pmd = pmd_mkwrite(pmd);
469        return pmd;
470}
471
472static inline struct list_head *page_deferred_list(struct page *page)
473{
474        /*
475         * ->lru in the tail pages is occupied by compound_head.
476         * Let's use ->mapping + ->index in the second tail page as list_head.
477         */
478        return (struct list_head *)&page[2].mapping;
479}
480
481void prep_transhuge_page(struct page *page)
482{
483        /*
484         * we use page->mapping and page->indexlru in second tail page
485         * as list_head: assuming THP order >= 2
486         */
487
488        INIT_LIST_HEAD(page_deferred_list(page));
489        set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
490}
491
492unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long len,
493                loff_t off, unsigned long flags, unsigned long size)
494{
495        unsigned long addr;
496        loff_t off_end = off + len;
497        loff_t off_align = round_up(off, size);
498        unsigned long len_pad;
499
500        if (off_end <= off_align || (off_end - off_align) < size)
501                return 0;
502
503        len_pad = len + size;
504        if (len_pad < len || (off + len_pad) < off)
505                return 0;
506
507        addr = current->mm->get_unmapped_area(filp, 0, len_pad,
508                                              off >> PAGE_SHIFT, flags);
509        if (IS_ERR_VALUE(addr))
510                return 0;
511
512        addr += (off - addr) & (size - 1);
513        return addr;
514}
515
516unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr,
517                unsigned long len, unsigned long pgoff, unsigned long flags)
518{
519        loff_t off = (loff_t)pgoff << PAGE_SHIFT;
520
521        if (addr)
522                goto out;
523        if (!IS_DAX(filp->f_mapping->host) || !IS_ENABLED(CONFIG_FS_DAX_PMD))
524                goto out;
525
526        addr = __thp_get_unmapped_area(filp, len, off, flags, PMD_SIZE);
527        if (addr)
528                return addr;
529
530 out:
531        return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
532}
533EXPORT_SYMBOL_GPL(thp_get_unmapped_area);
534
535static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
536                gfp_t gfp)
537{
538        struct vm_area_struct *vma = fe->vma;
539        struct mem_cgroup *memcg;
540        pgtable_t pgtable;
541        unsigned long haddr = fe->address & HPAGE_PMD_MASK;
542
543        VM_BUG_ON_PAGE(!PageCompound(page), page);
544
545        if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
546                put_page(page);
547                count_vm_event(THP_FAULT_FALLBACK);
548                return VM_FAULT_FALLBACK;
549        }
550
551        pgtable = pte_alloc_one(vma->vm_mm, haddr);
552        if (unlikely(!pgtable)) {
553                mem_cgroup_cancel_charge(page, memcg, true);
554                put_page(page);
555                return VM_FAULT_OOM;
556        }
557
558        clear_huge_page(page, haddr, HPAGE_PMD_NR);
559        /*
560         * The memory barrier inside __SetPageUptodate makes sure that
561         * clear_huge_page writes become visible before the set_pmd_at()
562         * write.
563         */
564        __SetPageUptodate(page);
565
566        fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
567        if (unlikely(!pmd_none(*fe->pmd))) {
568                spin_unlock(fe->ptl);
569                mem_cgroup_cancel_charge(page, memcg, true);
570                put_page(page);
571                pte_free(vma->vm_mm, pgtable);
572        } else {
573                pmd_t entry;
574
575                /* Deliver the page fault to userland */
576                if (userfaultfd_missing(vma)) {
577                        int ret;
578
579                        spin_unlock(fe->ptl);
580                        mem_cgroup_cancel_charge(page, memcg, true);
581                        put_page(page);
582                        pte_free(vma->vm_mm, pgtable);
583                        ret = handle_userfault(fe, VM_UFFD_MISSING);
584                        VM_BUG_ON(ret & VM_FAULT_FALLBACK);
585                        return ret;
586                }
587
588                entry = mk_huge_pmd(page, vma->vm_page_prot);
589                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
590                page_add_new_anon_rmap(page, vma, haddr, true);
591                mem_cgroup_commit_charge(page, memcg, false, true);
592                lru_cache_add_active_or_unevictable(page, vma);
593                pgtable_trans_huge_deposit(vma->vm_mm, fe->pmd, pgtable);
594                set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
595                add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
596                atomic_long_inc(&vma->vm_mm->nr_ptes);
597                spin_unlock(fe->ptl);
598                count_vm_event(THP_FAULT_ALLOC);
599        }
600
601        return 0;
602}
603
604/*
605 * If THP defrag is set to always then directly reclaim/compact as necessary
606 * If set to defer then do only background reclaim/compact and defer to khugepaged
607 * If set to madvise and the VMA is flagged then directly reclaim/compact
608 * When direct reclaim/compact is allowed, don't retry except for flagged VMA's
609 */
610static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
611{
612        bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
613
614        if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
615                                &transparent_hugepage_flags) && vma_madvised)
616                return GFP_TRANSHUGE;
617        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
618                                                &transparent_hugepage_flags))
619                return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
620        else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
621                                                &transparent_hugepage_flags))
622                return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
623
624        return GFP_TRANSHUGE_LIGHT;
625}
626
627/* Caller must hold page table lock. */
628static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
629                struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
630                struct page *zero_page)
631{
632        pmd_t entry;
633        if (!pmd_none(*pmd))
634                return false;
635        entry = mk_pmd(zero_page, vma->vm_page_prot);
636        entry = pmd_mkhuge(entry);
637        if (pgtable)
638                pgtable_trans_huge_deposit(mm, pmd, pgtable);
639        set_pmd_at(mm, haddr, pmd, entry);
640        atomic_long_inc(&mm->nr_ptes);
641        return true;
642}
643
644int do_huge_pmd_anonymous_page(struct fault_env *fe)
645{
646        struct vm_area_struct *vma = fe->vma;
647        gfp_t gfp;
648        struct page *page;
649        unsigned long haddr = fe->address & HPAGE_PMD_MASK;
650
651        if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
652                return VM_FAULT_FALLBACK;
653        if (unlikely(anon_vma_prepare(vma)))
654                return VM_FAULT_OOM;
655        if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
656                return VM_FAULT_OOM;
657        if (!(fe->flags & FAULT_FLAG_WRITE) &&
658                        !mm_forbids_zeropage(vma->vm_mm) &&
659                        transparent_hugepage_use_zero_page()) {
660                pgtable_t pgtable;
661                struct page *zero_page;
662                bool set;
663                int ret;
664                pgtable = pte_alloc_one(vma->vm_mm, haddr);
665                if (unlikely(!pgtable))
666                        return VM_FAULT_OOM;
667                zero_page = mm_get_huge_zero_page(vma->vm_mm);
668                if (unlikely(!zero_page)) {
669                        pte_free(vma->vm_mm, pgtable);
670                        count_vm_event(THP_FAULT_FALLBACK);
671                        return VM_FAULT_FALLBACK;
672                }
673                fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
674                ret = 0;
675                set = false;
676                if (pmd_none(*fe->pmd)) {
677                        if (userfaultfd_missing(vma)) {
678                                spin_unlock(fe->ptl);
679                                ret = handle_userfault(fe, VM_UFFD_MISSING);
680                                VM_BUG_ON(ret & VM_FAULT_FALLBACK);
681                        } else {
682                                set_huge_zero_page(pgtable, vma->vm_mm, vma,
683                                                   haddr, fe->pmd, zero_page);
684                                spin_unlock(fe->ptl);
685                                set = true;
686                        }
687                } else
688                        spin_unlock(fe->ptl);
689                if (!set)
690                        pte_free(vma->vm_mm, pgtable);
691                return ret;
692        }
693        gfp = alloc_hugepage_direct_gfpmask(vma);
694        page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
695        if (unlikely(!page)) {
696                count_vm_event(THP_FAULT_FALLBACK);
697                return VM_FAULT_FALLBACK;
698        }
699        prep_transhuge_page(page);
700        return __do_huge_pmd_anonymous_page(fe, page, gfp);
701}
702
703static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
704                pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write)
705{
706        struct mm_struct *mm = vma->vm_mm;
707        pmd_t entry;
708        spinlock_t *ptl;
709
710        ptl = pmd_lock(mm, pmd);
711        entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
712        if (pfn_t_devmap(pfn))
713                entry = pmd_mkdevmap(entry);
714        if (write) {
715                entry = pmd_mkyoung(pmd_mkdirty(entry));
716                entry = maybe_pmd_mkwrite(entry, vma);
717        }
718        set_pmd_at(mm, addr, pmd, entry);
719        update_mmu_cache_pmd(vma, addr, pmd);
720        spin_unlock(ptl);
721}
722
723int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
724                        pmd_t *pmd, pfn_t pfn, bool write)
725{
726        pgprot_t pgprot = vma->vm_page_prot;
727        /*
728         * If we had pmd_special, we could avoid all these restrictions,
729         * but we need to be consistent with PTEs and architectures that
730         * can't support a 'special' bit.
731         */
732        BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
733        BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
734                                                (VM_PFNMAP|VM_MIXEDMAP));
735        BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
736        BUG_ON(!pfn_t_devmap(pfn));
737
738        if (addr < vma->vm_start || addr >= vma->vm_end)
739                return VM_FAULT_SIGBUS;
740        if (track_pfn_insert(vma, &pgprot, pfn))
741                return VM_FAULT_SIGBUS;
742        insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
743        return VM_FAULT_NOPAGE;
744}
745EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
746
747static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
748                pmd_t *pmd)
749{
750        pmd_t _pmd;
751
752        /*
753         * We should set the dirty bit only for FOLL_WRITE but for now
754         * the dirty bit in the pmd is meaningless.  And if the dirty
755         * bit will become meaningful and we'll only set it with
756         * FOLL_WRITE, an atomic set_bit will be required on the pmd to
757         * set the young bit, instead of the current set_pmd_at.
758         */
759        _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
760        if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
761                                pmd, _pmd,  1))
762                update_mmu_cache_pmd(vma, addr, pmd);
763}
764
765struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
766                pmd_t *pmd, int flags)
767{
768        unsigned long pfn = pmd_pfn(*pmd);
769        struct mm_struct *mm = vma->vm_mm;
770        struct dev_pagemap *pgmap;
771        struct page *page;
772
773        assert_spin_locked(pmd_lockptr(mm, pmd));
774
775        /*
776         * When we COW a devmap PMD entry, we split it into PTEs, so we should
777         * not be in this function with `flags & FOLL_COW` set.
778         */
779        WARN_ONCE(flags & FOLL_COW, "mm: In follow_devmap_pmd with FOLL_COW set");
780
781        if (flags & FOLL_WRITE && !pmd_write(*pmd))
782                return NULL;
783
784        if (pmd_present(*pmd) && pmd_devmap(*pmd))
785                /* pass */;
786        else
787                return NULL;
788
789        if (flags & FOLL_TOUCH)
790                touch_pmd(vma, addr, pmd);
791
792        /*
793         * device mapped pages can only be returned if the
794         * caller will manage the page reference count.
795         */
796        if (!(flags & FOLL_GET))
797                return ERR_PTR(-EEXIST);
798
799        pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
800        pgmap = get_dev_pagemap(pfn, NULL);
801        if (!pgmap)
802                return ERR_PTR(-EFAULT);
803        page = pfn_to_page(pfn);
804        get_page(page);
805        put_dev_pagemap(pgmap);
806
807        return page;
808}
809
810int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
811                  pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
812                  struct vm_area_struct *vma)
813{
814        spinlock_t *dst_ptl, *src_ptl;
815        struct page *src_page;
816        pmd_t pmd;
817        pgtable_t pgtable = NULL;
818        int ret = -ENOMEM;
819
820        /* Skip if can be re-fill on fault */
821        if (!vma_is_anonymous(vma))
822                return 0;
823
824        pgtable = pte_alloc_one(dst_mm, addr);
825        if (unlikely(!pgtable))
826                goto out;
827
828        dst_ptl = pmd_lock(dst_mm, dst_pmd);
829        src_ptl = pmd_lockptr(src_mm, src_pmd);
830        spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
831
832        ret = -EAGAIN;
833        pmd = *src_pmd;
834        if (unlikely(!pmd_trans_huge(pmd))) {
835                pte_free(dst_mm, pgtable);
836                goto out_unlock;
837        }
838        /*
839         * When page table lock is held, the huge zero pmd should not be
840         * under splitting since we don't split the page itself, only pmd to
841         * a page table.
842         */
843        if (is_huge_zero_pmd(pmd)) {
844                struct page *zero_page;
845                /*
846                 * get_huge_zero_page() will never allocate a new page here,
847                 * since we already have a zero page to copy. It just takes a
848                 * reference.
849                 */
850                zero_page = mm_get_huge_zero_page(dst_mm);
851                set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
852                                zero_page);
853                ret = 0;
854                goto out_unlock;
855        }
856
857        src_page = pmd_page(pmd);
858        VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
859        get_page(src_page);
860        page_dup_rmap(src_page, true);
861        add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
862        atomic_long_inc(&dst_mm->nr_ptes);
863        pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
864
865        pmdp_set_wrprotect(src_mm, addr, src_pmd);
866        pmd = pmd_mkold(pmd_wrprotect(pmd));
867        set_pmd_at(dst_mm, addr, dst_pmd, pmd);
868
869        ret = 0;
870out_unlock:
871        spin_unlock(src_ptl);
872        spin_unlock(dst_ptl);
873out:
874        return ret;
875}
876
877void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd)
878{
879        pmd_t entry;
880        unsigned long haddr;
881        bool write = fe->flags & FAULT_FLAG_WRITE;
882
883        fe->ptl = pmd_lock(fe->vma->vm_mm, fe->pmd);
884        if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
885                goto unlock;
886
887        entry = pmd_mkyoung(orig_pmd);
888        if (write)
889                entry = pmd_mkdirty(entry);
890        haddr = fe->address & HPAGE_PMD_MASK;
891        if (pmdp_set_access_flags(fe->vma, haddr, fe->pmd, entry, write))
892                update_mmu_cache_pmd(fe->vma, fe->address, fe->pmd);
893
894unlock:
895        spin_unlock(fe->ptl);
896}
897
898static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
899                struct page *page)
900{
901        struct vm_area_struct *vma = fe->vma;
902        unsigned long haddr = fe->address & HPAGE_PMD_MASK;
903        struct mem_cgroup *memcg;
904        pgtable_t pgtable;
905        pmd_t _pmd;
906        int ret = 0, i;
907        struct page **pages;
908        unsigned long mmun_start;       /* For mmu_notifiers */
909        unsigned long mmun_end;         /* For mmu_notifiers */
910
911        pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
912                        GFP_KERNEL);
913        if (unlikely(!pages)) {
914                ret |= VM_FAULT_OOM;
915                goto out;
916        }
917
918        for (i = 0; i < HPAGE_PMD_NR; i++) {
919                pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
920                                               __GFP_OTHER_NODE, vma,
921                                               fe->address, page_to_nid(page));
922                if (unlikely(!pages[i] ||
923                             mem_cgroup_try_charge(pages[i], vma->vm_mm,
924                                     GFP_KERNEL, &memcg, false))) {
925                        if (pages[i])
926                                put_page(pages[i]);
927                        while (--i >= 0) {
928                                memcg = (void *)page_private(pages[i]);
929                                set_page_private(pages[i], 0);
930                                mem_cgroup_cancel_charge(pages[i], memcg,
931                                                false);
932                                put_page(pages[i]);
933                        }
934                        kfree(pages);
935                        ret |= VM_FAULT_OOM;
936                        goto out;
937                }
938                set_page_private(pages[i], (unsigned long)memcg);
939        }
940
941        for (i = 0; i < HPAGE_PMD_NR; i++) {
942                copy_user_highpage(pages[i], page + i,
943                                   haddr + PAGE_SIZE * i, vma);
944                __SetPageUptodate(pages[i]);
945                cond_resched();
946        }
947
948        mmun_start = haddr;
949        mmun_end   = haddr + HPAGE_PMD_SIZE;
950        mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
951
952        fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
953        if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
954                goto out_free_pages;
955        VM_BUG_ON_PAGE(!PageHead(page), page);
956
957        pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
958        /* leave pmd empty until pte is filled */
959
960        pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, fe->pmd);
961        pmd_populate(vma->vm_mm, &_pmd, pgtable);
962
963        for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
964                pte_t entry;
965                entry = mk_pte(pages[i], vma->vm_page_prot);
966                entry = maybe_mkwrite(pte_mkdirty(entry), vma);
967                memcg = (void *)page_private(pages[i]);
968                set_page_private(pages[i], 0);
969                page_add_new_anon_rmap(pages[i], fe->vma, haddr, false);
970                mem_cgroup_commit_charge(pages[i], memcg, false, false);
971                lru_cache_add_active_or_unevictable(pages[i], vma);
972                fe->pte = pte_offset_map(&_pmd, haddr);
973                VM_BUG_ON(!pte_none(*fe->pte));
974                set_pte_at(vma->vm_mm, haddr, fe->pte, entry);
975                pte_unmap(fe->pte);
976        }
977        kfree(pages);
978
979        smp_wmb(); /* make pte visible before pmd */
980        pmd_populate(vma->vm_mm, fe->pmd, pgtable);
981        page_remove_rmap(page, true);
982        spin_unlock(fe->ptl);
983
984        mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
985
986        ret |= VM_FAULT_WRITE;
987        put_page(page);
988
989out:
990        return ret;
991
992out_free_pages:
993        spin_unlock(fe->ptl);
994        mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
995        for (i = 0; i < HPAGE_PMD_NR; i++) {
996                memcg = (void *)page_private(pages[i]);
997                set_page_private(pages[i], 0);
998                mem_cgroup_cancel_charge(pages[i], memcg, false);
999                put_page(pages[i]);
1000        }
1001        kfree(pages);
1002        goto out;
1003}
1004
1005int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd)
1006{
1007        struct vm_area_struct *vma = fe->vma;
1008        struct page *page = NULL, *new_page;
1009        struct mem_cgroup *memcg;
1010        unsigned long haddr = fe->address & HPAGE_PMD_MASK;
1011        unsigned long mmun_start;       /* For mmu_notifiers */
1012        unsigned long mmun_end;         /* For mmu_notifiers */
1013        gfp_t huge_gfp;                 /* for allocation and charge */
1014        int ret = 0;
1015
1016        fe->ptl = pmd_lockptr(vma->vm_mm, fe->pmd);
1017        VM_BUG_ON_VMA(!vma->anon_vma, vma);
1018        if (is_huge_zero_pmd(orig_pmd))
1019                goto alloc;
1020        spin_lock(fe->ptl);
1021        if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
1022                goto out_unlock;
1023
1024        page = pmd_page(orig_pmd);
1025        VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
1026        /*
1027         * We can only reuse the page if nobody else maps the huge page or it's
1028         * part.
1029         */
1030        if (page_trans_huge_mapcount(page, NULL) == 1) {
1031                pmd_t entry;
1032                entry = pmd_mkyoung(orig_pmd);
1033                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1034                if (pmdp_set_access_flags(vma, haddr, fe->pmd, entry,  1))
1035                        update_mmu_cache_pmd(vma, fe->address, fe->pmd);
1036                ret |= VM_FAULT_WRITE;
1037                goto out_unlock;
1038        }
1039        get_page(page);
1040        spin_unlock(fe->ptl);
1041alloc:
1042        if (transparent_hugepage_enabled(vma) &&
1043            !transparent_hugepage_debug_cow()) {
1044                huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1045                new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1046        } else
1047                new_page = NULL;
1048
1049        if (likely(new_page)) {
1050                prep_transhuge_page(new_page);
1051        } else {
1052                if (!page) {
1053                        split_huge_pmd(vma, fe->pmd, fe->address);
1054                        ret |= VM_FAULT_FALLBACK;
1055                } else {
1056                        ret = do_huge_pmd_wp_page_fallback(fe, orig_pmd, page);
1057                        if (ret & VM_FAULT_OOM) {
1058                                split_huge_pmd(vma, fe->pmd, fe->address);
1059                                ret |= VM_FAULT_FALLBACK;
1060                        }
1061                        put_page(page);
1062                }
1063                count_vm_event(THP_FAULT_FALLBACK);
1064                goto out;
1065        }
1066
1067        if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
1068                                        huge_gfp, &memcg, true))) {
1069                put_page(new_page);
1070                split_huge_pmd(vma, fe->pmd, fe->address);
1071                if (page)
1072                        put_page(page);
1073                ret |= VM_FAULT_FALLBACK;
1074                count_vm_event(THP_FAULT_FALLBACK);
1075                goto out;
1076        }
1077
1078        count_vm_event(THP_FAULT_ALLOC);
1079
1080        if (!page)
1081                clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
1082        else
1083                copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
1084        __SetPageUptodate(new_page);
1085
1086        mmun_start = haddr;
1087        mmun_end   = haddr + HPAGE_PMD_SIZE;
1088        mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
1089
1090        spin_lock(fe->ptl);
1091        if (page)
1092                put_page(page);
1093        if (unlikely(!pmd_same(*fe->pmd, orig_pmd))) {
1094                spin_unlock(fe->ptl);
1095                mem_cgroup_cancel_charge(new_page, memcg, true);
1096                put_page(new_page);
1097                goto out_mn;
1098        } else {
1099                pmd_t entry;
1100                entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1101                entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1102                pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
1103                page_add_new_anon_rmap(new_page, vma, haddr, true);
1104                mem_cgroup_commit_charge(new_page, memcg, false, true);
1105                lru_cache_add_active_or_unevictable(new_page, vma);
1106                set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
1107                update_mmu_cache_pmd(vma, fe->address, fe->pmd);
1108                if (!page) {
1109                        add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1110                } else {
1111                        VM_BUG_ON_PAGE(!PageHead(page), page);
1112                        page_remove_rmap(page, true);
1113                        put_page(page);
1114                }
1115                ret |= VM_FAULT_WRITE;
1116        }
1117        spin_unlock(fe->ptl);
1118out_mn:
1119        mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
1120out:
1121        return ret;
1122out_unlock:
1123        spin_unlock(fe->ptl);
1124        return ret;
1125}
1126
1127/*
1128 * FOLL_FORCE can write to even unwritable pmd's, but only
1129 * after we've gone through a COW cycle and they are dirty.
1130 */
1131static inline bool can_follow_write_pmd(pmd_t pmd, unsigned int flags)
1132{
1133        return pmd_write(pmd) ||
1134               ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pmd_dirty(pmd));
1135}
1136
1137struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
1138                                   unsigned long addr,
1139                                   pmd_t *pmd,
1140                                   unsigned int flags)
1141{
1142        struct mm_struct *mm = vma->vm_mm;
1143        struct page *page = NULL;
1144
1145        assert_spin_locked(pmd_lockptr(mm, pmd));
1146
1147        if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
1148                goto out;
1149
1150        /* Avoid dumping huge zero page */
1151        if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1152                return ERR_PTR(-EFAULT);
1153
1154        /* Full NUMA hinting faults to serialise migration in fault paths */
1155        if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
1156                goto out;
1157
1158        page = pmd_page(*pmd);
1159        VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);
1160        if (flags & FOLL_TOUCH)
1161                touch_pmd(vma, addr, pmd);
1162        if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
1163                /*
1164                 * We don't mlock() pte-mapped THPs. This way we can avoid
1165                 * leaking mlocked pages into non-VM_LOCKED VMAs.
1166                 *
1167                 * For anon THP:
1168                 *
1169                 * In most cases the pmd is the only mapping of the page as we
1170                 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1171                 * writable private mappings in populate_vma_page_range().
1172                 *
1173                 * The only scenario when we have the page shared here is if we
1174                 * mlocking read-only mapping shared over fork(). We skip
1175                 * mlocking such pages.
1176                 *
1177                 * For file THP:
1178                 *
1179                 * We can expect PageDoubleMap() to be stable under page lock:
1180                 * for file pages we set it in page_add_file_rmap(), which
1181                 * requires page to be locked.
1182                 */
1183
1184                if (PageAnon(page) && compound_mapcount(page) != 1)
1185                        goto skip_mlock;
1186                if (PageDoubleMap(page) || !page->mapping)
1187                        goto skip_mlock;
1188                if (!trylock_page(page))
1189                        goto skip_mlock;
1190                lru_add_drain();
1191                if (page->mapping && !PageDoubleMap(page))
1192                        mlock_vma_page(page);
1193                unlock_page(page);
1194        }
1195skip_mlock:
1196        page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
1197        VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);
1198        if (flags & FOLL_GET)
1199                get_page(page);
1200
1201out:
1202        return page;
1203}
1204
1205/* NUMA hinting page fault entry point for trans huge pmds */
1206int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
1207{
1208        struct vm_area_struct *vma = fe->vma;
1209        struct anon_vma *anon_vma = NULL;
1210        struct page *page;
1211        unsigned long haddr = fe->address & HPAGE_PMD_MASK;
1212        int page_nid = -1, this_nid = numa_node_id();
1213        int target_nid, last_cpupid = -1;
1214        bool page_locked;
1215        bool migrated = false;
1216        bool was_writable;
1217        int flags = 0;
1218
1219        fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
1220        if (unlikely(!pmd_same(pmd, *fe->pmd)))
1221                goto out_unlock;
1222
1223        /*
1224         * If there are potential migrations, wait for completion and retry
1225         * without disrupting NUMA hinting information. Do not relock and
1226         * check_same as the page may no longer be mapped.
1227         */
1228        if (unlikely(pmd_trans_migrating(*fe->pmd))) {
1229                page = pmd_page(*fe->pmd);
1230                spin_unlock(fe->ptl);
1231                wait_on_page_locked(page);
1232                goto out;
1233        }
1234
1235        page = pmd_page(pmd);
1236        BUG_ON(is_huge_zero_page(page));
1237        page_nid = page_to_nid(page);
1238        last_cpupid = page_cpupid_last(page);
1239        count_vm_numa_event(NUMA_HINT_FAULTS);
1240        if (page_nid == this_nid) {
1241                count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
1242                flags |= TNF_FAULT_LOCAL;
1243        }
1244
1245        /* See similar comment in do_numa_page for explanation */
1246        if (!pmd_write(pmd))
1247                flags |= TNF_NO_GROUP;
1248
1249        /*
1250         * Acquire the page lock to serialise THP migrations but avoid dropping
1251         * page_table_lock if at all possible
1252         */
1253        page_locked = trylock_page(page);
1254        target_nid = mpol_misplaced(page, vma, haddr);
1255        if (target_nid == -1) {
1256                /* If the page was locked, there are no parallel migrations */
1257                if (page_locked)
1258                        goto clear_pmdnuma;
1259        }
1260
1261        /* Migration could have started since the pmd_trans_migrating check */
1262        if (!page_locked) {
1263                spin_unlock(fe->ptl);
1264                wait_on_page_locked(page);
1265                page_nid = -1;
1266                goto out;
1267        }
1268
1269        /*
1270         * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1271         * to serialises splits
1272         */
1273        get_page(page);
1274        spin_unlock(fe->ptl);
1275        anon_vma = page_lock_anon_vma_read(page);
1276
1277        /* Confirm the PMD did not change while page_table_lock was released */
1278        spin_lock(fe->ptl);
1279        if (unlikely(!pmd_same(pmd, *fe->pmd))) {
1280                unlock_page(page);
1281                put_page(page);
1282                page_nid = -1;
1283                goto out_unlock;
1284        }
1285
1286        /* Bail if we fail to protect against THP splits for any reason */
1287        if (unlikely(!anon_vma)) {
1288                put_page(page);
1289                page_nid = -1;
1290                goto clear_pmdnuma;
1291        }
1292
1293        /*
1294         * Migrate the THP to the requested node, returns with page unlocked
1295         * and access rights restored.
1296         */
1297        spin_unlock(fe->ptl);
1298        migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
1299                                fe->pmd, pmd, fe->address, page, target_nid);
1300        if (migrated) {
1301                flags |= TNF_MIGRATED;
1302                page_nid = target_nid;
1303        } else
1304                flags |= TNF_MIGRATE_FAIL;
1305
1306        goto out;
1307clear_pmdnuma:
1308        BUG_ON(!PageLocked(page));
1309        was_writable = pmd_write(pmd);
1310        pmd = pmd_modify(pmd, vma->vm_page_prot);
1311        pmd = pmd_mkyoung(pmd);
1312        if (was_writable)
1313                pmd = pmd_mkwrite(pmd);
1314        set_pmd_at(vma->vm_mm, haddr, fe->pmd, pmd);
1315        update_mmu_cache_pmd(vma, fe->address, fe->pmd);
1316        unlock_page(page);
1317out_unlock:
1318        spin_unlock(fe->ptl);
1319
1320out:
1321        if (anon_vma)
1322                page_unlock_anon_vma_read(anon_vma);
1323
1324        if (page_nid != -1)
1325                task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, fe->flags);
1326
1327        return 0;
1328}
1329
1330/*
1331 * Return true if we do MADV_FREE successfully on entire pmd page.
1332 * Otherwise, return false.
1333 */
1334bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1335                pmd_t *pmd, unsigned long addr, unsigned long next)
1336{
1337        spinlock_t *ptl;
1338        pmd_t orig_pmd;
1339        struct page *page;
1340        struct mm_struct *mm = tlb->mm;
1341        bool ret = false;
1342
1343        ptl = pmd_trans_huge_lock(pmd, vma);
1344        if (!ptl)
1345                goto out_unlocked;
1346
1347        orig_pmd = *pmd;
1348        if (is_huge_zero_pmd(orig_pmd))
1349                goto out;
1350
1351        page = pmd_page(orig_pmd);
1352        /*
1353         * If other processes are mapping this page, we couldn't discard
1354         * the page unless they all do MADV_FREE so let's skip the page.
1355         */
1356        if (page_mapcount(page) != 1)
1357                goto out;
1358
1359        if (!trylock_page(page))
1360                goto out;
1361
1362        /*
1363         * If user want to discard part-pages of THP, split it so MADV_FREE
1364         * will deactivate only them.
1365         */
1366        if (next - addr != HPAGE_PMD_SIZE) {
1367                get_page(page);
1368                spin_unlock(ptl);
1369                split_huge_page(page);
1370                put_page(page);
1371                unlock_page(page);
1372                goto out_unlocked;
1373        }
1374
1375        if (PageDirty(page))
1376                ClearPageDirty(page);
1377        unlock_page(page);
1378
1379        if (PageActive(page))
1380                deactivate_page(page);
1381
1382        if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1383                pmdp_invalidate(vma, addr, pmd);
1384                orig_pmd = pmd_mkold(orig_pmd);
1385                orig_pmd = pmd_mkclean(orig_pmd);
1386
1387                set_pmd_at(mm, addr, pmd, orig_pmd);
1388                tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1389        }
1390        ret = true;
1391out:
1392        spin_unlock(ptl);
1393out_unlocked:
1394        return ret;
1395}
1396
1397int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1398                 pmd_t *pmd, unsigned long addr)
1399{
1400        pmd_t orig_pmd;
1401        spinlock_t *ptl;
1402
1403        ptl = __pmd_trans_huge_lock(pmd, vma);
1404        if (!ptl)
1405                return 0;
1406        /*
1407         * For architectures like ppc64 we look at deposited pgtable
1408         * when calling pmdp_huge_get_and_clear. So do the
1409         * pgtable_trans_huge_withdraw after finishing pmdp related
1410         * operations.
1411         */
1412        orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1413                        tlb->fullmm);
1414        tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1415        if (vma_is_dax(vma)) {
1416                spin_unlock(ptl);
1417                if (is_huge_zero_pmd(orig_pmd))
1418                        tlb_remove_page(tlb, pmd_page(orig_pmd));
1419        } else if (is_huge_zero_pmd(orig_pmd)) {
1420                pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1421                atomic_long_dec(&tlb->mm->nr_ptes);
1422                spin_unlock(ptl);
1423                tlb_remove_page(tlb, pmd_page(orig_pmd));
1424        } else {
1425                struct page *page = pmd_page(orig_pmd);
1426                page_remove_rmap(page, true);
1427                VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1428                VM_BUG_ON_PAGE(!PageHead(page), page);
1429                if (PageAnon(page)) {
1430                        pgtable_t pgtable;
1431                        pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
1432                        pte_free(tlb->mm, pgtable);
1433                        atomic_long_dec(&tlb->mm->nr_ptes);
1434                        add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1435                } else {
1436                        add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);
1437                }
1438                spin_unlock(ptl);
1439                tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
1440        }
1441        return 1;
1442}
1443
1444bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
1445                  unsigned long new_addr, unsigned long old_end,
1446                  pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush)
1447{
1448        spinlock_t *old_ptl, *new_ptl;
1449        pmd_t pmd;
1450        struct mm_struct *mm = vma->vm_mm;
1451        bool force_flush = false;
1452
1453        if ((old_addr & ~HPAGE_PMD_MASK) ||
1454            (new_addr & ~HPAGE_PMD_MASK) ||
1455            old_end - old_addr < HPAGE_PMD_SIZE)
1456                return false;
1457
1458        /*
1459         * The destination pmd shouldn't be established, free_pgtables()
1460         * should have release it.
1461         */
1462        if (WARN_ON(!pmd_none(*new_pmd))) {
1463                VM_BUG_ON(pmd_trans_huge(*new_pmd));
1464                return false;
1465        }
1466
1467        /*
1468         * We don't have to worry about the ordering of src and dst
1469         * ptlocks because exclusive mmap_sem prevents deadlock.
1470         */
1471        old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1472        if (old_ptl) {
1473                new_ptl = pmd_lockptr(mm, new_pmd);
1474                if (new_ptl != old_ptl)
1475                        spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
1476                pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
1477                if (pmd_present(pmd) && pmd_dirty(pmd))
1478                        force_flush = true;
1479                VM_BUG_ON(!pmd_none(*new_pmd));
1480
1481                if (pmd_move_must_withdraw(new_ptl, old_ptl) &&
1482                                vma_is_anonymous(vma)) {
1483                        pgtable_t pgtable;
1484                        pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1485                        pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
1486                }
1487                set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
1488                if (new_ptl != old_ptl)
1489                        spin_unlock(new_ptl);
1490                if (force_flush)
1491                        flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
1492                else
1493                        *need_flush = true;
1494                spin_unlock(old_ptl);
1495                return true;
1496        }
1497        return false;
1498}
1499
1500/*
1501 * Returns
1502 *  - 0 if PMD could not be locked
1503 *  - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1504 *  - HPAGE_PMD_NR is protections changed and TLB flush necessary
1505 */
1506int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1507                unsigned long addr, pgprot_t newprot, int prot_numa)
1508{
1509        struct mm_struct *mm = vma->vm_mm;
1510        spinlock_t *ptl;
1511        int ret = 0;
1512
1513        ptl = __pmd_trans_huge_lock(pmd, vma);
1514        if (ptl) {
1515                pmd_t entry;
1516                bool preserve_write = prot_numa && pmd_write(*pmd);
1517                ret = 1;
1518
1519                /*
1520                 * Avoid trapping faults against the zero page. The read-only
1521                 * data is likely to be read-cached on the local CPU and
1522                 * local/remote hits to the zero page are not interesting.
1523                 */
1524                if (prot_numa && is_huge_zero_pmd(*pmd)) {
1525                        spin_unlock(ptl);
1526                        return ret;
1527                }
1528
1529                if (!prot_numa || !pmd_protnone(*pmd)) {
1530                        entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);
1531                        entry = pmd_modify(entry, newprot);
1532                        if (preserve_write)
1533                                entry = pmd_mkwrite(entry);
1534                        ret = HPAGE_PMD_NR;
1535                        set_pmd_at(mm, addr, pmd, entry);
1536                        BUG_ON(vma_is_anonymous(vma) && !preserve_write &&
1537                                        pmd_write(entry));
1538                }
1539                spin_unlock(ptl);
1540        }
1541
1542        return ret;
1543}
1544
1545/*
1546 * Returns page table lock pointer if a given pmd maps a thp, NULL otherwise.
1547 *
1548 * Note that if it returns page table lock pointer, this routine returns without
1549 * unlocking page table lock. So callers must unlock it.
1550 */
1551spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
1552{
1553        spinlock_t *ptl;
1554        ptl = pmd_lock(vma->vm_mm, pmd);
1555        if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd)))
1556                return ptl;
1557        spin_unlock(ptl);
1558        return NULL;
1559}
1560
1561static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
1562                unsigned long haddr, pmd_t *pmd)
1563{
1564        struct mm_struct *mm = vma->vm_mm;
1565        pgtable_t pgtable;
1566        pmd_t _pmd;
1567        int i;
1568
1569        /* leave pmd empty until pte is filled */
1570        pmdp_huge_clear_flush_notify(vma, haddr, pmd);
1571
1572        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1573        pmd_populate(mm, &_pmd, pgtable);
1574
1575        for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1576                pte_t *pte, entry;
1577                entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
1578                entry = pte_mkspecial(entry);
1579                pte = pte_offset_map(&_pmd, haddr);
1580                VM_BUG_ON(!pte_none(*pte));
1581                set_pte_at(mm, haddr, pte, entry);
1582                pte_unmap(pte);
1583        }
1584        smp_wmb(); /* make pte visible before pmd */
1585        pmd_populate(mm, pmd, pgtable);
1586}
1587
1588static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
1589                unsigned long haddr, bool freeze)
1590{
1591        struct mm_struct *mm = vma->vm_mm;
1592        struct page *page;
1593        pgtable_t pgtable;
1594        pmd_t _pmd;
1595        bool young, write, dirty, soft_dirty;
1596        unsigned long addr;
1597        int i;
1598
1599        VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
1600        VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
1601        VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
1602        VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd));
1603
1604        count_vm_event(THP_SPLIT_PMD);
1605
1606        if (!vma_is_anonymous(vma)) {
1607                _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
1608                if (vma_is_dax(vma))
1609                        return;
1610                page = pmd_page(_pmd);
1611                if (!PageReferenced(page) && pmd_young(_pmd))
1612                        SetPageReferenced(page);
1613                page_remove_rmap(page, true);
1614                put_page(page);
1615                add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
1616                return;
1617        } else if (is_huge_zero_pmd(*pmd)) {
1618                return __split_huge_zero_page_pmd(vma, haddr, pmd);
1619        }
1620
1621        page = pmd_page(*pmd);
1622        VM_BUG_ON_PAGE(!page_count(page), page);
1623        page_ref_add(page, HPAGE_PMD_NR - 1);
1624        write = pmd_write(*pmd);
1625        young = pmd_young(*pmd);
1626        dirty = pmd_dirty(*pmd);
1627        soft_dirty = pmd_soft_dirty(*pmd);
1628
1629        pmdp_huge_split_prepare(vma, haddr, pmd);
1630        pgtable = pgtable_trans_huge_withdraw(mm, pmd);
1631        pmd_populate(mm, &_pmd, pgtable);
1632
1633        for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
1634                pte_t entry, *pte;
1635                /*
1636                 * Note that NUMA hinting access restrictions are not
1637                 * transferred to avoid any possibility of altering
1638                 * permissions across VMAs.
1639                 */
1640                if (freeze) {
1641                        swp_entry_t swp_entry;
1642                        swp_entry = make_migration_entry(page + i, write);
1643                        entry = swp_entry_to_pte(swp_entry);
1644                        if (soft_dirty)
1645                                entry = pte_swp_mksoft_dirty(entry);
1646                } else {
1647                        entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
1648                        entry = maybe_mkwrite(entry, vma);
1649                        if (!write)
1650                                entry = pte_wrprotect(entry);
1651                        if (!young)
1652                                entry = pte_mkold(entry);
1653                        if (soft_dirty)
1654                                entry = pte_mksoft_dirty(entry);
1655                }
1656                if (dirty)
1657                        SetPageDirty(page + i);
1658                pte = pte_offset_map(&_pmd, addr);
1659                BUG_ON(!pte_none(*pte));
1660                set_pte_at(mm, addr, pte, entry);
1661                atomic_inc(&page[i]._mapcount);
1662                pte_unmap(pte);
1663        }
1664
1665        /*
1666         * Set PG_double_map before dropping compound_mapcount to avoid
1667         * false-negative page_mapped().
1668         */
1669        if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
1670                for (i = 0; i < HPAGE_PMD_NR; i++)
1671                        atomic_inc(&page[i]._mapcount);
1672        }
1673
1674        if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
1675                /* Last compound_mapcount is gone. */
1676                __dec_node_page_state(page, NR_ANON_THPS);
1677                if (TestClearPageDoubleMap(page)) {
1678                        /* No need in mapcount reference anymore */
1679                        for (i = 0; i < HPAGE_PMD_NR; i++)
1680                                atomic_dec(&page[i]._mapcount);
1681                }
1682        }
1683
1684        smp_wmb(); /* make pte visible before pmd */
1685        /*
1686         * Up to this point the pmd is present and huge and userland has the
1687         * whole access to the hugepage during the split (which happens in
1688         * place). If we overwrite the pmd with the not-huge version pointing
1689         * to the pte here (which of course we could if all CPUs were bug
1690         * free), userland could trigger a small page size TLB miss on the
1691         * small sized TLB while the hugepage TLB entry is still established in
1692         * the huge TLB. Some CPU doesn't like that.
1693         * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
1694         * 383 on page 93. Intel should be safe but is also warns that it's
1695         * only safe if the permission and cache attributes of the two entries
1696         * loaded in the two TLB is identical (which should be the case here).
1697         * But it is generally safer to never allow small and huge TLB entries
1698         * for the same virtual address to be loaded simultaneously. So instead
1699         * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
1700         * current pmd notpresent (atomically because here the pmd_trans_huge
1701         * and pmd_trans_splitting must remain set at all times on the pmd
1702         * until the split is complete for this pmd), then we flush the SMP TLB
1703         * and finally we write the non-huge version of the pmd entry with
1704         * pmd_populate.
1705         */
1706        pmdp_invalidate(vma, haddr, pmd);
1707        pmd_populate(mm, pmd, pgtable);
1708
1709        if (freeze) {
1710                for (i = 0; i < HPAGE_PMD_NR; i++) {
1711                        page_remove_rmap(page + i, false);
1712                        put_page(page + i);
1713                }
1714        }
1715}
1716
1717void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
1718                unsigned long address, bool freeze, struct page *page)
1719{
1720        spinlock_t *ptl;
1721        struct mm_struct *mm = vma->vm_mm;
1722        unsigned long haddr = address & HPAGE_PMD_MASK;
1723
1724        mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
1725        ptl = pmd_lock(mm, pmd);
1726
1727        /*
1728         * If caller asks to setup a migration entries, we need a page to check
1729         * pmd against. Otherwise we can end up replacing wrong page.
1730         */
1731        VM_BUG_ON(freeze && !page);
1732        if (page && page != pmd_page(*pmd))
1733                goto out;
1734
1735        if (pmd_trans_huge(*pmd)) {
1736                page = pmd_page(*pmd);
1737                if (PageMlocked(page))
1738                        clear_page_mlock(page);
1739        } else if (!pmd_devmap(*pmd))
1740                goto out;
1741        __split_huge_pmd_locked(vma, pmd, haddr, freeze);
1742out:
1743        spin_unlock(ptl);
1744        mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
1745}
1746
1747void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
1748                bool freeze, struct page *page)
1749{
1750        pgd_t *pgd;
1751        pud_t *pud;
1752        pmd_t *pmd;
1753
1754        pgd = pgd_offset(vma->vm_mm, address);
1755        if (!pgd_present(*pgd))
1756                return;
1757
1758        pud = pud_offset(pgd, address);
1759        if (!pud_present(*pud))
1760                return;
1761
1762        pmd = pmd_offset(pud, address);
1763
1764        __split_huge_pmd(vma, pmd, address, freeze, page);
1765}
1766
1767void vma_adjust_trans_huge(struct vm_area_struct *vma,
1768                             unsigned long start,
1769                             unsigned long end,
1770                             long adjust_next)
1771{
1772        /*
1773         * If the new start address isn't hpage aligned and it could
1774         * previously contain an hugepage: check if we need to split
1775         * an huge pmd.
1776         */
1777        if (start & ~HPAGE_PMD_MASK &&
1778            (start & HPAGE_PMD_MASK) >= vma->vm_start &&
1779            (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
1780                split_huge_pmd_address(vma, start, false, NULL);
1781
1782        /*
1783         * If the new end address isn't hpage aligned and it could
1784         * previously contain an hugepage: check if we need to split
1785         * an huge pmd.
1786         */
1787        if (end & ~HPAGE_PMD_MASK &&
1788            (end & HPAGE_PMD_MASK) >= vma->vm_start &&
1789            (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
1790                split_huge_pmd_address(vma, end, false, NULL);
1791
1792        /*
1793         * If we're also updating the vma->vm_next->vm_start, if the new
1794         * vm_next->vm_start isn't page aligned and it could previously
1795         * contain an hugepage: check if we need to split an huge pmd.
1796         */
1797        if (adjust_next > 0) {
1798                struct vm_area_struct *next = vma->vm_next;
1799                unsigned long nstart = next->vm_start;
1800                nstart += adjust_next << PAGE_SHIFT;
1801                if (nstart & ~HPAGE_PMD_MASK &&
1802                    (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
1803                    (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
1804                        split_huge_pmd_address(next, nstart, false, NULL);
1805        }
1806}
1807
1808static void freeze_page(struct page *page)
1809{
1810        enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
1811                TTU_RMAP_LOCKED;
1812        int i, ret;
1813
1814        VM_BUG_ON_PAGE(!PageHead(page), page);
1815
1816        if (PageAnon(page))
1817                ttu_flags |= TTU_MIGRATION;
1818
1819        /* We only need TTU_SPLIT_HUGE_PMD once */
1820        ret = try_to_unmap(page, ttu_flags | TTU_SPLIT_HUGE_PMD);
1821        for (i = 1; !ret && i < HPAGE_PMD_NR; i++) {
1822                /* Cut short if the page is unmapped */
1823                if (page_count(page) == 1)
1824                        return;
1825
1826                ret = try_to_unmap(page + i, ttu_flags);
1827        }
1828        VM_BUG_ON_PAGE(ret, page + i - 1);
1829}
1830
1831static void unfreeze_page(struct page *page)
1832{
1833        int i;
1834
1835        for (i = 0; i < HPAGE_PMD_NR; i++)
1836                remove_migration_ptes(page + i, page + i, true);
1837}
1838
1839static void __split_huge_page_tail(struct page *head, int tail,
1840                struct lruvec *lruvec, struct list_head *list)
1841{
1842        struct page *page_tail = head + tail;
1843
1844        VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
1845        VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
1846
1847        /*
1848         * tail_page->_refcount is zero and not changing from under us. But
1849         * get_page_unless_zero() may be running from under us on the
1850         * tail_page. If we used atomic_set() below instead of atomic_inc() or
1851         * atomic_add(), we would then run atomic_set() concurrently with
1852         * get_page_unless_zero(), and atomic_set() is implemented in C not
1853         * using locked ops. spin_unlock on x86 sometime uses locked ops
1854         * because of PPro errata 66, 92, so unless somebody can guarantee
1855         * atomic_set() here would be safe on all archs (and not only on x86),
1856         * it's safer to use atomic_inc()/atomic_add().
1857         */
1858        if (PageAnon(head)) {
1859                page_ref_inc(page_tail);
1860        } else {
1861                /* Additional pin to radix tree */
1862                page_ref_add(page_tail, 2);
1863        }
1864
1865        page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
1866        page_tail->flags |= (head->flags &
1867                        ((1L << PG_referenced) |
1868                         (1L << PG_swapbacked) |
1869                         (1L << PG_mlocked) |
1870                         (1L << PG_uptodate) |
1871                         (1L << PG_active) |
1872                         (1L << PG_locked) |
1873                         (1L << PG_unevictable) |
1874                         (1L << PG_dirty)));
1875
1876        /*
1877         * After clearing PageTail the gup refcount can be released.
1878         * Page flags also must be visible before we make the page non-compound.
1879         */
1880        smp_wmb();
1881
1882        clear_compound_head(page_tail);
1883
1884        if (page_is_young(head))
1885                set_page_young(page_tail);
1886        if (page_is_idle(head))
1887                set_page_idle(page_tail);
1888
1889        /* ->mapping in first tail page is compound_mapcount */
1890        VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
1891                        page_tail);
1892        page_tail->mapping = head->mapping;
1893
1894        page_tail->index = head->index + tail;
1895        page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
1896        lru_add_page_tail(head, page_tail, lruvec, list);
1897}
1898
1899static void __split_huge_page(struct page *page, struct list_head *list,
1900                unsigned long flags)
1901{
1902        struct page *head = compound_head(page);
1903        struct zone *zone = page_zone(head);
1904        struct lruvec *lruvec;
1905        pgoff_t end = -1;
1906        int i;
1907
1908        lruvec = mem_cgroup_page_lruvec(head, zone->zone_pgdat);
1909
1910        /* complete memcg works before add pages to LRU */
1911        mem_cgroup_split_huge_fixup(head);
1912
1913        if (!PageAnon(page))
1914                end = DIV_ROUND_UP(i_size_read(head->mapping->host), PAGE_SIZE);
1915
1916        for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
1917                __split_huge_page_tail(head, i, lruvec, list);
1918                /* Some pages can be beyond i_size: drop them from page cache */
1919                if (head[i].index >= end) {
1920                        __ClearPageDirty(head + i);
1921                        __delete_from_page_cache(head + i, NULL);
1922                        if (IS_ENABLED(CONFIG_SHMEM) && PageSwapBacked(head))
1923                                shmem_uncharge(head->mapping->host, 1);
1924                        put_page(head + i);
1925                }
1926        }
1927
1928        ClearPageCompound(head);
1929        /* See comment in __split_huge_page_tail() */
1930        if (PageAnon(head)) {
1931                page_ref_inc(head);
1932        } else {
1933                /* Additional pin to radix tree */
1934                page_ref_add(head, 2);
1935                spin_unlock(&head->mapping->tree_lock);
1936        }
1937
1938        spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
1939
1940        unfreeze_page(head);
1941
1942        for (i = 0; i < HPAGE_PMD_NR; i++) {
1943                struct page *subpage = head + i;
1944                if (subpage == page)
1945                        continue;
1946                unlock_page(subpage);
1947
1948                /*
1949                 * Subpages may be freed if there wasn't any mapping
1950                 * like if add_to_swap() is running on a lru page that
1951                 * had its mapping zapped. And freeing these pages
1952                 * requires taking the lru_lock so we do the put_page
1953                 * of the tail pages after the split is complete.
1954                 */
1955                put_page(subpage);
1956        }
1957}
1958
1959int total_mapcount(struct page *page)
1960{
1961        int i, compound, ret;
1962
1963        VM_BUG_ON_PAGE(PageTail(page), page);
1964
1965        if (likely(!PageCompound(page)))
1966                return atomic_read(&page->_mapcount) + 1;
1967
1968        compound = compound_mapcount(page);
1969        if (PageHuge(page))
1970                return compound;
1971        ret = compound;
1972        for (i = 0; i < HPAGE_PMD_NR; i++)
1973                ret += atomic_read(&page[i]._mapcount) + 1;
1974        /* File pages has compound_mapcount included in _mapcount */
1975        if (!PageAnon(page))
1976                return ret - compound * HPAGE_PMD_NR;
1977        if (PageDoubleMap(page))
1978                ret -= HPAGE_PMD_NR;
1979        return ret;
1980}
1981
1982/*
1983 * This calculates accurately how many mappings a transparent hugepage
1984 * has (unlike page_mapcount() which isn't fully accurate). This full
1985 * accuracy is primarily needed to know if copy-on-write faults can
1986 * reuse the page and change the mapping to read-write instead of
1987 * copying them. At the same time this returns the total_mapcount too.
1988 *
1989 * The function returns the highest mapcount any one of the subpages
1990 * has. If the return value is one, even if different processes are
1991 * mapping different subpages of the transparent hugepage, they can
1992 * all reuse it, because each process is reusing a different subpage.
1993 *
1994 * The total_mapcount is instead counting all virtual mappings of the
1995 * subpages. If the total_mapcount is equal to "one", it tells the
1996 * caller all mappings belong to the same "mm" and in turn the
1997 * anon_vma of the transparent hugepage can become the vma->anon_vma
1998 * local one as no other process may be mapping any of the subpages.
1999 *
2000 * It would be more accurate to replace page_mapcount() with
2001 * page_trans_huge_mapcount(), however we only use
2002 * page_trans_huge_mapcount() in the copy-on-write faults where we
2003 * need full accuracy to avoid breaking page pinning, because
2004 * page_trans_huge_mapcount() is slower than page_mapcount().
2005 */
2006int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
2007{
2008        int i, ret, _total_mapcount, mapcount;
2009
2010        /* hugetlbfs shouldn't call it */
2011        VM_BUG_ON_PAGE(PageHuge(page), page);
2012
2013        if (likely(!PageTransCompound(page))) {
2014                mapcount = atomic_read(&page->_mapcount) + 1;
2015                if (total_mapcount)
2016                        *total_mapcount = mapcount;
2017                return mapcount;
2018        }
2019
2020        page = compound_head(page);
2021
2022        _total_mapcount = ret = 0;
2023        for (i = 0; i < HPAGE_PMD_NR; i++) {
2024                mapcount = atomic_read(&page[i]._mapcount) + 1;
2025                ret = max(ret, mapcount);
2026                _total_mapcount += mapcount;
2027        }
2028        if (PageDoubleMap(page)) {
2029                ret -= 1;
2030                _total_mapcount -= HPAGE_PMD_NR;
2031        }
2032        mapcount = compound_mapcount(page);
2033        ret += mapcount;
2034        _total_mapcount += mapcount;
2035        if (total_mapcount)
2036                *total_mapcount = _total_mapcount;
2037        return ret;
2038}
2039
2040/*
2041 * This function splits huge page into normal pages. @page can point to any
2042 * subpage of huge page to split. Split doesn't change the position of @page.
2043 *
2044 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
2045 * The huge page must be locked.
2046 *
2047 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
2048 *
2049 * Both head page and tail pages will inherit mapping, flags, and so on from
2050 * the hugepage.
2051 *
2052 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
2053 * they are not mapped.
2054 *
2055 * Returns 0 if the hugepage is split successfully.
2056 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
2057 * us.
2058 */
2059int split_huge_page_to_list(struct page *page, struct list_head *list)
2060{
2061        struct page *head = compound_head(page);
2062        struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
2063        struct anon_vma *anon_vma = NULL;
2064        struct address_space *mapping = NULL;
2065        int count, mapcount, extra_pins, ret;
2066        bool mlocked;
2067        unsigned long flags;
2068
2069        VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
2070        VM_BUG_ON_PAGE(!PageLocked(page), page);
2071        VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
2072        VM_BUG_ON_PAGE(!PageCompound(page), page);
2073
2074        if (PageAnon(head)) {
2075                /*
2076                 * The caller does not necessarily hold an mmap_sem that would
2077                 * prevent the anon_vma disappearing so we first we take a
2078                 * reference to it and then lock the anon_vma for write. This
2079                 * is similar to page_lock_anon_vma_read except the write lock
2080                 * is taken to serialise against parallel split or collapse
2081                 * operations.
2082                 */
2083                anon_vma = page_get_anon_vma(head);
2084                if (!anon_vma) {
2085                        ret = -EBUSY;
2086                        goto out;
2087                }
2088                extra_pins = 0;
2089                mapping = NULL;
2090                anon_vma_lock_write(anon_vma);
2091        } else {
2092                mapping = head->mapping;
2093
2094                /* Truncated ? */
2095                if (!mapping) {
2096                        ret = -EBUSY;
2097                        goto out;
2098                }
2099
2100                /* Addidional pins from radix tree */
2101                extra_pins = HPAGE_PMD_NR;
2102                anon_vma = NULL;
2103                i_mmap_lock_read(mapping);
2104        }
2105
2106        /*
2107         * Racy check if we can split the page, before freeze_page() will
2108         * split PMDs
2109         */
2110        if (total_mapcount(head) != page_count(head) - extra_pins - 1) {
2111                ret = -EBUSY;
2112                goto out_unlock;
2113        }
2114
2115        mlocked = PageMlocked(page);
2116        freeze_page(head);
2117        VM_BUG_ON_PAGE(compound_mapcount(head), head);
2118
2119        /* Make sure the page is not on per-CPU pagevec as it takes pin */
2120        if (mlocked)
2121                lru_add_drain();
2122
2123        /* prevent PageLRU to go away from under us, and freeze lru stats */
2124        spin_lock_irqsave(zone_lru_lock(page_zone(head)), flags);
2125
2126        if (mapping) {
2127                void **pslot;
2128
2129                spin_lock(&mapping->tree_lock);
2130                pslot = radix_tree_lookup_slot(&mapping->page_tree,
2131                                page_index(head));
2132                /*
2133                 * Check if the head page is present in radix tree.
2134                 * We assume all tail are present too, if head is there.
2135                 */
2136                if (radix_tree_deref_slot_protected(pslot,
2137                                        &mapping->tree_lock) != head)
2138                        goto fail;
2139        }
2140
2141        /* Prevent deferred_split_scan() touching ->_refcount */
2142        spin_lock(&pgdata->split_queue_lock);
2143        count = page_count(head);
2144        mapcount = total_mapcount(head);
2145        if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
2146                if (!list_empty(page_deferred_list(head))) {
2147                        pgdata->split_queue_len--;
2148                        list_del(page_deferred_list(head));
2149                }
2150                if (mapping)
2151                        __dec_node_page_state(page, NR_SHMEM_THPS);
2152                spin_unlock(&pgdata->split_queue_lock);
2153                __split_huge_page(page, list, flags);
2154                ret = 0;
2155        } else {
2156                if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
2157                        pr_alert("total_mapcount: %u, page_count(): %u\n",
2158                                        mapcount, count);
2159                        if (PageTail(page))
2160                                dump_page(head, NULL);
2161                        dump_page(page, "total_mapcount(head) > 0");
2162                        BUG();
2163                }
2164                spin_unlock(&pgdata->split_queue_lock);
2165fail:           if (mapping)
2166                        spin_unlock(&mapping->tree_lock);
2167                spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2168                unfreeze_page(head);
2169                ret = -EBUSY;
2170        }
2171
2172out_unlock:
2173        if (anon_vma) {
2174                anon_vma_unlock_write(anon_vma);
2175                put_anon_vma(anon_vma);
2176        }
2177        if (mapping)
2178                i_mmap_unlock_read(mapping);
2179out:
2180        count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
2181        return ret;
2182}
2183
2184void free_transhuge_page(struct page *page)
2185{
2186        struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2187        unsigned long flags;
2188
2189        spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2190        if (!list_empty(page_deferred_list(page))) {
2191                pgdata->split_queue_len--;
2192                list_del(page_deferred_list(page));
2193        }
2194        spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2195        free_compound_page(page);
2196}
2197
2198void deferred_split_huge_page(struct page *page)
2199{
2200        struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
2201        unsigned long flags;
2202
2203        VM_BUG_ON_PAGE(!PageTransHuge(page), page);
2204
2205        spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2206        if (list_empty(page_deferred_list(page))) {
2207                count_vm_event(THP_DEFERRED_SPLIT_PAGE);
2208                list_add_tail(page_deferred_list(page), &pgdata->split_queue);
2209                pgdata->split_queue_len++;
2210        }
2211        spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2212}
2213
2214static unsigned long deferred_split_count(struct shrinker *shrink,
2215                struct shrink_control *sc)
2216{
2217        struct pglist_data *pgdata = NODE_DATA(sc->nid);
2218        return ACCESS_ONCE(pgdata->split_queue_len);
2219}
2220
2221static unsigned long deferred_split_scan(struct shrinker *shrink,
2222                struct shrink_control *sc)
2223{
2224        struct pglist_data *pgdata = NODE_DATA(sc->nid);
2225        unsigned long flags;
2226        LIST_HEAD(list), *pos, *next;
2227        struct page *page;
2228        int split = 0;
2229
2230        spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2231        /* Take pin on all head pages to avoid freeing them under us */
2232        list_for_each_safe(pos, next, &pgdata->split_queue) {
2233                page = list_entry((void *)pos, struct page, mapping);
2234                page = compound_head(page);
2235                if (get_page_unless_zero(page)) {
2236                        list_move(page_deferred_list(page), &list);
2237                } else {
2238                        /* We lost race with put_compound_page() */
2239                        list_del_init(page_deferred_list(page));
2240                        pgdata->split_queue_len--;
2241                }
2242                if (!--sc->nr_to_scan)
2243                        break;
2244        }
2245        spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2246
2247        list_for_each_safe(pos, next, &list) {
2248                page = list_entry((void *)pos, struct page, mapping);
2249                lock_page(page);
2250                /* split_huge_page() removes page from list on success */
2251                if (!split_huge_page(page))
2252                        split++;
2253                unlock_page(page);
2254                put_page(page);
2255        }
2256
2257        spin_lock_irqsave(&pgdata->split_queue_lock, flags);
2258        list_splice_tail(&list, &pgdata->split_queue);
2259        spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
2260
2261        /*
2262         * Stop shrinker if we didn't split any page, but the queue is empty.
2263         * This can happen if pages were freed under us.
2264         */
2265        if (!split && list_empty(&pgdata->split_queue))
2266                return SHRINK_STOP;
2267        return split;
2268}
2269
2270static struct shrinker deferred_split_shrinker = {
2271        .count_objects = deferred_split_count,
2272        .scan_objects = deferred_split_scan,
2273        .seeks = DEFAULT_SEEKS,
2274        .flags = SHRINKER_NUMA_AWARE,
2275};
2276
2277#ifdef CONFIG_DEBUG_FS
2278static int split_huge_pages_set(void *data, u64 val)
2279{
2280        struct zone *zone;
2281        struct page *page;
2282        unsigned long pfn, max_zone_pfn;
2283        unsigned long total = 0, split = 0;
2284
2285        if (val != 1)
2286                return -EINVAL;
2287
2288        for_each_populated_zone(zone) {
2289                max_zone_pfn = zone_end_pfn(zone);
2290                for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
2291                        if (!pfn_valid(pfn))
2292                                continue;
2293
2294                        page = pfn_to_page(pfn);
2295                        if (!get_page_unless_zero(page))
2296                                continue;
2297
2298                        if (zone != page_zone(page))
2299                                goto next;
2300
2301                        if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
2302                                goto next;
2303
2304                        total++;
2305                        lock_page(page);
2306                        if (!split_huge_page(page))
2307                                split++;
2308                        unlock_page(page);
2309next:
2310                        put_page(page);
2311                }
2312        }
2313
2314        pr_info("%lu of %lu THP split\n", split, total);
2315
2316        return 0;
2317}
2318DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
2319                "%llu\n");
2320
2321static int __init split_huge_pages_debugfs(void)
2322{
2323        void *ret;
2324
2325        ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
2326                        &split_huge_pages_fops);
2327        if (!ret)
2328                pr_warn("Failed to create split_huge_pages in debugfs");
2329        return 0;
2330}
2331late_initcall(split_huge_pages_debugfs);
2332#endif
Note: See TracBrowser for help on using the repository browser.