// SPDX-License-Identifier: GPL-2.0-only

  /*
   * Generic hugetlb support.

   * (C) Nadia Yvette Chambers, April 2004

   */

  #include <linux/list.h>
  #include <linux/init.h>

  #include <linux/mm.h>

  #include <linux/seq_file.h>

  #include <linux/sysctl.h>
  #include <linux/highmem.h>

  #include <linux/mmu_notifier.h>

  #include <linux/nodemask.h>

  #include <linux/pagemap.h>

  #include <linux/mempolicy.h>

  #include <linux/compiler.h>

  #include <linux/cpuset.h>

  #include <linux/mutex.h>

  #include <linux/memblock.h>

  #include <linux/sysfs.h>

  #include <linux/slab.h>

  #include <linux/mmdebug.h>

  #include <linux/sched/signal.h>

  #include <linux/rmap.h>

  #include <linux/string_helpers.h>

  #include <linux/swap.h>
  #include <linux/swapops.h>

  #include <linux/jhash.h>

  #include <linux/numa.h>

  #include <linux/llist.h>

  #include <asm/page.h>
  #include <asm/pgtable.h>

  #include <asm/tlb.h>

  #include <linux/io.h>

  #include <linux/hugetlb.h>

  #include <linux/hugetlb_cgroup.h>

  #include <linux/node.h>

  #include <linux/userfaultfd_k.h>

  #include <linux/page_owner.h>

  #include "internal.h"

  int hugetlb_max_hstate __read_mostly;

  unsigned int default_hstate_idx;
  struct hstate hstates[HUGE_MAX_HSTATE];

  /*
   * Minimum page order among possible hugepage sizes, set to a proper value
   * at boot time.
   */
  static unsigned int minimum_order __read_mostly = UINT_MAX;

  __initdata LIST_HEAD(huge_boot_pages);

  /* for command line parsing */
  static struct hstate * __initdata parsed_hstate;
  static unsigned long __initdata default_hstate_max_huge_pages;

  static unsigned long __initdata default_hstate_size;

  static bool __initdata parsed_valid_hugepagesz = true;

  /*

   * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
   * free_huge_pages, and surplus_huge_pages.

   */

  DEFINE_SPINLOCK(hugetlb_lock);

  /*
   * Serializes faults on the same logical page.  This is used to
   * prevent spurious OOMs when the hugepage pool is fully utilized.
   */
  static int num_fault_mutexes;

  struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;

  /* Forward declaration */
  static int hugetlb_acct_memory(struct hstate *h, long delta);

  static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  {
  	bool free = (spool->count == 0) && (spool->used_hpages == 0);
  
  	spin_unlock(&spool->lock);
  
  	/* If no pages are used, and no other handles to the subpool

  	 * remain, give up any reservations mased on minimum size and
  	 * free the subpool */
  	if (free) {
  		if (spool->min_hpages != -1)
  			hugetlb_acct_memory(spool->hstate,
  						-spool->min_hpages);

  		kfree(spool);

  	}

  }

  struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
  						long min_hpages)

  {
  	struct hugepage_subpool *spool;

  	spool = kzalloc(sizeof(*spool), GFP_KERNEL);

  	if (!spool)
  		return NULL;
  
  	spin_lock_init(&spool->lock);
  	spool->count = 1;

  	spool->max_hpages = max_hpages;
  	spool->hstate = h;
  	spool->min_hpages = min_hpages;
  
  	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
  		kfree(spool);
  		return NULL;
  	}
  	spool->rsv_hpages = min_hpages;

  
  	return spool;
  }
  
  void hugepage_put_subpool(struct hugepage_subpool *spool)
  {
  	spin_lock(&spool->lock);
  	BUG_ON(!spool->count);
  	spool->count--;
  	unlock_or_release_subpool(spool);
  }

  /*
   * Subpool accounting for allocating and reserving pages.
   * Return -ENOMEM if there are not enough resources to satisfy the
   * the request.  Otherwise, return the number of pages by which the
   * global pools must be adjusted (upward).  The returned value may
   * only be different than the passed value (delta) in the case where
   * a subpool minimum size must be manitained.
   */
  static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,

  				      long delta)
  {

  	long ret = delta;

  
  	if (!spool)

  		return ret;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1) {		/* maximum size accounting */
  		if ((spool->used_hpages + delta) <= spool->max_hpages)
  			spool->used_hpages += delta;
  		else {
  			ret = -ENOMEM;
  			goto unlock_ret;
  		}

  	}

  	/* minimum size accounting */
  	if (spool->min_hpages != -1 && spool->rsv_hpages) {

  		if (delta > spool->rsv_hpages) {
  			/*
  			 * Asking for more reserves than those already taken on
  			 * behalf of subpool.  Return difference.
  			 */
  			ret = delta - spool->rsv_hpages;
  			spool->rsv_hpages = 0;
  		} else {
  			ret = 0;	/* reserves already accounted for */
  			spool->rsv_hpages -= delta;
  		}
  	}
  
  unlock_ret:
  	spin_unlock(&spool->lock);

  	return ret;
  }

  /*
   * Subpool accounting for freeing and unreserving pages.
   * Return the number of global page reservations that must be dropped.
   * The return value may only be different than the passed value (delta)
   * in the case where a subpool minimum size must be maintained.
   */
  static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,

  				       long delta)
  {

  	long ret = delta;

  	if (!spool)

  		return delta;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1)		/* maximum size accounting */
  		spool->used_hpages -= delta;

  	 /* minimum size accounting */
  	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {

  		if (spool->rsv_hpages + delta <= spool->min_hpages)
  			ret = 0;
  		else
  			ret = spool->rsv_hpages + delta - spool->min_hpages;
  
  		spool->rsv_hpages += delta;
  		if (spool->rsv_hpages > spool->min_hpages)
  			spool->rsv_hpages = spool->min_hpages;
  	}
  
  	/*
  	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
  	 * quota reference, free it now.
  	 */

  	unlock_or_release_subpool(spool);

  
  	return ret;

  }
  
  static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
  {
  	return HUGETLBFS_SB(inode->i_sb)->spool;
  }
  
  static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
  {

  	return subpool_inode(file_inode(vma->vm_file));

  }

  /*

   * Region tracking -- allows tracking of reservations and instantiated pages
   *                    across the pages in a mapping.

   *

   * The region data structures are embedded into a resv_map and protected
   * by a resv_map's lock.  The set of regions within the resv_map represent
   * reservations for huge pages, or huge pages that have already been
   * instantiated within the map.  The from and to elements are huge page
   * indicies into the associated mapping.  from indicates the starting index
   * of the region.  to represents the first index past the end of  the region.
   *
   * For example, a file region structure with from == 0 and to == 4 represents
   * four huge pages in a mapping.  It is important to note that the to element
   * represents the first element past the end of the region. This is used in
   * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
   *
   * Interval notation of the form [from, to) will be used to indicate that
   * the endpoint from is inclusive and to is exclusive.

   */
  struct file_region {
  	struct list_head link;
  	long from;
  	long to;
  };

  /*
   * Add the huge page range represented by [f, t) to the reserve

   * map.  In the normal case, existing regions will be expanded
   * to accommodate the specified range.  Sufficient regions should
   * exist for expansion due to the previous call to region_chg
   * with the same range.  However, it is possible that region_del
   * could have been called after region_chg and modifed the map
   * in such a way that no region exists to be expanded.  In this
   * case, pull a region descriptor from the cache associated with
   * the map and use that for the new range.

   *
   * Return the number of new huge pages added to the map.  This
   * number is greater than or equal to zero.

   */

  static long region_add(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg, *trg;

  	long add = 0;

  	spin_lock(&resv->lock);

  	/* Locate the region we are either in or before. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;

  	/*
  	 * If no region exists which can be expanded to include the
  	 * specified range, the list must have been modified by an
  	 * interleving call to region_del().  Pull a region descriptor
  	 * from the cache and use it for this range.
  	 */
  	if (&rg->link == head || t < rg->from) {
  		VM_BUG_ON(resv->region_cache_count <= 0);
  
  		resv->region_cache_count--;
  		nrg = list_first_entry(&resv->region_cache, struct file_region,
  					link);
  		list_del(&nrg->link);
  
  		nrg->from = f;
  		nrg->to = t;
  		list_add(&nrg->link, rg->link.prev);
  
  		add += t - f;
  		goto out_locked;
  	}

  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  
  	/* Check for and consume any regions we now overlap with. */
  	nrg = rg;
  	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)
  			break;
  
  		/* If this area reaches higher then extend our area to
  		 * include it completely.  If this is not the first area
  		 * which we intend to reuse, free it. */
  		if (rg->to > t)
  			t = rg->to;
  		if (rg != nrg) {

  			/* Decrement return value by the deleted range.
  			 * Another range will span this area so that by
  			 * end of routine add will be >= zero
  			 */
  			add -= (rg->to - rg->from);

  			list_del(&rg->link);
  			kfree(rg);
  		}
  	}

  
  	add += (nrg->from - f);		/* Added to beginning of region */

  	nrg->from = f;

  	add += t - nrg->to;		/* Added to end of region */

  	nrg->to = t;

  out_locked:
  	resv->adds_in_progress--;

  	spin_unlock(&resv->lock);

  	VM_BUG_ON(add < 0);
  	return add;

  }

  /*
   * Examine the existing reserve map and determine how many
   * huge pages in the specified range [f, t) are NOT currently
   * represented.  This routine is called before a subsequent
   * call to region_add that will actually modify the reserve
   * map to add the specified range [f, t).  region_chg does
   * not change the number of huge pages represented by the
   * map.  However, if the existing regions in the map can not
   * be expanded to represent the new range, a new file_region
   * structure is added to the map as a placeholder.  This is
   * so that the subsequent region_add call will have all the
   * regions it needs and will not fail.
   *

   * Upon entry, region_chg will also examine the cache of region descriptors
   * associated with the map.  If there are not enough descriptors cached, one
   * will be allocated for the in progress add operation.
   *
   * Returns the number of huge pages that need to be added to the existing
   * reservation map for the range [f, t).  This number is greater or equal to
   * zero.  -ENOMEM is returned if a new file_region structure or cache entry
   * is needed and can not be allocated.

   */

  static long region_chg(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg = NULL;

  	long chg = 0;

  retry:
  	spin_lock(&resv->lock);

  retry_locked:
  	resv->adds_in_progress++;
  
  	/*
  	 * Check for sufficient descriptors in the cache to accommodate
  	 * the number of in progress add operations.
  	 */
  	if (resv->adds_in_progress > resv->region_cache_count) {
  		struct file_region *trg;
  
  		VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
  		/* Must drop lock to allocate a new descriptor. */
  		resv->adds_in_progress--;
  		spin_unlock(&resv->lock);
  
  		trg = kmalloc(sizeof(*trg), GFP_KERNEL);

  		if (!trg) {
  			kfree(nrg);

  			return -ENOMEM;

  		}

  
  		spin_lock(&resv->lock);
  		list_add(&trg->link, &resv->region_cache);
  		resv->region_cache_count++;
  		goto retry_locked;
  	}

  	/* Locate the region we are before or in. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;
  
  	/* If we are below the current region then a new region is required.
  	 * Subtle, allocate a new region at the position but make it zero
  	 * size such that we can guarantee to record the reservation. */
  	if (&rg->link == head || t < rg->from) {

  		if (!nrg) {

  			resv->adds_in_progress--;

  			spin_unlock(&resv->lock);
  			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  			if (!nrg)
  				return -ENOMEM;
  
  			nrg->from = f;
  			nrg->to   = f;
  			INIT_LIST_HEAD(&nrg->link);
  			goto retry;
  		}

  		list_add(&nrg->link, rg->link.prev);
  		chg = t - f;
  		goto out_nrg;

  	}
  
  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  	chg = t - f;
  
  	/* Check for and consume any regions we now overlap with. */
  	list_for_each_entry(rg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)

  			goto out;

  		/* We overlap with this area, if it extends further than

  		 * us then we must extend ourselves.  Account for its
  		 * existing reservation. */
  		if (rg->to > t) {
  			chg += rg->to - t;
  			t = rg->to;
  		}
  		chg -= rg->to - rg->from;
  	}

  
  out:
  	spin_unlock(&resv->lock);
  	/*  We already know we raced and no longer need the new region */
  	kfree(nrg);
  	return chg;
  out_nrg:
  	spin_unlock(&resv->lock);

  	return chg;
  }

  /*

   * Abort the in progress add operation.  The adds_in_progress field
   * of the resv_map keeps track of the operations in progress between
   * calls to region_chg and region_add.  Operations are sometimes
   * aborted after the call to region_chg.  In such cases, region_abort
   * is called to decrement the adds_in_progress counter.
   *
   * NOTE: The range arguments [f, t) are not needed or used in this
   * routine.  They are kept to make reading the calling code easier as
   * arguments will match the associated region_chg call.
   */
  static void region_abort(struct resv_map *resv, long f, long t)
  {
  	spin_lock(&resv->lock);
  	VM_BUG_ON(!resv->region_cache_count);
  	resv->adds_in_progress--;
  	spin_unlock(&resv->lock);
  }
  
  /*

   * Delete the specified range [f, t) from the reserve map.  If the
   * t parameter is LONG_MAX, this indicates that ALL regions after f
   * should be deleted.  Locate the regions which intersect [f, t)
   * and either trim, delete or split the existing regions.
   *
   * Returns the number of huge pages deleted from the reserve map.
   * In the normal case, the return value is zero or more.  In the
   * case where a region must be split, a new region descriptor must
   * be allocated.  If the allocation fails, -ENOMEM will be returned.
   * NOTE: If the parameter t == LONG_MAX, then we will never split
   * a region and possibly return -ENOMEM.  Callers specifying
   * t == LONG_MAX do not need to check for -ENOMEM error.

   */

  static long region_del(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *trg;

  	struct file_region *nrg = NULL;
  	long del = 0;

  retry:

  	spin_lock(&resv->lock);

  	list_for_each_entry_safe(rg, trg, head, link) {

  		/*
  		 * Skip regions before the range to be deleted.  file_region
  		 * ranges are normally of the form [from, to).  However, there
  		 * may be a "placeholder" entry in the map which is of the form
  		 * (from, to) with from == to.  Check for placeholder entries
  		 * at the beginning of the range to be deleted.
  		 */
  		if (rg->to <= f && (rg->to != rg->from || rg->to != f))

  			continue;

  		if (rg->from >= t)

  			break;

  		if (f > rg->from && t < rg->to) { /* Must split region */
  			/*
  			 * Check for an entry in the cache before dropping
  			 * lock and attempting allocation.
  			 */
  			if (!nrg &&
  			    resv->region_cache_count > resv->adds_in_progress) {
  				nrg = list_first_entry(&resv->region_cache,
  							struct file_region,
  							link);
  				list_del(&nrg->link);
  				resv->region_cache_count--;
  			}

  			if (!nrg) {
  				spin_unlock(&resv->lock);
  				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  				if (!nrg)
  					return -ENOMEM;
  				goto retry;
  			}
  
  			del += t - f;
  
  			/* New entry for end of split region */
  			nrg->from = t;
  			nrg->to = rg->to;
  			INIT_LIST_HEAD(&nrg->link);
  
  			/* Original entry is trimmed */
  			rg->to = f;
  
  			list_add(&nrg->link, &rg->link);
  			nrg = NULL;

  			break;

  		}
  
  		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
  			del += rg->to - rg->from;
  			list_del(&rg->link);
  			kfree(rg);
  			continue;
  		}
  
  		if (f <= rg->from) {	/* Trim beginning of region */
  			del += t - rg->from;
  			rg->from = t;
  		} else {		/* Trim end of region */
  			del += rg->to - f;
  			rg->to = f;
  		}

  	}

  	spin_unlock(&resv->lock);

  	kfree(nrg);
  	return del;

  }

  /*

   * A rare out of memory error was encountered which prevented removal of
   * the reserve map region for a page.  The huge page itself was free'ed
   * and removed from the page cache.  This routine will adjust the subpool
   * usage count, and the global reserve count if needed.  By incrementing
   * these counts, the reserve map entry which could not be deleted will
   * appear as a "reserved" entry instead of simply dangling with incorrect
   * counts.
   */

  void hugetlb_fix_reserve_counts(struct inode *inode)

  {
  	struct hugepage_subpool *spool = subpool_inode(inode);
  	long rsv_adjust;
  
  	rsv_adjust = hugepage_subpool_get_pages(spool, 1);

  	if (rsv_adjust) {

  		struct hstate *h = hstate_inode(inode);
  
  		hugetlb_acct_memory(h, 1);
  	}
  }
  
  /*

   * Count and return the number of huge pages in the reserve map
   * that intersect with the range [f, t).
   */

  static long region_count(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg;
  	long chg = 0;

  	spin_lock(&resv->lock);

  	/* Locate each segment we overlap with, and count that overlap. */
  	list_for_each_entry(rg, head, link) {

  		long seg_from;
  		long seg_to;

  
  		if (rg->to <= f)
  			continue;
  		if (rg->from >= t)
  			break;
  
  		seg_from = max(rg->from, f);
  		seg_to = min(rg->to, t);
  
  		chg += seg_to - seg_from;
  	}

  	spin_unlock(&resv->lock);

  
  	return chg;
  }

  /*

   * Convert the address within this vma to the page offset within

   * the mapping, in pagecache page units; huge pages here.
   */

  static pgoff_t vma_hugecache_offset(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long address)

  {

  	return ((address - vma->vm_start) >> huge_page_shift(h)) +
  			(vma->vm_pgoff >> huge_page_order(h));

  }

  pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
  				     unsigned long address)
  {
  	return vma_hugecache_offset(hstate_vma(vma), vma, address);
  }

  EXPORT_SYMBOL_GPL(linear_hugepage_index);

  /*

   * Return the size of the pages allocated when backing a VMA. In the majority
   * cases this will be same size as used by the page table entries.
   */
  unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
  {

  	if (vma->vm_ops && vma->vm_ops->pagesize)
  		return vma->vm_ops->pagesize(vma);
  	return PAGE_SIZE;

  }

  EXPORT_SYMBOL_GPL(vma_kernel_pagesize);

  
  /*

   * Return the page size being used by the MMU to back a VMA. In the majority
   * of cases, the page size used by the kernel matches the MMU size. On

   * architectures where it differs, an architecture-specific 'strong'
   * version of this symbol is required.

   */

  __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)

  {
  	return vma_kernel_pagesize(vma);
  }

  
  /*

   * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
   * bits of the reservation map pointer, which are always clear due to
   * alignment.
   */
  #define HPAGE_RESV_OWNER    (1UL << 0)
  #define HPAGE_RESV_UNMAPPED (1UL << 1)

  #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)

  /*
   * These helpers are used to track how many pages are reserved for
   * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
   * is guaranteed to have their future faults succeed.
   *
   * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
   * the reserve counters are updated with the hugetlb_lock held. It is safe
   * to reset the VMA at fork() time as it is not in use yet and there is no
   * chance of the global counters getting corrupted as a result of the values.

   *
   * The private mapping reservation is represented in a subtly different
   * manner to a shared mapping.  A shared mapping has a region map associated
   * with the underlying file, this region map represents the backing file
   * pages which have ever had a reservation assigned which this persists even
   * after the page is instantiated.  A private mapping has a region map
   * associated with the original mmap which is attached to all VMAs which
   * reference it, this region map represents those offsets which have consumed
   * reservation ie. where pages have been instantiated.

   */

  static unsigned long get_vma_private_data(struct vm_area_struct *vma)
  {
  	return (unsigned long)vma->vm_private_data;
  }
  
  static void set_vma_private_data(struct vm_area_struct *vma,
  							unsigned long value)
  {
  	vma->vm_private_data = (void *)value;
  }

  struct resv_map *resv_map_alloc(void)

  {
  	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);

  	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
  
  	if (!resv_map || !rg) {
  		kfree(resv_map);
  		kfree(rg);

  		return NULL;

  	}

  
  	kref_init(&resv_map->refs);

  	spin_lock_init(&resv_map->lock);

  	INIT_LIST_HEAD(&resv_map->regions);

  	resv_map->adds_in_progress = 0;
  
  	INIT_LIST_HEAD(&resv_map->region_cache);
  	list_add(&rg->link, &resv_map->region_cache);
  	resv_map->region_cache_count = 1;

  	return resv_map;
  }

  void resv_map_release(struct kref *ref)

  {
  	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);

  	struct list_head *head = &resv_map->region_cache;
  	struct file_region *rg, *trg;

  
  	/* Clear out any active regions before we release the map. */

  	region_del(resv_map, 0, LONG_MAX);

  
  	/* ... and any entries left in the cache */
  	list_for_each_entry_safe(rg, trg, head, link) {
  		list_del(&rg->link);
  		kfree(rg);
  	}
  
  	VM_BUG_ON(resv_map->adds_in_progress);

  	kfree(resv_map);
  }

  static inline struct resv_map *inode_resv_map(struct inode *inode)
  {

  	/*
  	 * At inode evict time, i_mapping may not point to the original
  	 * address space within the inode.  This original address space
  	 * contains the pointer to the resv_map.  So, always use the
  	 * address space embedded within the inode.
  	 * The VERY common case is inode->mapping == &inode->i_data but,
  	 * this may not be true for device special inodes.
  	 */
  	return (struct resv_map *)(&inode->i_data)->private_data;

  }

  static struct resv_map *vma_resv_map(struct vm_area_struct *vma)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (vma->vm_flags & VM_MAYSHARE) {
  		struct address_space *mapping = vma->vm_file->f_mapping;
  		struct inode *inode = mapping->host;
  
  		return inode_resv_map(inode);
  
  	} else {

  		return (struct resv_map *)(get_vma_private_data(vma) &
  							~HPAGE_RESV_MASK);

  	}

  }

  static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  	set_vma_private_data(vma, (get_vma_private_data(vma) &
  				HPAGE_RESV_MASK) | (unsigned long)map);

  }
  
  static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  
  	set_vma_private_data(vma, get_vma_private_data(vma) | flags);

  }
  
  static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  
  	return (get_vma_private_data(vma) & flag) != 0;

  }

  /* Reset counters to 0 and clear all HPAGE_RESV_* flags */

  void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (!(vma->vm_flags & VM_MAYSHARE))

  		vma->vm_private_data = (void *)0;
  }
  
  /* Returns true if the VMA has associated reserve pages */

  static bool vma_has_reserves(struct vm_area_struct *vma, long chg)

  {

  	if (vma->vm_flags & VM_NORESERVE) {
  		/*
  		 * This address is already reserved by other process(chg == 0),
  		 * so, we should decrement reserved count. Without decrementing,
  		 * reserve count remains after releasing inode, because this
  		 * allocated page will go into page cache and is regarded as
  		 * coming from reserved pool in releasing step.  Currently, we
  		 * don't have any other solution to deal with this situation
  		 * properly, so add work-around here.
  		 */
  		if (vma->vm_flags & VM_MAYSHARE && chg == 0)

  			return true;

  		else

  			return false;

  	}

  
  	/* Shared mappings always use reserves */

  	if (vma->vm_flags & VM_MAYSHARE) {
  		/*
  		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
  		 * be a region map for all pages.  The only situation where
  		 * there is no region map is if a hole was punched via
  		 * fallocate.  In this case, there really are no reverves to
  		 * use.  This situation is indicated if chg != 0.
  		 */
  		if (chg)
  			return false;
  		else
  			return true;
  	}

  
  	/*
  	 * Only the process that called mmap() has reserves for
  	 * private mappings.
  	 */

  	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
  		/*
  		 * Like the shared case above, a hole punch or truncate
  		 * could have been performed on the private mapping.
  		 * Examine the value of chg to determine if reserves
  		 * actually exist or were previously consumed.
  		 * Very Subtle - The value of chg comes from a previous
  		 * call to vma_needs_reserves().  The reserve map for
  		 * private mappings has different (opposite) semantics
  		 * than that of shared mappings.  vma_needs_reserves()
  		 * has already taken this difference in semantics into
  		 * account.  Therefore, the meaning of chg is the same
  		 * as in the shared case above.  Code could easily be
  		 * combined, but keeping it separate draws attention to
  		 * subtle differences.
  		 */
  		if (chg)
  			return false;
  		else
  			return true;
  	}

  	return false;

  }

  static void enqueue_huge_page(struct hstate *h, struct page *page)

  {
  	int nid = page_to_nid(page);

  	list_move(&page->lru, &h->hugepage_freelists[nid]);

  	h->free_huge_pages++;
  	h->free_huge_pages_node[nid]++;

  }

  static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)

  {
  	struct page *page;

  	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)

  		if (!PageHWPoison(page))

  			break;
  	/*
  	 * if 'non-isolated free hugepage' not found on the list,
  	 * the allocation fails.
  	 */
  	if (&h->hugepage_freelists[nid] == &page->lru)

  		return NULL;

  	list_move(&page->lru, &h->hugepage_activelist);

  	set_page_refcounted(page);

  	h->free_huge_pages--;
  	h->free_huge_pages_node[nid]--;
  	return page;
  }

  static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
  		nodemask_t *nmask)

  {

  	unsigned int cpuset_mems_cookie;
  	struct zonelist *zonelist;
  	struct zone *zone;
  	struct zoneref *z;

  	int node = NUMA_NO_NODE;

  	zonelist = node_zonelist(nid, gfp_mask);
  
  retry_cpuset:
  	cpuset_mems_cookie = read_mems_allowed_begin();
  	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
  		struct page *page;
  
  		if (!cpuset_zone_allowed(zone, gfp_mask))
  			continue;
  		/*
  		 * no need to ask again on the same node. Pool is node rather than
  		 * zone aware
  		 */
  		if (zone_to_nid(zone) == node)
  			continue;
  		node = zone_to_nid(zone);

  		page = dequeue_huge_page_node_exact(h, node);
  		if (page)
  			return page;
  	}

  	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
  		goto retry_cpuset;

  	return NULL;
  }

  /* Movability of hugepages depends on migration support. */
  static inline gfp_t htlb_alloc_mask(struct hstate *h)
  {

  	if (hugepage_movable_supported(h))

  		return GFP_HIGHUSER_MOVABLE;
  	else
  		return GFP_HIGHUSER;
  }

  static struct page *dequeue_huge_page_vma(struct hstate *h,
  				struct vm_area_struct *vma,

  				unsigned long address, int avoid_reserve,
  				long chg)

  {

  	struct page *page;

  	struct mempolicy *mpol;

  	gfp_t gfp_mask;

  	nodemask_t *nodemask;

  	int nid;

  	/*
  	 * A child process with MAP_PRIVATE mappings created by their parent
  	 * have no page reserves. This check ensures that reservations are
  	 * not "stolen". The child may still get SIGKILLed
  	 */

  	if (!vma_has_reserves(vma, chg) &&

  			h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  	/* If reserves cannot be used, ensure enough pages are in the pool */

  	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  	gfp_mask = htlb_alloc_mask(h);
  	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);

  	page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
  	if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
  		SetPagePrivate(page);
  		h->resv_huge_pages--;

  	}

  	mpol_cond_put(mpol);

  	return page;

  
  err:

  	return NULL;

  }

  /*
   * common helper functions for hstate_next_node_to_{alloc|free}.
   * We may have allocated or freed a huge page based on a different
   * nodes_allowed previously, so h->next_node_to_{alloc|free} might
   * be outside of *nodes_allowed.  Ensure that we use an allowed
   * node for alloc or free.
   */
  static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
  {

  	nid = next_node_in(nid, *nodes_allowed);

  	VM_BUG_ON(nid >= MAX_NUMNODES);
  
  	return nid;
  }
  
  static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
  {
  	if (!node_isset(nid, *nodes_allowed))
  		nid = next_node_allowed(nid, nodes_allowed);
  	return nid;
  }
  
  /*
   * returns the previously saved node ["this node"] from which to
   * allocate a persistent huge page for the pool and advance the
   * next node from which to allocate, handling wrap at end of node
   * mask.
   */
  static int hstate_next_node_to_alloc(struct hstate *h,
  					nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
  	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  /*
   * helper for free_pool_huge_page() - return the previously saved
   * node ["this node"] from which to free a huge page.  Advance the
   * next node id whether or not we find a free huge page to free so
   * that the next attempt to free addresses the next node.
   */
  static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
  	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
  		nr_nodes--)
  
  #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
  		nr_nodes--)

  #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE

  static void destroy_compound_gigantic_page(struct page *page,

  					unsigned int order)

  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;

  	atomic_set(compound_mapcount_ptr(page), 0);

  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

  		clear_compound_head(p);

  		set_page_refcounted(p);

  	}
  
  	set_compound_order(page, 0);
  	__ClearPageHead(page);
  }

  static void free_gigantic_page(struct page *page, unsigned int order)

  {
  	free_contig_range(page_to_pfn(page), 1 << order);
  }

  #ifdef CONFIG_CONTIG_ALLOC

  static int __alloc_gigantic_page(unsigned long start_pfn,

  				unsigned long nr_pages, gfp_t gfp_mask)

  {
  	unsigned long end_pfn = start_pfn + nr_pages;

  	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,

  				  gfp_mask);

  }

  static bool pfn_range_valid_gigantic(struct zone *z,
  			unsigned long start_pfn, unsigned long nr_pages)

  {
  	unsigned long i, end_pfn = start_pfn + nr_pages;
  	struct page *page;
  
  	for (i = start_pfn; i < end_pfn; i++) {

  		page = pfn_to_online_page(i);
  		if (!page)

  			return false;

  		if (page_zone(page) != z)
  			return false;

  		if (PageReserved(page))
  			return false;
  
  		if (page_count(page) > 0)
  			return false;
  
  		if (PageHuge(page))
  			return false;
  	}
  
  	return true;
  }
  
  static bool zone_spans_last_pfn(const struct zone *zone,
  			unsigned long start_pfn, unsigned long nr_pages)
  {
  	unsigned long last_pfn = start_pfn + nr_pages - 1;
  	return zone_spans_pfn(zone, last_pfn);
  }

  static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
  		int nid, nodemask_t *nodemask)

  {

  	unsigned int order = huge_page_order(h);

  	unsigned long nr_pages = 1 << order;
  	unsigned long ret, pfn, flags;

  	struct zonelist *zonelist;
  	struct zone *zone;
  	struct zoneref *z;

  	zonelist = node_zonelist(nid, gfp_mask);

  	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nodemask) {

  		spin_lock_irqsave(&zone->lock, flags);

  		pfn = ALIGN(zone->zone_start_pfn, nr_pages);
  		while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
  			if (pfn_range_valid_gigantic(zone, pfn, nr_pages)) {

  				/*
  				 * We release the zone lock here because
  				 * alloc_contig_range() will also lock the zone
  				 * at some point. If there's an allocation
  				 * spinning on this lock, it may win the race
  				 * and cause alloc_contig_range() to fail...
  				 */

  				spin_unlock_irqrestore(&zone->lock, flags);
  				ret = __alloc_gigantic_page(pfn, nr_pages, gfp_mask);

  				if (!ret)
  					return pfn_to_page(pfn);

  				spin_lock_irqsave(&zone->lock, flags);

  			}
  			pfn += nr_pages;
  		}

  		spin_unlock_irqrestore(&zone->lock, flags);

  	}
  
  	return NULL;
  }
  
  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);

  static void prep_compound_gigantic_page(struct page *page, unsigned int order);

  #else /* !CONFIG_CONTIG_ALLOC */
  static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
  					int nid, nodemask_t *nodemask)
  {
  	return NULL;
  }
  #endif /* CONFIG_CONTIG_ALLOC */

  #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */

  static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,

  					int nid, nodemask_t *nodemask)
  {
  	return NULL;
  }

  static inline void free_gigantic_page(struct page *page, unsigned int order) { }

  static inline void destroy_compound_gigantic_page(struct page *page,

  						unsigned int order) { }

  #endif

  static void update_and_free_page(struct hstate *h, struct page *page)

  {
  	int i;

  	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())

  		return;

  	h->nr_huge_pages--;
  	h->nr_huge_pages_node[page_to_nid(page)]--;
  	for (i = 0; i < pages_per_huge_page(h); i++) {

  		page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
  				1 << PG_referenced | 1 << PG_dirty |

  				1 << PG_active | 1 << PG_private |
  				1 << PG_writeback);

  	}

  	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);

  	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);

  	set_page_refcounted(page);

  	if (hstate_is_gigantic(h)) {
  		destroy_compound_gigantic_page(page, huge_page_order(h));
  		free_gigantic_page(page, huge_page_order(h));
  	} else {

  		__free_pages(page, huge_page_order(h));
  	}

  }

  struct hstate *size_to_hstate(unsigned long size)
  {
  	struct hstate *h;
  
  	for_each_hstate(h) {
  		if (huge_page_size(h) == size)
  			return h;
  	}
  	return NULL;
  }

  /*
   * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
   * to hstate->hugepage_activelist.)
   *
   * This function can be called for tail pages, but never returns true for them.
   */
  bool page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHuge(page), page);
  	return PageHead(page) && PagePrivate(&page[1]);
  }
  
  /* never called for tail page */
  static void set_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	SetPagePrivate(&page[1]);
  }
  
  static void clear_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	ClearPagePrivate(&page[1]);
  }

  /*
   * Internal hugetlb specific page flag. Do not use outside of the hugetlb
   * code
   */
  static inline bool PageHugeTemporary(struct page *page)
  {
  	if (!PageHuge(page))
  		return false;
  
  	return (unsigned long)page[2].mapping == -1U;
  }
  
  static inline void SetPageHugeTemporary(struct page *page)
  {
  	page[2].mapping = (void *)-1U;
  }
  
  static inline void ClearPageHugeTemporary(struct page *page)
  {
  	page[2].mapping = NULL;
  }

  static void __free_huge_page(struct page *page)

  {

  	/*
  	 * Can't pass hstate in here because it is called from the
  	 * compound page destructor.
  	 */

  	struct hstate *h = page_hstate(page);

  	int nid = page_to_nid(page);

  	struct hugepage_subpool *spool =
  		(struct hugepage_subpool *)page_private(page);

  	bool restore_reserve;

  	VM_BUG_ON_PAGE(page_count(page), page);
  	VM_BUG_ON_PAGE(page_mapcount(page), page);

  
  	set_page_private(page, 0);
  	page->mapping = NULL;

  	restore_reserve = PagePrivate(page);

  	ClearPagePrivate(page);

  	/*

  	 * If PagePrivate() was set on page, page allocation consumed a
  	 * reservation.  If the page was associated with a subpool, there
  	 * would have been a page reserved in the subpool before allocation
  	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
  	 * reservtion, do not call hugepage_subpool_put_pages() as this will
  	 * remove the reserved page from the subpool.

  	 */

  	if (!restore_reserve) {
  		/*
  		 * A return code of zero implies that the subpool will be
  		 * under its minimum size if the reservation is not restored
  		 * after page is free.  Therefore, force restore_reserve
  		 * operation.
  		 */
  		if (hugepage_subpool_put_pages(spool, 1) == 0)
  			restore_reserve = true;
  	}

  	spin_lock(&hugetlb_lock);

  	clear_page_huge_active(page);

  	hugetlb_cgroup_uncharge_page(hstate_index(h),
  				     pages_per_huge_page(h), page);

  	if (restore_reserve)
  		h->resv_huge_pages++;

  	if (PageHugeTemporary(page)) {
  		list_del(&page->lru);
  		ClearPageHugeTemporary(page);
  		update_and_free_page(h, page);
  	} else if (h->surplus_huge_pages_node[nid]) {

  		/* remove the page from active list */
  		list_del(&page->lru);

  		update_and_free_page(h, page);
  		h->surplus_huge_pages--;
  		h->surplus_huge_pages_node[nid]--;

  	} else {

  		arch_clear_hugepage_flags(page);

  		enqueue_huge_page(h, page);

  	}

  	spin_unlock(&hugetlb_lock);
  }

  /*
   * As free_huge_page() can be called from a non-task context, we have
   * to defer the actual freeing in a workqueue to prevent potential
   * hugetlb_lock deadlock.
   *
   * free_hpage_workfn() locklessly retrieves the linked list of pages to
   * be freed and frees them one-by-one. As the page->mapping pointer is
   * going to be cleared in __free_huge_page() anyway, it is reused as the
   * llist_node structure of a lockless linked list of huge pages to be freed.
   */
  static LLIST_HEAD(hpage_freelist);
  
  static void free_hpage_workfn(struct work_struct *work)
  {
  	struct llist_node *node;
  	struct page *page;
  
  	node = llist_del_all(&hpage_freelist);
  
  	while (node) {
  		page = container_of((struct address_space **)node,
  				     struct page, mapping);
  		node = node->next;
  		__free_huge_page(page);
  	}
  }
  static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
  
  void free_huge_page(struct page *page)
  {
  	/*
  	 * Defer freeing if in non-task context to avoid hugetlb_lock deadlock.
  	 */
  	if (!in_task()) {
  		/*
  		 * Only call schedule_work() if hpage_freelist is previously
  		 * empty. Otherwise, schedule_work() had been called but the
  		 * workfn hasn't retrieved the list yet.
  		 */
  		if (llist_add((struct llist_node *)&page->mapping,
  			      &hpage_freelist))
  			schedule_work(&free_hpage_work);
  		return;
  	}
  
  	__free_huge_page(page);
  }

  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)

  {

  	INIT_LIST_HEAD(&page->lru);

  	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);

  	spin_lock(&hugetlb_lock);

  	set_hugetlb_cgroup(page, NULL);

  	h->nr_huge_pages++;
  	h->nr_huge_pages_node[nid]++;

  	spin_unlock(&hugetlb_lock);

  }

  static void prep_compound_gigantic_page(struct page *page, unsigned int order)

  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;
  
  	/* we rely on prep_new_huge_page to set the destructor */
  	set_compound_order(page, order);

  	__ClearPageReserved(page);

  	__SetPageHead(page);

  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

  		/*
  		 * For gigantic hugepages allocated through bootmem at
  		 * boot, it's safer to be consistent with the not-gigantic
  		 * hugepages and clear the PG_reserved bit from all tail pages
  		 * too.  Otherwse drivers using get_user_pages() to access tail
  		 * pages may get the reference counting wrong if they see
  		 * PG_reserved set on a tail page (despite the head page not
  		 * having PG_reserved set).  Enforcing this consistency between
  		 * head and tail pages allows drivers to optimize away a check
  		 * on the head page when they need know if put_page() is needed
  		 * after get_user_pages().
  		 */
  		__ClearPageReserved(p);

  		set_page_count(p, 0);

  		set_compound_head(p, page);

  	}

  	atomic_set(compound_mapcount_ptr(page), -1);

  }

  /*
   * PageHuge() only returns true for hugetlbfs pages, but not for normal or
   * transparent huge pages.  See the PageTransHuge() documentation for more
   * details.
   */

  int PageHuge(struct page *page)
  {

  	if (!PageCompound(page))
  		return 0;
  
  	page = compound_head(page);

  	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;

  }

  EXPORT_SYMBOL_GPL(PageHuge);

  /*
   * PageHeadHuge() only returns true for hugetlbfs head page, but not for
   * normal or transparent huge pages.
   */
  int PageHeadHuge(struct page *page_head)
  {

  	if (!PageHead(page_head))
  		return 0;

  	return get_compound_page_dtor(page_head) == free_huge_page;

  }

  pgoff_t __basepage_index(struct page *page)
  {
  	struct page *page_head = compound_head(page);
  	pgoff_t index = page_index(page_head);
  	unsigned long compound_idx;
  
  	if (!PageHuge(page_head))
  		return page_index(page);
  
  	if (compound_order(page_head) >= MAX_ORDER)
  		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
  	else
  		compound_idx = page - page_head;
  
  	return (index << compound_order(page_head)) + compound_idx;
  }

  static struct page *alloc_buddy_huge_page(struct hstate *h,

  		gfp_t gfp_mask, int nid, nodemask_t *nmask,
  		nodemask_t *node_alloc_noretry)

  {

  	int order = huge_page_order(h);

  	struct page *page;

  	bool alloc_try_hard = true;

  	/*
  	 * By default we always try hard to allocate the page with
  	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
  	 * a loop (to adjust global huge page counts) and previous allocation
  	 * failed, do not continue to try hard on the same node.  Use the
  	 * node_alloc_noretry bitmap to manage this state information.
  	 */
  	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
  		alloc_try_hard = false;
  	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
  	if (alloc_try_hard)
  		gfp_mask |= __GFP_RETRY_MAYFAIL;

  	if (nid == NUMA_NO_NODE)
  		nid = numa_mem_id();
  	page = __alloc_pages_nodemask(gfp_mask, order, nid, nmask);
  	if (page)
  		__count_vm_event(HTLB_BUDDY_PGALLOC);
  	else
  		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);

  	/*
  	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
  	 * indicates an overall state change.  Clear bit so that we resume
  	 * normal 'try hard' allocations.
  	 */
  	if (node_alloc_noretry && page && !alloc_try_hard)
  		node_clear(nid, *node_alloc_noretry);
  
  	/*
  	 * If we tried hard to get a page but failed, set bit so that
  	 * subsequent attempts will not try as hard until there is an
  	 * overall state change.
  	 */
  	if (node_alloc_noretry && !page && alloc_try_hard)
  		node_set(nid, *node_alloc_noretry);

  	return page;
  }

  /*

   * Common helper to allocate a fresh hugetlb page. All specific allocators
   * should use this function to get new hugetlb pages
   */
  static struct page *alloc_fresh_huge_page(struct hstate *h,

  		gfp_t gfp_mask, int nid, nodemask_t *nmask,
  		nodemask_t *node_alloc_noretry)

  {
  	struct page *page;
  
  	if (hstate_is_gigantic(h))
  		page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
  	else
  		page = alloc_buddy_huge_page(h, gfp_mask,

  				nid, nmask, node_alloc_noretry);

  	if (!page)
  		return NULL;
  
  	if (hstate_is_gigantic(h))
  		prep_compound_gigantic_page(page, huge_page_order(h));
  	prep_new_huge_page(h, page, page_to_nid(page));
  
  	return page;
  }
  
  /*

   * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
   * manner.
   */

  static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
  				nodemask_t *node_alloc_noretry)

  {
  	struct page *page;
  	int nr_nodes, node;

  	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;

  
  	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {

  		page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
  						node_alloc_noretry);

  		if (page)

  			break;

  	}

  	if (!page)
  		return 0;

  	put_page(page); /* free it into the hugepage allocator */
  
  	return 1;

  }

  /*
   * Free huge page from pool from next node to free.
   * Attempt to keep persistent huge pages more or less
   * balanced over allowed nodes.
   * Called with hugetlb_lock locked.
   */

  static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
  							 bool acct_surplus)

  {

  	int nr_nodes, node;

  	int ret = 0;

  	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {

  		/*
  		 * If we're returning unused surplus pages, only examine
  		 * nodes with surplus pages.
  		 */

  		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
  		    !list_empty(&h->hugepage_freelists[node])) {

  			struct page *page =

  				list_entry(h->hugepage_freelists[node].next,

  					  struct page, lru);
  			list_del(&page->lru);
  			h->free_huge_pages--;

  			h->free_huge_pages_node[node]--;

  			if (acct_surplus) {
  				h->surplus_huge_pages--;

  				h->surplus_huge_pages_node[node]--;

  			}

  			update_and_free_page(h, page);
  			ret = 1;

  			break;

  		}

  	}

  
  	return ret;
  }

  /*
   * Dissolve a given free hugepage into free buddy pages. This function does

   * nothing for in-use hugepages and non-hugepages.
   * This function returns values like below:
   *
   *  -EBUSY: failed to dissolved free hugepages or the hugepage is in-use
   *          (allocated or reserved.)
   *       0: successfully dissolved free hugepages or the page is not a
   *          hugepage (considered as already dissolved)

   */

  int dissolve_free_huge_page(struct page *page)

  {

  	int rc = -EBUSY;

  	/* Not to disrupt normal path by vainly holding hugetlb_lock */
  	if (!PageHuge(page))
  		return 0;

  	spin_lock(&hugetlb_lock);

  	if (!PageHuge(page)) {
  		rc = 0;
  		goto out;
  	}
  
  	if (!page_count(page)) {

  		struct page *head = compound_head(page);
  		struct hstate *h = page_hstate(head);
  		int nid = page_to_nid(head);

  		if (h->free_huge_pages - h->resv_huge_pages == 0)

  			goto out;

  		/*
  		 * Move PageHWPoison flag from head page to the raw error page,
  		 * which makes any subpages rather than the error page reusable.
  		 */
  		if (PageHWPoison(head) && page != head) {
  			SetPageHWPoison(page);
  			ClearPageHWPoison(head);
  		}

  		list_del(&head->lru);

  		h->free_huge_pages--;
  		h->free_huge_pages_node[nid]--;

  		h->max_huge_pages--;

  		update_and_free_page(h, head);

  		rc = 0;

  	}

  out:

  	spin_unlock(&hugetlb_lock);

  	return rc;

  }
  
  /*
   * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
   * make specified memory blocks removable from the system.

   * Note that this will dissolve a free gigantic hugepage completely, if any
   * part of it lies within the given range.

   * Also note that if dissolve_free_huge_page() returns with an error, all
   * free hugepages that were dissolved before that error are lost.

   */

  int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)

  {

  	unsigned long pfn;

  	struct page *page;

  	int rc = 0;

  	if (!hugepages_supported())

  		return rc;

  	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
  		page = pfn_to_page(pfn);

  		rc = dissolve_free_huge_page(page);
  		if (rc)
  			break;

  	}

  
  	return rc;

  }

  /*
   * Allocates a fresh surplus page from the page allocator.
   */

  static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,

  		int nid, nodemask_t *nmask)

  {

  	struct page *page = NULL;

  	if (hstate_is_gigantic(h))

  		return NULL;

  	spin_lock(&hugetlb_lock);

  	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
  		goto out_unlock;

  	spin_unlock(&hugetlb_lock);

  	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);

  	if (!page)

  		return NULL;

  
  	spin_lock(&hugetlb_lock);

  	/*
  	 * We could have raced with the pool size change.
  	 * Double check that and simply deallocate the new page
  	 * if we would end up overcommiting the surpluses. Abuse
  	 * temporary page to workaround the nasty free_huge_page
  	 * codeflow
  	 */
  	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
  		SetPageHugeTemporary(page);

  		spin_unlock(&hugetlb_lock);

  		put_page(page);

  		return NULL;

  	} else {

  		h->surplus_huge_pages++;

  		h->surplus_huge_pages_node[page_to_nid(page)]++;

  	}

  
  out_unlock:

  	spin_unlock(&hugetlb_lock);

  
  	return page;
  }

  struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
  				     int nid, nodemask_t *nmask)

  {
  	struct page *page;
  
  	if (hstate_is_gigantic(h))
  		return NULL;

  	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);

  	if (!page)
  		return NULL;
  
  	/*
  	 * We do not account these pages as surplus because they are only
  	 * temporary and will be released properly on the last reference
  	 */

  	SetPageHugeTemporary(page);
  
  	return page;
  }

  /*

   * Use the VMA's mpolicy to allocate a huge page from the buddy.
   */

  static

  struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,

  		struct vm_area_struct *vma, unsigned long addr)
  {

  	struct page *page;
  	struct mempolicy *mpol;
  	gfp_t gfp_mask = htlb_alloc_mask(h);
  	int nid;
  	nodemask_t *nodemask;
  
  	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);

  	page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);

  	mpol_cond_put(mpol);
  
  	return page;

  }

  /* page migration callback function */

  struct page *alloc_huge_page_node(struct hstate *h, int nid)
  {

  	gfp_t gfp_mask = htlb_alloc_mask(h);

  	struct page *page = NULL;

  	if (nid != NUMA_NO_NODE)
  		gfp_mask |= __GFP_THISNODE;

  	spin_lock(&hugetlb_lock);

  	if (h->free_huge_pages - h->resv_huge_pages > 0)

  		page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL);

  	spin_unlock(&hugetlb_lock);

  	if (!page)

  		page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL);

  
  	return page;
  }

  /* page migration callback function */

  struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
  		nodemask_t *nmask)

  {

  	gfp_t gfp_mask = htlb_alloc_mask(h);

  
  	spin_lock(&hugetlb_lock);
  	if (h->free_huge_pages - h->resv_huge_pages > 0) {

  		struct page *page;
  
  		page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
  		if (page) {
  			spin_unlock(&hugetlb_lock);
  			return page;

  		}
  	}
  	spin_unlock(&hugetlb_lock);

  	return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);

  }

  /* mempolicy aware migration callback */

  struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
  		unsigned long address)

  {
  	struct mempolicy *mpol;
  	nodemask_t *nodemask;
  	struct page *page;

  	gfp_t gfp_mask;
  	int node;

  	gfp_mask = htlb_alloc_mask(h);
  	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
  	page = alloc_huge_page_nodemask(h, node, nodemask);
  	mpol_cond_put(mpol);
  
  	return page;
  }

  /*

   * Increase the hugetlb pool such that it can accommodate a reservation

   * of size 'delta'.
   */

  static int gather_surplus_pages(struct hstate *h, int delta)

  {
  	struct list_head surplus_list;
  	struct page *page, *tmp;
  	int ret, i;
  	int needed, allocated;

  	bool alloc_ok = true;

  	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;

  	if (needed <= 0) {

  		h->resv_huge_pages += delta;

  		return 0;

  	}

  
  	allocated = 0;
  	INIT_LIST_HEAD(&surplus_list);
  
  	ret = -ENOMEM;
  retry:
  	spin_unlock(&hugetlb_lock);
  	for (i = 0; i < needed; i++) {

  		page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),

  				NUMA_NO_NODE, NULL);

  		if (!page) {
  			alloc_ok = false;
  			break;
  		}

  		list_add(&page->lru, &surplus_list);

  		cond_resched();

  	}

  	allocated += i;

  
  	/*
  	 * After retaking hugetlb_lock, we need to recalculate 'needed'
  	 * because either resv_huge_pages or free_huge_pages may have changed.
  	 */
  	spin_lock(&hugetlb_lock);

  	needed = (h->resv_huge_pages + delta) -
  			(h->free_huge_pages + allocated);

  	if (needed > 0) {
  		if (alloc_ok)
  			goto retry;
  		/*
  		 * We were not able to allocate enough pages to
  		 * satisfy the entire reservation so we free what
  		 * we've allocated so far.
  		 */
  		goto free;
  	}

  	/*
  	 * The surplus_list now contains _at_least_ the number of extra pages

  	 * needed to accommodate the reservation.  Add the appropriate number

  	 * of pages to the hugetlb pool and free the extras back to the buddy

  	 * allocator.  Commit the entire reservation here to prevent another
  	 * process from stealing the pages as they are added to the pool but
  	 * before they are reserved.

  	 */
  	needed += allocated;

  	h->resv_huge_pages += delta;