/*
   * Generic hugetlb support.

   * (C) Nadia Yvette Chambers, April 2004

   */

  #include <linux/list.h>
  #include <linux/init.h>

  #include <linux/mm.h>

  #include <linux/seq_file.h>

  #include <linux/sysctl.h>
  #include <linux/highmem.h>

  #include <linux/mmu_notifier.h>

  #include <linux/nodemask.h>

  #include <linux/pagemap.h>

  #include <linux/mempolicy.h>

  #include <linux/compiler.h>

  #include <linux/cpuset.h>

  #include <linux/mutex.h>

  #include <linux/bootmem.h>

  #include <linux/sysfs.h>

  #include <linux/slab.h>

  #include <linux/rmap.h>

  #include <linux/swap.h>
  #include <linux/swapops.h>

  #include <linux/page-isolation.h>

  #include <linux/jhash.h>

  #include <asm/page.h>
  #include <asm/pgtable.h>

  #include <asm/tlb.h>

  #include <linux/io.h>

  #include <linux/hugetlb.h>

  #include <linux/hugetlb_cgroup.h>

  #include <linux/node.h>

  #include "internal.h"

  int hugepages_treat_as_movable;

  int hugetlb_max_hstate __read_mostly;

  unsigned int default_hstate_idx;
  struct hstate hstates[HUGE_MAX_HSTATE];

  /*
   * Minimum page order among possible hugepage sizes, set to a proper value
   * at boot time.
   */
  static unsigned int minimum_order __read_mostly = UINT_MAX;

  __initdata LIST_HEAD(huge_boot_pages);

  /* for command line parsing */
  static struct hstate * __initdata parsed_hstate;
  static unsigned long __initdata default_hstate_max_huge_pages;

  static unsigned long __initdata default_hstate_size;

  static bool __initdata parsed_valid_hugepagesz = true;

  /*

   * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
   * free_huge_pages, and surplus_huge_pages.

   */

  DEFINE_SPINLOCK(hugetlb_lock);

  /*
   * Serializes faults on the same logical page.  This is used to
   * prevent spurious OOMs when the hugepage pool is fully utilized.
   */
  static int num_fault_mutexes;

  struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;

  /* Forward declaration */
  static int hugetlb_acct_memory(struct hstate *h, long delta);

  static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  {
  	bool free = (spool->count == 0) && (spool->used_hpages == 0);
  
  	spin_unlock(&spool->lock);
  
  	/* If no pages are used, and no other handles to the subpool

  	 * remain, give up any reservations mased on minimum size and
  	 * free the subpool */
  	if (free) {
  		if (spool->min_hpages != -1)
  			hugetlb_acct_memory(spool->hstate,
  						-spool->min_hpages);

  		kfree(spool);

  	}

  }

  struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
  						long min_hpages)

  {
  	struct hugepage_subpool *spool;

  	spool = kzalloc(sizeof(*spool), GFP_KERNEL);

  	if (!spool)
  		return NULL;
  
  	spin_lock_init(&spool->lock);
  	spool->count = 1;

  	spool->max_hpages = max_hpages;
  	spool->hstate = h;
  	spool->min_hpages = min_hpages;
  
  	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
  		kfree(spool);
  		return NULL;
  	}
  	spool->rsv_hpages = min_hpages;

  
  	return spool;
  }
  
  void hugepage_put_subpool(struct hugepage_subpool *spool)
  {
  	spin_lock(&spool->lock);
  	BUG_ON(!spool->count);
  	spool->count--;
  	unlock_or_release_subpool(spool);
  }

  /*
   * Subpool accounting for allocating and reserving pages.
   * Return -ENOMEM if there are not enough resources to satisfy the
   * the request.  Otherwise, return the number of pages by which the
   * global pools must be adjusted (upward).  The returned value may
   * only be different than the passed value (delta) in the case where
   * a subpool minimum size must be manitained.
   */
  static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,

  				      long delta)
  {

  	long ret = delta;

  
  	if (!spool)

  		return ret;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1) {		/* maximum size accounting */
  		if ((spool->used_hpages + delta) <= spool->max_hpages)
  			spool->used_hpages += delta;
  		else {
  			ret = -ENOMEM;
  			goto unlock_ret;
  		}

  	}

  	/* minimum size accounting */
  	if (spool->min_hpages != -1 && spool->rsv_hpages) {

  		if (delta > spool->rsv_hpages) {
  			/*
  			 * Asking for more reserves than those already taken on
  			 * behalf of subpool.  Return difference.
  			 */
  			ret = delta - spool->rsv_hpages;
  			spool->rsv_hpages = 0;
  		} else {
  			ret = 0;	/* reserves already accounted for */
  			spool->rsv_hpages -= delta;
  		}
  	}
  
  unlock_ret:
  	spin_unlock(&spool->lock);

  	return ret;
  }

  /*
   * Subpool accounting for freeing and unreserving pages.
   * Return the number of global page reservations that must be dropped.
   * The return value may only be different than the passed value (delta)
   * in the case where a subpool minimum size must be maintained.
   */
  static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,

  				       long delta)
  {

  	long ret = delta;

  	if (!spool)

  		return delta;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1)		/* maximum size accounting */
  		spool->used_hpages -= delta;

  	 /* minimum size accounting */
  	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {

  		if (spool->rsv_hpages + delta <= spool->min_hpages)
  			ret = 0;
  		else
  			ret = spool->rsv_hpages + delta - spool->min_hpages;
  
  		spool->rsv_hpages += delta;
  		if (spool->rsv_hpages > spool->min_hpages)
  			spool->rsv_hpages = spool->min_hpages;
  	}
  
  	/*
  	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
  	 * quota reference, free it now.
  	 */

  	unlock_or_release_subpool(spool);

  
  	return ret;

  }
  
  static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
  {
  	return HUGETLBFS_SB(inode->i_sb)->spool;
  }
  
  static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
  {

  	return subpool_inode(file_inode(vma->vm_file));

  }

  /*

   * Region tracking -- allows tracking of reservations and instantiated pages
   *                    across the pages in a mapping.

   *

   * The region data structures are embedded into a resv_map and protected
   * by a resv_map's lock.  The set of regions within the resv_map represent
   * reservations for huge pages, or huge pages that have already been
   * instantiated within the map.  The from and to elements are huge page
   * indicies into the associated mapping.  from indicates the starting index
   * of the region.  to represents the first index past the end of  the region.
   *
   * For example, a file region structure with from == 0 and to == 4 represents
   * four huge pages in a mapping.  It is important to note that the to element
   * represents the first element past the end of the region. This is used in
   * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
   *
   * Interval notation of the form [from, to) will be used to indicate that
   * the endpoint from is inclusive and to is exclusive.

   */
  struct file_region {
  	struct list_head link;
  	long from;
  	long to;
  };

  /*
   * Add the huge page range represented by [f, t) to the reserve

   * map.  In the normal case, existing regions will be expanded
   * to accommodate the specified range.  Sufficient regions should
   * exist for expansion due to the previous call to region_chg
   * with the same range.  However, it is possible that region_del
   * could have been called after region_chg and modifed the map
   * in such a way that no region exists to be expanded.  In this
   * case, pull a region descriptor from the cache associated with
   * the map and use that for the new range.

   *
   * Return the number of new huge pages added to the map.  This
   * number is greater than or equal to zero.

   */

  static long region_add(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg, *trg;

  	long add = 0;

  	spin_lock(&resv->lock);

  	/* Locate the region we are either in or before. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;

  	/*
  	 * If no region exists which can be expanded to include the
  	 * specified range, the list must have been modified by an
  	 * interleving call to region_del().  Pull a region descriptor
  	 * from the cache and use it for this range.
  	 */
  	if (&rg->link == head || t < rg->from) {
  		VM_BUG_ON(resv->region_cache_count <= 0);
  
  		resv->region_cache_count--;
  		nrg = list_first_entry(&resv->region_cache, struct file_region,
  					link);
  		list_del(&nrg->link);
  
  		nrg->from = f;
  		nrg->to = t;
  		list_add(&nrg->link, rg->link.prev);
  
  		add += t - f;
  		goto out_locked;
  	}

  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  
  	/* Check for and consume any regions we now overlap with. */
  	nrg = rg;
  	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)
  			break;
  
  		/* If this area reaches higher then extend our area to
  		 * include it completely.  If this is not the first area
  		 * which we intend to reuse, free it. */
  		if (rg->to > t)
  			t = rg->to;
  		if (rg != nrg) {

  			/* Decrement return value by the deleted range.
  			 * Another range will span this area so that by
  			 * end of routine add will be >= zero
  			 */
  			add -= (rg->to - rg->from);

  			list_del(&rg->link);
  			kfree(rg);
  		}
  	}

  
  	add += (nrg->from - f);		/* Added to beginning of region */

  	nrg->from = f;

  	add += t - nrg->to;		/* Added to end of region */

  	nrg->to = t;

  out_locked:
  	resv->adds_in_progress--;

  	spin_unlock(&resv->lock);

  	VM_BUG_ON(add < 0);
  	return add;

  }

  /*
   * Examine the existing reserve map and determine how many
   * huge pages in the specified range [f, t) are NOT currently
   * represented.  This routine is called before a subsequent
   * call to region_add that will actually modify the reserve
   * map to add the specified range [f, t).  region_chg does
   * not change the number of huge pages represented by the
   * map.  However, if the existing regions in the map can not
   * be expanded to represent the new range, a new file_region
   * structure is added to the map as a placeholder.  This is
   * so that the subsequent region_add call will have all the
   * regions it needs and will not fail.
   *

   * Upon entry, region_chg will also examine the cache of region descriptors
   * associated with the map.  If there are not enough descriptors cached, one
   * will be allocated for the in progress add operation.
   *
   * Returns the number of huge pages that need to be added to the existing
   * reservation map for the range [f, t).  This number is greater or equal to
   * zero.  -ENOMEM is returned if a new file_region structure or cache entry
   * is needed and can not be allocated.

   */

  static long region_chg(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg = NULL;

  	long chg = 0;

  retry:
  	spin_lock(&resv->lock);

  retry_locked:
  	resv->adds_in_progress++;
  
  	/*
  	 * Check for sufficient descriptors in the cache to accommodate
  	 * the number of in progress add operations.
  	 */
  	if (resv->adds_in_progress > resv->region_cache_count) {
  		struct file_region *trg;
  
  		VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
  		/* Must drop lock to allocate a new descriptor. */
  		resv->adds_in_progress--;
  		spin_unlock(&resv->lock);
  
  		trg = kmalloc(sizeof(*trg), GFP_KERNEL);

  		if (!trg) {
  			kfree(nrg);

  			return -ENOMEM;

  		}

  
  		spin_lock(&resv->lock);
  		list_add(&trg->link, &resv->region_cache);
  		resv->region_cache_count++;
  		goto retry_locked;
  	}

  	/* Locate the region we are before or in. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;
  
  	/* If we are below the current region then a new region is required.
  	 * Subtle, allocate a new region at the position but make it zero
  	 * size such that we can guarantee to record the reservation. */
  	if (&rg->link == head || t < rg->from) {

  		if (!nrg) {

  			resv->adds_in_progress--;

  			spin_unlock(&resv->lock);
  			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  			if (!nrg)
  				return -ENOMEM;
  
  			nrg->from = f;
  			nrg->to   = f;
  			INIT_LIST_HEAD(&nrg->link);
  			goto retry;
  		}

  		list_add(&nrg->link, rg->link.prev);
  		chg = t - f;
  		goto out_nrg;

  	}
  
  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  	chg = t - f;
  
  	/* Check for and consume any regions we now overlap with. */
  	list_for_each_entry(rg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)

  			goto out;

  		/* We overlap with this area, if it extends further than

  		 * us then we must extend ourselves.  Account for its
  		 * existing reservation. */
  		if (rg->to > t) {
  			chg += rg->to - t;
  			t = rg->to;
  		}
  		chg -= rg->to - rg->from;
  	}

  
  out:
  	spin_unlock(&resv->lock);
  	/*  We already know we raced and no longer need the new region */
  	kfree(nrg);
  	return chg;
  out_nrg:
  	spin_unlock(&resv->lock);

  	return chg;
  }

  /*

   * Abort the in progress add operation.  The adds_in_progress field
   * of the resv_map keeps track of the operations in progress between
   * calls to region_chg and region_add.  Operations are sometimes
   * aborted after the call to region_chg.  In such cases, region_abort
   * is called to decrement the adds_in_progress counter.
   *
   * NOTE: The range arguments [f, t) are not needed or used in this
   * routine.  They are kept to make reading the calling code easier as
   * arguments will match the associated region_chg call.
   */
  static void region_abort(struct resv_map *resv, long f, long t)
  {
  	spin_lock(&resv->lock);
  	VM_BUG_ON(!resv->region_cache_count);
  	resv->adds_in_progress--;
  	spin_unlock(&resv->lock);
  }
  
  /*

   * Delete the specified range [f, t) from the reserve map.  If the
   * t parameter is LONG_MAX, this indicates that ALL regions after f
   * should be deleted.  Locate the regions which intersect [f, t)
   * and either trim, delete or split the existing regions.
   *
   * Returns the number of huge pages deleted from the reserve map.
   * In the normal case, the return value is zero or more.  In the
   * case where a region must be split, a new region descriptor must
   * be allocated.  If the allocation fails, -ENOMEM will be returned.
   * NOTE: If the parameter t == LONG_MAX, then we will never split
   * a region and possibly return -ENOMEM.  Callers specifying
   * t == LONG_MAX do not need to check for -ENOMEM error.

   */

  static long region_del(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *trg;

  	struct file_region *nrg = NULL;
  	long del = 0;

  retry:

  	spin_lock(&resv->lock);

  	list_for_each_entry_safe(rg, trg, head, link) {

  		/*
  		 * Skip regions before the range to be deleted.  file_region
  		 * ranges are normally of the form [from, to).  However, there
  		 * may be a "placeholder" entry in the map which is of the form
  		 * (from, to) with from == to.  Check for placeholder entries
  		 * at the beginning of the range to be deleted.
  		 */
  		if (rg->to <= f && (rg->to != rg->from || rg->to != f))

  			continue;

  		if (rg->from >= t)

  			break;

  		if (f > rg->from && t < rg->to) { /* Must split region */
  			/*
  			 * Check for an entry in the cache before dropping
  			 * lock and attempting allocation.
  			 */
  			if (!nrg &&
  			    resv->region_cache_count > resv->adds_in_progress) {
  				nrg = list_first_entry(&resv->region_cache,
  							struct file_region,
  							link);
  				list_del(&nrg->link);
  				resv->region_cache_count--;
  			}

  			if (!nrg) {
  				spin_unlock(&resv->lock);
  				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  				if (!nrg)
  					return -ENOMEM;
  				goto retry;
  			}
  
  			del += t - f;
  
  			/* New entry for end of split region */
  			nrg->from = t;
  			nrg->to = rg->to;
  			INIT_LIST_HEAD(&nrg->link);
  
  			/* Original entry is trimmed */
  			rg->to = f;
  
  			list_add(&nrg->link, &rg->link);
  			nrg = NULL;

  			break;

  		}
  
  		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
  			del += rg->to - rg->from;
  			list_del(&rg->link);
  			kfree(rg);
  			continue;
  		}
  
  		if (f <= rg->from) {	/* Trim beginning of region */
  			del += t - rg->from;
  			rg->from = t;
  		} else {		/* Trim end of region */
  			del += rg->to - f;
  			rg->to = f;
  		}

  	}

  	spin_unlock(&resv->lock);

  	kfree(nrg);
  	return del;

  }

  /*

   * A rare out of memory error was encountered which prevented removal of
   * the reserve map region for a page.  The huge page itself was free'ed
   * and removed from the page cache.  This routine will adjust the subpool
   * usage count, and the global reserve count if needed.  By incrementing
   * these counts, the reserve map entry which could not be deleted will
   * appear as a "reserved" entry instead of simply dangling with incorrect
   * counts.
   */

  void hugetlb_fix_reserve_counts(struct inode *inode)

  {
  	struct hugepage_subpool *spool = subpool_inode(inode);
  	long rsv_adjust;
  
  	rsv_adjust = hugepage_subpool_get_pages(spool, 1);

  	if (rsv_adjust) {

  		struct hstate *h = hstate_inode(inode);
  
  		hugetlb_acct_memory(h, 1);
  	}
  }
  
  /*

   * Count and return the number of huge pages in the reserve map
   * that intersect with the range [f, t).
   */

  static long region_count(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg;
  	long chg = 0;

  	spin_lock(&resv->lock);

  	/* Locate each segment we overlap with, and count that overlap. */
  	list_for_each_entry(rg, head, link) {

  		long seg_from;
  		long seg_to;

  
  		if (rg->to <= f)
  			continue;
  		if (rg->from >= t)
  			break;
  
  		seg_from = max(rg->from, f);
  		seg_to = min(rg->to, t);
  
  		chg += seg_to - seg_from;
  	}

  	spin_unlock(&resv->lock);

  
  	return chg;
  }

  /*

   * Convert the address within this vma to the page offset within

   * the mapping, in pagecache page units; huge pages here.
   */

  static pgoff_t vma_hugecache_offset(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long address)

  {

  	return ((address - vma->vm_start) >> huge_page_shift(h)) +
  			(vma->vm_pgoff >> huge_page_order(h));

  }

  pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
  				     unsigned long address)
  {
  	return vma_hugecache_offset(hstate_vma(vma), vma, address);
  }

  EXPORT_SYMBOL_GPL(linear_hugepage_index);

  /*

   * Return the size of the pages allocated when backing a VMA. In the majority
   * cases this will be same size as used by the page table entries.
   */
  unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
  {
  	struct hstate *hstate;
  
  	if (!is_vm_hugetlb_page(vma))
  		return PAGE_SIZE;
  
  	hstate = hstate_vma(vma);

  	return 1UL << huge_page_shift(hstate);

  }

  EXPORT_SYMBOL_GPL(vma_kernel_pagesize);

  
  /*

   * Return the page size being used by the MMU to back a VMA. In the majority
   * of cases, the page size used by the kernel matches the MMU size. On
   * architectures where it differs, an architecture-specific version of this
   * function is required.
   */
  #ifndef vma_mmu_pagesize
  unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
  {
  	return vma_kernel_pagesize(vma);
  }
  #endif
  
  /*

   * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
   * bits of the reservation map pointer, which are always clear due to
   * alignment.
   */
  #define HPAGE_RESV_OWNER    (1UL << 0)
  #define HPAGE_RESV_UNMAPPED (1UL << 1)

  #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)

  /*
   * These helpers are used to track how many pages are reserved for
   * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
   * is guaranteed to have their future faults succeed.
   *
   * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
   * the reserve counters are updated with the hugetlb_lock held. It is safe
   * to reset the VMA at fork() time as it is not in use yet and there is no
   * chance of the global counters getting corrupted as a result of the values.

   *
   * The private mapping reservation is represented in a subtly different
   * manner to a shared mapping.  A shared mapping has a region map associated
   * with the underlying file, this region map represents the backing file
   * pages which have ever had a reservation assigned which this persists even
   * after the page is instantiated.  A private mapping has a region map
   * associated with the original mmap which is attached to all VMAs which
   * reference it, this region map represents those offsets which have consumed
   * reservation ie. where pages have been instantiated.

   */

  static unsigned long get_vma_private_data(struct vm_area_struct *vma)
  {
  	return (unsigned long)vma->vm_private_data;
  }
  
  static void set_vma_private_data(struct vm_area_struct *vma,
  							unsigned long value)
  {
  	vma->vm_private_data = (void *)value;
  }

  struct resv_map *resv_map_alloc(void)

  {
  	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);

  	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
  
  	if (!resv_map || !rg) {
  		kfree(resv_map);
  		kfree(rg);

  		return NULL;

  	}

  
  	kref_init(&resv_map->refs);

  	spin_lock_init(&resv_map->lock);

  	INIT_LIST_HEAD(&resv_map->regions);

  	resv_map->adds_in_progress = 0;
  
  	INIT_LIST_HEAD(&resv_map->region_cache);
  	list_add(&rg->link, &resv_map->region_cache);
  	resv_map->region_cache_count = 1;

  	return resv_map;
  }

  void resv_map_release(struct kref *ref)

  {
  	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);

  	struct list_head *head = &resv_map->region_cache;
  	struct file_region *rg, *trg;

  
  	/* Clear out any active regions before we release the map. */

  	region_del(resv_map, 0, LONG_MAX);

  
  	/* ... and any entries left in the cache */
  	list_for_each_entry_safe(rg, trg, head, link) {
  		list_del(&rg->link);
  		kfree(rg);
  	}
  
  	VM_BUG_ON(resv_map->adds_in_progress);

  	kfree(resv_map);
  }

  static inline struct resv_map *inode_resv_map(struct inode *inode)
  {
  	return inode->i_mapping->private_data;
  }

  static struct resv_map *vma_resv_map(struct vm_area_struct *vma)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (vma->vm_flags & VM_MAYSHARE) {
  		struct address_space *mapping = vma->vm_file->f_mapping;
  		struct inode *inode = mapping->host;
  
  		return inode_resv_map(inode);
  
  	} else {

  		return (struct resv_map *)(get_vma_private_data(vma) &
  							~HPAGE_RESV_MASK);

  	}

  }

  static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  	set_vma_private_data(vma, (get_vma_private_data(vma) &
  				HPAGE_RESV_MASK) | (unsigned long)map);

  }
  
  static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  
  	set_vma_private_data(vma, get_vma_private_data(vma) | flags);

  }
  
  static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  
  	return (get_vma_private_data(vma) & flag) != 0;

  }

  /* Reset counters to 0 and clear all HPAGE_RESV_* flags */

  void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (!(vma->vm_flags & VM_MAYSHARE))

  		vma->vm_private_data = (void *)0;
  }
  
  /* Returns true if the VMA has associated reserve pages */

  static bool vma_has_reserves(struct vm_area_struct *vma, long chg)

  {

  	if (vma->vm_flags & VM_NORESERVE) {
  		/*
  		 * This address is already reserved by other process(chg == 0),
  		 * so, we should decrement reserved count. Without decrementing,
  		 * reserve count remains after releasing inode, because this
  		 * allocated page will go into page cache and is regarded as
  		 * coming from reserved pool in releasing step.  Currently, we
  		 * don't have any other solution to deal with this situation
  		 * properly, so add work-around here.
  		 */
  		if (vma->vm_flags & VM_MAYSHARE && chg == 0)

  			return true;

  		else

  			return false;

  	}

  
  	/* Shared mappings always use reserves */

  	if (vma->vm_flags & VM_MAYSHARE) {
  		/*
  		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
  		 * be a region map for all pages.  The only situation where
  		 * there is no region map is if a hole was punched via
  		 * fallocate.  In this case, there really are no reverves to
  		 * use.  This situation is indicated if chg != 0.
  		 */
  		if (chg)
  			return false;
  		else
  			return true;
  	}

  
  	/*
  	 * Only the process that called mmap() has reserves for
  	 * private mappings.
  	 */

  	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
  		/*
  		 * Like the shared case above, a hole punch or truncate
  		 * could have been performed on the private mapping.
  		 * Examine the value of chg to determine if reserves
  		 * actually exist or were previously consumed.
  		 * Very Subtle - The value of chg comes from a previous
  		 * call to vma_needs_reserves().  The reserve map for
  		 * private mappings has different (opposite) semantics
  		 * than that of shared mappings.  vma_needs_reserves()
  		 * has already taken this difference in semantics into
  		 * account.  Therefore, the meaning of chg is the same
  		 * as in the shared case above.  Code could easily be
  		 * combined, but keeping it separate draws attention to
  		 * subtle differences.
  		 */
  		if (chg)
  			return false;
  		else
  			return true;
  	}

  	return false;

  }

  static void enqueue_huge_page(struct hstate *h, struct page *page)

  {
  	int nid = page_to_nid(page);

  	list_move(&page->lru, &h->hugepage_freelists[nid]);

  	h->free_huge_pages++;
  	h->free_huge_pages_node[nid]++;

  }

  static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
  {
  	struct page *page;

  	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
  		if (!is_migrate_isolate_page(page))
  			break;
  	/*
  	 * if 'non-isolated free hugepage' not found on the list,
  	 * the allocation fails.
  	 */
  	if (&h->hugepage_freelists[nid] == &page->lru)

  		return NULL;

  	list_move(&page->lru, &h->hugepage_activelist);

  	set_page_refcounted(page);

  	h->free_huge_pages--;
  	h->free_huge_pages_node[nid]--;
  	return page;
  }

  /* Movability of hugepages depends on migration support. */
  static inline gfp_t htlb_alloc_mask(struct hstate *h)
  {

  	if (hugepages_treat_as_movable || hugepage_migration_supported(h))

  		return GFP_HIGHUSER_MOVABLE;
  	else
  		return GFP_HIGHUSER;
  }

  static struct page *dequeue_huge_page_vma(struct hstate *h,
  				struct vm_area_struct *vma,

  				unsigned long address, int avoid_reserve,
  				long chg)

  {

  	struct page *page = NULL;

  	struct mempolicy *mpol;

  	nodemask_t *nodemask;

  	struct zonelist *zonelist;

  	struct zone *zone;
  	struct zoneref *z;

  	unsigned int cpuset_mems_cookie;

  	/*
  	 * A child process with MAP_PRIVATE mappings created by their parent
  	 * have no page reserves. This check ensures that reservations are
  	 * not "stolen". The child may still get SIGKILLed
  	 */

  	if (!vma_has_reserves(vma, chg) &&

  			h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  	/* If reserves cannot be used, ensure enough pages are in the pool */

  	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  retry_cpuset:

  	cpuset_mems_cookie = read_mems_allowed_begin();

  	zonelist = huge_zonelist(vma, address,

  					htlb_alloc_mask(h), &mpol, &nodemask);

  	for_each_zone_zonelist_nodemask(zone, z, zonelist,
  						MAX_NR_ZONES - 1, nodemask) {

  		if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {

  			page = dequeue_huge_page_node(h, zone_to_nid(zone));
  			if (page) {

  				if (avoid_reserve)
  					break;
  				if (!vma_has_reserves(vma, chg))
  					break;

  				SetPagePrivate(page);

  				h->resv_huge_pages--;

  				break;
  			}

  		}

  	}

  	mpol_cond_put(mpol);

  	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))

  		goto retry_cpuset;

  	return page;

  
  err:

  	return NULL;

  }

  /*
   * common helper functions for hstate_next_node_to_{alloc|free}.
   * We may have allocated or freed a huge page based on a different
   * nodes_allowed previously, so h->next_node_to_{alloc|free} might
   * be outside of *nodes_allowed.  Ensure that we use an allowed
   * node for alloc or free.
   */
  static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
  {

  	nid = next_node_in(nid, *nodes_allowed);

  	VM_BUG_ON(nid >= MAX_NUMNODES);
  
  	return nid;
  }
  
  static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
  {
  	if (!node_isset(nid, *nodes_allowed))
  		nid = next_node_allowed(nid, nodes_allowed);
  	return nid;
  }
  
  /*
   * returns the previously saved node ["this node"] from which to
   * allocate a persistent huge page for the pool and advance the
   * next node from which to allocate, handling wrap at end of node
   * mask.
   */
  static int hstate_next_node_to_alloc(struct hstate *h,
  					nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
  	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  /*
   * helper for free_pool_huge_page() - return the previously saved
   * node ["this node"] from which to free a huge page.  Advance the
   * next node id whether or not we find a free huge page to free so
   * that the next attempt to free addresses the next node.
   */
  static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
  	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
  		nr_nodes--)
  
  #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
  		nr_nodes--)

  #if defined(CONFIG_ARCH_HAS_GIGANTIC_PAGE) && \

  	((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
  	defined(CONFIG_CMA))

  static void destroy_compound_gigantic_page(struct page *page,

  					unsigned int order)

  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;

  	atomic_set(compound_mapcount_ptr(page), 0);

  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

  		clear_compound_head(p);

  		set_page_refcounted(p);

  	}
  
  	set_compound_order(page, 0);
  	__ClearPageHead(page);
  }

  static void free_gigantic_page(struct page *page, unsigned int order)

  {
  	free_contig_range(page_to_pfn(page), 1 << order);
  }
  
  static int __alloc_gigantic_page(unsigned long start_pfn,
  				unsigned long nr_pages)
  {
  	unsigned long end_pfn = start_pfn + nr_pages;
  	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  }

  static bool pfn_range_valid_gigantic(struct zone *z,
  			unsigned long start_pfn, unsigned long nr_pages)

  {
  	unsigned long i, end_pfn = start_pfn + nr_pages;
  	struct page *page;
  
  	for (i = start_pfn; i < end_pfn; i++) {
  		if (!pfn_valid(i))
  			return false;
  
  		page = pfn_to_page(i);

  		if (page_zone(page) != z)
  			return false;

  		if (PageReserved(page))
  			return false;
  
  		if (page_count(page) > 0)
  			return false;
  
  		if (PageHuge(page))
  			return false;
  	}
  
  	return true;
  }
  
  static bool zone_spans_last_pfn(const struct zone *zone,
  			unsigned long start_pfn, unsigned long nr_pages)
  {
  	unsigned long last_pfn = start_pfn + nr_pages - 1;
  	return zone_spans_pfn(zone, last_pfn);
  }

  static struct page *alloc_gigantic_page(int nid, unsigned int order)

  {
  	unsigned long nr_pages = 1 << order;
  	unsigned long ret, pfn, flags;
  	struct zone *z;
  
  	z = NODE_DATA(nid)->node_zones;
  	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
  		spin_lock_irqsave(&z->lock, flags);
  
  		pfn = ALIGN(z->zone_start_pfn, nr_pages);
  		while (zone_spans_last_pfn(z, pfn, nr_pages)) {

  			if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {

  				/*
  				 * We release the zone lock here because
  				 * alloc_contig_range() will also lock the zone
  				 * at some point. If there's an allocation
  				 * spinning on this lock, it may win the race
  				 * and cause alloc_contig_range() to fail...
  				 */
  				spin_unlock_irqrestore(&z->lock, flags);
  				ret = __alloc_gigantic_page(pfn, nr_pages);
  				if (!ret)
  					return pfn_to_page(pfn);
  				spin_lock_irqsave(&z->lock, flags);
  			}
  			pfn += nr_pages;
  		}
  
  		spin_unlock_irqrestore(&z->lock, flags);
  	}
  
  	return NULL;
  }
  
  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);

  static void prep_compound_gigantic_page(struct page *page, unsigned int order);

  
  static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
  {
  	struct page *page;
  
  	page = alloc_gigantic_page(nid, huge_page_order(h));
  	if (page) {
  		prep_compound_gigantic_page(page, huge_page_order(h));
  		prep_new_huge_page(h, page, nid);
  	}
  
  	return page;
  }
  
  static int alloc_fresh_gigantic_page(struct hstate *h,
  				nodemask_t *nodes_allowed)
  {
  	struct page *page = NULL;
  	int nr_nodes, node;
  
  	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  		page = alloc_fresh_gigantic_page_node(h, node);
  		if (page)
  			return 1;
  	}
  
  	return 0;
  }
  
  static inline bool gigantic_page_supported(void) { return true; }
  #else
  static inline bool gigantic_page_supported(void) { return false; }

  static inline void free_gigantic_page(struct page *page, unsigned int order) { }

  static inline void destroy_compound_gigantic_page(struct page *page,

  						unsigned int order) { }

  static inline int alloc_fresh_gigantic_page(struct hstate *h,
  					nodemask_t *nodes_allowed) { return 0; }
  #endif

  static void update_and_free_page(struct hstate *h, struct page *page)

  {
  	int i;

  	if (hstate_is_gigantic(h) && !gigantic_page_supported())
  		return;

  	h->nr_huge_pages--;
  	h->nr_huge_pages_node[page_to_nid(page)]--;
  	for (i = 0; i < pages_per_huge_page(h); i++) {

  		page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
  				1 << PG_referenced | 1 << PG_dirty |

  				1 << PG_active | 1 << PG_private |
  				1 << PG_writeback);

  	}

  	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);

  	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);

  	set_page_refcounted(page);

  	if (hstate_is_gigantic(h)) {
  		destroy_compound_gigantic_page(page, huge_page_order(h));
  		free_gigantic_page(page, huge_page_order(h));
  	} else {

  		__free_pages(page, huge_page_order(h));
  	}

  }

  struct hstate *size_to_hstate(unsigned long size)
  {
  	struct hstate *h;
  
  	for_each_hstate(h) {
  		if (huge_page_size(h) == size)
  			return h;
  	}
  	return NULL;
  }

  /*
   * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
   * to hstate->hugepage_activelist.)
   *
   * This function can be called for tail pages, but never returns true for them.
   */
  bool page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHuge(page), page);
  	return PageHead(page) && PagePrivate(&page[1]);
  }
  
  /* never called for tail page */
  static void set_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	SetPagePrivate(&page[1]);
  }
  
  static void clear_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	ClearPagePrivate(&page[1]);
  }

  void free_huge_page(struct page *page)

  {

  	/*
  	 * Can't pass hstate in here because it is called from the
  	 * compound page destructor.
  	 */

  	struct hstate *h = page_hstate(page);

  	int nid = page_to_nid(page);

  	struct hugepage_subpool *spool =
  		(struct hugepage_subpool *)page_private(page);

  	bool restore_reserve;

  	set_page_private(page, 0);

  	page->mapping = NULL;

  	VM_BUG_ON_PAGE(page_count(page), page);
  	VM_BUG_ON_PAGE(page_mapcount(page), page);

  	restore_reserve = PagePrivate(page);

  	ClearPagePrivate(page);

  	/*
  	 * A return code of zero implies that the subpool will be under its
  	 * minimum size if the reservation is not restored after page is free.
  	 * Therefore, force restore_reserve operation.
  	 */
  	if (hugepage_subpool_put_pages(spool, 1) == 0)
  		restore_reserve = true;

  	spin_lock(&hugetlb_lock);

  	clear_page_huge_active(page);

  	hugetlb_cgroup_uncharge_page(hstate_index(h),
  				     pages_per_huge_page(h), page);

  	if (restore_reserve)
  		h->resv_huge_pages++;

  	if (h->surplus_huge_pages_node[nid]) {

  		/* remove the page from active list */
  		list_del(&page->lru);

  		update_and_free_page(h, page);
  		h->surplus_huge_pages--;
  		h->surplus_huge_pages_node[nid]--;

  	} else {

  		arch_clear_hugepage_flags(page);

  		enqueue_huge_page(h, page);

  	}

  	spin_unlock(&hugetlb_lock);
  }

  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)

  {

  	INIT_LIST_HEAD(&page->lru);

  	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);

  	spin_lock(&hugetlb_lock);

  	set_hugetlb_cgroup(page, NULL);

  	h->nr_huge_pages++;
  	h->nr_huge_pages_node[nid]++;

  	spin_unlock(&hugetlb_lock);
  	put_page(page); /* free it into the hugepage allocator */
  }

  static void prep_compound_gigantic_page(struct page *page, unsigned int order)

  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;
  
  	/* we rely on prep_new_huge_page to set the destructor */
  	set_compound_order(page, order);

  	__ClearPageReserved(page);

  	__SetPageHead(page);

  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

  		/*
  		 * For gigantic hugepages allocated through bootmem at
  		 * boot, it's safer to be consistent with the not-gigantic
  		 * hugepages and clear the PG_reserved bit from all tail pages
  		 * too.  Otherwse drivers using get_user_pages() to access tail
  		 * pages may get the reference counting wrong if they see
  		 * PG_reserved set on a tail page (despite the head page not
  		 * having PG_reserved set).  Enforcing this consistency between
  		 * head and tail pages allows drivers to optimize away a check
  		 * on the head page when they need know if put_page() is needed
  		 * after get_user_pages().
  		 */
  		__ClearPageReserved(p);

  		set_page_count(p, 0);

  		set_compound_head(p, page);

  	}

  	atomic_set(compound_mapcount_ptr(page), -1);

  }

  /*
   * PageHuge() only returns true for hugetlbfs pages, but not for normal or
   * transparent huge pages.  See the PageTransHuge() documentation for more
   * details.
   */

  int PageHuge(struct page *page)
  {

  	if (!PageCompound(page))
  		return 0;
  
  	page = compound_head(page);

  	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;

  }

  EXPORT_SYMBOL_GPL(PageHuge);

  /*
   * PageHeadHuge() only returns true for hugetlbfs head page, but not for
   * normal or transparent huge pages.
   */
  int PageHeadHuge(struct page *page_head)
  {

  	if (!PageHead(page_head))
  		return 0;

  	return get_compound_page_dtor(page_head) == free_huge_page;

  }

  pgoff_t __basepage_index(struct page *page)
  {
  	struct page *page_head = compound_head(page);
  	pgoff_t index = page_index(page_head);
  	unsigned long compound_idx;
  
  	if (!PageHuge(page_head))
  		return page_index(page);
  
  	if (compound_order(page_head) >= MAX_ORDER)
  		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
  	else
  		compound_idx = page - page_head;
  
  	return (index << compound_order(page_head)) + compound_idx;
  }

  static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)

  {

  	struct page *page;

  	page = __alloc_pages_node(nid,

  		htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|

  						__GFP_REPEAT|__GFP_NOWARN,

  		huge_page_order(h));

  	if (page) {

  		prep_new_huge_page(h, page, nid);

  	}

  
  	return page;
  }

  static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
  {
  	struct page *page;
  	int nr_nodes, node;
  	int ret = 0;
  
  	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  		page = alloc_fresh_huge_page_node(h, node);
  		if (page) {
  			ret = 1;
  			break;
  		}
  	}
  
  	if (ret)
  		count_vm_event(HTLB_BUDDY_PGALLOC);
  	else
  		count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
  
  	return ret;
  }

  /*
   * Free huge page from pool from next node to free.
   * Attempt to keep persistent huge pages more or less
   * balanced over allowed nodes.
   * Called with hugetlb_lock locked.
   */

  static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
  							 bool acct_surplus)

  {

  	int nr_nodes, node;

  	int ret = 0;

  	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {

  		/*
  		 * If we're returning unused surplus pages, only examine
  		 * nodes with surplus pages.
  		 */

  		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
  		    !list_empty(&h->hugepage_freelists[node])) {

  			struct page *page =

  				list_entry(h->hugepage_freelists[node].next,

  					  struct page, lru);
  			list_del(&page->lru);
  			h->free_huge_pages--;

  			h->free_huge_pages_node[node]--;

  			if (acct_surplus) {
  				h->surplus_huge_pages--;

  				h->surplus_huge_pages_node[node]--;

  			}

  			update_and_free_page(h, page);
  			ret = 1;

  			break;

  		}

  	}

  
  	return ret;
  }

  /*
   * Dissolve a given free hugepage into free buddy pages. This function does

   * nothing for in-use (including surplus) hugepages. Returns -EBUSY if the
   * number of free hugepages would be reduced below the number of reserved
   * hugepages.

   */

  static int dissolve_free_huge_page(struct page *page)

  {

  	int rc = 0;

  	spin_lock(&hugetlb_lock);
  	if (PageHuge(page) && !page_count(page)) {

  		struct page *head = compound_head(page);
  		struct hstate *h = page_hstate(head);
  		int nid = page_to_nid(head);

  		if (h->free_huge_pages - h->resv_huge_pages == 0) {
  			rc = -EBUSY;
  			goto out;
  		}

  		list_del(&head->lru);

  		h->free_huge_pages--;
  		h->free_huge_pages_node[nid]--;

  		h->max_huge_pages--;

  		update_and_free_page(h, head);

  	}

  out:

  	spin_unlock(&hugetlb_lock);

  	return rc;

  }
  
  /*
   * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
   * make specified memory blocks removable from the system.

   * Note that this will dissolve a free gigantic hugepage completely, if any
   * part of it lies within the given range.

   * Also note that if dissolve_free_huge_page() returns with an error, all
   * free hugepages that were dissolved before that error are lost.

   */

  int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)

  {

  	unsigned long pfn;

  	struct page *page;

  	int rc = 0;

  	if (!hugepages_supported())

  		return rc;

  	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
  		page = pfn_to_page(pfn);
  		if (PageHuge(page) && !page_count(page)) {
  			rc = dissolve_free_huge_page(page);
  			if (rc)
  				break;
  		}
  	}

  
  	return rc;

  }

  /*
   * There are 3 ways this can get called:
   * 1. With vma+addr: we use the VMA's memory policy
   * 2. With !vma, but nid=NUMA_NO_NODE:  We try to allocate a huge
   *    page from any node, and let the buddy allocator itself figure
   *    it out.
   * 3. With !vma, but nid!=NUMA_NO_NODE.  We allocate a huge page
   *    strictly from 'nid'
   */
  static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
  		struct vm_area_struct *vma, unsigned long addr, int nid)
  {
  	int order = huge_page_order(h);
  	gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
  	unsigned int cpuset_mems_cookie;
  
  	/*
  	 * We need a VMA to get a memory policy.  If we do not

  	 * have one, we use the 'nid' argument.
  	 *
  	 * The mempolicy stuff below has some non-inlined bits
  	 * and calls ->vm_ops.  That makes it hard to optimize at
  	 * compile-time, even when NUMA is off and it does
  	 * nothing.  This helps the compiler optimize it out.

  	 */

  	if (!IS_ENABLED(CONFIG_NUMA) || !vma) {

  		/*
  		 * If a specific node is requested, make sure to
  		 * get memory from there, but only when a node
  		 * is explicitly specified.
  		 */
  		if (nid != NUMA_NO_NODE)
  			gfp |= __GFP_THISNODE;
  		/*
  		 * Make sure to call something that can handle
  		 * nid=NUMA_NO_NODE
  		 */
  		return alloc_pages_node(nid, gfp, order);
  	}
  
  	/*
  	 * OK, so we have a VMA.  Fetch the mempolicy and try to

  	 * allocate a huge page with it.  We will only reach this
  	 * when CONFIG_NUMA=y.

  	 */
  	do {
  		struct page *page;
  		struct mempolicy *mpol;
  		struct zonelist *zl;
  		nodemask_t *nodemask;
  
  		cpuset_mems_cookie = read_mems_allowed_begin();
  		zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
  		mpol_cond_put(mpol);
  		page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
  		if (page)
  			return page;
  	} while (read_mems_allowed_retry(cpuset_mems_cookie));
  
  	return NULL;
  }
  
  /*
   * There are two ways to allocate a huge page:
   * 1. When you have a VMA and an address (like a fault)
   * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
   *
   * 'vma' and 'addr' are only for (1).  'nid' is always NUMA_NO_NODE in
   * this case which signifies that the allocation should be done with
   * respect for the VMA's memory policy.
   *
   * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
   * implies that memory policies will not be taken in to account.
   */
  static struct page *__alloc_buddy_huge_page(struct hstate *h,
  		struct vm_area_struct *vma, unsigned long addr, int nid)

  {
  	struct page *page;

  	unsigned int r_nid;

  	if (hstate_is_gigantic(h))

  		return NULL;

  	/*

  	 * Make sure that anyone specifying 'nid' is not also specifying a VMA.
  	 * This makes sure the caller is picking _one_ of the modes with which
  	 * we can call this function, not both.
  	 */
  	if (vma || (addr != -1)) {

  		VM_WARN_ON_ONCE(addr == -1);
  		VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);

  	}
  	/*

  	 * Assume we will successfully allocate the surplus page to
  	 * prevent racing processes from causing the surplus to exceed
  	 * overcommit
  	 *
  	 * This however introduces a different race, where a process B
  	 * tries to grow the static hugepage pool while alloc_pages() is
  	 * called by process A. B will only examine the per-node
  	 * counters in determining if surplus huge pages can be
  	 * converted to normal huge pages in adjust_pool_surplus(). A
  	 * won't be able to increment the per-node counter, until the
  	 * lock is dropped by B, but B doesn't drop hugetlb_lock until
  	 * no more huge pages can be converted from surplus to normal
  	 * state (and doesn't try to convert again). Thus, we have a
  	 * case where a surplus huge page exists, the pool is grown, and
  	 * the surplus huge page still exists after, even though it
  	 * should just have been converted to a normal huge page. This
  	 * does not leak memory, though, as the hugepage will be freed
  	 * once it is out of use. It also does not allow the counters to
  	 * go out of whack in adjust_pool_surplus() as we don't modify
  	 * the node values until we've gotten the hugepage and only the
  	 * per-node value is checked there.
  	 */
  	spin_lock(&hugetlb_lock);

  	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {

  		spin_unlock(&hugetlb_lock);
  		return NULL;
  	} else {

  		h->nr_huge_pages++;
  		h->surplus_huge_pages++;

  	}
  	spin_unlock(&hugetlb_lock);

  	page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);

  
  	spin_lock(&hugetlb_lock);

  	if (page) {

  		INIT_LIST_HEAD(&page->lru);

  		r_nid = page_to_nid(page);

  		set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);

  		set_hugetlb_cgroup(page, NULL);

  		/*
  		 * We incremented the global counters already
  		 */

  		h->nr_huge_pages_node[r_nid]++;
  		h->surplus_huge_pages_node[r_nid]++;

  		__count_vm_event(HTLB_BUDDY_PGALLOC);

  	} else {

  		h->nr_huge_pages--;
  		h->surplus_huge_pages--;

  		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);

  	}

  	spin_unlock(&hugetlb_lock);

  
  	return page;
  }

  /*

   * Allocate a huge page from 'nid'.  Note, 'nid' may be
   * NUMA_NO_NODE, which means that it may be allocated
   * anywhere.
   */

  static

  struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
  {
  	unsigned long addr = -1;
  
  	return __alloc_buddy_huge_page(h, NULL, addr, nid);
  }
  
  /*
   * Use the VMA's mpolicy to allocate a huge page from the buddy.
   */

  static

  struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
  		struct vm_area_struct *vma, unsigned long addr)
  {
  	return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
  }
  
  /*

   * This allocation function is useful in the context where vma is irrelevant.
   * E.g. soft-offlining uses this function because it only cares physical
   * address of error page.
   */
  struct page *alloc_huge_page_node(struct hstate *h, int nid)
  {

  	struct page *page = NULL;

  
  	spin_lock(&hugetlb_lock);

  	if (h->free_huge_pages - h->resv_huge_pages > 0)
  		page = dequeue_huge_page_node(h, nid);

  	spin_unlock(&hugetlb_lock);

  	if (!page)

  		page = __alloc_buddy_huge_page_no_mpol(h, nid);

  
  	return page;
  }
  
  /*

   * Increase the hugetlb pool such that it can accommodate a reservation

   * of size 'delta'.
   */

  static int gather_surplus_pages(struct hstate *h, int delta)

  {
  	struct list_head surplus_list;
  	struct page *page, *tmp;
  	int ret, i;
  	int needed, allocated;

  	bool alloc_ok = true;

  	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;

  	if (needed <= 0) {

  		h->resv_huge_pages += delta;

  		return 0;

  	}

  
  	allocated = 0;
  	INIT_LIST_HEAD(&surplus_list);
  
  	ret = -ENOMEM;
  retry:
  	spin_unlock(&hugetlb_lock);
  	for (i = 0; i < needed; i++) {

  		page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);

  		if (!page) {
  			alloc_ok = false;
  			break;
  		}

  		list_add(&page->lru, &surplus_list);
  	}

  	allocated += i;

  
  	/*
  	 * After retaking hugetlb_lock, we need to recalculate 'needed'
  	 * because either resv_huge_pages or free_huge_pages may have changed.
  	 */
  	spin_lock(&hugetlb_lock);

  	needed = (h->resv_huge_pages + delta) -
  			(h->free_huge_pages + allocated);

  	if (needed > 0) {
  		if (alloc_ok)
  			goto retry;
  		/*
  		 * We were not able to allocate enough pages to
  		 * satisfy the entire reservation so we free what
  		 * we've allocated so far.
  		 */
  		goto free;
  	}

  	/*
  	 * The surplus_list now contains _at_least_ the number of extra pages

  	 * needed to accommodate the reservation.  Add the appropriate number

  	 * of pages to the hugetlb pool and free the extras back to the buddy

  	 * allocator.  Commit the entire reservation here to prevent another
  	 * process from stealing the pages as they are added to the pool but
  	 * before they are reserved.

  	 */
  	needed += allocated;

  	h->resv_huge_pages += delta;

  	ret = 0;

  	/* Free the needed pages to the hugetlb pool */

  	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

  		if ((--needed) < 0)
  			break;

  		/*
  		 * This page is now managed by the hugetlb allocator and has
  		 * no users -- drop the buddy allocator's reference.
  		 */
  		put_page_testzero(page);

  		VM_BUG_ON_PAGE(page_count(page), page);

  		enqueue_huge_page(h, page);

  	}

  free:

  	spin_unlock(&hugetlb_lock);

  
  	/* Free unnecessary surplus pages to the buddy allocator */

  	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
  		put_page(page);

  	spin_lock(&hugetlb_lock);

  
  	return ret;
  }
  
  /*
   * When releasing a hugetlb pool reservation, any surplus pages that were
   * allocated to satisfy the reservation must be explicitly freed if they were
   * never used.

   * Called with hugetlb_lock held.

   */

  static void return_unused_surplus_pages(struct hstate *h,
  					unsigned long unused_resv_pages)

  {

  	unsigned long nr_pages;

  	/* Uncommit the reservation */

  	h->resv_huge_pages -= unused_resv_pages;

  	/* Cannot return gigantic pages currently */

  	if (hstate_is_gigantic(h))

  		return;

  	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);

  	/*
  	 * We want to release as many surplus pages as possible, spread

  	 * evenly across all nodes with memory. Iterate across these nodes
  	 * until we can no longer free unreserved surplus pages. This occurs
  	 * when the nodes with surplus pages have no free pages.
  	 * free_pool_huge_page() will balance the the freed pages across the
  	 * on-line nodes with memory and will handle the hstate accounting.

  	 */
  	while (nr_pages--) {

  		if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))

  			break;

  		cond_resched_lock(&hugetlb_lock);

  	}
  }

  /*

   * vma_needs_reservation, vma_commit_reservation and vma_end_reservation

   * are used by the huge page allocation routines to manage reservations.

   *
   * vma_needs_reservation is called to determine if the huge page at addr
   * within the vma has an associated reservation.  If a reservation is
   * needed, the value 1 is returned.  The caller is then responsible for
   * managing the global reservation and subpool usage counts.  After
   * the huge page has been allocated, vma_commit_reservation is called

   * to add the page to the reservation map.  If the page allocation fails,
   * the reservation must be ended instead of committed.  vma_end_reservation
   * is called in such cases.

   *
   * In the normal case, vma_commit_reservation returns the same value
   * as the preceding vma_needs_reservation call.  The only time this
   * is not the case is if a reserve map was changed between calls.  It
   * is the responsibility of the caller to notice the difference and
   * take appropriate action.

   *
   * vma_add_reservation is used in error paths where a reservation must
   * be restored when a newly allocated huge page must be freed.  It is
   * to be called after calling vma_needs_reservation to determine if a
   * reservation exists.

   */

  enum vma_resv_mode {
  	VMA_NEEDS_RESV,
  	VMA_COMMIT_RESV,

  	VMA_END_RESV,

  	VMA_ADD_RESV,

  };

  static long __vma_reservation_common(struct hstate *h,
  				struct vm_area_struct *vma, unsigned long addr,

  				enum vma_resv_mode mode)

  {

  	struct resv_map *resv;
  	pgoff_t idx;

  	long ret;

  	resv = vma_resv_map(vma);
  	if (!resv)

  		return 1;

  	idx = vma_hugecache_offset(h, vma, addr);

  	switch (mode) {
  	case VMA_NEEDS_RESV:

  		ret = region_chg(resv, idx, idx + 1);

  		break;
  	case VMA_COMMIT_RESV:
  		ret = region_add(resv, idx, idx + 1);
  		break;

  	case VMA_END_RESV:

  		region_abort(resv, idx, idx + 1);
  		ret = 0;
  		break;

  	case VMA_ADD_RESV:
  		if (vma->vm_flags & VM_MAYSHARE)
  			ret = region_add(resv, idx, idx + 1);
  		else {
  			region_abort(resv, idx, idx + 1);
  			ret = region_del(resv, idx, idx + 1);
  		}
  		break;

  	default:
  		BUG();
  	}

  	if (vma->vm_flags & VM_MAYSHARE)

  		return ret;

  	else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
  		/*
  		 * In most cases, reserves always exist for private mappings.
  		 * However, a file associated with mapping could have been
  		 * hole punched or truncated after reserves were consumed.
  		 * As subsequent fault on such a range will not use reserves.
  		 * Subtle - The reserve map for private mappings has the
  		 * opposite meaning than that of shared mappings.  If NO
  		 * entry is in the reserve map, it means a reservation exists.
  		 * If an entry exists in the reserve map, it means the
  		 * reservation has already been consumed.  As a result, the
  		 * return value of this routine is the opposite of the
  		 * value returned from reserve map manipulation routines above.
  		 */
  		if (ret)
  			return 0;
  		else
  			return 1;
  	}

  	else

  		return ret < 0 ? ret : 0;

  }

  
  static long vma_needs_reservation(struct hstate *h,

  			struct vm_area_struct *vma, unsigned long addr)

  {

  	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);

  }

  static long vma_commit_reservation(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long addr)
  {

  	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
  }

  static void vma_end_reservation(struct hstate *h,

  			struct vm_area_struct *vma, unsigned long addr)
  {

  	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);

  }

  static long vma_add_reservation(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long addr)
  {
  	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
  }
  
  /*
   * This routine is called to restore a reservation on error paths.  In the
   * specific error paths, a huge page was allocated (via alloc_huge_page)
   * and is about to be freed.  If a reservation for the page existed,
   * alloc_huge_page would have consumed the reservation and set PagePrivate
   * in the newly allocated page.  When the page is freed via free_huge_page,
   * the global reservation count will be incremented if PagePrivate is set.
   * However, free_huge_page can not adjust the reserve map.  Adjust the
   * reserve map here to be consistent with global reserve count adjustments
   * to be made by free_huge_page.
   */
  static void restore_reserve_on_error(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long address,
  			struct page *page)
  {
  	if (unlikely(PagePrivate(page))) {
  		long rc = vma_needs_reservation(h, vma, address);
  
  		if (unlikely(rc < 0)) {
  			/*
  			 * Rare out of memory condition in reserve map
  			 * manipulation.  Clear PagePrivate so that
  			 * global reserve count will not be incremented
  			 * by free_huge_page.  This will make it appear
  			 * as though the reservation for this page was
  			 * consumed.  This may prevent the task from
  			 * faulting in the page at a later time.  This
  			 * is better than inconsistent global huge page
  			 * accounting of reserve counts.
  			 */
  			ClearPagePrivate(page);
  		} else if (rc) {
  			rc = vma_add_reservation(h, vma, address);
  			if (unlikely(rc < 0))
  				/*
  				 * See above comment about rare out of
  				 * memory condition.
  				 */
  				ClearPagePrivate(page);
  		} else
  			vma_end_reservation(h, vma, address);
  	}
  }

  struct page *alloc_huge_page(struct vm_area_struct *vma,

  				    unsigned long addr, int avoid_reserve)

  {

  	struct hugepage_subpool *spool = subpool_vma(vma);

  	struct hstate *h = hstate_vma(vma);

  	struct page *page;