/*
   * Generic hugetlb support.

   * (C) Nadia Yvette Chambers, April 2004

   */

  #include <linux/list.h>
  #include <linux/init.h>
  #include <linux/module.h>
  #include <linux/mm.h>

  #include <linux/seq_file.h>

  #include <linux/sysctl.h>
  #include <linux/highmem.h>

  #include <linux/mmu_notifier.h>

  #include <linux/nodemask.h>

  #include <linux/pagemap.h>

  #include <linux/mempolicy.h>

  #include <linux/compiler.h>

  #include <linux/cpuset.h>

  #include <linux/mutex.h>

  #include <linux/bootmem.h>

  #include <linux/sysfs.h>

  #include <linux/slab.h>

  #include <linux/rmap.h>

  #include <linux/swap.h>
  #include <linux/swapops.h>

  #include <linux/page-isolation.h>

  #include <linux/jhash.h>

  #include <asm/page.h>
  #include <asm/pgtable.h>

  #include <asm/tlb.h>

  #include <linux/io.h>

  #include <linux/hugetlb.h>

  #include <linux/hugetlb_cgroup.h>

  #include <linux/node.h>

  #include "internal.h"

  int hugepages_treat_as_movable;

  int hugetlb_max_hstate __read_mostly;

  unsigned int default_hstate_idx;
  struct hstate hstates[HUGE_MAX_HSTATE];

  /*
   * Minimum page order among possible hugepage sizes, set to a proper value
   * at boot time.
   */
  static unsigned int minimum_order __read_mostly = UINT_MAX;

  __initdata LIST_HEAD(huge_boot_pages);

  /* for command line parsing */
  static struct hstate * __initdata parsed_hstate;
  static unsigned long __initdata default_hstate_max_huge_pages;

  static unsigned long __initdata default_hstate_size;

  /*

   * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
   * free_huge_pages, and surplus_huge_pages.

   */

  DEFINE_SPINLOCK(hugetlb_lock);

  /*
   * Serializes faults on the same logical page.  This is used to
   * prevent spurious OOMs when the hugepage pool is fully utilized.
   */
  static int num_fault_mutexes;
  static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;

  /* Forward declaration */
  static int hugetlb_acct_memory(struct hstate *h, long delta);

  static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
  {
  	bool free = (spool->count == 0) && (spool->used_hpages == 0);
  
  	spin_unlock(&spool->lock);
  
  	/* If no pages are used, and no other handles to the subpool

  	 * remain, give up any reservations mased on minimum size and
  	 * free the subpool */
  	if (free) {
  		if (spool->min_hpages != -1)
  			hugetlb_acct_memory(spool->hstate,
  						-spool->min_hpages);

  		kfree(spool);

  	}

  }

  struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
  						long min_hpages)

  {
  	struct hugepage_subpool *spool;

  	spool = kzalloc(sizeof(*spool), GFP_KERNEL);

  	if (!spool)
  		return NULL;
  
  	spin_lock_init(&spool->lock);
  	spool->count = 1;

  	spool->max_hpages = max_hpages;
  	spool->hstate = h;
  	spool->min_hpages = min_hpages;
  
  	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
  		kfree(spool);
  		return NULL;
  	}
  	spool->rsv_hpages = min_hpages;

  
  	return spool;
  }
  
  void hugepage_put_subpool(struct hugepage_subpool *spool)
  {
  	spin_lock(&spool->lock);
  	BUG_ON(!spool->count);
  	spool->count--;
  	unlock_or_release_subpool(spool);
  }

  /*
   * Subpool accounting for allocating and reserving pages.
   * Return -ENOMEM if there are not enough resources to satisfy the
   * the request.  Otherwise, return the number of pages by which the
   * global pools must be adjusted (upward).  The returned value may
   * only be different than the passed value (delta) in the case where
   * a subpool minimum size must be manitained.
   */
  static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,

  				      long delta)
  {

  	long ret = delta;

  
  	if (!spool)

  		return ret;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1) {		/* maximum size accounting */
  		if ((spool->used_hpages + delta) <= spool->max_hpages)
  			spool->used_hpages += delta;
  		else {
  			ret = -ENOMEM;
  			goto unlock_ret;
  		}

  	}

  	if (spool->min_hpages != -1) {		/* minimum size accounting */
  		if (delta > spool->rsv_hpages) {
  			/*
  			 * Asking for more reserves than those already taken on
  			 * behalf of subpool.  Return difference.
  			 */
  			ret = delta - spool->rsv_hpages;
  			spool->rsv_hpages = 0;
  		} else {
  			ret = 0;	/* reserves already accounted for */
  			spool->rsv_hpages -= delta;
  		}
  	}
  
  unlock_ret:
  	spin_unlock(&spool->lock);

  	return ret;
  }

  /*
   * Subpool accounting for freeing and unreserving pages.
   * Return the number of global page reservations that must be dropped.
   * The return value may only be different than the passed value (delta)
   * in the case where a subpool minimum size must be maintained.
   */
  static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,

  				       long delta)
  {

  	long ret = delta;

  	if (!spool)

  		return delta;

  
  	spin_lock(&spool->lock);

  
  	if (spool->max_hpages != -1)		/* maximum size accounting */
  		spool->used_hpages -= delta;
  
  	if (spool->min_hpages != -1) {		/* minimum size accounting */
  		if (spool->rsv_hpages + delta <= spool->min_hpages)
  			ret = 0;
  		else
  			ret = spool->rsv_hpages + delta - spool->min_hpages;
  
  		spool->rsv_hpages += delta;
  		if (spool->rsv_hpages > spool->min_hpages)
  			spool->rsv_hpages = spool->min_hpages;
  	}
  
  	/*
  	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
  	 * quota reference, free it now.
  	 */

  	unlock_or_release_subpool(spool);

  
  	return ret;

  }
  
  static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
  {
  	return HUGETLBFS_SB(inode->i_sb)->spool;
  }
  
  static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
  {

  	return subpool_inode(file_inode(vma->vm_file));

  }

  /*

   * Region tracking -- allows tracking of reservations and instantiated pages
   *                    across the pages in a mapping.

   *

   * The region data structures are embedded into a resv_map and
   * protected by a resv_map's lock

   */
  struct file_region {
  	struct list_head link;
  	long from;
  	long to;
  };

  static long region_add(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg, *trg;

  	spin_lock(&resv->lock);

  	/* Locate the region we are either in or before. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;
  
  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  
  	/* Check for and consume any regions we now overlap with. */
  	nrg = rg;
  	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)
  			break;
  
  		/* If this area reaches higher then extend our area to
  		 * include it completely.  If this is not the first area
  		 * which we intend to reuse, free it. */
  		if (rg->to > t)
  			t = rg->to;
  		if (rg != nrg) {
  			list_del(&rg->link);
  			kfree(rg);
  		}
  	}
  	nrg->from = f;
  	nrg->to = t;

  	spin_unlock(&resv->lock);

  	return 0;
  }

  static long region_chg(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *nrg = NULL;

  	long chg = 0;

  retry:
  	spin_lock(&resv->lock);

  	/* Locate the region we are before or in. */
  	list_for_each_entry(rg, head, link)
  		if (f <= rg->to)
  			break;
  
  	/* If we are below the current region then a new region is required.
  	 * Subtle, allocate a new region at the position but make it zero
  	 * size such that we can guarantee to record the reservation. */
  	if (&rg->link == head || t < rg->from) {

  		if (!nrg) {
  			spin_unlock(&resv->lock);
  			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  			if (!nrg)
  				return -ENOMEM;
  
  			nrg->from = f;
  			nrg->to   = f;
  			INIT_LIST_HEAD(&nrg->link);
  			goto retry;
  		}

  		list_add(&nrg->link, rg->link.prev);
  		chg = t - f;
  		goto out_nrg;

  	}
  
  	/* Round our left edge to the current segment if it encloses us. */
  	if (f > rg->from)
  		f = rg->from;
  	chg = t - f;
  
  	/* Check for and consume any regions we now overlap with. */
  	list_for_each_entry(rg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		if (rg->from > t)

  			goto out;

  		/* We overlap with this area, if it extends further than

  		 * us then we must extend ourselves.  Account for its
  		 * existing reservation. */
  		if (rg->to > t) {
  			chg += rg->to - t;
  			t = rg->to;
  		}
  		chg -= rg->to - rg->from;
  	}

  
  out:
  	spin_unlock(&resv->lock);
  	/*  We already know we raced and no longer need the new region */
  	kfree(nrg);
  	return chg;
  out_nrg:
  	spin_unlock(&resv->lock);

  	return chg;
  }

  static long region_truncate(struct resv_map *resv, long end)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg, *trg;
  	long chg = 0;

  	spin_lock(&resv->lock);

  	/* Locate the region we are either in or before. */
  	list_for_each_entry(rg, head, link)
  		if (end <= rg->to)
  			break;
  	if (&rg->link == head)

  		goto out;

  
  	/* If we are in the middle of a region then adjust it. */
  	if (end > rg->from) {
  		chg = rg->to - end;
  		rg->to = end;
  		rg = list_entry(rg->link.next, typeof(*rg), link);
  	}
  
  	/* Drop any remaining regions. */
  	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  		if (&rg->link == head)
  			break;
  		chg += rg->to - rg->from;
  		list_del(&rg->link);
  		kfree(rg);
  	}

  
  out:
  	spin_unlock(&resv->lock);

  	return chg;
  }

  static long region_count(struct resv_map *resv, long f, long t)

  {

  	struct list_head *head = &resv->regions;

  	struct file_region *rg;
  	long chg = 0;

  	spin_lock(&resv->lock);

  	/* Locate each segment we overlap with, and count that overlap. */
  	list_for_each_entry(rg, head, link) {

  		long seg_from;
  		long seg_to;

  
  		if (rg->to <= f)
  			continue;
  		if (rg->from >= t)
  			break;
  
  		seg_from = max(rg->from, f);
  		seg_to = min(rg->to, t);
  
  		chg += seg_to - seg_from;
  	}

  	spin_unlock(&resv->lock);

  
  	return chg;
  }

  /*

   * Convert the address within this vma to the page offset within

   * the mapping, in pagecache page units; huge pages here.
   */

  static pgoff_t vma_hugecache_offset(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long address)

  {

  	return ((address - vma->vm_start) >> huge_page_shift(h)) +
  			(vma->vm_pgoff >> huge_page_order(h));

  }

  pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
  				     unsigned long address)
  {
  	return vma_hugecache_offset(hstate_vma(vma), vma, address);
  }

  /*

   * Return the size of the pages allocated when backing a VMA. In the majority
   * cases this will be same size as used by the page table entries.
   */
  unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
  {
  	struct hstate *hstate;
  
  	if (!is_vm_hugetlb_page(vma))
  		return PAGE_SIZE;
  
  	hstate = hstate_vma(vma);

  	return 1UL << huge_page_shift(hstate);

  }

  EXPORT_SYMBOL_GPL(vma_kernel_pagesize);

  
  /*

   * Return the page size being used by the MMU to back a VMA. In the majority
   * of cases, the page size used by the kernel matches the MMU size. On
   * architectures where it differs, an architecture-specific version of this
   * function is required.
   */
  #ifndef vma_mmu_pagesize
  unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
  {
  	return vma_kernel_pagesize(vma);
  }
  #endif
  
  /*

   * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
   * bits of the reservation map pointer, which are always clear due to
   * alignment.
   */
  #define HPAGE_RESV_OWNER    (1UL << 0)
  #define HPAGE_RESV_UNMAPPED (1UL << 1)

  #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)

  /*
   * These helpers are used to track how many pages are reserved for
   * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
   * is guaranteed to have their future faults succeed.
   *
   * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
   * the reserve counters are updated with the hugetlb_lock held. It is safe
   * to reset the VMA at fork() time as it is not in use yet and there is no
   * chance of the global counters getting corrupted as a result of the values.

   *
   * The private mapping reservation is represented in a subtly different
   * manner to a shared mapping.  A shared mapping has a region map associated
   * with the underlying file, this region map represents the backing file
   * pages which have ever had a reservation assigned which this persists even
   * after the page is instantiated.  A private mapping has a region map
   * associated with the original mmap which is attached to all VMAs which
   * reference it, this region map represents those offsets which have consumed
   * reservation ie. where pages have been instantiated.

   */

  static unsigned long get_vma_private_data(struct vm_area_struct *vma)
  {
  	return (unsigned long)vma->vm_private_data;
  }
  
  static void set_vma_private_data(struct vm_area_struct *vma,
  							unsigned long value)
  {
  	vma->vm_private_data = (void *)value;
  }

  struct resv_map *resv_map_alloc(void)

  {
  	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
  	if (!resv_map)
  		return NULL;
  
  	kref_init(&resv_map->refs);

  	spin_lock_init(&resv_map->lock);

  	INIT_LIST_HEAD(&resv_map->regions);
  
  	return resv_map;
  }

  void resv_map_release(struct kref *ref)

  {
  	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
  
  	/* Clear out any active regions before we release the map. */

  	region_truncate(resv_map, 0);

  	kfree(resv_map);
  }

  static inline struct resv_map *inode_resv_map(struct inode *inode)
  {
  	return inode->i_mapping->private_data;
  }

  static struct resv_map *vma_resv_map(struct vm_area_struct *vma)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (vma->vm_flags & VM_MAYSHARE) {
  		struct address_space *mapping = vma->vm_file->f_mapping;
  		struct inode *inode = mapping->host;
  
  		return inode_resv_map(inode);
  
  	} else {

  		return (struct resv_map *)(get_vma_private_data(vma) &
  							~HPAGE_RESV_MASK);

  	}

  }

  static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)

  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  	set_vma_private_data(vma, (get_vma_private_data(vma) &
  				HPAGE_RESV_MASK) | (unsigned long)map);

  }
  
  static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
  	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);

  
  	set_vma_private_data(vma, get_vma_private_data(vma) | flags);

  }
  
  static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  
  	return (get_vma_private_data(vma) & flag) != 0;

  }

  /* Reset counters to 0 and clear all HPAGE_RESV_* flags */

  void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
  {

  	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);

  	if (!(vma->vm_flags & VM_MAYSHARE))

  		vma->vm_private_data = (void *)0;
  }
  
  /* Returns true if the VMA has associated reserve pages */

  static int vma_has_reserves(struct vm_area_struct *vma, long chg)

  {

  	if (vma->vm_flags & VM_NORESERVE) {
  		/*
  		 * This address is already reserved by other process(chg == 0),
  		 * so, we should decrement reserved count. Without decrementing,
  		 * reserve count remains after releasing inode, because this
  		 * allocated page will go into page cache and is regarded as
  		 * coming from reserved pool in releasing step.  Currently, we
  		 * don't have any other solution to deal with this situation
  		 * properly, so add work-around here.
  		 */
  		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
  			return 1;
  		else
  			return 0;
  	}

  
  	/* Shared mappings always use reserves */

  	if (vma->vm_flags & VM_MAYSHARE)

  		return 1;

  
  	/*
  	 * Only the process that called mmap() has reserves for
  	 * private mappings.
  	 */

  	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
  		return 1;

  	return 0;

  }

  static void enqueue_huge_page(struct hstate *h, struct page *page)

  {
  	int nid = page_to_nid(page);

  	list_move(&page->lru, &h->hugepage_freelists[nid]);

  	h->free_huge_pages++;
  	h->free_huge_pages_node[nid]++;

  }

  static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
  {
  	struct page *page;

  	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
  		if (!is_migrate_isolate_page(page))
  			break;
  	/*
  	 * if 'non-isolated free hugepage' not found on the list,
  	 * the allocation fails.
  	 */
  	if (&h->hugepage_freelists[nid] == &page->lru)

  		return NULL;

  	list_move(&page->lru, &h->hugepage_activelist);

  	set_page_refcounted(page);

  	h->free_huge_pages--;
  	h->free_huge_pages_node[nid]--;
  	return page;
  }

  /* Movability of hugepages depends on migration support. */
  static inline gfp_t htlb_alloc_mask(struct hstate *h)
  {

  	if (hugepages_treat_as_movable || hugepage_migration_supported(h))

  		return GFP_HIGHUSER_MOVABLE;
  	else
  		return GFP_HIGHUSER;
  }

  static struct page *dequeue_huge_page_vma(struct hstate *h,
  				struct vm_area_struct *vma,

  				unsigned long address, int avoid_reserve,
  				long chg)

  {

  	struct page *page = NULL;

  	struct mempolicy *mpol;

  	nodemask_t *nodemask;

  	struct zonelist *zonelist;

  	struct zone *zone;
  	struct zoneref *z;

  	unsigned int cpuset_mems_cookie;

  	/*
  	 * A child process with MAP_PRIVATE mappings created by their parent
  	 * have no page reserves. This check ensures that reservations are
  	 * not "stolen". The child may still get SIGKILLed
  	 */

  	if (!vma_has_reserves(vma, chg) &&

  			h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  	/* If reserves cannot be used, ensure enough pages are in the pool */

  	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)

  		goto err;

  retry_cpuset:

  	cpuset_mems_cookie = read_mems_allowed_begin();

  	zonelist = huge_zonelist(vma, address,

  					htlb_alloc_mask(h), &mpol, &nodemask);

  	for_each_zone_zonelist_nodemask(zone, z, zonelist,
  						MAX_NR_ZONES - 1, nodemask) {

  		if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {

  			page = dequeue_huge_page_node(h, zone_to_nid(zone));
  			if (page) {

  				if (avoid_reserve)
  					break;
  				if (!vma_has_reserves(vma, chg))
  					break;

  				SetPagePrivate(page);

  				h->resv_huge_pages--;

  				break;
  			}

  		}

  	}

  	mpol_cond_put(mpol);

  	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))

  		goto retry_cpuset;

  	return page;

  
  err:

  	return NULL;

  }

  /*
   * common helper functions for hstate_next_node_to_{alloc|free}.
   * We may have allocated or freed a huge page based on a different
   * nodes_allowed previously, so h->next_node_to_{alloc|free} might
   * be outside of *nodes_allowed.  Ensure that we use an allowed
   * node for alloc or free.
   */
  static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
  {
  	nid = next_node(nid, *nodes_allowed);
  	if (nid == MAX_NUMNODES)
  		nid = first_node(*nodes_allowed);
  	VM_BUG_ON(nid >= MAX_NUMNODES);
  
  	return nid;
  }
  
  static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
  {
  	if (!node_isset(nid, *nodes_allowed))
  		nid = next_node_allowed(nid, nodes_allowed);
  	return nid;
  }
  
  /*
   * returns the previously saved node ["this node"] from which to
   * allocate a persistent huge page for the pool and advance the
   * next node from which to allocate, handling wrap at end of node
   * mask.
   */
  static int hstate_next_node_to_alloc(struct hstate *h,
  					nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
  	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  /*
   * helper for free_pool_huge_page() - return the previously saved
   * node ["this node"] from which to free a huge page.  Advance the
   * next node id whether or not we find a free huge page to free so
   * that the next attempt to free addresses the next node.
   */
  static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
  {
  	int nid;
  
  	VM_BUG_ON(!nodes_allowed);
  
  	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
  	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
  
  	return nid;
  }
  
  #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
  		nr_nodes--)
  
  #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
  	for (nr_nodes = nodes_weight(*mask);				\
  		nr_nodes > 0 &&						\
  		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
  		nr_nodes--)

  #if defined(CONFIG_CMA) && defined(CONFIG_X86_64)
  static void destroy_compound_gigantic_page(struct page *page,
  					unsigned long order)
  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;
  
  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
  		__ClearPageTail(p);
  		set_page_refcounted(p);
  		p->first_page = NULL;
  	}
  
  	set_compound_order(page, 0);
  	__ClearPageHead(page);
  }
  
  static void free_gigantic_page(struct page *page, unsigned order)
  {
  	free_contig_range(page_to_pfn(page), 1 << order);
  }
  
  static int __alloc_gigantic_page(unsigned long start_pfn,
  				unsigned long nr_pages)
  {
  	unsigned long end_pfn = start_pfn + nr_pages;
  	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
  }
  
  static bool pfn_range_valid_gigantic(unsigned long start_pfn,
  				unsigned long nr_pages)
  {
  	unsigned long i, end_pfn = start_pfn + nr_pages;
  	struct page *page;
  
  	for (i = start_pfn; i < end_pfn; i++) {
  		if (!pfn_valid(i))
  			return false;
  
  		page = pfn_to_page(i);
  
  		if (PageReserved(page))
  			return false;
  
  		if (page_count(page) > 0)
  			return false;
  
  		if (PageHuge(page))
  			return false;
  	}
  
  	return true;
  }
  
  static bool zone_spans_last_pfn(const struct zone *zone,
  			unsigned long start_pfn, unsigned long nr_pages)
  {
  	unsigned long last_pfn = start_pfn + nr_pages - 1;
  	return zone_spans_pfn(zone, last_pfn);
  }
  
  static struct page *alloc_gigantic_page(int nid, unsigned order)
  {
  	unsigned long nr_pages = 1 << order;
  	unsigned long ret, pfn, flags;
  	struct zone *z;
  
  	z = NODE_DATA(nid)->node_zones;
  	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
  		spin_lock_irqsave(&z->lock, flags);
  
  		pfn = ALIGN(z->zone_start_pfn, nr_pages);
  		while (zone_spans_last_pfn(z, pfn, nr_pages)) {
  			if (pfn_range_valid_gigantic(pfn, nr_pages)) {
  				/*
  				 * We release the zone lock here because
  				 * alloc_contig_range() will also lock the zone
  				 * at some point. If there's an allocation
  				 * spinning on this lock, it may win the race
  				 * and cause alloc_contig_range() to fail...
  				 */
  				spin_unlock_irqrestore(&z->lock, flags);
  				ret = __alloc_gigantic_page(pfn, nr_pages);
  				if (!ret)
  					return pfn_to_page(pfn);
  				spin_lock_irqsave(&z->lock, flags);
  			}
  			pfn += nr_pages;
  		}
  
  		spin_unlock_irqrestore(&z->lock, flags);
  	}
  
  	return NULL;
  }
  
  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
  static void prep_compound_gigantic_page(struct page *page, unsigned long order);
  
  static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
  {
  	struct page *page;
  
  	page = alloc_gigantic_page(nid, huge_page_order(h));
  	if (page) {
  		prep_compound_gigantic_page(page, huge_page_order(h));
  		prep_new_huge_page(h, page, nid);
  	}
  
  	return page;
  }
  
  static int alloc_fresh_gigantic_page(struct hstate *h,
  				nodemask_t *nodes_allowed)
  {
  	struct page *page = NULL;
  	int nr_nodes, node;
  
  	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  		page = alloc_fresh_gigantic_page_node(h, node);
  		if (page)
  			return 1;
  	}
  
  	return 0;
  }
  
  static inline bool gigantic_page_supported(void) { return true; }
  #else
  static inline bool gigantic_page_supported(void) { return false; }
  static inline void free_gigantic_page(struct page *page, unsigned order) { }
  static inline void destroy_compound_gigantic_page(struct page *page,
  						unsigned long order) { }
  static inline int alloc_fresh_gigantic_page(struct hstate *h,
  					nodemask_t *nodes_allowed) { return 0; }
  #endif

  static void update_and_free_page(struct hstate *h, struct page *page)

  {
  	int i;

  	if (hstate_is_gigantic(h) && !gigantic_page_supported())
  		return;

  	h->nr_huge_pages--;
  	h->nr_huge_pages_node[page_to_nid(page)]--;
  	for (i = 0; i < pages_per_huge_page(h); i++) {

  		page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
  				1 << PG_referenced | 1 << PG_dirty |

  				1 << PG_active | 1 << PG_private |
  				1 << PG_writeback);

  	}

  	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);

  	set_compound_page_dtor(page, NULL);
  	set_page_refcounted(page);

  	if (hstate_is_gigantic(h)) {
  		destroy_compound_gigantic_page(page, huge_page_order(h));
  		free_gigantic_page(page, huge_page_order(h));
  	} else {
  		arch_release_hugepage(page);
  		__free_pages(page, huge_page_order(h));
  	}

  }

  struct hstate *size_to_hstate(unsigned long size)
  {
  	struct hstate *h;
  
  	for_each_hstate(h) {
  		if (huge_page_size(h) == size)
  			return h;
  	}
  	return NULL;
  }

  /*
   * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
   * to hstate->hugepage_activelist.)
   *
   * This function can be called for tail pages, but never returns true for them.
   */
  bool page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHuge(page), page);
  	return PageHead(page) && PagePrivate(&page[1]);
  }
  
  /* never called for tail page */
  static void set_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	SetPagePrivate(&page[1]);
  }
  
  static void clear_page_huge_active(struct page *page)
  {
  	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
  	ClearPagePrivate(&page[1]);
  }

  void free_huge_page(struct page *page)

  {

  	/*
  	 * Can't pass hstate in here because it is called from the
  	 * compound page destructor.
  	 */

  	struct hstate *h = page_hstate(page);

  	int nid = page_to_nid(page);

  	struct hugepage_subpool *spool =
  		(struct hugepage_subpool *)page_private(page);

  	bool restore_reserve;

  	set_page_private(page, 0);

  	page->mapping = NULL;

  	BUG_ON(page_count(page));

  	BUG_ON(page_mapcount(page));

  	restore_reserve = PagePrivate(page);

  	ClearPagePrivate(page);

  	/*
  	 * A return code of zero implies that the subpool will be under its
  	 * minimum size if the reservation is not restored after page is free.
  	 * Therefore, force restore_reserve operation.
  	 */
  	if (hugepage_subpool_put_pages(spool, 1) == 0)
  		restore_reserve = true;

  	spin_lock(&hugetlb_lock);

  	clear_page_huge_active(page);

  	hugetlb_cgroup_uncharge_page(hstate_index(h),
  				     pages_per_huge_page(h), page);

  	if (restore_reserve)
  		h->resv_huge_pages++;

  	if (h->surplus_huge_pages_node[nid]) {

  		/* remove the page from active list */
  		list_del(&page->lru);

  		update_and_free_page(h, page);
  		h->surplus_huge_pages--;
  		h->surplus_huge_pages_node[nid]--;

  	} else {

  		arch_clear_hugepage_flags(page);

  		enqueue_huge_page(h, page);

  	}

  	spin_unlock(&hugetlb_lock);
  }

  static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)

  {

  	INIT_LIST_HEAD(&page->lru);

  	set_compound_page_dtor(page, free_huge_page);
  	spin_lock(&hugetlb_lock);

  	set_hugetlb_cgroup(page, NULL);

  	h->nr_huge_pages++;
  	h->nr_huge_pages_node[nid]++;

  	spin_unlock(&hugetlb_lock);
  	put_page(page); /* free it into the hugepage allocator */
  }

  static void prep_compound_gigantic_page(struct page *page, unsigned long order)

  {
  	int i;
  	int nr_pages = 1 << order;
  	struct page *p = page + 1;
  
  	/* we rely on prep_new_huge_page to set the destructor */
  	set_compound_order(page, order);
  	__SetPageHead(page);

  	__ClearPageReserved(page);

  	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {

  		/*
  		 * For gigantic hugepages allocated through bootmem at
  		 * boot, it's safer to be consistent with the not-gigantic
  		 * hugepages and clear the PG_reserved bit from all tail pages
  		 * too.  Otherwse drivers using get_user_pages() to access tail
  		 * pages may get the reference counting wrong if they see
  		 * PG_reserved set on a tail page (despite the head page not
  		 * having PG_reserved set).  Enforcing this consistency between
  		 * head and tail pages allows drivers to optimize away a check
  		 * on the head page when they need know if put_page() is needed
  		 * after get_user_pages().
  		 */
  		__ClearPageReserved(p);

  		set_page_count(p, 0);

  		p->first_page = page;

  		/* Make sure p->first_page is always valid for PageTail() */
  		smp_wmb();
  		__SetPageTail(p);

  	}
  }

  /*
   * PageHuge() only returns true for hugetlbfs pages, but not for normal or
   * transparent huge pages.  See the PageTransHuge() documentation for more
   * details.
   */

  int PageHuge(struct page *page)
  {

  	if (!PageCompound(page))
  		return 0;
  
  	page = compound_head(page);

  	return get_compound_page_dtor(page) == free_huge_page;

  }

  EXPORT_SYMBOL_GPL(PageHuge);

  /*
   * PageHeadHuge() only returns true for hugetlbfs head page, but not for
   * normal or transparent huge pages.
   */
  int PageHeadHuge(struct page *page_head)
  {

  	if (!PageHead(page_head))
  		return 0;

  	return get_compound_page_dtor(page_head) == free_huge_page;

  }

  pgoff_t __basepage_index(struct page *page)
  {
  	struct page *page_head = compound_head(page);
  	pgoff_t index = page_index(page_head);
  	unsigned long compound_idx;
  
  	if (!PageHuge(page_head))
  		return page_index(page);
  
  	if (compound_order(page_head) >= MAX_ORDER)
  		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
  	else
  		compound_idx = page - page_head;
  
  	return (index << compound_order(page_head)) + compound_idx;
  }

  static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)

  {

  	struct page *page;

  	page = alloc_pages_exact_node(nid,

  		htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|

  						__GFP_REPEAT|__GFP_NOWARN,

  		huge_page_order(h));

  	if (page) {

  		if (arch_prepare_hugepage(page)) {

  			__free_pages(page, huge_page_order(h));

  			return NULL;

  		}

  		prep_new_huge_page(h, page, nid);

  	}

  
  	return page;
  }

  static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
  {
  	struct page *page;
  	int nr_nodes, node;
  	int ret = 0;
  
  	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  		page = alloc_fresh_huge_page_node(h, node);
  		if (page) {
  			ret = 1;
  			break;
  		}
  	}
  
  	if (ret)
  		count_vm_event(HTLB_BUDDY_PGALLOC);
  	else
  		count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
  
  	return ret;
  }

  /*
   * Free huge page from pool from next node to free.
   * Attempt to keep persistent huge pages more or less
   * balanced over allowed nodes.
   * Called with hugetlb_lock locked.
   */

  static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
  							 bool acct_surplus)

  {

  	int nr_nodes, node;

  	int ret = 0;

  	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {

  		/*
  		 * If we're returning unused surplus pages, only examine
  		 * nodes with surplus pages.
  		 */

  		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
  		    !list_empty(&h->hugepage_freelists[node])) {

  			struct page *page =

  				list_entry(h->hugepage_freelists[node].next,

  					  struct page, lru);
  			list_del(&page->lru);
  			h->free_huge_pages--;

  			h->free_huge_pages_node[node]--;

  			if (acct_surplus) {
  				h->surplus_huge_pages--;

  				h->surplus_huge_pages_node[node]--;

  			}

  			update_and_free_page(h, page);
  			ret = 1;

  			break;

  		}

  	}

  
  	return ret;
  }

  /*
   * Dissolve a given free hugepage into free buddy pages. This function does
   * nothing for in-use (including surplus) hugepages.
   */
  static void dissolve_free_huge_page(struct page *page)
  {
  	spin_lock(&hugetlb_lock);
  	if (PageHuge(page) && !page_count(page)) {
  		struct hstate *h = page_hstate(page);
  		int nid = page_to_nid(page);
  		list_del(&page->lru);
  		h->free_huge_pages--;
  		h->free_huge_pages_node[nid]--;
  		update_and_free_page(h, page);
  	}
  	spin_unlock(&hugetlb_lock);
  }
  
  /*
   * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
   * make specified memory blocks removable from the system.
   * Note that start_pfn should aligned with (minimum) hugepage size.
   */
  void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
  {

  	unsigned long pfn;

  	if (!hugepages_supported())
  		return;

  	VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order));
  	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order)

  		dissolve_free_huge_page(pfn_to_page(pfn));
  }

  static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)

  {
  	struct page *page;

  	unsigned int r_nid;

  	if (hstate_is_gigantic(h))

  		return NULL;

  	/*
  	 * Assume we will successfully allocate the surplus page to
  	 * prevent racing processes from causing the surplus to exceed
  	 * overcommit
  	 *
  	 * This however introduces a different race, where a process B
  	 * tries to grow the static hugepage pool while alloc_pages() is
  	 * called by process A. B will only examine the per-node
  	 * counters in determining if surplus huge pages can be
  	 * converted to normal huge pages in adjust_pool_surplus(). A
  	 * won't be able to increment the per-node counter, until the
  	 * lock is dropped by B, but B doesn't drop hugetlb_lock until
  	 * no more huge pages can be converted from surplus to normal
  	 * state (and doesn't try to convert again). Thus, we have a
  	 * case where a surplus huge page exists, the pool is grown, and
  	 * the surplus huge page still exists after, even though it
  	 * should just have been converted to a normal huge page. This
  	 * does not leak memory, though, as the hugepage will be freed
  	 * once it is out of use. It also does not allow the counters to
  	 * go out of whack in adjust_pool_surplus() as we don't modify
  	 * the node values until we've gotten the hugepage and only the
  	 * per-node value is checked there.
  	 */
  	spin_lock(&hugetlb_lock);

  	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {

  		spin_unlock(&hugetlb_lock);
  		return NULL;
  	} else {

  		h->nr_huge_pages++;
  		h->surplus_huge_pages++;

  	}
  	spin_unlock(&hugetlb_lock);

  	if (nid == NUMA_NO_NODE)

  		page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|

  				   __GFP_REPEAT|__GFP_NOWARN,
  				   huge_page_order(h));
  	else
  		page = alloc_pages_exact_node(nid,

  			htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|

  			__GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));

  	if (page && arch_prepare_hugepage(page)) {
  		__free_pages(page, huge_page_order(h));

  		page = NULL;

  	}

  	spin_lock(&hugetlb_lock);

  	if (page) {

  		INIT_LIST_HEAD(&page->lru);

  		r_nid = page_to_nid(page);

  		set_compound_page_dtor(page, free_huge_page);

  		set_hugetlb_cgroup(page, NULL);

  		/*
  		 * We incremented the global counters already
  		 */

  		h->nr_huge_pages_node[r_nid]++;
  		h->surplus_huge_pages_node[r_nid]++;

  		__count_vm_event(HTLB_BUDDY_PGALLOC);

  	} else {

  		h->nr_huge_pages--;
  		h->surplus_huge_pages--;

  		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);

  	}

  	spin_unlock(&hugetlb_lock);

  
  	return page;
  }

  /*

   * This allocation function is useful in the context where vma is irrelevant.
   * E.g. soft-offlining uses this function because it only cares physical
   * address of error page.
   */
  struct page *alloc_huge_page_node(struct hstate *h, int nid)
  {

  	struct page *page = NULL;

  
  	spin_lock(&hugetlb_lock);

  	if (h->free_huge_pages - h->resv_huge_pages > 0)
  		page = dequeue_huge_page_node(h, nid);

  	spin_unlock(&hugetlb_lock);

  	if (!page)

  		page = alloc_buddy_huge_page(h, nid);
  
  	return page;
  }
  
  /*

   * Increase the hugetlb pool such that it can accommodate a reservation

   * of size 'delta'.
   */

  static int gather_surplus_pages(struct hstate *h, int delta)

  {
  	struct list_head surplus_list;
  	struct page *page, *tmp;
  	int ret, i;
  	int needed, allocated;

  	bool alloc_ok = true;

  	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;

  	if (needed <= 0) {

  		h->resv_huge_pages += delta;

  		return 0;

  	}

  
  	allocated = 0;
  	INIT_LIST_HEAD(&surplus_list);
  
  	ret = -ENOMEM;
  retry:
  	spin_unlock(&hugetlb_lock);
  	for (i = 0; i < needed; i++) {

  		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);

  		if (!page) {
  			alloc_ok = false;
  			break;
  		}

  		list_add(&page->lru, &surplus_list);
  	}

  	allocated += i;

  
  	/*
  	 * After retaking hugetlb_lock, we need to recalculate 'needed'
  	 * because either resv_huge_pages or free_huge_pages may have changed.
  	 */
  	spin_lock(&hugetlb_lock);

  	needed = (h->resv_huge_pages + delta) -
  			(h->free_huge_pages + allocated);

  	if (needed > 0) {
  		if (alloc_ok)
  			goto retry;
  		/*
  		 * We were not able to allocate enough pages to
  		 * satisfy the entire reservation so we free what
  		 * we've allocated so far.
  		 */
  		goto free;
  	}

  	/*
  	 * The surplus_list now contains _at_least_ the number of extra pages

  	 * needed to accommodate the reservation.  Add the appropriate number

  	 * of pages to the hugetlb pool and free the extras back to the buddy

  	 * allocator.  Commit the entire reservation here to prevent another
  	 * process from stealing the pages as they are added to the pool but
  	 * before they are reserved.

  	 */
  	needed += allocated;

  	h->resv_huge_pages += delta;

  	ret = 0;

  	/* Free the needed pages to the hugetlb pool */

  	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {

  		if ((--needed) < 0)
  			break;

  		/*
  		 * This page is now managed by the hugetlb allocator and has
  		 * no users -- drop the buddy allocator's reference.
  		 */
  		put_page_testzero(page);

  		VM_BUG_ON_PAGE(page_count(page), page);

  		enqueue_huge_page(h, page);

  	}

  free:

  	spin_unlock(&hugetlb_lock);

  
  	/* Free unnecessary surplus pages to the buddy allocator */

  	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
  		put_page(page);

  	spin_lock(&hugetlb_lock);

  
  	return ret;
  }
  
  /*
   * When releasing a hugetlb pool reservation, any surplus pages that were
   * allocated to satisfy the reservation must be explicitly freed if they were
   * never used.

   * Called with hugetlb_lock held.

   */

  static void return_unused_surplus_pages(struct hstate *h,
  					unsigned long unused_resv_pages)

  {

  	unsigned long nr_pages;

  	/* Uncommit the reservation */

  	h->resv_huge_pages -= unused_resv_pages;

  	/* Cannot return gigantic pages currently */

  	if (hstate_is_gigantic(h))

  		return;

  	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);

  	/*
  	 * We want to release as many surplus pages as possible, spread

  	 * evenly across all nodes with memory. Iterate across these nodes
  	 * until we can no longer free unreserved surplus pages. This occurs
  	 * when the nodes with surplus pages have no free pages.
  	 * free_pool_huge_page() will balance the the freed pages across the
  	 * on-line nodes with memory and will handle the hstate accounting.

  	 */
  	while (nr_pages--) {

  		if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))

  			break;

  		cond_resched_lock(&hugetlb_lock);

  	}
  }

  /*
   * Determine if the huge page at addr within the vma has an associated
   * reservation.  Where it does not we will need to logically increase

   * reservation and actually increase subpool usage before an allocation
   * can occur.  Where any new reservation would be required the
   * reservation change is prepared, but not committed.  Once the page
   * has been allocated from the subpool and instantiated the change should
   * be committed via vma_commit_reservation.  No action is required on
   * failure.

   */

  static long vma_needs_reservation(struct hstate *h,

  			struct vm_area_struct *vma, unsigned long addr)

  {

  	struct resv_map *resv;
  	pgoff_t idx;
  	long chg;

  	resv = vma_resv_map(vma);
  	if (!resv)

  		return 1;

  	idx = vma_hugecache_offset(h, vma, addr);
  	chg = region_chg(resv, idx, idx + 1);

  	if (vma->vm_flags & VM_MAYSHARE)
  		return chg;
  	else
  		return chg < 0 ? chg : 0;

  }

  static void vma_commit_reservation(struct hstate *h,
  			struct vm_area_struct *vma, unsigned long addr)

  {

  	struct resv_map *resv;
  	pgoff_t idx;

  	resv = vma_resv_map(vma);
  	if (!resv)
  		return;

  	idx = vma_hugecache_offset(h, vma, addr);
  	region_add(resv, idx, idx + 1);

  }

  static struct page *alloc_huge_page(struct vm_area_struct *vma,

  				    unsigned long addr, int avoid_reserve)

  {

  	struct hugepage_subpool *spool = subpool_vma(vma);

  	struct hstate *h = hstate_vma(vma);

  	struct page *page;

  	long chg;

  	int ret, idx;
  	struct hugetlb_cgroup *h_cg;

  	idx = hstate_index(h);

  	/*

  	 * Processes that did not create the mapping will have no
  	 * reserves and will not have accounted against subpool
  	 * limit. Check that the subpool limit can be made before
  	 * satisfying the allocation MAP_NORESERVE mappings may also
  	 * need pages and subpool limit allocated allocated if no reserve
  	 * mapping overlaps.

  	 */

  	chg = vma_needs_reservation(h, vma, addr);

  	if (chg < 0)

  		return ERR_PTR(-ENOMEM);

  	if (chg || avoid_reserve)

  		if (hugepage_subpool_get_pages(spool, 1) < 0)

  			return ERR_PTR(-ENOSPC);

  	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);

  	if (ret)
  		goto out_subpool_put;

  	spin_lock(&hugetlb_lock);

  	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);

  	if (!page) {

  		spin_unlock(&hugetlb_lock);

  		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);

  		if (!page)
  			goto out_uncharge_cgroup;

  		spin_lock(&hugetlb_lock);
  		list_move(&page->lru, &h->hugepage_activelist);

  		/* Fall through */

  	}

  	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
  	spin_unlock(&hugetlb_lock);

  	set_page_private(page, (unsigned long)spool);

  	vma_commit_reservation(h, vma, addr);

  	return page;

  
  out_uncharge_cgroup:
  	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
  out_subpool_put:
  	if (chg || avoid_reserve)
  		hugepage_subpool_put_pages(spool, 1);
  	return ERR_PTR(-ENOSPC);

  }

  /*
   * alloc_huge_page()'s wrapper which simply returns the page if allocation
   * succeeds, otherwise NULL. This function is called from new_vma_page(),
   * where no ERR_VALUE is expected to be returned.
   */
  struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
  				unsigned long addr, int avoid_reserve)
  {
  	struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
  	if (IS_ERR(page))
  		page = NULL;
  	return page;
  }

  int __weak alloc_bootmem_huge_page(struct hstate *h)

  {
  	struct huge_bootmem_page *m;

  	int nr_nodes, node;

  	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {

  		void *addr;

  		addr = memblock_virt_alloc_try_nid_nopanic(
  				huge_page_size(h), huge_page_size(h),
  				0, BOOTMEM_ALLOC_ACCESSIBLE, node);

  		if (addr) {
  			/*
  			 * Use the beginning of the huge page to store the
  			 * huge_bootmem_page struct (until gather_bootmem
  			 * puts them into the mem_map).
  			 */
  			m = addr;

  			goto found;

  		}

  	}
  	return 0;
  
  found:

  	BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));

  	/* Put them into a private list first because mem_map is not up yet */
  	list_add(&m->list, &huge_boot_pages);
  	m->hstate = h;
  	return 1;
  }

  static void __init prep_compound_huge_page(struct page *page, int order)

  {
  	if (unlikely(order > (MAX_ORDER - 1)))
  		prep_compound_gigantic_page(page, order);
  	else
  		prep_compound_page(page, order);
  }

  /* Put bootmem huge pages into the standard lists after mem_map is up */
  static void __init gather_bootmem_prealloc(void)
  {
  	struct huge_bootmem_page *m;
  
  	list_for_each_entry(m, &huge_boot_pages, list) {

  		struct hstate *h = m->hstate;

  		struct page *page;
  
  #ifdef CONFIG_HIGHMEM
  		page = pfn_to_page(m->phys >> PAGE_SHIFT);

  		memblock_free_late(__pa(m),
  				   sizeof(struct huge_bootmem_page));

  #else
  		page = virt_to_page(m);
  #endif

  		WARN_ON(page_count(page) != 1);

  		prep_compound_huge_page(page, h->order);

  		WARN_ON(PageReserved(page));

  		prep_new_huge_page(h, page, page_to_nid(page));

  		/*
  		 * If we had gigantic hugepages allocated at boot time, we need
  		 * to restore the 'stolen' pages to totalram_pages in order to
  		 * fix confusing memory reports from free(1) and another
  		 * side-effects, like CommitLimit going negative.
  		 */

  		if (hstate_is_gigantic(h))

  			adjust_managed_page_count(page, 1 << h->order);

  	}
  }

  static void __init hugetlb_hstate_alloc_pages(struct hstate *h)

  {
  	unsigned long i;

  	for (i = 0; i < h->max_huge_pages; ++i) {

  		if (hstate_is_gigantic(h)) {

  			if (!alloc_bootmem_huge_page(h))
  				break;

  		} else if (!alloc_fresh_huge_page(h,

  					 &node_states[N_MEMORY]))

  			break;

  	}

  	h->max_huge_pages = i;

  }
  
  static void __init hugetlb_init_hstates(void)
  {
  	struct hstate *h;
  
  	for_each_hstate(h) {

  		if (minimum_order > huge_page_order(h))
  			minimum_order = huge_page_order(h);

  		/* oversize hugepages were init'ed in early boot */

  		if (!hstate_is_gigantic(h))

  			hugetlb_hstate_alloc_pages(h);

  	}

  	VM_BUG_ON(minimum_order == UINT_MAX);

  }

  static char * __init memfmt(char *buf, unsigned long n)
  {
  	if (n >= (1UL << 30))
  		sprintf(buf, "%lu GB", n >> 30);
  	else if (n >= (1UL << 20))
  		sprintf(buf, "%lu MB", n >> 20);
  	else
  		sprintf(buf, "%lu KB", n >> 10);
  	return buf;
  }

  static void __init report_hugepages(void)
  {
  	struct hstate *h;
  
  	for_each_hstate(h) {

  		char buf[32];

  		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages
  ",

  			memfmt(buf, huge_page_size(h)),
  			h->free_huge_pages);

  	}
  }

  #ifdef CONFIG_HIGHMEM

  static void try_to_free_low(struct hstate *h, unsigned long count,
  						nodemask_t *nodes_allowed)

  {

  	int i;

  	if (hstate_is_gigantic(h))

  		return;

  	for_each_node_mask(i, *nodes_allowed) {

  		struct page *page, *next;

  		struct list_head *freel = &h->hugepage_freelists[i];
  		list_for_each_entry_safe(page, next, freel, lru) {
  			if (count >= h->nr_huge_pages)

  				return;

  			if (PageHighMem(page))
  				continue;
  			list_del(&page->lru);

  			update_and_free_page(h, page);

  			h->free_huge_pages--;
  			h->free_huge_pages_node[page_to_nid(page)]--;

  		}
  	}
  }
  #else

  static inline void try_to_free_low(struct hstate *h, unsigned long count,
  						nodemask_t *nodes_allowed)

  {
  }
  #endif

  /*
   * Increment or decrement surplus_huge_pages.  Keep node-specific counters
   * balanced by operating on them in a round-robin fashion.
   * Returns 1 if an adjustment was made.
   */

  static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
  				int delta)

  {

  	int nr_nodes, node;

  
  	VM_BUG_ON(delta != -1 && delta != 1);

  	if (delta < 0) {
  		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
  			if (h->surplus_huge_pages_node[node])
  				goto found;

  		}

  	} else {
  		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
  			if (h->surplus_huge_pages_node[node] <
  					h->nr_huge_pages_node[node])
  				goto found;

  		}

  	}
  	return 0;

  found:
  	h->surplus_huge_pages += delta;
  	h->surplus_huge_pages_node[node] += delta;
  	return 1;

  }

  #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)

  static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
  						nodemask_t *nodes_allowed)

  {

  	unsigned long min_count, ret;

  	if (hstate_is_gigantic(h) && !gigantic_page_supported())

  		return h->max_huge_pages;

  	/*
  	 * Increase the pool size
  	 * First take pages out of surplus state.  Then make up the
  	 * remaining difference by allocating fresh huge pages.

  	 *
  	 * We might race with alloc_buddy_huge_page() here and be unable
  	 * to convert a surplus huge page to a normal huge page. That is
  	 * not critical, though, it just means the overall size of the
  	 * pool might be one hugepage larger than it needs to be, but
  	 * within all the constraints specified by the sysctls.

  	 */

  	spin_lock(&hugetlb_lock);

  	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {

  		if (!adjust_pool_surplus(h, nodes_allowed, -1))

  			break;
  	}

  	while (count > persistent_huge_pages(h)) {

  		/*
  		 * If this allocation races such that we no longer need the
  		 * page, free_huge_page will handle it by freeing the page
  		 * and reducing the surplus.
  		 */
  		spin_unlock(&hugetlb_lock);

  		if (hstate_is_gigantic(h))
  			ret = alloc_fresh_gigantic_page(h, nodes_allowed);
  		else
  			ret = alloc_fresh_huge_page(h, nodes_allowed);

  		spin_lock(&hugetlb_lock);
  		if (!ret)
  			goto out;

  		/* Bail for signals. Probably ctrl-c from user */
  		if (signal_pending(current))
  			goto out;

  	}

  
  	/*
  	 * Decrease the pool size
  	 * First return free pages to the buddy allocator (being careful
  	 * to keep enough around to satisfy reservations).  Then place
  	 * pages into surplus state as needed so the pool will shrink
  	 * to the desired size as pages become free.

  	 *
  	 * By placing pages into the surplus state independent of the
  	 * overcommit value, we are allowing the surplus pool size to
  	 * exceed overcommit. There are few sane options here. Since
  	 * alloc_buddy_huge_page() is checking the global counter,
  	 * though, we'll note that we're not allowed to exceed surplus
  	 * and won't grow the pool anywhere else. Not until one of the
  	 * sysctls are changed, or the surplus pages go out of use.

  	 */

  	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;

  	min_count = max(count, min_count);

  	try_to_free_low(h, min_count, nodes_allowed);

  	while (min_count < persistent_huge_pages(h)) {

  		if (!free_pool_huge_page(h, nodes_allowed, 0))

  			break;

  		cond_resched_lock(&hugetlb_lock);

  	}

  	while (count < persistent_huge_pages(h)) {

  		if (!adjust_pool_surplus(h, nodes_allowed, 1))

  			break;
  	}
  out:

  	ret = persistent_huge_pages(h);

  	spin_unlock(&hugetlb_lock);

  	return ret;

  }

  #define HSTATE_ATTR_RO(_name) \
  	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
  
  #define HSTATE_ATTR(_name) \
  	static struct kobj_attribute _name##_attr = \
  		__ATTR(_name, 0644, _name##_show, _name##_store)
  
  static struct kobject *hugepages_kobj;
  static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];

  static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
  
  static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)

  {
  	int i;

  	for (i = 0; i < HUGE_MAX_HSTATE; i++)

  		if (hstate_kobjs[i] == kobj) {
  			if (nidp)
  				*nidp = NUMA_NO_NODE;

  			return &hstates[i];

  		}
  
  	return kobj_to_node_hstate(kobj, nidp);