#include <linux/gfp.h>
  #include <linux/highmem.h>
  #include <linux/kernel.h>
  #include <linux/mmdebug.h>
  #include <linux/mm_types.h>
  #include <linux/pagemap.h>
  #include <linux/rcupdate.h>
  #include <linux/smp.h>
  #include <linux/swap.h>
  
  #include <asm/pgalloc.h>
  #include <asm/tlb.h>

  #ifndef CONFIG_MMU_GATHER_NO_GATHER

  static bool tlb_next_batch(struct mmu_gather *tlb)
  {
  	struct mmu_gather_batch *batch;
  
  	batch = tlb->active;
  	if (batch->next) {
  		tlb->active = batch->next;
  		return true;
  	}
  
  	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
  		return false;
  
  	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
  	if (!batch)
  		return false;
  
  	tlb->batch_count++;
  	batch->next = NULL;
  	batch->nr   = 0;
  	batch->max  = MAX_GATHER_BATCH;
  
  	tlb->active->next = batch;
  	tlb->active = batch;
  
  	return true;
  }

  static void tlb_batch_pages_flush(struct mmu_gather *tlb)
  {
  	struct mmu_gather_batch *batch;
  
  	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
  		free_pages_and_swap_cache(batch->pages, batch->nr);
  		batch->nr = 0;
  	}
  	tlb->active = &tlb->local;
  }
  
  static void tlb_batch_list_free(struct mmu_gather *tlb)
  {
  	struct mmu_gather_batch *batch, *next;
  
  	for (batch = tlb->local.next; batch; batch = next) {
  		next = batch->next;
  		free_pages((unsigned long)batch, 0);
  	}
  	tlb->local.next = NULL;
  }
  
  bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
  {
  	struct mmu_gather_batch *batch;
  
  	VM_BUG_ON(!tlb->end);

  #ifdef CONFIG_MMU_GATHER_PAGE_SIZE

  	VM_WARN_ON(tlb->page_size != page_size);
  #endif
  
  	batch = tlb->active;
  	/*
  	 * Add the page and check if we are full. If so
  	 * force a flush.
  	 */
  	batch->pages[batch->nr++] = page;
  	if (batch->nr == batch->max) {
  		if (!tlb_next_batch(tlb))
  			return true;
  		batch = tlb->active;
  	}
  	VM_BUG_ON_PAGE(batch->nr > batch->max, page);
  
  	return false;
  }

  #endif /* MMU_GATHER_NO_GATHER */

  #ifdef CONFIG_MMU_GATHER_TABLE_FREE

  static void __tlb_remove_table_free(struct mmu_table_batch *batch)
  {
  	int i;
  
  	for (i = 0; i < batch->nr; i++)
  		__tlb_remove_table(batch->tables[i]);
  
  	free_page((unsigned long)batch);
  }
  
  #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE

  
  /*

   * Semi RCU freeing of the page directories.
   *
   * This is needed by some architectures to implement software pagetable walkers.
   *
   * gup_fast() and other software pagetable walkers do a lockless page-table
   * walk and therefore needs some synchronization with the freeing of the page
   * directories. The chosen means to accomplish that is by disabling IRQs over
   * the walk.
   *
   * Architectures that use IPIs to flush TLBs will then automagically DTRT,
   * since we unlink the page, flush TLBs, free the page. Since the disabling of
   * IRQs delays the completion of the TLB flush we can never observe an already
   * freed page.
   *
   * Architectures that do not have this (PPC) need to delay the freeing by some
   * other means, this is that means.
   *
   * What we do is batch the freed directory pages (tables) and RCU free them.
   * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
   * holds off grace periods.
   *
   * However, in order to batch these pages we need to allocate storage, this
   * allocation is deep inside the MM code and can thus easily fail on memory
   * pressure. To guarantee progress we fall back to single table freeing, see
   * the implementation of tlb_remove_table_one().
   *

   */

  
  static void tlb_remove_table_smp_sync(void *arg)
  {
  	/* Simply deliver the interrupt */
  }

  static void tlb_remove_table_sync_one(void)

  {
  	/*
  	 * This isn't an RCU grace period and hence the page-tables cannot be
  	 * assumed to be actually RCU-freed.
  	 *
  	 * It is however sufficient for software page-table walkers that rely on

  	 * IRQ disabling.

  	 */
  	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);

  }
  
  static void tlb_remove_table_rcu(struct rcu_head *head)
  {

  	__tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
  }

  static void tlb_remove_table_free(struct mmu_table_batch *batch)
  {
  	call_rcu(&batch->rcu, tlb_remove_table_rcu);
  }

  #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */

  static void tlb_remove_table_sync_one(void) { }
  
  static void tlb_remove_table_free(struct mmu_table_batch *batch)
  {
  	__tlb_remove_table_free(batch);
  }
  
  #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
  
  /*
   * If we want tlb_remove_table() to imply TLB invalidates.
   */
  static inline void tlb_table_invalidate(struct mmu_gather *tlb)
  {
  	if (tlb_needs_table_invalidate()) {
  		/*
  		 * Invalidate page-table caches used by hardware walkers. Then
  		 * we still need to RCU-sched wait while freeing the pages
  		 * because software walkers can still be in-flight.
  		 */
  		tlb_flush_mmu_tlbonly(tlb);
  	}
  }
  
  static void tlb_remove_table_one(void *table)
  {
  	tlb_remove_table_sync_one();
  	__tlb_remove_table(table);

  }

  static void tlb_table_flush(struct mmu_gather *tlb)

  {
  	struct mmu_table_batch **batch = &tlb->batch;
  
  	if (*batch) {
  		tlb_table_invalidate(tlb);

  		tlb_remove_table_free(*batch);

  		*batch = NULL;
  	}
  }
  
  void tlb_remove_table(struct mmu_gather *tlb, void *table)
  {
  	struct mmu_table_batch **batch = &tlb->batch;
  
  	if (*batch == NULL) {
  		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
  		if (*batch == NULL) {
  			tlb_table_invalidate(tlb);
  			tlb_remove_table_one(table);
  			return;
  		}
  		(*batch)->nr = 0;
  	}
  
  	(*batch)->tables[(*batch)->nr++] = table;
  	if ((*batch)->nr == MAX_TABLE_BATCH)
  		tlb_table_flush(tlb);
  }

  static inline void tlb_table_init(struct mmu_gather *tlb)
  {
  	tlb->batch = NULL;
  }
  
  #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
  
  static inline void tlb_table_flush(struct mmu_gather *tlb) { }
  static inline void tlb_table_init(struct mmu_gather *tlb) { }
  
  #endif /* CONFIG_MMU_GATHER_TABLE_FREE */

  static void tlb_flush_mmu_free(struct mmu_gather *tlb)
  {

  	tlb_table_flush(tlb);

  #ifndef CONFIG_MMU_GATHER_NO_GATHER

  	tlb_batch_pages_flush(tlb);
  #endif
  }
  
  void tlb_flush_mmu(struct mmu_gather *tlb)
  {
  	tlb_flush_mmu_tlbonly(tlb);
  	tlb_flush_mmu_free(tlb);
  }

  /**
   * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
   * @tlb: the mmu_gather structure to initialize
   * @mm: the mm_struct of the target address space
   * @start: start of the region that will be removed from the page-table
   * @end: end of the region that will be removed from the page-table
   *
   * Called to initialize an (on-stack) mmu_gather structure for page-table
   * tear-down from @mm. The @start and @end are set to 0 and -1
   * respectively when @mm is without users and we're going to destroy
   * the full address space (exit/execve).
   */
  void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
  			unsigned long start, unsigned long end)
  {

  	tlb->mm = mm;
  
  	/* Is it from 0 to ~0? */
  	tlb->fullmm     = !(start | (end+1));

  #ifndef CONFIG_MMU_GATHER_NO_GATHER

  	tlb->need_flush_all = 0;
  	tlb->local.next = NULL;
  	tlb->local.nr   = 0;
  	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
  	tlb->active     = &tlb->local;
  	tlb->batch_count = 0;
  #endif

  	tlb_table_init(tlb);

  #ifdef CONFIG_MMU_GATHER_PAGE_SIZE

  	tlb->page_size = 0;
  #endif
  
  	__tlb_reset_range(tlb);

  	inc_tlb_flush_pending(tlb->mm);
  }

  /**
   * tlb_finish_mmu - finish an mmu_gather structure
   * @tlb: the mmu_gather structure to finish
   * @start: start of the region that will be removed from the page-table
   * @end: end of the region that will be removed from the page-table
   *
   * Called at the end of the shootdown operation to free up any resources that
   * were required.
   */

  void tlb_finish_mmu(struct mmu_gather *tlb,
  		unsigned long start, unsigned long end)
  {
  	/*
  	 * If there are parallel threads are doing PTE changes on same range

  	 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB

  	 * flush by batching, one thread may end up seeing inconsistent PTEs
  	 * and result in having stale TLB entries.  So flush TLB forcefully
  	 * if we detect parallel PTE batching threads.
  	 *
  	 * However, some syscalls, e.g. munmap(), may free page tables, this
  	 * needs force flush everything in the given range. Otherwise this
  	 * may result in having stale TLB entries for some architectures,
  	 * e.g. aarch64, that could specify flush what level TLB.

  	 */

  	if (mm_tlb_flush_nested(tlb->mm)) {

  		/*
  		 * The aarch64 yields better performance with fullmm by
  		 * avoiding multiple CPUs spamming TLBI messages at the
  		 * same time.
  		 *
  		 * On x86 non-fullmm doesn't yield significant difference
  		 * against fullmm.
  		 */
  		tlb->fullmm = 1;

  		__tlb_reset_range(tlb);

  		tlb->freed_tables = 1;

  	}

  	tlb_flush_mmu(tlb);

  #ifndef CONFIG_MMU_GATHER_NO_GATHER

  	tlb_batch_list_free(tlb);
  #endif

  	dec_tlb_flush_pending(tlb->mm);
  }