// SPDX-License-Identifier: GPL-2.0-only

  /*

   * mm/page-writeback.c

   *
   * Copyright (C) 2002, Linus Torvalds.

   * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra

   *
   * Contains functions related to writing back dirty pages at the
   * address_space level.
   *

   * 10Apr2002	Andrew Morton

   *		Initial version
   */
  
  #include <linux/kernel.h>

  #include <linux/export.h>

  #include <linux/spinlock.h>
  #include <linux/fs.h>
  #include <linux/mm.h>
  #include <linux/swap.h>
  #include <linux/slab.h>
  #include <linux/pagemap.h>
  #include <linux/writeback.h>
  #include <linux/init.h>
  #include <linux/backing-dev.h>

  #include <linux/task_io_accounting_ops.h>

  #include <linux/blkdev.h>
  #include <linux/mpage.h>

  #include <linux/rmap.h>

  #include <linux/percpu.h>

  #include <linux/smp.h>
  #include <linux/sysctl.h>
  #include <linux/cpu.h>
  #include <linux/syscalls.h>

  #include <linux/buffer_head.h> /* __set_page_dirty_buffers */

  #include <linux/pagevec.h>

  #include <linux/timer.h>

  #include <linux/sched/rt.h>

  #include <linux/sched/signal.h>

  #include <linux/mm_inline.h>

  #include <trace/events/writeback.h>

  #include "internal.h"

  /*

   * Sleep at most 200ms at a time in balance_dirty_pages().
   */
  #define MAX_PAUSE		max(HZ/5, 1)
  
  /*

   * Try to keep balance_dirty_pages() call intervals higher than this many pages
   * by raising pause time to max_pause when falls below it.
   */
  #define DIRTY_POLL_THRESH	(128 >> (PAGE_SHIFT - 10))
  
  /*

   * Estimate write bandwidth at 200ms intervals.
   */
  #define BANDWIDTH_INTERVAL	max(HZ/5, 1)

  #define RATELIMIT_CALC_SHIFT	10

  /*

   * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
   * will look to see if it needs to force writeback or throttling.
   */
  static long ratelimit_pages = 32;

  /* The following parameters are exported via /proc/sys/vm */
  
  /*

   * Start background writeback (via writeback threads) at this percentage

   */

  int dirty_background_ratio = 10;

  
  /*

   * dirty_background_bytes starts at 0 (disabled) so that it is a function of
   * dirty_background_ratio * the amount of dirtyable memory
   */
  unsigned long dirty_background_bytes;
  
  /*

   * free highmem will not be subtracted from the total free memory
   * for calculating free ratios if vm_highmem_is_dirtyable is true
   */
  int vm_highmem_is_dirtyable;
  
  /*

   * The generator of dirty data starts writeback at this percentage
   */

  int vm_dirty_ratio = 20;

  
  /*

   * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
   * vm_dirty_ratio * the amount of dirtyable memory
   */
  unsigned long vm_dirty_bytes;
  
  /*

   * The interval between `kupdate'-style writebacks

   */

  unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */

  EXPORT_SYMBOL_GPL(dirty_writeback_interval);

  /*

   * The longest time for which data is allowed to remain dirty

   */

  unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */

  
  /*
   * Flag that makes the machine dump writes/reads and block dirtyings.
   */
  int block_dump;
  
  /*

   * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
   * a full sync is triggered after this time elapses without any disk activity.

   */
  int laptop_mode;
  
  EXPORT_SYMBOL(laptop_mode);
  
  /* End of sysctl-exported parameters */

  struct wb_domain global_wb_domain;

  /* consolidated parameters for balance_dirty_pages() and its subroutines */
  struct dirty_throttle_control {

  #ifdef CONFIG_CGROUP_WRITEBACK
  	struct wb_domain	*dom;

  	struct dirty_throttle_control *gdtc;	/* only set in memcg dtc's */

  #endif

  	struct bdi_writeback	*wb;

  	struct fprop_local_percpu *wb_completions;

  	unsigned long		avail;		/* dirtyable */

  	unsigned long		dirty;		/* file_dirty + write + nfs */
  	unsigned long		thresh;		/* dirty threshold */
  	unsigned long		bg_thresh;	/* dirty background threshold */
  
  	unsigned long		wb_dirty;	/* per-wb counterparts */
  	unsigned long		wb_thresh;

  	unsigned long		wb_bg_thresh;

  
  	unsigned long		pos_ratio;

  };

  /*
   * Length of period for aging writeout fractions of bdis. This is an
   * arbitrarily chosen number. The longer the period, the slower fractions will
   * reflect changes in current writeout rate.
   */
  #define VM_COMPLETIONS_PERIOD_LEN (3*HZ)

  #ifdef CONFIG_CGROUP_WRITEBACK

  #define GDTC_INIT(__wb)		.wb = (__wb),				\
  				.dom = &global_wb_domain,		\
  				.wb_completions = &(__wb)->completions

  #define GDTC_INIT_NO_WB		.dom = &global_wb_domain

  
  #define MDTC_INIT(__wb, __gdtc)	.wb = (__wb),				\
  				.dom = mem_cgroup_wb_domain(__wb),	\
  				.wb_completions = &(__wb)->memcg_completions, \
  				.gdtc = __gdtc

  
  static bool mdtc_valid(struct dirty_throttle_control *dtc)
  {
  	return dtc->dom;
  }

  
  static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
  {
  	return dtc->dom;
  }

  static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
  {
  	return mdtc->gdtc;
  }

  static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
  {
  	return &wb->memcg_completions;
  }

  static void wb_min_max_ratio(struct bdi_writeback *wb,
  			     unsigned long *minp, unsigned long *maxp)
  {
  	unsigned long this_bw = wb->avg_write_bandwidth;
  	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
  	unsigned long long min = wb->bdi->min_ratio;
  	unsigned long long max = wb->bdi->max_ratio;
  
  	/*
  	 * @wb may already be clean by the time control reaches here and
  	 * the total may not include its bw.
  	 */
  	if (this_bw < tot_bw) {
  		if (min) {
  			min *= this_bw;

  			min = div64_ul(min, tot_bw);

  		}
  		if (max < 100) {
  			max *= this_bw;

  			max = div64_ul(max, tot_bw);

  		}
  	}
  
  	*minp = min;
  	*maxp = max;
  }
  
  #else	/* CONFIG_CGROUP_WRITEBACK */

  #define GDTC_INIT(__wb)		.wb = (__wb),                           \
  				.wb_completions = &(__wb)->completions

  #define GDTC_INIT_NO_WB

  #define MDTC_INIT(__wb, __gdtc)
  
  static bool mdtc_valid(struct dirty_throttle_control *dtc)
  {
  	return false;
  }

  
  static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
  {
  	return &global_wb_domain;
  }

  static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
  {
  	return NULL;
  }

  static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
  {
  	return NULL;
  }

  static void wb_min_max_ratio(struct bdi_writeback *wb,
  			     unsigned long *minp, unsigned long *maxp)
  {
  	*minp = wb->bdi->min_ratio;
  	*maxp = wb->bdi->max_ratio;
  }
  
  #endif	/* CONFIG_CGROUP_WRITEBACK */

  /*

   * In a memory zone, there is a certain amount of pages we consider
   * available for the page cache, which is essentially the number of
   * free and reclaimable pages, minus some zone reserves to protect
   * lowmem and the ability to uphold the zone's watermarks without
   * requiring writeback.
   *
   * This number of dirtyable pages is the base value of which the

   * user-configurable dirty ratio is the effective number of pages that

   * are allowed to be actually dirtied.  Per individual zone, or
   * globally by using the sum of dirtyable pages over all zones.
   *
   * Because the user is allowed to specify the dirty limit globally as
   * absolute number of bytes, calculating the per-zone dirty limit can
   * require translating the configured limit into a percentage of
   * global dirtyable memory first.
   */

  /**

   * node_dirtyable_memory - number of dirtyable pages in a node
   * @pgdat: the node

   *

   * Return: the node's number of pages potentially available for dirty

   * page cache.  This is the base value for the per-node dirty limits.

   */

  static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)

  {

  	unsigned long nr_pages = 0;
  	int z;
  
  	for (z = 0; z < MAX_NR_ZONES; z++) {
  		struct zone *zone = pgdat->node_zones + z;
  
  		if (!populated_zone(zone))
  			continue;
  
  		nr_pages += zone_page_state(zone, NR_FREE_PAGES);
  	}

  	/*
  	 * Pages reserved for the kernel should not be considered
  	 * dirtyable, to prevent a situation where reclaim has to
  	 * clean pages in order to balance the zones.
  	 */

  	nr_pages -= min(nr_pages, pgdat->totalreserve_pages);

  	nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
  	nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);

  
  	return nr_pages;
  }

  static unsigned long highmem_dirtyable_memory(unsigned long total)
  {
  #ifdef CONFIG_HIGHMEM
  	int node;

  	unsigned long x = 0;

  	int i;

  
  	for_each_node_state(node, N_HIGH_MEMORY) {

  		for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
  			struct zone *z;

  			unsigned long nr_pages;

  
  			if (!is_highmem_idx(i))
  				continue;
  
  			z = &NODE_DATA(node)->node_zones[i];

  			if (!populated_zone(z))
  				continue;

  			nr_pages = zone_page_state(z, NR_FREE_PAGES);

  			/* watch for underflows */

  			nr_pages -= min(nr_pages, high_wmark_pages(z));

  			nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
  			nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
  			x += nr_pages;

  		}

  	}

  	/*

  	 * Unreclaimable memory (kernel memory or anonymous memory
  	 * without swap) can bring down the dirtyable pages below
  	 * the zone's dirty balance reserve and the above calculation
  	 * will underflow.  However we still want to add in nodes
  	 * which are below threshold (negative values) to get a more
  	 * accurate calculation but make sure that the total never
  	 * underflows.
  	 */
  	if ((long)x < 0)
  		x = 0;
  
  	/*

  	 * Make sure that the number of highmem pages is never larger
  	 * than the number of the total dirtyable memory. This can only
  	 * occur in very strange VM situations but we want to make sure
  	 * that this does not occur.
  	 */
  	return min(x, total);
  #else
  	return 0;
  #endif
  }
  
  /**

   * global_dirtyable_memory - number of globally dirtyable pages

   *

   * Return: the global number of pages potentially available for dirty

   * page cache.  This is the base value for the global dirty limits.

   */

  static unsigned long global_dirtyable_memory(void)

  {
  	unsigned long x;

  	x = global_zone_page_state(NR_FREE_PAGES);

  	/*
  	 * Pages reserved for the kernel should not be considered
  	 * dirtyable, to prevent a situation where reclaim has to
  	 * clean pages in order to balance the zones.
  	 */
  	x -= min(x, totalreserve_pages);

  	x += global_node_page_state(NR_INACTIVE_FILE);
  	x += global_node_page_state(NR_ACTIVE_FILE);

  	if (!vm_highmem_is_dirtyable)
  		x -= highmem_dirtyable_memory(x);
  
  	return x + 1;	/* Ensure that we never return 0 */
  }

  /**
   * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
   * @dtc: dirty_throttle_control of interest

   *

   * Calculate @dtc->thresh and ->bg_thresh considering
   * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}.  The caller
   * must ensure that @dtc->avail is set before calling this function.  The

   * dirty limits will be lifted by 1/4 for real-time tasks.

   */

  static void domain_dirty_limits(struct dirty_throttle_control *dtc)

  {

  	const unsigned long available_memory = dtc->avail;
  	struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
  	unsigned long bytes = vm_dirty_bytes;
  	unsigned long bg_bytes = dirty_background_bytes;

  	/* convert ratios to per-PAGE_SIZE for higher precision */
  	unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
  	unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;

  	unsigned long thresh;
  	unsigned long bg_thresh;

  	struct task_struct *tsk;

  	/* gdtc is !NULL iff @dtc is for memcg domain */
  	if (gdtc) {
  		unsigned long global_avail = gdtc->avail;
  
  		/*
  		 * The byte settings can't be applied directly to memcg
  		 * domains.  Convert them to ratios by scaling against

  		 * globally available memory.  As the ratios are in
  		 * per-PAGE_SIZE, they can be obtained by dividing bytes by
  		 * number of pages.

  		 */
  		if (bytes)

  			ratio = min(DIV_ROUND_UP(bytes, global_avail),
  				    PAGE_SIZE);

  		if (bg_bytes)

  			bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
  				       PAGE_SIZE);

  		bytes = bg_bytes = 0;
  	}
  
  	if (bytes)
  		thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);

  	else

  		thresh = (ratio * available_memory) / PAGE_SIZE;

  	if (bg_bytes)
  		bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);

  	else

  		bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;

  	if (bg_thresh >= thresh)

  		bg_thresh = thresh / 2;

  	tsk = current;

  	if (rt_task(tsk)) {

  		bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
  		thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;

  	}

  	dtc->thresh = thresh;
  	dtc->bg_thresh = bg_thresh;
  
  	/* we should eventually report the domain in the TP */
  	if (!gdtc)
  		trace_global_dirty_state(bg_thresh, thresh);
  }
  
  /**
   * global_dirty_limits - background-writeback and dirty-throttling thresholds
   * @pbackground: out parameter for bg_thresh
   * @pdirty: out parameter for thresh
   *
   * Calculate bg_thresh and thresh for global_wb_domain.  See
   * domain_dirty_limits() for details.
   */
  void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
  {
  	struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
  
  	gdtc.avail = global_dirtyable_memory();
  	domain_dirty_limits(&gdtc);
  
  	*pbackground = gdtc.bg_thresh;
  	*pdirty = gdtc.thresh;

  }

  /**

   * node_dirty_limit - maximum number of dirty pages allowed in a node
   * @pgdat: the node

   *

   * Return: the maximum number of dirty pages allowed in a node, based

   * on the node's dirtyable memory.

   */

  static unsigned long node_dirty_limit(struct pglist_data *pgdat)

  {

  	unsigned long node_memory = node_dirtyable_memory(pgdat);

  	struct task_struct *tsk = current;
  	unsigned long dirty;
  
  	if (vm_dirty_bytes)
  		dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *

  			node_memory / global_dirtyable_memory();

  	else

  		dirty = vm_dirty_ratio * node_memory / 100;

  	if (rt_task(tsk))

  		dirty += dirty / 4;
  
  	return dirty;
  }
  
  /**

   * node_dirty_ok - tells whether a node is within its dirty limits
   * @pgdat: the node to check

   *

   * Return: %true when the dirty pages in @pgdat are within the node's

   * dirty limit, %false if the limit is exceeded.
   */

  bool node_dirty_ok(struct pglist_data *pgdat)

  {

  	unsigned long limit = node_dirty_limit(pgdat);
  	unsigned long nr_pages = 0;

  	nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);

  	nr_pages += node_page_state(pgdat, NR_WRITEBACK);

  	return nr_pages <= limit;

  }

  int dirty_background_ratio_handler(struct ctl_table *table, int write,

  		void *buffer, size_t *lenp, loff_t *ppos)

  {
  	int ret;

  	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

  	if (ret == 0 && write)
  		dirty_background_bytes = 0;
  	return ret;
  }
  
  int dirty_background_bytes_handler(struct ctl_table *table, int write,

  		void *buffer, size_t *lenp, loff_t *ppos)

  {
  	int ret;

  	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

  	if (ret == 0 && write)
  		dirty_background_ratio = 0;
  	return ret;
  }

  int dirty_ratio_handler(struct ctl_table *table, int write, void *buffer,
  		size_t *lenp, loff_t *ppos)

  {
  	int old_ratio = vm_dirty_ratio;

  	int ret;

  	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);

  	if (ret == 0 && write && vm_dirty_ratio != old_ratio) {

  		writeback_set_ratelimit();

  		vm_dirty_bytes = 0;
  	}
  	return ret;
  }

  int dirty_bytes_handler(struct ctl_table *table, int write,

  		void *buffer, size_t *lenp, loff_t *ppos)

  {

  	unsigned long old_bytes = vm_dirty_bytes;

  	int ret;

  	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

  	if (ret == 0 && write && vm_dirty_bytes != old_bytes) {

  		writeback_set_ratelimit();

  		vm_dirty_ratio = 0;

  	}
  	return ret;
  }

  static unsigned long wp_next_time(unsigned long cur_time)
  {
  	cur_time += VM_COMPLETIONS_PERIOD_LEN;
  	/* 0 has a special meaning... */
  	if (!cur_time)
  		return 1;
  	return cur_time;
  }

  static void wb_domain_writeout_inc(struct wb_domain *dom,
  				   struct fprop_local_percpu *completions,
  				   unsigned int max_prop_frac)

  {

  	__fprop_inc_percpu_max(&dom->completions, completions,
  			       max_prop_frac);

  	/* First event after period switching was turned off? */

  	if (unlikely(!dom->period_time)) {

  		/*
  		 * We can race with other __bdi_writeout_inc calls here but
  		 * it does not cause any harm since the resulting time when
  		 * timer will fire and what is in writeout_period_time will be
  		 * roughly the same.
  		 */

  		dom->period_time = wp_next_time(jiffies);
  		mod_timer(&dom->period_timer, dom->period_time);

  	}

  }

  /*
   * Increment @wb's writeout completion count and the global writeout
   * completion count. Called from test_clear_page_writeback().
   */
  static inline void __wb_writeout_inc(struct bdi_writeback *wb)

  {

  	struct wb_domain *cgdom;

  	inc_wb_stat(wb, WB_WRITTEN);

  	wb_domain_writeout_inc(&global_wb_domain, &wb->completions,
  			       wb->bdi->max_prop_frac);

  
  	cgdom = mem_cgroup_wb_domain(wb);
  	if (cgdom)
  		wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb),
  				       wb->bdi->max_prop_frac);

  }

  void wb_writeout_inc(struct bdi_writeback *wb)

  {

  	unsigned long flags;
  
  	local_irq_save(flags);

  	__wb_writeout_inc(wb);

  	local_irq_restore(flags);

  }

  EXPORT_SYMBOL_GPL(wb_writeout_inc);

  /*

   * On idle system, we can be called long after we scheduled because we use
   * deferred timers so count with missed periods.
   */

  static void writeout_period(struct timer_list *t)

  {

  	struct wb_domain *dom = from_timer(dom, t, period_timer);

  	int miss_periods = (jiffies - dom->period_time) /

  						 VM_COMPLETIONS_PERIOD_LEN;

  	if (fprop_new_period(&dom->completions, miss_periods + 1)) {
  		dom->period_time = wp_next_time(dom->period_time +

  				miss_periods * VM_COMPLETIONS_PERIOD_LEN);

  		mod_timer(&dom->period_timer, dom->period_time);

  	} else {
  		/*
  		 * Aging has zeroed all fractions. Stop wasting CPU on period
  		 * updates.
  		 */

  		dom->period_time = 0;

  	}
  }

  int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
  {
  	memset(dom, 0, sizeof(*dom));

  
  	spin_lock_init(&dom->lock);

  	timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);

  
  	dom->dirty_limit_tstamp = jiffies;

  	return fprop_global_init(&dom->completions, gfp);
  }

  #ifdef CONFIG_CGROUP_WRITEBACK
  void wb_domain_exit(struct wb_domain *dom)
  {
  	del_timer_sync(&dom->period_timer);
  	fprop_global_destroy(&dom->completions);
  }
  #endif

  /*

   * bdi_min_ratio keeps the sum of the minimum dirty shares of all
   * registered backing devices, which, for obvious reasons, can not
   * exceed 100%.

   */

  static unsigned int bdi_min_ratio;
  
  int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
  {
  	int ret = 0;

  	spin_lock_bh(&bdi_lock);

  	if (min_ratio > bdi->max_ratio) {

  		ret = -EINVAL;

  	} else {
  		min_ratio -= bdi->min_ratio;
  		if (bdi_min_ratio + min_ratio < 100) {
  			bdi_min_ratio += min_ratio;
  			bdi->min_ratio += min_ratio;
  		} else {
  			ret = -EINVAL;
  		}
  	}

  	spin_unlock_bh(&bdi_lock);

  
  	return ret;
  }
  
  int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
  {

  	int ret = 0;
  
  	if (max_ratio > 100)
  		return -EINVAL;

  	spin_lock_bh(&bdi_lock);

  	if (bdi->min_ratio > max_ratio) {
  		ret = -EINVAL;
  	} else {
  		bdi->max_ratio = max_ratio;

  		bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;

  	}

  	spin_unlock_bh(&bdi_lock);

  
  	return ret;
  }

  EXPORT_SYMBOL(bdi_set_max_ratio);

  static unsigned long dirty_freerun_ceiling(unsigned long thresh,
  					   unsigned long bg_thresh)
  {
  	return (thresh + bg_thresh) / 2;
  }

  static unsigned long hard_dirty_limit(struct wb_domain *dom,
  				      unsigned long thresh)

  {

  	return max(thresh, dom->dirty_limit);

  }

  /*
   * Memory which can be further allocated to a memcg domain is capped by
   * system-wide clean memory excluding the amount being used in the domain.
   */
  static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
  			    unsigned long filepages, unsigned long headroom)

  {
  	struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);

  	unsigned long clean = filepages - min(filepages, mdtc->dirty);
  	unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
  	unsigned long other_clean = global_clean - min(global_clean, clean);

  	mdtc->avail = filepages + min(headroom, other_clean);

  }

  /**

   * __wb_calc_thresh - @wb's share of dirty throttling threshold
   * @dtc: dirty_throttle_context of interest

   *

   * Note that balance_dirty_pages() will only seriously take it as a hard limit
   * when sleeping max_pause per page is not enough to keep the dirty pages under
   * control. For example, when the device is completely stalled due to some error
   * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
   * In the other normal situations, it acts more gently by throttling the tasks

   * more (rather than completely block them) when the wb dirty pages go high.

   *

   * It allocates high/low dirty limits to fast/slow devices, in order to prevent

   * - starving fast devices
   * - piling up dirty pages (that will take long time to sync) on slow devices
   *

   * The wb's share of dirty limit will be adapting to its throughput and

   * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.

   *
   * Return: @wb's dirty limit in pages. The term "dirty" in the context of

   * dirty balancing includes all PG_dirty and PG_writeback pages.

   */

  static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc)

  {

  	struct wb_domain *dom = dtc_dom(dtc);

  	unsigned long thresh = dtc->thresh;

  	u64 wb_thresh;

  	unsigned long numerator, denominator;

  	unsigned long wb_min_ratio, wb_max_ratio;

  	/*

  	 * Calculate this BDI's share of the thresh ratio.

  	 */

  	fprop_fraction_percpu(&dom->completions, dtc->wb_completions,

  			      &numerator, &denominator);

  	wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100;
  	wb_thresh *= numerator;

  	wb_thresh = div64_ul(wb_thresh, denominator);

  	wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio);

  	wb_thresh += (thresh * wb_min_ratio) / 100;
  	if (wb_thresh > (thresh * wb_max_ratio) / 100)
  		wb_thresh = thresh * wb_max_ratio / 100;

  	return wb_thresh;

  }

  unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
  {
  	struct dirty_throttle_control gdtc = { GDTC_INIT(wb),
  					       .thresh = thresh };
  	return __wb_calc_thresh(&gdtc);

  }

  /*

   *                           setpoint - dirty 3
   *        f(dirty) := 1.0 + (----------------)
   *                           limit - setpoint
   *
   * it's a 3rd order polynomial that subjects to
   *
   * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
   * (2) f(setpoint) = 1.0 => the balance point
   * (3) f(limit)    = 0   => the hard limit
   * (4) df/dx      <= 0	 => negative feedback control
   * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
   *     => fast response on large errors; small oscillation near setpoint
   */

  static long long pos_ratio_polynom(unsigned long setpoint,

  					  unsigned long dirty,
  					  unsigned long limit)
  {
  	long long pos_ratio;
  	long x;

  	x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,

  		      (limit - setpoint) | 1);

  	pos_ratio = x;
  	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
  	pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
  	pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
  
  	return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
  }
  
  /*

   * Dirty position control.
   *
   * (o) global/bdi setpoints
   *

   * We want the dirty pages be balanced around the global/wb setpoints.

   * When the number of dirty pages is higher/lower than the setpoint, the
   * dirty position control ratio (and hence task dirty ratelimit) will be
   * decreased/increased to bring the dirty pages back to the setpoint.
   *
   *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
   *
   *     if (dirty < setpoint) scale up   pos_ratio
   *     if (dirty > setpoint) scale down pos_ratio
   *

   *     if (wb_dirty < wb_setpoint) scale up   pos_ratio
   *     if (wb_dirty > wb_setpoint) scale down pos_ratio

   *
   *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
   *
   * (o) global control line
   *
   *     ^ pos_ratio
   *     |
   *     |            |<===== global dirty control scope ======>|
   * 2.0 .............*
   *     |            .*
   *     |            . *
   *     |            .   *
   *     |            .     *
   *     |            .        *
   *     |            .            *
   * 1.0 ................................*
   *     |            .                  .     *
   *     |            .                  .          *
   *     |            .                  .              *
   *     |            .                  .                 *
   *     |            .                  .                    *
   *   0 +------------.------------------.----------------------*------------->
   *           freerun^          setpoint^                 limit^   dirty pages
   *

   * (o) wb control line

   *
   *     ^ pos_ratio
   *     |
   *     |            *
   *     |              *
   *     |                *
   *     |                  *
   *     |                    * |<=========== span ============>|
   * 1.0 .......................*
   *     |                      . *
   *     |                      .   *
   *     |                      .     *
   *     |                      .       *
   *     |                      .         *
   *     |                      .           *
   *     |                      .             *
   *     |                      .               *
   *     |                      .                 *
   *     |                      .                   *
   *     |                      .                     *
   * 1/4 ...............................................* * * * * * * * * * * *
   *     |                      .                         .
   *     |                      .                           .
   *     |                      .                             .
   *   0 +----------------------.-------------------------------.------------->

   *                wb_setpoint^                    x_intercept^

   *

   * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can

   * be smoothly throttled down to normal if it starts high in situations like
   * - start writing to a slow SD card and a fast disk at the same time. The SD

   *   card's wb_dirty may rush to many times higher than wb_setpoint.
   * - the wb dirty thresh drops quickly due to change of JBOD workload

   */

  static void wb_position_ratio(struct dirty_throttle_control *dtc)

  {

  	struct bdi_writeback *wb = dtc->wb;

  	unsigned long write_bw = wb->avg_write_bandwidth;

  	unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);

  	unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);

  	unsigned long wb_thresh = dtc->wb_thresh;

  	unsigned long x_intercept;
  	unsigned long setpoint;		/* dirty pages' target balance point */

  	unsigned long wb_setpoint;

  	unsigned long span;
  	long long pos_ratio;		/* for scaling up/down the rate limit */
  	long x;

  	dtc->pos_ratio = 0;

  	if (unlikely(dtc->dirty >= limit))

  		return;

  
  	/*
  	 * global setpoint
  	 *

  	 * See comment for pos_ratio_polynom().
  	 */
  	setpoint = (freerun + limit) / 2;

  	pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);

  
  	/*
  	 * The strictlimit feature is a tool preventing mistrusted filesystems
  	 * from growing a large number of dirty pages before throttling. For

  	 * such filesystems balance_dirty_pages always checks wb counters
  	 * against wb limits. Even if global "nr_dirty" is under "freerun".

  	 * This is especially important for fuse which sets bdi->max_ratio to
  	 * 1% by default. Without strictlimit feature, fuse writeback may
  	 * consume arbitrary amount of RAM because it is accounted in
  	 * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty".

  	 *

  	 * Here, in wb_position_ratio(), we calculate pos_ratio based on

  	 * two values: wb_dirty and wb_thresh. Let's consider an example:

  	 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
  	 * limits are set by default to 10% and 20% (background and throttle).

  	 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.

  	 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is

  	 * about ~6K pages (as the average of background and throttle wb

  	 * limits). The 3rd order polynomial will provide positive feedback if

  	 * wb_dirty is under wb_setpoint and vice versa.

  	 *

  	 * Note, that we cannot use global counters in these calculations

  	 * because we want to throttle process writing to a strictlimit wb

  	 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
  	 * in the example above).

  	 */

  	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {

  		long long wb_pos_ratio;

  		if (dtc->wb_dirty < 8) {
  			dtc->pos_ratio = min_t(long long, pos_ratio * 2,
  					   2 << RATELIMIT_CALC_SHIFT);
  			return;
  		}

  		if (dtc->wb_dirty >= wb_thresh)

  			return;

  		wb_setpoint = dirty_freerun_ceiling(wb_thresh,
  						    dtc->wb_bg_thresh);

  		if (wb_setpoint == 0 || wb_setpoint == wb_thresh)

  			return;

  		wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,

  						 wb_thresh);

  
  		/*

  		 * Typically, for strictlimit case, wb_setpoint << setpoint
  		 * and pos_ratio >> wb_pos_ratio. In the other words global

  		 * state ("dirty") is not limiting factor and we have to

  		 * make decision based on wb counters. But there is an

  		 * important case when global pos_ratio should get precedence:
  		 * global limits are exceeded (e.g. due to activities on other

  		 * wb's) while given strictlimit wb is below limit.

  		 *

  		 * "pos_ratio * wb_pos_ratio" would work for the case above,

  		 * but it would look too non-natural for the case of all

  		 * activity in the system coming from a single strictlimit wb

  		 * with bdi->max_ratio == 100%.
  		 *
  		 * Note that min() below somewhat changes the dynamics of the
  		 * control system. Normally, pos_ratio value can be well over 3

  		 * (when globally we are at freerun and wb is well below wb

  		 * setpoint). Now the maximum pos_ratio in the same situation
  		 * is 2. We might want to tweak this if we observe the control
  		 * system is too slow to adapt.
  		 */

  		dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
  		return;

  	}

  
  	/*
  	 * We have computed basic pos_ratio above based on global situation. If

  	 * the wb is over/under its share of dirty pages, we want to scale

  	 * pos_ratio further down/up. That is done by the following mechanism.
  	 */
  
  	/*

  	 * wb setpoint

  	 *

  	 *        f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)

  	 *

  	 *                        x_intercept - wb_dirty

  	 *                     := --------------------------

  	 *                        x_intercept - wb_setpoint

  	 *

  	 * The main wb control line is a linear function that subjects to

  	 *

  	 * (1) f(wb_setpoint) = 1.0
  	 * (2) k = - 1 / (8 * write_bw)  (in single wb case)
  	 *     or equally: x_intercept = wb_setpoint + 8 * write_bw

  	 *

  	 * For single wb case, the dirty pages are observed to fluctuate

  	 * regularly within range

  	 *        [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]

  	 * for various filesystems, where (2) can yield in a reasonable 12.5%
  	 * fluctuation range for pos_ratio.
  	 *

  	 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its

  	 * own size, so move the slope over accordingly and choose a slope that

  	 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.

  	 */

  	if (unlikely(wb_thresh > dtc->thresh))
  		wb_thresh = dtc->thresh;

  	/*

  	 * It's very possible that wb_thresh is close to 0 not because the

  	 * device is slow, but that it has remained inactive for long time.
  	 * Honour such devices a reasonable good (hopefully IO efficient)
  	 * threshold, so that the occasional writes won't be blocked and active
  	 * writes can rampup the threshold quickly.
  	 */

  	wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8);

  	/*

  	 * scale global setpoint to wb's:
  	 *	wb_setpoint = setpoint * wb_thresh / thresh

  	 */

  	x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);

  	wb_setpoint = setpoint * (u64)x >> 16;

  	/*

  	 * Use span=(8*write_bw) in single wb case as indicated by
  	 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.

  	 *

  	 *        wb_thresh                    thresh - wb_thresh
  	 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh
  	 *         thresh                           thresh

  	 */

  	span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;

  	x_intercept = wb_setpoint + span;

  	if (dtc->wb_dirty < x_intercept - span / 4) {
  		pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),

  				      (x_intercept - wb_setpoint) | 1);

  	} else
  		pos_ratio /= 4;

  	/*

  	 * wb reserve area, safeguard against dirty pool underrun and disk idle

  	 * It may push the desired control point of global dirty pages higher
  	 * than setpoint.
  	 */

  	x_intercept = wb_thresh / 2;

  	if (dtc->wb_dirty < x_intercept) {
  		if (dtc->wb_dirty > x_intercept / 8)
  			pos_ratio = div_u64(pos_ratio * x_intercept,
  					    dtc->wb_dirty);

  		else

  			pos_ratio *= 8;
  	}

  	dtc->pos_ratio = pos_ratio;

  }

  static void wb_update_write_bandwidth(struct bdi_writeback *wb,
  				      unsigned long elapsed,
  				      unsigned long written)

  {
  	const unsigned long period = roundup_pow_of_two(3 * HZ);

  	unsigned long avg = wb->avg_write_bandwidth;
  	unsigned long old = wb->write_bandwidth;

  	u64 bw;
  
  	/*
  	 * bw = written * HZ / elapsed
  	 *
  	 *                   bw * elapsed + write_bandwidth * (period - elapsed)
  	 * write_bandwidth = ---------------------------------------------------
  	 *                                          period

  	 *
  	 * @written may have decreased due to account_page_redirty().
  	 * Avoid underflowing @bw calculation.

  	 */

  	bw = written - min(written, wb->written_stamp);

  	bw *= HZ;
  	if (unlikely(elapsed > period)) {

  		bw = div64_ul(bw, elapsed);

  		avg = bw;
  		goto out;
  	}

  	bw += (u64)wb->write_bandwidth * (period - elapsed);

  	bw >>= ilog2(period);
  
  	/*
  	 * one more level of smoothing, for filtering out sudden spikes
  	 */
  	if (avg > old && old >= (unsigned long)bw)
  		avg -= (avg - old) >> 3;
  
  	if (avg < old && old <= (unsigned long)bw)
  		avg += (old - avg) >> 3;
  
  out:

  	/* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
  	avg = max(avg, 1LU);
  	if (wb_has_dirty_io(wb)) {
  		long delta = avg - wb->avg_write_bandwidth;
  		WARN_ON_ONCE(atomic_long_add_return(delta,
  					&wb->bdi->tot_write_bandwidth) <= 0);
  	}

  	wb->write_bandwidth = bw;
  	wb->avg_write_bandwidth = avg;

  }

  static void update_dirty_limit(struct dirty_throttle_control *dtc)

  {

  	struct wb_domain *dom = dtc_dom(dtc);

  	unsigned long thresh = dtc->thresh;

  	unsigned long limit = dom->dirty_limit;

  
  	/*
  	 * Follow up in one step.
  	 */
  	if (limit < thresh) {
  		limit = thresh;
  		goto update;
  	}
  
  	/*
  	 * Follow down slowly. Use the higher one as the target, because thresh
  	 * may drop below dirty. This is exactly the reason to introduce

  	 * dom->dirty_limit which is guaranteed to lie above the dirty pages.

  	 */

  	thresh = max(thresh, dtc->dirty);

  	if (limit > thresh) {
  		limit -= (limit - thresh) >> 5;
  		goto update;
  	}
  	return;
  update:

  	dom->dirty_limit = limit;

  }

  static void domain_update_bandwidth(struct dirty_throttle_control *dtc,

  				    unsigned long now)
  {

  	struct wb_domain *dom = dtc_dom(dtc);

  
  	/*
  	 * check locklessly first to optimize away locking for the most time
  	 */

  	if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))

  		return;

  	spin_lock(&dom->lock);
  	if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {

  		update_dirty_limit(dtc);

  		dom->dirty_limit_tstamp = now;

  	}

  	spin_unlock(&dom->lock);

  }

  /*

   * Maintain wb->dirty_ratelimit, the base dirty throttle rate.

   *

   * Normal wb tasks will be curbed at or below it in long term.

   * Obviously it should be around (write_bw / N) when there are N dd tasks.
   */

  static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,

  				      unsigned long dirtied,
  				      unsigned long elapsed)

  {

  	struct bdi_writeback *wb = dtc->wb;
  	unsigned long dirty = dtc->dirty;
  	unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);

  	unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);

  	unsigned long setpoint = (freerun + limit) / 2;

  	unsigned long write_bw = wb->avg_write_bandwidth;
  	unsigned long dirty_ratelimit = wb->dirty_ratelimit;

  	unsigned long dirty_rate;
  	unsigned long task_ratelimit;
  	unsigned long balanced_dirty_ratelimit;

  	unsigned long step;
  	unsigned long x;

  	unsigned long shift;

  
  	/*
  	 * The dirty rate will match the writeout rate in long term, except
  	 * when dirty pages are truncated by userspace or re-dirtied by FS.
  	 */

  	dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;

  	/*
  	 * task_ratelimit reflects each dd's dirty rate for the past 200ms.
  	 */
  	task_ratelimit = (u64)dirty_ratelimit *

  					dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;

  	task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
  
  	/*
  	 * A linear estimation of the "balanced" throttle rate. The theory is,

  	 * if there are N dd tasks, each throttled at task_ratelimit, the wb's

  	 * dirty_rate will be measured to be (N * task_ratelimit). So the below
  	 * formula will yield the balanced rate limit (write_bw / N).
  	 *
  	 * Note that the expanded form is not a pure rate feedback:
  	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate)		     (1)
  	 * but also takes pos_ratio into account:
  	 *	rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio  (2)
  	 *
  	 * (1) is not realistic because pos_ratio also takes part in balancing
  	 * the dirty rate.  Consider the state
  	 *	pos_ratio = 0.5						     (3)
  	 *	rate = 2 * (write_bw / N)				     (4)
  	 * If (1) is used, it will stuck in that state! Because each dd will
  	 * be throttled at
  	 *	task_ratelimit = pos_ratio * rate = (write_bw / N)	     (5)
  	 * yielding
  	 *	dirty_rate = N * task_ratelimit = write_bw		     (6)
  	 * put (6) into (1) we get
  	 *	rate_(i+1) = rate_(i)					     (7)
  	 *
  	 * So we end up using (2) to always keep
  	 *	rate_(i+1) ~= (write_bw / N)				     (8)
  	 * regardless of the value of pos_ratio. As long as (8) is satisfied,
  	 * pos_ratio is able to drive itself to 1.0, which is not only where
  	 * the dirty count meet the setpoint, but also where the slope of
  	 * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
  	 */
  	balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
  					   dirty_rate | 1);

  	/*
  	 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
  	 */
  	if (unlikely(balanced_dirty_ratelimit > write_bw))
  		balanced_dirty_ratelimit = write_bw;

  	/*
  	 * We could safely do this and return immediately:
  	 *

  	 *	wb->dirty_ratelimit = balanced_dirty_ratelimit;

  	 *
  	 * However to get a more stable dirty_ratelimit, the below elaborated

  	 * code makes use of task_ratelimit to filter out singular points and

  	 * limit the step size.
  	 *
  	 * The below code essentially only uses the relative value of
  	 *
  	 *	task_ratelimit - dirty_ratelimit
  	 *	= (pos_ratio - 1) * dirty_ratelimit
  	 *
  	 * which reflects the direction and size of dirty position error.
  	 */
  
  	/*
  	 * dirty_ratelimit will follow balanced_dirty_ratelimit iff
  	 * task_ratelimit is on the same side of dirty_ratelimit, too.
  	 * For example, when
  	 * - dirty_ratelimit > balanced_dirty_ratelimit
  	 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
  	 * lowering dirty_ratelimit will help meet both the position and rate
  	 * control targets. Otherwise, don't update dirty_ratelimit if it will
  	 * only help meet the rate target. After all, what the users ultimately
  	 * feel and care are stable dirty rate and small position error.
  	 *
  	 * |task_ratelimit - dirty_ratelimit| is used to limit the step size

  	 * and filter out the singular points of balanced_dirty_ratelimit. Which

  	 * keeps jumping around randomly and can even leap far away at times
  	 * due to the small 200ms estimation period of dirty_rate (we want to
  	 * keep that period small to reduce time lags).
  	 */
  	step = 0;

  
  	/*

  	 * For strictlimit case, calculations above were based on wb counters

  	 * and limits (starting from pos_ratio = wb_position_ratio() and up to

  	 * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate).

  	 * Hence, to calculate "step" properly, we have to use wb_dirty as
  	 * "dirty" and wb_setpoint as "setpoint".

  	 *

  	 * We rampup dirty_ratelimit forcibly if wb_dirty is low because
  	 * it's possible that wb_thresh is close to zero due to inactivity

  	 * of backing device.

  	 */

  	if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {

  		dirty = dtc->wb_dirty;
  		if (dtc->wb_dirty < 8)
  			setpoint = dtc->wb_dirty + 1;

  		else

  			setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;

  	}

  	if (dirty < setpoint) {

  		x = min3(wb->balanced_dirty_ratelimit,

  			 balanced_dirty_ratelimit, task_ratelimit);

  		if (dirty_ratelimit < x)
  			step = x - dirty_ratelimit;
  	} else {

  		x = max3(wb->balanced_dirty_ratelimit,

  			 balanced_dirty_ratelimit, task_ratelimit);

  		if (dirty_ratelimit > x)
  			step = dirty_ratelimit - x;
  	}
  
  	/*
  	 * Don't pursue 100% rate matching. It's impossible since the balanced
  	 * rate itself is constantly fluctuating. So decrease the track speed
  	 * when it gets close to the target. Helps eliminate pointless tremors.
  	 */

  	shift = dirty_ratelimit / (2 * step + 1);
  	if (shift < BITS_PER_LONG)
  		step = DIV_ROUND_UP(step >> shift, 8);
  	else
  		step = 0;

  
  	if (dirty_ratelimit < balanced_dirty_ratelimit)
  		dirty_ratelimit += step;
  	else
  		dirty_ratelimit -= step;

  	wb->dirty_ratelimit = max(dirty_ratelimit, 1UL);
  	wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;

  	trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);

  }

  static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
  				  struct dirty_throttle_control *mdtc,

  				  unsigned long start_time,
  				  bool update_ratelimit)

  {

  	struct bdi_writeback *wb = gdtc->wb;

  	unsigned long now = jiffies;

  	unsigned long elapsed = now - wb->bw_time_stamp;

  	unsigned long dirtied;

  	unsigned long written;

  	lockdep_assert_held(&wb->list_lock);

  	/*
  	 * rate-limit, only update once every 200ms.
  	 */
  	if (elapsed < BANDWIDTH_INTERVAL)
  		return;

  	dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);
  	written = percpu_counter_read(&wb->stat[WB_WRITTEN]);

  
  	/*
  	 * Skip quiet periods when disk bandwidth is under-utilized.
  	 * (at least 1s idle time between two flusher runs)
  	 */

  	if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time))

  		goto snapshot;

  	if (update_ratelimit) {

  		domain_update_bandwidth(gdtc, now);
  		wb_update_dirty_ratelimit(gdtc, dirtied, elapsed);
  
  		/*
  		 * @mdtc is always NULL if !CGROUP_WRITEBACK but the
  		 * compiler has no way to figure that out.  Help it.
  		 */
  		if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
  			domain_update_bandwidth(mdtc, now);
  			wb_update_dirty_ratelimit(mdtc, dirtied, elapsed);
  		}

  	}

  	wb_update_write_bandwidth(wb, elapsed, written);

  
  snapshot:

  	wb->dirtied_stamp = dirtied;
  	wb->written_stamp = written;
  	wb->bw_time_stamp = now;

  }

  void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time)

  {

  	struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };

  	__wb_update_bandwidth(&gdtc, NULL, start_time, false);

  }

  /*

   * After a task dirtied this many pages, balance_dirty_pages_ratelimited()

   * will look to see if it needs to start dirty throttling.
   *
   * If dirty_poll_interval is too low, big NUMA machines will call the expensive

   * global_zone_page_state() too often. So scale it near-sqrt to the safety margin

   * (the number of pages we may dirty without exceeding the dirty limits).
   */
  static unsigned long dirty_poll_interval(unsigned long dirty,
  					 unsigned long thresh)
  {
  	if (thresh > dirty)
  		return 1UL << (ilog2(thresh - dirty) >> 1);
  
  	return 1;
  }

  static unsigned long wb_max_pause(struct bdi_writeback *wb,

  				  unsigned long wb_dirty)

  {

  	unsigned long bw = wb->avg_write_bandwidth;

  	unsigned long t;

  	/*
  	 * Limit pause time for small memory systems. If sleeping for too long
  	 * time, a small pool of dirty/writeback pages may go empty and disk go
  	 * idle.
  	 *
  	 * 8 serves as the safety ratio.
  	 */

  	t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));

  	t++;

  	return min_t(unsigned long, t, MAX_PAUSE);

  }

  static long wb_min_pause(struct bdi_writeback *wb,
  			 long max_pause,
  			 unsigned long task_ratelimit,
  			 unsigned long dirty_ratelimit,
  			 int *nr_dirtied_pause)

  {

  	long hi = ilog2(wb->avg_write_bandwidth);
  	long lo = ilog2(wb->dirty_ratelimit);

  	long t;		/* target pause */
  	long pause;	/* estimated next pause */
  	int pages;	/* target nr_dirtied_pause */

  	/* target for 10ms pause on 1-dd case */
  	t = max(1, HZ / 100);

  
  	/*
  	 * Scale up pause time for concurrent dirtiers in order to reduce CPU
  	 * overheads.
  	 *

  	 * (N * 10ms) on 2^N concurrent tasks.

  	 */
  	if (hi > lo)

  		t += (hi - lo) * (10 * HZ) / 1024;

  
  	/*

  	 * This is a bit convoluted. We try to base the next nr_dirtied_pause
  	 * on the much more stable dirty_ratelimit. However the next pause time
  	 * will be computed based on task_ratelimit and the two rate limits may
  	 * depart considerably at some time. Especially if task_ratelimit goes
  	 * below dirty_ratelimit/2 and the target pause is max_pause, the next
  	 * pause time will be max_pause*2 _trimmed down_ to max_pause.  As a
  	 * result task_ratelimit won't be executed faithfully, which could
  	 * eventually bring down dirty_ratelimit.

  	 *

  	 * We apply two rules to fix it up:
  	 * 1) try to estimate the next pause time and if necessary, use a lower
  	 *    nr_dirtied_pause so as not to exceed max_pause. When this happens,
  	 *    nr_dirtied_pause will be "dancing" with task_ratelimit.
  	 * 2) limit the target pause time to max_pause/2, so that the normal
  	 *    small fluctuations of task_ratelimit won't trigger rule (1) and
  	 *    nr_dirtied_pause will remain as stable as dirty_ratelimit.

  	 */

  	t = min(t, 1 + max_pause / 2);
  	pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);

  
  	/*

  	 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test
  	 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
  	 * When the 16 consecutive reads are often interrupted by some dirty
  	 * throttling pause during the async writes, cfq will go into idles
  	 * (deadline is fine). So push nr_dirtied_pause as high as possible
  	 * until reaches DIRTY_POLL_THRESH=32 pages.

  	 */

  	if (pages < DIRTY_POLL_THRESH) {
  		t = max_pause;
  		pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
  		if (pages > DIRTY_POLL_THRESH) {
  			pages = DIRTY_POLL_THRESH;
  			t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
  		}
  	}

  	pause = HZ * pages / (task_ratelimit + 1);
  	if (pause > max_pause) {
  		t = max_pause;
  		pages = task_ratelimit * t / roundup_pow_of_two(HZ);
  	}

  	*nr_dirtied_pause = pages;

  	/*

  	 * The minimal pause time will normally be half the target pause time.

  	 */

  	return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;

  }

  static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)

  {

  	struct bdi_writeback *wb = dtc->wb;

  	unsigned long wb_reclaimable;

  
  	/*

  	 * wb_thresh is not treated as some limiting factor as

  	 * dirty_thresh, due to reasons

  	 * - in JBOD setup, wb_thresh can fluctuate a lot

  	 * - in a system with HDD and USB key, the USB key may somehow

  	 *   go into state (wb_dirty >> wb_thresh) either because
  	 *   wb_dirty starts high, or because wb_thresh drops low.

  	 *   In this case we don't want to hard throttle the USB key

  	 *   dirtiers for 100 seconds until wb_dirty drops under
  	 *   wb_thresh. Instead the auxiliary wb control line in

  	 *   wb_position_ratio() will let the dirtier task progress

  	 *   at some rate <= (write_bw / 2) for bringing down wb_dirty.

  	 */

  	dtc->wb_thresh = __wb_calc_thresh(dtc);