ioat: switch watchdog and reset handler from workqueue to timer

In order to support dynamic resizing of the descriptor ring or polling for a descriptor in the presence of a hung channel the reset handler needs to make progress while in a non-preemptible context. The current workqueue implementation precludes polling channel reset completion under spin_lock(). This conversion also allows us to return to opportunistic cleanup in the ioat2 case as the timer implementation guarantees at least one cleanup after every descriptor is submitted. This means the worst case completion latency becomes the timer frequency (for exceptional circumstances), but with the benefit of avoiding busy waiting when the lock is contended. Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com>

ioat: switch watchdog and reset handler from workqueue to timer
In order to support dynamic resizing of the descriptor ring or polling for a descriptor in the presence of a hung channel the reset handler needs to make progress while in a non-preemptible context. The current workqueue implementation precludes polling channel reset completion under spin_lock(). This conversion also allows us to return to opportunistic cleanup in the ioat2 case as the timer implementation guarantees at least one cleanup after every descriptor is submitted. This means the worst case completion latency becomes the timer frequency (for exceptional circumstances), but with the benefit of avoiding busy waiting when the lock is contended. Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Dan Williams
1 parent ad643f54c8
Showing 6 changed files with 396 additions and 436 deletions Side-by-side Diff
drivers/dma/ioat/dma.c
drivers/dma/ioat/dma.h
drivers/dma/ioat/dma_v2.c
drivers/dma/ioat/dma_v2.h
drivers/dma/ioat/registers.h
drivers/idle/i7300_idle.c
@@ -99,23 +99,26 @@
 /* common channel initialization */
 void ioat_init_channel(struct ioatdma_device *device,
 		       struct ioat_chan_common *chan, int idx,
-		       work_func_t work_fn, void (*tasklet)(unsigned long),
-		       unsigned long tasklet_data)
+		       void (*timer_fn)(unsigned long),
+		       void (*tasklet)(unsigned long),
+		       unsigned long ioat)
 {
 	struct dma_device *dma = &device->common;
  
 	chan->device = device;
 	chan->reg_base = device->reg_base + (0x80 * (idx + 1));
-	INIT_DELAYED_WORK(&chan->work, work_fn);
 	spin_lock_init(&chan->cleanup_lock);
 	chan->common.device = dma;
 	list_add_tail(&chan->common.device_node, &dma->channels);
 	device->idx[idx] = chan;
-	tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
+	init_timer(&chan->timer);
+	chan->timer.function = timer_fn;
+	chan->timer.data = ioat;
+	tasklet_init(&chan->cleanup_task, tasklet, ioat);
 	tasklet_disable(&chan->cleanup_task);
 }
  
-static void ioat1_reset_part2(struct work_struct *work);
+static void ioat1_timer_event(unsigned long data);
  
 /**
  * ioat1_dma_enumerate_channels - find and initialize the device's channels
@@ -153,7 +156,7 @@
 			break;
  
 		ioat_init_channel(device, &ioat->base, i,
-				  ioat1_reset_part2,
+				  ioat1_timer_event,
 				  ioat1_cleanup_tasklet,
 				  (unsigned long) ioat);
 		ioat->xfercap = xfercap;
@@ -193,61 +196,6 @@
 }
  
 /**
- * ioat1_reset_part2 - reinit the channel after a reset
- */
-static void ioat1_reset_part2(struct work_struct *work)
-{
-	struct ioat_chan_common *chan;
-	struct ioat_dma_chan *ioat;
-	struct ioat_desc_sw *desc;
-	int dmacount;
-	bool start_null = false;
-
-	chan = container_of(work, struct ioat_chan_common, work.work);
-	ioat = container_of(chan, struct ioat_dma_chan, base);
-	spin_lock_bh(&chan->cleanup_lock);
-	spin_lock_bh(&ioat->desc_lock);
-
-	*chan->completion = 0;
-	ioat->pending = 0;
-
-	/* count the descriptors waiting */
-	dmacount = 0;
-	if (ioat->used_desc.prev) {
-		desc = to_ioat_desc(ioat->used_desc.prev);
-		do {
-			dmacount++;
-			desc = to_ioat_desc(desc->node.next);
-		} while (&desc->node != ioat->used_desc.next);
-	}
-
-	if (dmacount) {
-		/*
-		 * write the new starting descriptor address
-		 * this puts channel engine into ARMED state
-		 */
-		desc = to_ioat_desc(ioat->used_desc.prev);
-		writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
-		       chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-		writel(((u64) desc->txd.phys) >> 32,
-		       chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-		writeb(IOAT_CHANCMD_START, chan->reg_base
-			+ IOAT_CHANCMD_OFFSET(chan->device->version));
-	} else
-		start_null = true;
-	spin_unlock_bh(&ioat->desc_lock);
-	spin_unlock_bh(&chan->cleanup_lock);
-
-	dev_err(to_dev(chan),
-		"chan%d reset - %d descs waiting, %d total desc\n",
-		chan_num(chan), dmacount, ioat->desccount);
-
-	if (start_null)
-		ioat1_dma_start_null_desc(ioat);
-}
-
-/**
  * ioat1_reset_channel - restart a channel
  * @ioat: IOAT DMA channel handle
  */
  
@@ -257,12 +205,9 @@
 	void __iomem *reg_base = chan->reg_base;
 	u32 chansts, chanerr;
  
-	if (!ioat->used_desc.prev)
-		return;
-
-	dev_dbg(to_dev(chan), "%s\n", __func__);
+	dev_warn(to_dev(chan), "reset\n");
 	chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
-	chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+	chansts = *chan->completion & IOAT_CHANSTS_STATUS;
 	if (chanerr) {
 		dev_err(to_dev(chan),
 			"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
  
  
  
@@ -278,100 +223,19 @@
 	 * while we're waiting.
 	 */
  
-	spin_lock_bh(&ioat->desc_lock);
 	ioat->pending = INT_MIN;
 	writeb(IOAT_CHANCMD_RESET,
 	       reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
-	spin_unlock_bh(&ioat->desc_lock);
-
-	/* schedule the 2nd half instead of sleeping a long time */
-	schedule_delayed_work(&chan->work, RESET_DELAY);
+	set_bit(IOAT_RESET_PENDING, &chan->state);
+	mod_timer(&chan->timer, jiffies + RESET_DELAY);
 }
  
-/**
- * ioat1_chan_watchdog - watch for stuck channels
- */
-static void ioat1_chan_watchdog(struct work_struct *work)
-{
-	struct ioatdma_device *device =
-		container_of(work, struct ioatdma_device, work.work);
-	struct ioat_dma_chan *ioat;
-	struct ioat_chan_common *chan;
-	int i;
-	u64 completion;
-	u32 completion_low;
-	unsigned long compl_desc_addr_hw;
-
-	for (i = 0; i < device->common.chancnt; i++) {
-		chan = ioat_chan_by_index(device, i);
-		ioat = container_of(chan, struct ioat_dma_chan, base);
-
-		if (/* have we started processing anything yet */
-		    chan->last_completion
-		    /* have we completed any since last watchdog cycle? */
-		    && (chan->last_completion == chan->watchdog_completion)
-		    /* has TCP stuck on one cookie since last watchdog? */
-		    && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
-		    && (chan->watchdog_tcp_cookie != chan->completed_cookie)
-		    /* is there something in the chain to be processed? */
-		    /* CB1 chain always has at least the last one processed */
-		    && (ioat->used_desc.prev != ioat->used_desc.next)
-		    && ioat->pending == 0) {
-
-			/*
-			 * check CHANSTS register for completed
-			 * descriptor address.
-			 * if it is different than completion writeback,
-			 * it is not zero
-			 * and it has changed since the last watchdog
-			 *     we can assume that channel
-			 *     is still working correctly
-			 *     and the problem is in completion writeback.
-			 *     update completion writeback
-			 *     with actual CHANSTS value
-			 * else
-			 *     try resetting the channel
-			 */
-
-			/* we need to read the low address first as this
-			 * causes the chipset to latch the upper bits
-			 * for the subsequent read
-			 */
-			completion_low = readl(chan->reg_base +
-				IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
-			completion = readl(chan->reg_base +
-				IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
-			completion <<= 32;
-			completion |= completion_low;
-			compl_desc_addr_hw = completion &
-					IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
-
-			if ((compl_desc_addr_hw != 0)
-			   && (compl_desc_addr_hw != chan->watchdog_completion)
-			   && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
-				chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
-				*chan->completion = completion;
-			} else {
-				ioat1_reset_channel(ioat);
-				chan->watchdog_completion = 0;
-				chan->last_compl_desc_addr_hw = 0;
-			}
-		} else {
-			chan->last_compl_desc_addr_hw = 0;
-			chan->watchdog_completion = chan->last_completion;
-		}
-
-		chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
-	}
-
-	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-}
-
 static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct dma_chan *c = tx->chan;
 	struct ioat_dma_chan *ioat = to_ioat_chan(c);
 	struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+	struct ioat_chan_common *chan = &ioat->base;
 	struct ioat_desc_sw *first;
 	struct ioat_desc_sw *chain_tail;
 	dma_cookie_t cookie;
@@ -396,6 +260,9 @@
 	dump_desc_dbg(ioat, chain_tail);
 	dump_desc_dbg(ioat, first);
  
+	if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
 	ioat->pending += desc->hw->tx_cnt;
 	if (ioat->pending >= ioat_pending_level)
 		__ioat1_dma_memcpy_issue_pending(ioat);
@@ -520,6 +387,7 @@
 		return;
  
 	tasklet_disable(&chan->cleanup_task);
+	del_timer_sync(&chan->timer);
 	ioat1_cleanup(ioat);
  
 	/* Delay 100ms after reset to allow internal DMA logic to quiesce
@@ -560,9 +428,6 @@
  
 	chan->last_completion = 0;
 	chan->completion_dma = 0;
-	chan->watchdog_completion = 0;
-	chan->last_compl_desc_addr_hw = 0;
-	chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
 	ioat->pending = 0;
 	ioat->desccount = 0;
 }
  
  
@@ -705,15 +570,15 @@
 	u64 completion;
  
 	completion = *chan->completion;
-	phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+	phys_complete = ioat_chansts_to_addr(completion);
  
 	dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
 		(unsigned long long) phys_complete);
  
-	if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
-				IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
+	if (is_ioat_halted(completion)) {
+		u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
 		dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
-			readl(chan->reg_base + IOAT_CHANERR_OFFSET));
+			chanerr);
  
 		/* TODO do something to salvage the situation */
 	}
  
  
  
  
@@ -721,48 +586,31 @@
 	return phys_complete;
 }
  
-/**
- * ioat1_cleanup - cleanup up finished descriptors
- * @chan: ioat channel to be cleaned up
- */
-static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+			   unsigned long *phys_complete)
 {
+	*phys_complete = ioat_get_current_completion(chan);
+	if (*phys_complete == chan->last_completion)
+		return false;
+	clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+	mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+	return true;
+}
+
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
 	struct ioat_chan_common *chan = &ioat->base;
-	unsigned long phys_complete;
-	struct ioat_desc_sw *desc, *_desc;
-	dma_cookie_t cookie = 0;
+	struct list_head *_desc, *n;
 	struct dma_async_tx_descriptor *tx;
  
-	prefetch(chan->completion);
-
-	if (!spin_trylock_bh(&chan->cleanup_lock))
-		return;
-
-	phys_complete = ioat_get_current_completion(chan);
-	if (phys_complete == chan->last_completion) {
-		spin_unlock_bh(&chan->cleanup_lock);
-		/*
-		 * perhaps we're stuck so hard that the watchdog can't go off?
-		 * try to catch it after 2 seconds
-		 */
-		if (time_after(jiffies,
-			       chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
-			ioat1_chan_watchdog(&(chan->device->work.work));
-			chan->last_completion_time = jiffies;
-		}
-		return;
-	}
-	chan->last_completion_time = jiffies;
-
-	cookie = 0;
-	if (!spin_trylock_bh(&ioat->desc_lock)) {
-		spin_unlock_bh(&chan->cleanup_lock);
-		return;
-	}
-
 	dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
 		 __func__, phys_complete);
-	list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+	list_for_each_safe(_desc, n, &ioat->used_desc) {
+		struct ioat_desc_sw *desc;
+
+		prefetch(n);
+		desc = list_entry(_desc, typeof(*desc), node);
 		tx = &desc->txd;
 		/*
 		 * Incoming DMA requests may use multiple descriptors,
@@ -771,7 +619,8 @@
 		 */
 		dump_desc_dbg(ioat, desc);
 		if (tx->cookie) {
-			cookie = tx->cookie;
+			chan->completed_cookie = tx->cookie;
+			tx->cookie = 0;
 			ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
 			if (tx->callback) {
 				tx->callback(tx->callback_param);
  
  
  
  
  
  
  
@@ -786,27 +635,110 @@
 			 */
 			if (async_tx_test_ack(tx))
 				list_move_tail(&desc->node, &ioat->free_desc);
-			else
-				tx->cookie = 0;
 		} else {
 			/*
 			 * last used desc. Do not remove, so we can
-			 * append from it, but don't look at it next
-			 * time, either
+			 * append from it.
 			 */
-			tx->cookie = 0;
  
+			/* if nothing else is pending, cancel the
+			 * completion timeout
+			 */
+			if (n == &ioat->used_desc) {
+				dev_dbg(to_dev(chan),
+					"%s cancel completion timeout\n",
+					__func__);
+				clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+			}
+
 			/* TODO check status bits? */
 			break;
 		}
 	}
  
+	chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+	struct ioat_chan_common *chan = &ioat->base;
+	unsigned long phys_complete;
+
+	prefetch(chan->completion);
+
+	if (!spin_trylock_bh(&chan->cleanup_lock))
+		return;
+
+	if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+		spin_unlock_bh(&chan->cleanup_lock);
+		return;
+	}
+
+	if (!spin_trylock_bh(&ioat->desc_lock)) {
+		spin_unlock_bh(&chan->cleanup_lock);
+		return;
+	}
+
+	__cleanup(ioat, phys_complete);
+
 	spin_unlock_bh(&ioat->desc_lock);
+	spin_unlock_bh(&chan->cleanup_lock);
+}
  
-	chan->last_completion = phys_complete;
-	if (cookie != 0)
-		chan->completed_cookie = cookie;
+static void ioat1_timer_event(unsigned long data)
+{
+	struct ioat_dma_chan *ioat = (void *) data;
+	struct ioat_chan_common *chan = &ioat->base;
  
+	dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+	spin_lock_bh(&chan->cleanup_lock);
+	if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+		struct ioat_desc_sw *desc;
+
+		spin_lock_bh(&ioat->desc_lock);
+
+		/* restart active descriptors */
+		desc = to_ioat_desc(ioat->used_desc.prev);
+		ioat_set_chainaddr(ioat, desc->txd.phys);
+		ioat_start(chan);
+
+		ioat->pending = 0;
+		set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+		spin_unlock_bh(&ioat->desc_lock);
+	} else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+		unsigned long phys_complete;
+
+		spin_lock_bh(&ioat->desc_lock);
+		/* if we haven't made progress and we have already
+		 * acknowledged a pending completion once, then be more
+		 * forceful with a restart
+		 */
+		if (ioat_cleanup_preamble(chan, &phys_complete))
+			__cleanup(ioat, phys_complete);
+		else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+			ioat1_reset_channel(ioat);
+		else {
+			u64 status = ioat_chansts(chan);
+
+			/* manually update the last completion address */
+			if (ioat_chansts_to_addr(status) != 0)
+				*chan->completion = status;
+
+			set_bit(IOAT_COMPLETION_ACK, &chan->state);
+			mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+		}
+		spin_unlock_bh(&ioat->desc_lock);
+	}
 	spin_unlock_bh(&chan->cleanup_lock);
 }
  
@@ -855,13 +787,8 @@
 	list_add_tail(&desc->node, &ioat->used_desc);
 	dump_desc_dbg(ioat, desc);
  
-	writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
-	       chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
-	writel(((u64) desc->txd.phys) >> 32,
-	       chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
-
-	writeb(IOAT_CHANCMD_START, chan->reg_base
-		+ IOAT_CHANCMD_OFFSET(chan->device->version));
+	ioat_set_chainaddr(ioat, desc->txd.phys);
+	ioat_start(chan);
 	spin_unlock_bh(&ioat->desc_lock);
 }
  
  
@@ -1194,18 +1121,12 @@
 	if (dca)
 		device->dca = ioat_dca_init(pdev, device->reg_base);
  
-	INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
-	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-
 	return err;
 }
  
 void __devexit ioat_dma_remove(struct ioatdma_device *device)
 {
 	struct dma_device *dma = &device->common;
-
-	if (device->version != IOAT_VER_3_0)
-		cancel_delayed_work(&device->work);
  
 	ioat_disable_interrupts(device);
  
@@ -23,6 +23,7 @@
  
 #include <linux/dmaengine.h>
 #include "hw.h"
+#include "registers.h"
 #include <linux/init.h>
 #include <linux/dmapool.h>
 #include <linux/cache.h>
@@ -33,7 +34,6 @@
  
 #define IOAT_LOW_COMPLETION_MASK	0xffffffc0
 #define IOAT_DMA_DCA_ANY_CPU		~0
-#define IOAT_WATCHDOG_PERIOD		(2 * HZ)
  
 #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
 #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
@@ -42,9 +42,6 @@
  
 #define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
  
-#define RESET_DELAY  msecs_to_jiffies(100)
-#define WATCHDOG_DELAY  round_jiffies(msecs_to_jiffies(2000))
-
 /*
  * workaround for IOAT ver.3.0 null descriptor issue
  * (channel returns error when size is 0)
@@ -72,7 +69,6 @@
 	struct pci_pool *completion_pool;
 	struct dma_device common;
 	u8 version;
-	struct delayed_work work;
 	struct msix_entry msix_entries[4];
 	struct ioat_chan_common *idx[4];
 	struct dca_provider *dca;
  
  
  
  
  
@@ -81,24 +77,21 @@
 };
  
 struct ioat_chan_common {
+	struct dma_chan common;
 	void __iomem *reg_base;
-
 	unsigned long last_completion;
-	unsigned long last_completion_time;
-
 	spinlock_t cleanup_lock;
 	dma_cookie_t completed_cookie;
-	unsigned long watchdog_completion;
-	int watchdog_tcp_cookie;
-	u32 watchdog_last_tcp_cookie;
-	struct delayed_work work;
-
+	unsigned long state;
+	#define IOAT_COMPLETION_PENDING 0
+	#define IOAT_COMPLETION_ACK 1
+	#define IOAT_RESET_PENDING 2
+	struct timer_list timer;
+	#define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+	#define RESET_DELAY msecs_to_jiffies(100)
 	struct ioatdma_device *device;
-	struct dma_chan common;
-
 	dma_addr_t completion_dma;
 	u64 *completion;
-	unsigned long last_compl_desc_addr_hw;
 	struct tasklet_struct cleanup_task;
 };
  
@@ -148,7 +141,6 @@
  
 	last_used = c->cookie;
 	last_complete = chan->completed_cookie;
-	chan->watchdog_tcp_cookie = cookie;
  
 	if (done)
 		*done = last_complete;
@@ -215,6 +207,85 @@
 	return device->idx[index];
 }
  
+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
+{
+	u8 ver = chan->device->version;
+	u64 status;
+	u32 status_lo;
+
+	/* We need to read the low address first as this causes the
+	 * chipset to latch the upper bits for the subsequent read
+	 */
+	status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
+	status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
+	status <<= 32;
+	status |= status_lo;
+
+	return status;
+}
+
+static inline void ioat_start(struct ioat_chan_common *chan)
+{
+	u8 ver = chan->device->version;
+
+	writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline u64 ioat_chansts_to_addr(u64 status)
+{
+	return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+}
+
+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
+{
+	return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+}
+
+static inline void ioat_suspend(struct ioat_chan_common *chan)
+{
+	u8 ver = chan->device->version;
+
+	writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
+{
+	struct ioat_chan_common *chan = &ioat->base;
+
+	writel(addr & 0x00000000FFFFFFFF,
+	       chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
+	writel(addr >> 32,
+	       chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
+}
+
+static inline bool is_ioat_active(unsigned long status)
+{
+	return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
+}
+
+static inline bool is_ioat_idle(unsigned long status)
+{
+	return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
+}
+
+static inline bool is_ioat_halted(unsigned long status)
+{
+	return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
+}
+
+static inline bool is_ioat_suspended(unsigned long status)
+{
+	return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
+}
+
+/* channel was fatally programmed */
+static inline bool is_ioat_bug(unsigned long err)
+{
+	return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
+			 IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
+			 IOAT_CHANERR_LENGTH_ERR));
+}
+
 int __devinit ioat_probe(struct ioatdma_device *device);
 int __devinit ioat_register(struct ioatdma_device *device);
 int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
  
@@ -224,9 +295,12 @@
 unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
 void ioat_init_channel(struct ioatdma_device *device,
 		       struct ioat_chan_common *chan, int idx,
-		       work_func_t work_fn, void (*tasklet)(unsigned long),
-		       unsigned long tasklet_data);
+		       void (*timer_fn)(unsigned long),
+		       void (*tasklet)(unsigned long),
+		       unsigned long ioat);
 void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
 		    size_t len, struct ioat_dma_descriptor *hw);
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+			   unsigned long *phys_complete);
 #endif /* IOATDMA_H */
@@ -49,7 +49,7 @@
 	void * __iomem reg_base = ioat->base.reg_base;
  
 	ioat->pending = 0;
-	ioat->dmacount += ioat2_ring_pending(ioat);
+	ioat->dmacount += ioat2_ring_pending(ioat);;
 	ioat->issued = ioat->head;
 	/* make descriptor updates globally visible before notifying channel */
 	wmb();
@@ -92,7 +92,6 @@
  
 static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
 {
-	void __iomem *reg_base = ioat->base.reg_base;
 	struct ioat_ring_ent *desc;
 	struct ioat_dma_descriptor *hw;
 	int idx;
@@ -118,10 +117,7 @@
 	hw->src_addr = 0;
 	hw->dst_addr = 0;
 	async_tx_ack(&desc->txd);
-	writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
-	       reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-	writel(((u64) desc->txd.phys) >> 32,
-	       reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+	ioat2_set_chainaddr(ioat, desc->txd.phys);
 	dump_desc_dbg(ioat, desc);
 	__ioat2_issue_pending(ioat);
 }
  
  
  
  
  
  
  
  
  
  
@@ -133,33 +129,97 @@
 	spin_unlock_bh(&ioat->ring_lock);
 }
  
-static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+	struct ioat_chan_common *chan = &ioat->base;
+	struct dma_async_tx_descriptor *tx;
+	struct ioat_ring_ent *desc;
+	bool seen_current = false;
+	u16 active;
+	int i;
  
+	dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+		__func__, ioat->head, ioat->tail, ioat->issued);
+
+	active = ioat2_ring_active(ioat);
+	for (i = 0; i < active && !seen_current; i++) {
+		prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+		desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+		tx = &desc->txd;
+		dump_desc_dbg(ioat, desc);
+		if (tx->cookie) {
+			ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+			chan->completed_cookie = tx->cookie;
+			tx->cookie = 0;
+			if (tx->callback) {
+				tx->callback(tx->callback_param);
+				tx->callback = NULL;
+			}
+		}
+
+		if (tx->phys == phys_complete)
+			seen_current = true;
+	}
+	ioat->tail += i;
+	BUG_ON(!seen_current); /* no active descs have written a completion? */
+
+	chan->last_completion = phys_complete;
+	if (ioat->head == ioat->tail) {
+		dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+			__func__);
+		clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+	}
+}
+
 /**
- * ioat2_reset_part2 - reinit the channel after a reset
+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
+ * @chan: ioat channel to be cleaned up
  */
-static void ioat2_reset_part2(struct work_struct *work)
+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
 {
-	struct ioat_chan_common *chan;
-	struct ioat2_dma_chan *ioat;
+	struct ioat_chan_common *chan = &ioat->base;
+	unsigned long phys_complete;
  
-	chan = container_of(work, struct ioat_chan_common, work.work);
-	ioat = container_of(chan, struct ioat2_dma_chan, base);
+	prefetch(chan->completion);
  
-	/* ensure that ->tail points to the stalled descriptor
-	 * (ioat->pending is set to 2 at this point so no new
-	 * descriptors will be issued while we perform this cleanup)
-	 */
+	if (!spin_trylock_bh(&chan->cleanup_lock))
+		return;
+
+	if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+		spin_unlock_bh(&chan->cleanup_lock);
+		return;
+	}
+
+	if (!spin_trylock_bh(&ioat->ring_lock)) {
+		spin_unlock_bh(&chan->cleanup_lock);
+		return;
+	}
+
+	__cleanup(ioat, phys_complete);
+
+	spin_unlock_bh(&ioat->ring_lock);
+	spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat2_cleanup_tasklet(unsigned long data)
+{
+	struct ioat2_dma_chan *ioat = (void *) data;
+
 	ioat2_cleanup(ioat);
+	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
  
-	spin_lock_bh(&chan->cleanup_lock);
-	spin_lock_bh(&ioat->ring_lock);
+static void __restart_chan(struct ioat2_dma_chan *ioat)
+{
+	struct ioat_chan_common *chan = &ioat->base;
  
 	/* set the tail to be re-issued */
 	ioat->issued = ioat->tail;
 	ioat->dmacount = 0;
+	set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+	mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
  
-	dev_dbg(to_dev(&ioat->base),
+	dev_dbg(to_dev(chan),
 		"%s: head: %#x tail: %#x issued: %#x count: %#x\n",
 		__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
@@ -167,183 +227,72 @@
 		struct ioat_ring_ent *desc;
  
 		desc = ioat2_get_ring_ent(ioat, ioat->tail);
-		writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
-		       chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
-		writel(((u64) desc->txd.phys) >> 32,
-		       chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+		ioat2_set_chainaddr(ioat, desc->txd.phys);
 		__ioat2_issue_pending(ioat);
 	} else
 		__ioat2_start_null_desc(ioat);
-
-	spin_unlock_bh(&ioat->ring_lock);
-	spin_unlock_bh(&chan->cleanup_lock);
-
-	dev_info(to_dev(chan),
-		 "chan%d reset - %d descs waiting, %d total desc\n",
-		 chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
 }
  
-/**
- * ioat2_reset_channel - restart a channel
- * @ioat: IOAT DMA channel handle
- */
-static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
 {
-	u32 chansts, chanerr;
 	struct ioat_chan_common *chan = &ioat->base;
-	u16 active;
+	unsigned long phys_complete;
+	u32 status;
  
-	spin_lock_bh(&ioat->ring_lock);
-	active = ioat2_ring_active(ioat);
-	spin_unlock_bh(&ioat->ring_lock);
-	if (!active)
-		return;
-
-	chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
-	chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
-	if (chanerr) {
-		dev_err(to_dev(chan),
-			"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
-			chan_num(chan), chansts, chanerr);
-		writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+	status = ioat_chansts(chan);
+	if (is_ioat_active(status) || is_ioat_idle(status))
+		ioat_suspend(chan);
+	while (is_ioat_active(status) || is_ioat_idle(status)) {
+		status = ioat_chansts(chan);
+		cpu_relax();
 	}
  
-	spin_lock_bh(&ioat->ring_lock);
-	ioat->pending = 2;
-	writeb(IOAT_CHANCMD_RESET,
-	       chan->reg_base
-	       + IOAT_CHANCMD_OFFSET(chan->device->version));
-	spin_unlock_bh(&ioat->ring_lock);
-	schedule_delayed_work(&chan->work, RESET_DELAY);
+	if (ioat_cleanup_preamble(chan, &phys_complete))
+		__cleanup(ioat, phys_complete);
+
+	__restart_chan(ioat);
 }
  
-/**
- * ioat2_chan_watchdog - watch for stuck channels
- */
-static void ioat2_chan_watchdog(struct work_struct *work)
+static void ioat2_timer_event(unsigned long data)
 {
-	struct ioatdma_device *device =
-		container_of(work, struct ioatdma_device, work.work);
-	struct ioat2_dma_chan *ioat;
-	struct ioat_chan_common *chan;
-	u16 active;
-	int i;
+	struct ioat2_dma_chan *ioat = (void *) data;
+	struct ioat_chan_common *chan = &ioat->base;
  
-	dev_dbg(&device->pdev->dev, "%s\n", __func__);
+	spin_lock_bh(&chan->cleanup_lock);
+	if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+		unsigned long phys_complete;
+		u64 status;
  
-	for (i = 0; i < device->common.chancnt; i++) {
-		chan = ioat_chan_by_index(device, i);
-		ioat = container_of(chan, struct ioat2_dma_chan, base);
+		spin_lock_bh(&ioat->ring_lock);
+		status = ioat_chansts(chan);
  
-		/*
-		 * for version 2.0 if there are descriptors yet to be processed
-		 * and the last completed hasn't changed since the last watchdog
-		 *      if they haven't hit the pending level
-		 *          issue the pending to push them through
-		 *      else
-		 *          try resetting the channel
+		/* when halted due to errors check for channel
+		 * programming errors before advancing the completion state
 		 */
-		spin_lock_bh(&ioat->ring_lock);
-		active = ioat2_ring_active(ioat);
-		spin_unlock_bh(&ioat->ring_lock);
+		if (is_ioat_halted(status)) {
+			u32 chanerr;
  
-		if (active &&
-		    chan->last_completion &&
-		    chan->last_completion == chan->watchdog_completion) {
-
-			if (ioat->pending == 1)
-				ioat2_issue_pending(&chan->common);
-			else {
-				ioat2_reset_channel(ioat);
-				chan->watchdog_completion = 0;
-			}
-		} else {
-			chan->last_compl_desc_addr_hw = 0;
-			chan->watchdog_completion = chan->last_completion;
+			chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+			BUG_ON(is_ioat_bug(chanerr));
 		}
-		chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
-	}
-	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
-}
  
-/**
- * ioat2_cleanup - clean finished descriptors (advance tail pointer)
- * @chan: ioat channel to be cleaned up
- */
-static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
-{
-	struct ioat_chan_common *chan = &ioat->base;
-	unsigned long phys_complete;
-	struct ioat_ring_ent *desc;
-	bool seen_current = false;
-	u16 active;
-	int i;
-	struct dma_async_tx_descriptor *tx;
-
-	prefetch(chan->completion);
-
-	spin_lock_bh(&chan->cleanup_lock);
-	phys_complete = ioat_get_current_completion(chan);
-	if (phys_complete == chan->last_completion) {
-		spin_unlock_bh(&chan->cleanup_lock);
-		/*
-		 * perhaps we're stuck so hard that the watchdog can't go off?
-		 * try to catch it after WATCHDOG_DELAY seconds
+		/* if we haven't made progress and we have already
+		 * acknowledged a pending completion once, then be more
+		 * forceful with a restart
 		 */
-		if (chan->device->version < IOAT_VER_3_0) {
-			unsigned long tmo;
-
-			tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
-			if (time_after(jiffies, tmo)) {
-				ioat2_chan_watchdog(&(chan->device->work.work));
-				chan->last_completion_time = jiffies;
-			}
+		if (ioat_cleanup_preamble(chan, &phys_complete))
+			__cleanup(ioat, phys_complete);
+		else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+			ioat2_restart_channel(ioat);
+		else {
+			set_bit(IOAT_COMPLETION_ACK, &chan->state);
+			mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
 		}
-		return;
+		spin_unlock_bh(&ioat->ring_lock);
 	}
-	chan->last_completion_time = jiffies;
-
-	spin_lock_bh(&ioat->ring_lock);
-
-	dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
-		__func__, ioat->head, ioat->tail, ioat->issued);
-
-	active = ioat2_ring_active(ioat);
-	for (i = 0; i < active && !seen_current; i++) {
-		prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
-		desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
-		tx = &desc->txd;
-		dump_desc_dbg(ioat, desc);
-		if (tx->cookie) {
-			ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
-			chan->completed_cookie = tx->cookie;
-			tx->cookie = 0;
-			if (tx->callback) {
-				tx->callback(tx->callback_param);
-				tx->callback = NULL;
-			}
-		}
-
-		if (tx->phys == phys_complete)
-			seen_current = true;
-	}
-	ioat->tail += i;
-	BUG_ON(!seen_current); /* no active descs have written a completion? */
-	spin_unlock_bh(&ioat->ring_lock);
-
-	chan->last_completion = phys_complete;
-
 	spin_unlock_bh(&chan->cleanup_lock);
 }
  
-static void ioat2_cleanup_tasklet(unsigned long data)
-{
-	struct ioat2_dma_chan *ioat = (void *) data;
-
-	ioat2_cleanup(ioat);
-	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
-}
-
 /**
  * ioat2_enumerate_channels - find and initialize the device's channels
  * @device: the device to be enumerated
@@ -381,7 +330,7 @@
 			break;
  
 		ioat_init_channel(device, &ioat->base, i,
-				  ioat2_reset_part2,
+				  ioat2_timer_event,
 				  ioat2_cleanup_tasklet,
 				  (unsigned long) ioat);
 		ioat->xfercap_log = xfercap_log;
@@ -395,6 +344,7 @@
 {
 	struct dma_chan *c = tx->chan;
 	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+	struct ioat_chan_common *chan = &ioat->base;
 	dma_cookie_t cookie = c->cookie;
  
 	cookie++;
@@ -404,6 +354,8 @@
 	c->cookie = cookie;
 	dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
  
+	if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+		mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
 	ioat2_update_pending(ioat);
 	spin_unlock_bh(&ioat->ring_lock);
  
@@ -543,9 +495,18 @@
 				ioat->issued);
 		spin_unlock_bh(&ioat->ring_lock);
  
-		/* do direct reclaim in the allocation failure case */
-		ioat2_cleanup(ioat);
-
+		/* progress reclaim in the allocation failure case we
+		 * may be called under bh_disabled so we need to trigger
+		 * the timer event directly
+		 */
+		spin_lock_bh(&chan->cleanup_lock);
+		if (jiffies > chan->timer.expires &&
+		    timer_pending(&chan->timer)) {
+			mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+			spin_unlock_bh(&chan->cleanup_lock);
+			ioat2_timer_event((unsigned long) ioat);
+		} else
+			spin_unlock_bh(&chan->cleanup_lock);
 		return -ENOMEM;
 	}
  
@@ -624,6 +585,7 @@
 		return;
  
 	tasklet_disable(&chan->cleanup_task);
+	del_timer_sync(&chan->timer);
 	ioat2_cleanup(ioat);
  
 	/* Delay 100ms after reset to allow internal DMA logic to quiesce
@@ -663,10 +625,6 @@
 	chan->completion_dma = 0;
 	ioat->pending = 0;
 	ioat->dmacount = 0;
-	chan->watchdog_completion = 0;
-	chan->last_compl_desc_addr_hw = 0;
-	chan->watchdog_tcp_cookie = 0;
-	chan->watchdog_last_tcp_cookie = 0;
 }
  
 static enum dma_status
@@ -715,9 +673,6 @@
 		return err;
 	if (dca)
 		device->dca = ioat2_dca_init(pdev, device->reg_base);
-
-	INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
-	schedule_delayed_work(&device->work, WATCHDOG_DELAY);
  
 	return err;
 }
@@ -127,6 +127,16 @@
 	return ioat->ring[idx & ioat2_ring_mask(ioat)];
 }
  
+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
+{
+	struct ioat_chan_common *chan = &ioat->base;
+
+	writel(addr & 0x00000000FFFFFFFF,
+	       chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
+	writel(addr >> 32,
+	       chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+}
+
 int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
 int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
 struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
@@ -101,11 +101,11 @@
 #define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR	(~0x3fULL)
 #define IOAT_CHANSTS_SOFT_ERR			0x10ULL
 #define IOAT_CHANSTS_UNAFFILIATED_ERR		0x8ULL
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS	0x7ULL
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE	0x0
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE	0x1
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED	0x2
-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED	0x3
+#define IOAT_CHANSTS_STATUS	0x7ULL
+#define IOAT_CHANSTS_ACTIVE	0x0
+#define IOAT_CHANSTS_DONE	0x1
+#define IOAT_CHANSTS_SUSPENDED	0x2
+#define IOAT_CHANSTS_HALTED	0x3
  
  
  
  
@@ -208,18 +208,18 @@
 #define IOAT_CDAR_OFFSET_HIGH			0x24
  
 #define IOAT_CHANERR_OFFSET			0x28	/* 32-bit Channel Error Register */
-#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR	0x0001
-#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR	0x0002
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR	0x0004
-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR	0x0008
+#define IOAT_CHANERR_SRC_ADDR_ERR	0x0001
+#define IOAT_CHANERR_DEST_ADDR_ERR	0x0002
+#define IOAT_CHANERR_NEXT_ADDR_ERR	0x0004
+#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR	0x0008
 #define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR	0x0010
 #define IOAT_CHANERR_CHANCMD_ERR		0x0020
 #define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR	0x0040
 #define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR	0x0080
 #define IOAT_CHANERR_READ_DATA_ERR		0x0100
 #define IOAT_CHANERR_WRITE_DATA_ERR		0x0200
-#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR	0x0400
-#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR	0x0800
+#define IOAT_CHANERR_CONTROL_ERR	0x0400
+#define IOAT_CHANERR_LENGTH_ERR	0x0800
 #define IOAT_CHANERR_COMPLETION_ADDR_ERR	0x1000
 #define IOAT_CHANERR_INT_CONFIGURATION_ERR	0x2000
 #define IOAT_CHANERR_SOFT_ERR			0x4000
@@ -126,9 +126,9 @@
 		udelay(10);
  
 		sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-			IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+			IOAT_CHANSTS_STATUS;
  
-		if (sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE)
+		if (sts != IOAT_CHANSTS_ACTIVE)
 			break;
  
 	}
  
@@ -160,9 +160,9 @@
 	udelay(1000);
  
 	chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-			IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+			IOAT_CHANSTS_STATUS;
  
-	if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE) {
+	if (chan_sts != IOAT_CHANSTS_DONE) {
 		/* Not complete, reset the channel */
 		writeb(IOAT_CHANCMD_RESET,
 		       ioat_chanbase + IOAT1_CHANCMD_OFFSET);
  
@@ -288,9 +288,9 @@
 		       ioat_chanbase + IOAT1_CHANCMD_OFFSET);
  
 		chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-			IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+			IOAT_CHANSTS_STATUS;
  
-		if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
+		if (chan_sts != IOAT_CHANSTS_ACTIVE) {
 			writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET);
 			break;
 		}
  
@@ -298,14 +298,14 @@
 	}
  
 	chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
-			IOAT_CHANSTS_DMA_TRANSFER_STATUS;
+			IOAT_CHANSTS_STATUS;
  
 	/*
 	 * We tried to reset multiple times. If IO A/T channel is still active
 	 * flag an error and return without cleanup. Memory leak is better
 	 * than random corruption in that extreme error situation.
 	 */
-	if (chan_sts == IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
+	if (chan_sts == IOAT_CHANSTS_ACTIVE) {
 		printk(KERN_ERR I7300_PRINT "Unable to stop IO A/T channels."
 			" Not freeing resources\n");
 		return;
...	...	@@ -99,23 +99,26 @@
99	99	/* common channel initialization */
100	100	void ioat_init_channel(struct ioatdma_device *device,
101	101	struct ioat_chan_common *chan, int idx,
102		- work_func_t work_fn, void (*tasklet)(unsigned long),
103		- unsigned long tasklet_data)
	102	+ void (*timer_fn)(unsigned long),
	103	+ void (*tasklet)(unsigned long),
	104	+ unsigned long ioat)
104	105	{
105	106	struct dma_device *dma = &device->common;
106	107
107	108	chan->device = device;
108	109	chan->reg_base = device->reg_base + (0x80 * (idx + 1));
109		- INIT_DELAYED_WORK(&chan->work, work_fn);
110	110	spin_lock_init(&chan->cleanup_lock);
111	111	chan->common.device = dma;
112	112	list_add_tail(&chan->common.device_node, &dma->channels);
113	113	device->idx[idx] = chan;
114		- tasklet_init(&chan->cleanup_task, tasklet, tasklet_data);
	114	+ init_timer(&chan->timer);
	115	+ chan->timer.function = timer_fn;
	116	+ chan->timer.data = ioat;
	117	+ tasklet_init(&chan->cleanup_task, tasklet, ioat);
115	118	tasklet_disable(&chan->cleanup_task);
116	119	}
117	120
118		-static void ioat1_reset_part2(struct work_struct *work);
	121	+static void ioat1_timer_event(unsigned long data);
119	122
120	123	/**
121	124	* ioat1_dma_enumerate_channels - find and initialize the device's channels
...	...	@@ -153,7 +156,7 @@
153	156	break;
154	157
155	158	ioat_init_channel(device, &ioat->base, i,
156		- ioat1_reset_part2,
	159	+ ioat1_timer_event,
157	160	ioat1_cleanup_tasklet,
158	161	(unsigned long) ioat);
159	162	ioat->xfercap = xfercap;
...	...	@@ -193,61 +196,6 @@
193	196	}
194	197
195	198	/**
196		- * ioat1_reset_part2 - reinit the channel after a reset
197		- */
198		-static void ioat1_reset_part2(struct work_struct *work)
199		-{
200		- struct ioat_chan_common *chan;
201		- struct ioat_dma_chan *ioat;
202		- struct ioat_desc_sw *desc;
203		- int dmacount;
204		- bool start_null = false;
205		-
206		- chan = container_of(work, struct ioat_chan_common, work.work);
207		- ioat = container_of(chan, struct ioat_dma_chan, base);
208		- spin_lock_bh(&chan->cleanup_lock);
209		- spin_lock_bh(&ioat->desc_lock);
210		-
211		- *chan->completion = 0;
212		- ioat->pending = 0;
213		-
214		- /* count the descriptors waiting */
215		- dmacount = 0;
216		- if (ioat->used_desc.prev) {
217		- desc = to_ioat_desc(ioat->used_desc.prev);
218		- do {
219		- dmacount++;
220		- desc = to_ioat_desc(desc->node.next);
221		- } while (&desc->node != ioat->used_desc.next);
222		- }
223		-
224		- if (dmacount) {
225		- /*
226		- * write the new starting descriptor address
227		- * this puts channel engine into ARMED state
228		- */
229		- desc = to_ioat_desc(ioat->used_desc.prev);
230		- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
231		- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
232		- writel(((u64) desc->txd.phys) >> 32,
233		- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
234		-
235		- writeb(IOAT_CHANCMD_START, chan->reg_base
236		- + IOAT_CHANCMD_OFFSET(chan->device->version));
237		- } else
238		- start_null = true;
239		- spin_unlock_bh(&ioat->desc_lock);
240		- spin_unlock_bh(&chan->cleanup_lock);
241		-
242		- dev_err(to_dev(chan),
243		- "chan%d reset - %d descs waiting, %d total desc\n",
244		- chan_num(chan), dmacount, ioat->desccount);
245		-
246		- if (start_null)
247		- ioat1_dma_start_null_desc(ioat);
248		-}
249		-
250		-/**
251	199	* ioat1_reset_channel - restart a channel
252	200	* @ioat: IOAT DMA channel handle
253	201	*/
254	202
...	...	@@ -257,12 +205,9 @@
257	205	void __iomem *reg_base = chan->reg_base;
258	206	u32 chansts, chanerr;
259	207
260		- if (!ioat->used_desc.prev)
261		- return;
262		-
263		- dev_dbg(to_dev(chan), "%s\n", __func__);
	208	+ dev_warn(to_dev(chan), "reset\n");
264	209	chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
265		- chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
	210	+ chansts = *chan->completion & IOAT_CHANSTS_STATUS;
266	211	if (chanerr) {
267	212	dev_err(to_dev(chan),
268	213	"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
269	214
270	215
271	216
...	...	@@ -278,100 +223,19 @@
278	223	* while we're waiting.
279	224	*/
280	225
281		- spin_lock_bh(&ioat->desc_lock);
282	226	ioat->pending = INT_MIN;
283	227	writeb(IOAT_CHANCMD_RESET,
284	228	reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
285		- spin_unlock_bh(&ioat->desc_lock);
286		-
287		- /* schedule the 2nd half instead of sleeping a long time */
288		- schedule_delayed_work(&chan->work, RESET_DELAY);
	229	+ set_bit(IOAT_RESET_PENDING, &chan->state);
	230	+ mod_timer(&chan->timer, jiffies + RESET_DELAY);
289	231	}
290	232
291		-/**
292		- * ioat1_chan_watchdog - watch for stuck channels
293		- */
294		-static void ioat1_chan_watchdog(struct work_struct *work)
295		-{
296		- struct ioatdma_device *device =
297		- container_of(work, struct ioatdma_device, work.work);
298		- struct ioat_dma_chan *ioat;
299		- struct ioat_chan_common *chan;
300		- int i;
301		- u64 completion;
302		- u32 completion_low;
303		- unsigned long compl_desc_addr_hw;
304		-
305		- for (i = 0; i < device->common.chancnt; i++) {
306		- chan = ioat_chan_by_index(device, i);
307		- ioat = container_of(chan, struct ioat_dma_chan, base);
308		-
309		- if (/* have we started processing anything yet */
310		- chan->last_completion
311		- /* have we completed any since last watchdog cycle? */
312		- && (chan->last_completion == chan->watchdog_completion)
313		- /* has TCP stuck on one cookie since last watchdog? */
314		- && (chan->watchdog_tcp_cookie == chan->watchdog_last_tcp_cookie)
315		- && (chan->watchdog_tcp_cookie != chan->completed_cookie)
316		- /* is there something in the chain to be processed? */
317		- /* CB1 chain always has at least the last one processed */
318		- && (ioat->used_desc.prev != ioat->used_desc.next)
319		- && ioat->pending == 0) {
320		-
321		- /*
322		- * check CHANSTS register for completed
323		- * descriptor address.
324		- * if it is different than completion writeback,
325		- * it is not zero
326		- * and it has changed since the last watchdog
327		- * we can assume that channel
328		- * is still working correctly
329		- * and the problem is in completion writeback.
330		- * update completion writeback
331		- * with actual CHANSTS value
332		- * else
333		- * try resetting the channel
334		- */
335		-
336		- /* we need to read the low address first as this
337		- * causes the chipset to latch the upper bits
338		- * for the subsequent read
339		- */
340		- completion_low = readl(chan->reg_base +
341		- IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
342		- completion = readl(chan->reg_base +
343		- IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
344		- completion <<= 32;
345		- completion \|= completion_low;
346		- compl_desc_addr_hw = completion &
347		- IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
348		-
349		- if ((compl_desc_addr_hw != 0)
350		- && (compl_desc_addr_hw != chan->watchdog_completion)
351		- && (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
352		- chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
353		- *chan->completion = completion;
354		- } else {
355		- ioat1_reset_channel(ioat);
356		- chan->watchdog_completion = 0;
357		- chan->last_compl_desc_addr_hw = 0;
358		- }
359		- } else {
360		- chan->last_compl_desc_addr_hw = 0;
361		- chan->watchdog_completion = chan->last_completion;
362		- }
363		-
364		- chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
365		- }
366		-
367		- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
368		-}
369		-
370	233	static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
371	234	{
372	235	struct dma_chan *c = tx->chan;
373	236	struct ioat_dma_chan *ioat = to_ioat_chan(c);
374	237	struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
	238	+ struct ioat_chan_common *chan = &ioat->base;
375	239	struct ioat_desc_sw *first;
376	240	struct ioat_desc_sw *chain_tail;
377	241	dma_cookie_t cookie;
...	...	@@ -396,6 +260,9 @@
396	260	dump_desc_dbg(ioat, chain_tail);
397	261	dump_desc_dbg(ioat, first);
398	262
	263	+ if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
	264	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
	265	+
399	266	ioat->pending += desc->hw->tx_cnt;
400	267	if (ioat->pending >= ioat_pending_level)
401	268	__ioat1_dma_memcpy_issue_pending(ioat);
...	...	@@ -520,6 +387,7 @@
520	387	return;
521	388
522	389	tasklet_disable(&chan->cleanup_task);
	390	+ del_timer_sync(&chan->timer);
523	391	ioat1_cleanup(ioat);
524	392
525	393	/* Delay 100ms after reset to allow internal DMA logic to quiesce
...	...	@@ -560,9 +428,6 @@
560	428
561	429	chan->last_completion = 0;
562	430	chan->completion_dma = 0;
563		- chan->watchdog_completion = 0;
564		- chan->last_compl_desc_addr_hw = 0;
565		- chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
566	431	ioat->pending = 0;
567	432	ioat->desccount = 0;
568	433	}
569	434
570	435
...	...	@@ -705,15 +570,15 @@
705	570	u64 completion;
706	571
707	572	completion = *chan->completion;
708		- phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
	573	+ phys_complete = ioat_chansts_to_addr(completion);
709	574
710	575	dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
711	576	(unsigned long long) phys_complete);
712	577
713		- if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
714		- IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
	578	+ if (is_ioat_halted(completion)) {
	579	+ u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
715	580	dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
716		- readl(chan->reg_base + IOAT_CHANERR_OFFSET));
	581	+ chanerr);
717	582
718	583	/* TODO do something to salvage the situation */
719	584	}
720	585
721	586
722	587
723	588
...	...	@@ -721,48 +586,31 @@
721	586	return phys_complete;
722	587	}
723	588
724		-/**
725		- * ioat1_cleanup - cleanup up finished descriptors
726		- * @chan: ioat channel to be cleaned up
727		- */
728		-static void ioat1_cleanup(struct ioat_dma_chan *ioat)
	589	+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
	590	+ unsigned long *phys_complete)
729	591	{
	592	+ *phys_complete = ioat_get_current_completion(chan);
	593	+ if (*phys_complete == chan->last_completion)
	594	+ return false;
	595	+ clear_bit(IOAT_COMPLETION_ACK, &chan->state);
	596	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
	597	+
	598	+ return true;
	599	+}
	600	+
	601	+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
	602	+{
730	603	struct ioat_chan_common *chan = &ioat->base;
731		- unsigned long phys_complete;
732		- struct ioat_desc_sw desc, _desc;
733		- dma_cookie_t cookie = 0;
	604	+ struct list_head _desc, n;
734	605	struct dma_async_tx_descriptor *tx;
735	606
736		- prefetch(chan->completion);
737		-
738		- if (!spin_trylock_bh(&chan->cleanup_lock))
739		- return;
740		-
741		- phys_complete = ioat_get_current_completion(chan);
742		- if (phys_complete == chan->last_completion) {
743		- spin_unlock_bh(&chan->cleanup_lock);
744		- /*
745		- * perhaps we're stuck so hard that the watchdog can't go off?
746		- * try to catch it after 2 seconds
747		- */
748		- if (time_after(jiffies,
749		- chan->last_completion_time + HZ*WATCHDOG_DELAY)) {
750		- ioat1_chan_watchdog(&(chan->device->work.work));
751		- chan->last_completion_time = jiffies;
752		- }
753		- return;
754		- }
755		- chan->last_completion_time = jiffies;
756		-
757		- cookie = 0;
758		- if (!spin_trylock_bh(&ioat->desc_lock)) {
759		- spin_unlock_bh(&chan->cleanup_lock);
760		- return;
761		- }
762		-
763	607	dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
764	608	__func__, phys_complete);
765		- list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
	609	+ list_for_each_safe(_desc, n, &ioat->used_desc) {
	610	+ struct ioat_desc_sw *desc;
	611	+
	612	+ prefetch(n);
	613	+ desc = list_entry(_desc, typeof(*desc), node);
766	614	tx = &desc->txd;
767	615	/*
768	616	* Incoming DMA requests may use multiple descriptors,
...	...	@@ -771,7 +619,8 @@
771	619	*/
772	620	dump_desc_dbg(ioat, desc);
773	621	if (tx->cookie) {
774		- cookie = tx->cookie;
	622	+ chan->completed_cookie = tx->cookie;
	623	+ tx->cookie = 0;
775	624	ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
776	625	if (tx->callback) {
777	626	tx->callback(tx->callback_param);
778	627
779	628
780	629
781	630
782	631
783	632
784	633
...	...	@@ -786,27 +635,110 @@
786	635	*/
787	636	if (async_tx_test_ack(tx))
788	637	list_move_tail(&desc->node, &ioat->free_desc);
789		- else
790		- tx->cookie = 0;
791	638	} else {
792	639	/*
793	640	* last used desc. Do not remove, so we can
794		- * append from it, but don't look at it next
795		- * time, either
	641	+ * append from it.
796	642	*/
797		- tx->cookie = 0;
798	643
	644	+ /* if nothing else is pending, cancel the
	645	+ * completion timeout
	646	+ */
	647	+ if (n == &ioat->used_desc) {
	648	+ dev_dbg(to_dev(chan),
	649	+ "%s cancel completion timeout\n",
	650	+ __func__);
	651	+ clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
	652	+ }
	653	+
799	654	/* TODO check status bits? */
800	655	break;
801	656	}
802	657	}
803	658
	659	+ chan->last_completion = phys_complete;
	660	+}
	661	+
	662	+/**
	663	+ * ioat1_cleanup - cleanup up finished descriptors
	664	+ * @chan: ioat channel to be cleaned up
	665	+ *
	666	+ * To prevent lock contention we defer cleanup when the locks are
	667	+ * contended with a terminal timeout that forces cleanup and catches
	668	+ * completion notification errors.
	669	+ */
	670	+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
	671	+{
	672	+ struct ioat_chan_common *chan = &ioat->base;
	673	+ unsigned long phys_complete;
	674	+
	675	+ prefetch(chan->completion);
	676	+
	677	+ if (!spin_trylock_bh(&chan->cleanup_lock))
	678	+ return;
	679	+
	680	+ if (!ioat_cleanup_preamble(chan, &phys_complete)) {
	681	+ spin_unlock_bh(&chan->cleanup_lock);
	682	+ return;
	683	+ }
	684	+
	685	+ if (!spin_trylock_bh(&ioat->desc_lock)) {
	686	+ spin_unlock_bh(&chan->cleanup_lock);
	687	+ return;
	688	+ }
	689	+
	690	+ __cleanup(ioat, phys_complete);
	691	+
804	692	spin_unlock_bh(&ioat->desc_lock);
	693	+ spin_unlock_bh(&chan->cleanup_lock);
	694	+}
805	695
806		- chan->last_completion = phys_complete;
807		- if (cookie != 0)
808		- chan->completed_cookie = cookie;
	696	+static void ioat1_timer_event(unsigned long data)
	697	+{
	698	+ struct ioat_dma_chan ioat = (void ) data;
	699	+ struct ioat_chan_common *chan = &ioat->base;
809	700
	701	+ dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
	702	+
	703	+ spin_lock_bh(&chan->cleanup_lock);
	704	+ if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
	705	+ struct ioat_desc_sw *desc;
	706	+
	707	+ spin_lock_bh(&ioat->desc_lock);
	708	+
	709	+ /* restart active descriptors */
	710	+ desc = to_ioat_desc(ioat->used_desc.prev);
	711	+ ioat_set_chainaddr(ioat, desc->txd.phys);
	712	+ ioat_start(chan);
	713	+
	714	+ ioat->pending = 0;
	715	+ set_bit(IOAT_COMPLETION_PENDING, &chan->state);
	716	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
	717	+ spin_unlock_bh(&ioat->desc_lock);
	718	+ } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
	719	+ unsigned long phys_complete;
	720	+
	721	+ spin_lock_bh(&ioat->desc_lock);
	722	+ /* if we haven't made progress and we have already
	723	+ * acknowledged a pending completion once, then be more
	724	+ * forceful with a restart
	725	+ */
	726	+ if (ioat_cleanup_preamble(chan, &phys_complete))
	727	+ __cleanup(ioat, phys_complete);
	728	+ else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
	729	+ ioat1_reset_channel(ioat);
	730	+ else {
	731	+ u64 status = ioat_chansts(chan);
	732	+
	733	+ /* manually update the last completion address */
	734	+ if (ioat_chansts_to_addr(status) != 0)
	735	+ *chan->completion = status;
	736	+
	737	+ set_bit(IOAT_COMPLETION_ACK, &chan->state);
	738	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
	739	+ }
	740	+ spin_unlock_bh(&ioat->desc_lock);
	741	+ }
810	742	spin_unlock_bh(&chan->cleanup_lock);
811	743	}
812	744
...	...	@@ -855,13 +787,8 @@
855	787	list_add_tail(&desc->node, &ioat->used_desc);
856	788	dump_desc_dbg(ioat, desc);
857	789
858		- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
859		- chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
860		- writel(((u64) desc->txd.phys) >> 32,
861		- chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
862		-
863		- writeb(IOAT_CHANCMD_START, chan->reg_base
864		- + IOAT_CHANCMD_OFFSET(chan->device->version));
	790	+ ioat_set_chainaddr(ioat, desc->txd.phys);
	791	+ ioat_start(chan);
865	792	spin_unlock_bh(&ioat->desc_lock);
866	793	}
867	794
868	795
...	...	@@ -1194,18 +1121,12 @@
1194	1121	if (dca)
1195	1122	device->dca = ioat_dca_init(pdev, device->reg_base);
1196	1123
1197		- INIT_DELAYED_WORK(&device->work, ioat1_chan_watchdog);
1198		- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
1199		-
1200	1124	return err;
1201	1125	}
1202	1126
1203	1127	void __devexit ioat_dma_remove(struct ioatdma_device *device)
1204	1128	{
1205	1129	struct dma_device *dma = &device->common;
1206		-
1207		- if (device->version != IOAT_VER_3_0)
1208		- cancel_delayed_work(&device->work);
1209	1130
1210	1131	ioat_disable_interrupts(device);
1211	1132
...	...	@@ -23,6 +23,7 @@
23	23
24	24	#include <linux/dmaengine.h>
25	25	#include "hw.h"
	26	+#include "registers.h"
26	27	#include <linux/init.h>
27	28	#include <linux/dmapool.h>
28	29	#include <linux/cache.h>
...	...	@@ -33,7 +34,6 @@
33	34
34	35	#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
35	36	#define IOAT_DMA_DCA_ANY_CPU ~0
36		-#define IOAT_WATCHDOG_PERIOD (2 * HZ)
37	37
38	38	#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
39	39	#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
...	...	@@ -42,9 +42,6 @@
42	42
43	43	#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
44	44
45		-#define RESET_DELAY msecs_to_jiffies(100)
46		-#define WATCHDOG_DELAY round_jiffies(msecs_to_jiffies(2000))
47		-
48	45	/*
49	46	* workaround for IOAT ver.3.0 null descriptor issue
50	47	* (channel returns error when size is 0)
...	...	@@ -72,7 +69,6 @@
72	69	struct pci_pool *completion_pool;
73	70	struct dma_device common;
74	71	u8 version;
75		- struct delayed_work work;
76	72	struct msix_entry msix_entries[4];
77	73	struct ioat_chan_common *idx[4];
78	74	struct dca_provider *dca;
79	75
80	76
81	77
82	78
83	79
...	...	@@ -81,24 +77,21 @@
81	77	};
82	78
83	79	struct ioat_chan_common {
	80	+ struct dma_chan common;
84	81	void __iomem *reg_base;
85		-
86	82	unsigned long last_completion;
87		- unsigned long last_completion_time;
88		-
89	83	spinlock_t cleanup_lock;
90	84	dma_cookie_t completed_cookie;
91		- unsigned long watchdog_completion;
92		- int watchdog_tcp_cookie;
93		- u32 watchdog_last_tcp_cookie;
94		- struct delayed_work work;
95		-
	85	+ unsigned long state;
	86	+ #define IOAT_COMPLETION_PENDING 0
	87	+ #define IOAT_COMPLETION_ACK 1
	88	+ #define IOAT_RESET_PENDING 2
	89	+ struct timer_list timer;
	90	+ #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
	91	+ #define RESET_DELAY msecs_to_jiffies(100)
96	92	struct ioatdma_device *device;
97		- struct dma_chan common;
98		-
99	93	dma_addr_t completion_dma;
100	94	u64 *completion;
101		- unsigned long last_compl_desc_addr_hw;
102	95	struct tasklet_struct cleanup_task;
103	96	};
104	97
...	...	@@ -148,7 +141,6 @@
148	141
149	142	last_used = c->cookie;
150	143	last_complete = chan->completed_cookie;
151		- chan->watchdog_tcp_cookie = cookie;
152	144
153	145	if (done)
154	146	*done = last_complete;
...	...	@@ -215,6 +207,85 @@
215	207	return device->idx[index];
216	208	}
217	209
	210	+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
	211	+{
	212	+ u8 ver = chan->device->version;
	213	+ u64 status;
	214	+ u32 status_lo;
	215	+
	216	+ /* We need to read the low address first as this causes the
	217	+ * chipset to latch the upper bits for the subsequent read
	218	+ */
	219	+ status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
	220	+ status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
	221	+ status <<= 32;
	222	+ status \|= status_lo;
	223	+
	224	+ return status;
	225	+}
	226	+
	227	+static inline void ioat_start(struct ioat_chan_common *chan)
	228	+{
	229	+ u8 ver = chan->device->version;
	230	+
	231	+ writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
	232	+}
	233	+
	234	+static inline u64 ioat_chansts_to_addr(u64 status)
	235	+{
	236	+ return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
	237	+}
	238	+
	239	+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
	240	+{
	241	+ return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
	242	+}
	243	+
	244	+static inline void ioat_suspend(struct ioat_chan_common *chan)
	245	+{
	246	+ u8 ver = chan->device->version;
	247	+
	248	+ writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
	249	+}
	250	+
	251	+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
	252	+{
	253	+ struct ioat_chan_common *chan = &ioat->base;
	254	+
	255	+ writel(addr & 0x00000000FFFFFFFF,
	256	+ chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
	257	+ writel(addr >> 32,
	258	+ chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
	259	+}
	260	+
	261	+static inline bool is_ioat_active(unsigned long status)
	262	+{
	263	+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
	264	+}
	265	+
	266	+static inline bool is_ioat_idle(unsigned long status)
	267	+{
	268	+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
	269	+}
	270	+
	271	+static inline bool is_ioat_halted(unsigned long status)
	272	+{
	273	+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
	274	+}
	275	+
	276	+static inline bool is_ioat_suspended(unsigned long status)
	277	+{
	278	+ return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
	279	+}
	280	+
	281	+/* channel was fatally programmed */
	282	+static inline bool is_ioat_bug(unsigned long err)
	283	+{
	284	+ return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR\|IOAT_CHANERR_DEST_ADDR_ERR\|
	285	+ IOAT_CHANERR_NEXT_ADDR_ERR\|IOAT_CHANERR_CONTROL_ERR\|
	286	+ IOAT_CHANERR_LENGTH_ERR));
	287	+}
	288	+
218	289	int __devinit ioat_probe(struct ioatdma_device *device);
219	290	int __devinit ioat_register(struct ioatdma_device *device);
220	291	int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
221	292
...	...	@@ -224,9 +295,12 @@
224	295	unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
225	296	void ioat_init_channel(struct ioatdma_device *device,
226	297	struct ioat_chan_common *chan, int idx,
227		- work_func_t work_fn, void (*tasklet)(unsigned long),
228		- unsigned long tasklet_data);
	298	+ void (*timer_fn)(unsigned long),
	299	+ void (*tasklet)(unsigned long),
	300	+ unsigned long ioat);
229	301	void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
230	302	size_t len, struct ioat_dma_descriptor *hw);
	303	+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
	304	+ unsigned long *phys_complete);
231	305	#endif /* IOATDMA_H */
...	...	@@ -49,7 +49,7 @@
49	49	void * __iomem reg_base = ioat->base.reg_base;
50	50
51	51	ioat->pending = 0;
52		- ioat->dmacount += ioat2_ring_pending(ioat);
	52	+ ioat->dmacount += ioat2_ring_pending(ioat);;
53	53	ioat->issued = ioat->head;
54	54	/* make descriptor updates globally visible before notifying channel */
55	55	wmb();
...	...	@@ -92,7 +92,6 @@
92	92
93	93	static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
94	94	{
95		- void __iomem *reg_base = ioat->base.reg_base;
96	95	struct ioat_ring_ent *desc;
97	96	struct ioat_dma_descriptor *hw;
98	97	int idx;
...	...	@@ -118,10 +117,7 @@
118	117	hw->src_addr = 0;
119	118	hw->dst_addr = 0;
120	119	async_tx_ack(&desc->txd);
121		- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
122		- reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
123		- writel(((u64) desc->txd.phys) >> 32,
124		- reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
	120	+ ioat2_set_chainaddr(ioat, desc->txd.phys);
125	121	dump_desc_dbg(ioat, desc);
126	122	__ioat2_issue_pending(ioat);
127	123	}
128	124
129	125
130	126
131	127
132	128
133	129
134	130
135	131
136	132
137	133
...	...	@@ -133,33 +129,97 @@
133	129	spin_unlock_bh(&ioat->ring_lock);
134	130	}
135	131
136		-static void ioat2_cleanup(struct ioat2_dma_chan *ioat);
	132	+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
	133	+{
	134	+ struct ioat_chan_common *chan = &ioat->base;
	135	+ struct dma_async_tx_descriptor *tx;
	136	+ struct ioat_ring_ent *desc;
	137	+ bool seen_current = false;
	138	+ u16 active;
	139	+ int i;
137	140
	141	+ dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
	142	+ __func__, ioat->head, ioat->tail, ioat->issued);
	143	+
	144	+ active = ioat2_ring_active(ioat);
	145	+ for (i = 0; i < active && !seen_current; i++) {
	146	+ prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
	147	+ desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
	148	+ tx = &desc->txd;
	149	+ dump_desc_dbg(ioat, desc);
	150	+ if (tx->cookie) {
	151	+ ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
	152	+ chan->completed_cookie = tx->cookie;
	153	+ tx->cookie = 0;
	154	+ if (tx->callback) {
	155	+ tx->callback(tx->callback_param);
	156	+ tx->callback = NULL;
	157	+ }
	158	+ }
	159	+
	160	+ if (tx->phys == phys_complete)
	161	+ seen_current = true;
	162	+ }
	163	+ ioat->tail += i;
	164	+ BUG_ON(!seen_current); /* no active descs have written a completion? */
	165	+
	166	+ chan->last_completion = phys_complete;
	167	+ if (ioat->head == ioat->tail) {
	168	+ dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
	169	+ __func__);
	170	+ clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
	171	+ }
	172	+}
	173	+
138	174	/**
139		- * ioat2_reset_part2 - reinit the channel after a reset
	175	+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
	176	+ * @chan: ioat channel to be cleaned up
140	177	*/
141		-static void ioat2_reset_part2(struct work_struct *work)
	178	+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
142	179	{
143		- struct ioat_chan_common *chan;
144		- struct ioat2_dma_chan *ioat;
	180	+ struct ioat_chan_common *chan = &ioat->base;
	181	+ unsigned long phys_complete;
145	182
146		- chan = container_of(work, struct ioat_chan_common, work.work);
147		- ioat = container_of(chan, struct ioat2_dma_chan, base);
	183	+ prefetch(chan->completion);
148	184
149		- /* ensure that ->tail points to the stalled descriptor
150		- * (ioat->pending is set to 2 at this point so no new
151		- * descriptors will be issued while we perform this cleanup)
152		- */
	185	+ if (!spin_trylock_bh(&chan->cleanup_lock))
	186	+ return;
	187	+
	188	+ if (!ioat_cleanup_preamble(chan, &phys_complete)) {
	189	+ spin_unlock_bh(&chan->cleanup_lock);
	190	+ return;
	191	+ }
	192	+
	193	+ if (!spin_trylock_bh(&ioat->ring_lock)) {
	194	+ spin_unlock_bh(&chan->cleanup_lock);
	195	+ return;
	196	+ }
	197	+
	198	+ __cleanup(ioat, phys_complete);
	199	+
	200	+ spin_unlock_bh(&ioat->ring_lock);
	201	+ spin_unlock_bh(&chan->cleanup_lock);
	202	+}
	203	+
	204	+static void ioat2_cleanup_tasklet(unsigned long data)
	205	+{
	206	+ struct ioat2_dma_chan ioat = (void ) data;
	207	+
153	208	ioat2_cleanup(ioat);
	209	+ writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
	210	+}
154	211
155		- spin_lock_bh(&chan->cleanup_lock);
156		- spin_lock_bh(&ioat->ring_lock);
	212	+static void __restart_chan(struct ioat2_dma_chan *ioat)
	213	+{
	214	+ struct ioat_chan_common *chan = &ioat->base;
157	215
158	216	/* set the tail to be re-issued */
159	217	ioat->issued = ioat->tail;
160	218	ioat->dmacount = 0;
	219	+ set_bit(IOAT_COMPLETION_PENDING, &chan->state);
	220	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
161	221
162		- dev_dbg(to_dev(&ioat->base),
	222	+ dev_dbg(to_dev(chan),
163	223	"%s: head: %#x tail: %#x issued: %#x count: %#x\n",
164	224	__func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
165	225
166	226
167	227
168	228
169	229
170	230
171	231
172	232
173	233
174	234
175	235
176	236
177	237
178	238
179	239
180	240
181	241
182	242
183	243
184	244
...	...	@@ -167,183 +227,72 @@
167	227	struct ioat_ring_ent *desc;
168	228
169	229	desc = ioat2_get_ring_ent(ioat, ioat->tail);
170		- writel(((u64) desc->txd.phys) & 0x00000000FFFFFFFF,
171		- chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
172		- writel(((u64) desc->txd.phys) >> 32,
173		- chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
	230	+ ioat2_set_chainaddr(ioat, desc->txd.phys);
174	231	__ioat2_issue_pending(ioat);
175	232	} else
176	233	__ioat2_start_null_desc(ioat);
177		-
178		- spin_unlock_bh(&ioat->ring_lock);
179		- spin_unlock_bh(&chan->cleanup_lock);
180		-
181		- dev_info(to_dev(chan),
182		- "chan%d reset - %d descs waiting, %d total desc\n",
183		- chan_num(chan), ioat->dmacount, 1 << ioat->alloc_order);
184	234	}
185	235
186		-/**
187		- * ioat2_reset_channel - restart a channel
188		- * @ioat: IOAT DMA channel handle
189		- */
190		-static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
	236	+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
191	237	{
192		- u32 chansts, chanerr;
193	238	struct ioat_chan_common *chan = &ioat->base;
194		- u16 active;
	239	+ unsigned long phys_complete;
	240	+ u32 status;
195	241
196		- spin_lock_bh(&ioat->ring_lock);
197		- active = ioat2_ring_active(ioat);
198		- spin_unlock_bh(&ioat->ring_lock);
199		- if (!active)
200		- return;
201		-
202		- chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
203		- chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
204		- if (chanerr) {
205		- dev_err(to_dev(chan),
206		- "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
207		- chan_num(chan), chansts, chanerr);
208		- writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
	242	+ status = ioat_chansts(chan);
	243	+ if (is_ioat_active(status) \|\| is_ioat_idle(status))
	244	+ ioat_suspend(chan);
	245	+ while (is_ioat_active(status) \|\| is_ioat_idle(status)) {
	246	+ status = ioat_chansts(chan);
	247	+ cpu_relax();
209	248	}
210	249
211		- spin_lock_bh(&ioat->ring_lock);
212		- ioat->pending = 2;
213		- writeb(IOAT_CHANCMD_RESET,
214		- chan->reg_base
215		- + IOAT_CHANCMD_OFFSET(chan->device->version));
216		- spin_unlock_bh(&ioat->ring_lock);
217		- schedule_delayed_work(&chan->work, RESET_DELAY);
	250	+ if (ioat_cleanup_preamble(chan, &phys_complete))
	251	+ __cleanup(ioat, phys_complete);
	252	+
	253	+ __restart_chan(ioat);
218	254	}
219	255
220		-/**
221		- * ioat2_chan_watchdog - watch for stuck channels
222		- */
223		-static void ioat2_chan_watchdog(struct work_struct *work)
	256	+static void ioat2_timer_event(unsigned long data)
224	257	{
225		- struct ioatdma_device *device =
226		- container_of(work, struct ioatdma_device, work.work);
227		- struct ioat2_dma_chan *ioat;
228		- struct ioat_chan_common *chan;
229		- u16 active;
230		- int i;
	258	+ struct ioat2_dma_chan ioat = (void ) data;
	259	+ struct ioat_chan_common *chan = &ioat->base;
231	260
232		- dev_dbg(&device->pdev->dev, "%s\n", __func__);
	261	+ spin_lock_bh(&chan->cleanup_lock);
	262	+ if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
	263	+ unsigned long phys_complete;
	264	+ u64 status;
233	265
234		- for (i = 0; i < device->common.chancnt; i++) {
235		- chan = ioat_chan_by_index(device, i);
236		- ioat = container_of(chan, struct ioat2_dma_chan, base);
	266	+ spin_lock_bh(&ioat->ring_lock);
	267	+ status = ioat_chansts(chan);
237	268
238		- /*
239		- * for version 2.0 if there are descriptors yet to be processed
240		- * and the last completed hasn't changed since the last watchdog
241		- * if they haven't hit the pending level
242		- * issue the pending to push them through
243		- * else
244		- * try resetting the channel
	269	+ /* when halted due to errors check for channel
	270	+ * programming errors before advancing the completion state
245	271	*/
246		- spin_lock_bh(&ioat->ring_lock);
247		- active = ioat2_ring_active(ioat);
248		- spin_unlock_bh(&ioat->ring_lock);
	272	+ if (is_ioat_halted(status)) {
	273	+ u32 chanerr;
249	274
250		- if (active &&
251		- chan->last_completion &&
252		- chan->last_completion == chan->watchdog_completion) {
253		-
254		- if (ioat->pending == 1)
255		- ioat2_issue_pending(&chan->common);
256		- else {
257		- ioat2_reset_channel(ioat);
258		- chan->watchdog_completion = 0;
259		- }
260		- } else {
261		- chan->last_compl_desc_addr_hw = 0;
262		- chan->watchdog_completion = chan->last_completion;
	275	+ chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
	276	+ BUG_ON(is_ioat_bug(chanerr));
263	277	}
264		- chan->watchdog_last_tcp_cookie = chan->watchdog_tcp_cookie;
265		- }
266		- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
267		-}
268	278
269		-/**
270		- * ioat2_cleanup - clean finished descriptors (advance tail pointer)
271		- * @chan: ioat channel to be cleaned up
272		- */
273		-static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
274		-{
275		- struct ioat_chan_common *chan = &ioat->base;
276		- unsigned long phys_complete;
277		- struct ioat_ring_ent *desc;
278		- bool seen_current = false;
279		- u16 active;
280		- int i;
281		- struct dma_async_tx_descriptor *tx;
282		-
283		- prefetch(chan->completion);
284		-
285		- spin_lock_bh(&chan->cleanup_lock);
286		- phys_complete = ioat_get_current_completion(chan);
287		- if (phys_complete == chan->last_completion) {
288		- spin_unlock_bh(&chan->cleanup_lock);
289		- /*
290		- * perhaps we're stuck so hard that the watchdog can't go off?
291		- * try to catch it after WATCHDOG_DELAY seconds
	279	+ /* if we haven't made progress and we have already
	280	+ * acknowledged a pending completion once, then be more
	281	+ * forceful with a restart
292	282	*/
293		- if (chan->device->version < IOAT_VER_3_0) {
294		- unsigned long tmo;
295		-
296		- tmo = chan->last_completion_time + HZ*WATCHDOG_DELAY;
297		- if (time_after(jiffies, tmo)) {
298		- ioat2_chan_watchdog(&(chan->device->work.work));
299		- chan->last_completion_time = jiffies;
300		- }
	283	+ if (ioat_cleanup_preamble(chan, &phys_complete))
	284	+ __cleanup(ioat, phys_complete);
	285	+ else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
	286	+ ioat2_restart_channel(ioat);
	287	+ else {
	288	+ set_bit(IOAT_COMPLETION_ACK, &chan->state);
	289	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
301	290	}
302		- return;
	291	+ spin_unlock_bh(&ioat->ring_lock);
303	292	}
304		- chan->last_completion_time = jiffies;
305		-
306		- spin_lock_bh(&ioat->ring_lock);
307		-
308		- dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
309		- __func__, ioat->head, ioat->tail, ioat->issued);
310		-
311		- active = ioat2_ring_active(ioat);
312		- for (i = 0; i < active && !seen_current; i++) {
313		- prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
314		- desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
315		- tx = &desc->txd;
316		- dump_desc_dbg(ioat, desc);
317		- if (tx->cookie) {
318		- ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
319		- chan->completed_cookie = tx->cookie;
320		- tx->cookie = 0;
321		- if (tx->callback) {
322		- tx->callback(tx->callback_param);
323		- tx->callback = NULL;
324		- }
325		- }
326		-
327		- if (tx->phys == phys_complete)
328		- seen_current = true;
329		- }
330		- ioat->tail += i;
331		- BUG_ON(!seen_current); /* no active descs have written a completion? */
332		- spin_unlock_bh(&ioat->ring_lock);
333		-
334		- chan->last_completion = phys_complete;
335		-
336	293	spin_unlock_bh(&chan->cleanup_lock);
337	294	}
338	295
339		-static void ioat2_cleanup_tasklet(unsigned long data)
340		-{
341		- struct ioat2_dma_chan ioat = (void ) data;
342		-
343		- ioat2_cleanup(ioat);
344		- writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
345		-}
346		-
347	296	/**
348	297	* ioat2_enumerate_channels - find and initialize the device's channels
349	298	* @device: the device to be enumerated
...	...	@@ -381,7 +330,7 @@
381	330	break;
382	331
383	332	ioat_init_channel(device, &ioat->base, i,
384		- ioat2_reset_part2,
	333	+ ioat2_timer_event,
385	334	ioat2_cleanup_tasklet,
386	335	(unsigned long) ioat);
387	336	ioat->xfercap_log = xfercap_log;
...	...	@@ -395,6 +344,7 @@
395	344	{
396	345	struct dma_chan *c = tx->chan;
397	346	struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
	347	+ struct ioat_chan_common *chan = &ioat->base;
398	348	dma_cookie_t cookie = c->cookie;
399	349
400	350	cookie++;
...	...	@@ -404,6 +354,8 @@
404	354	c->cookie = cookie;
405	355	dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
406	356
	357	+ if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
	358	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
407	359	ioat2_update_pending(ioat);
408	360	spin_unlock_bh(&ioat->ring_lock);
409	361
...	...	@@ -543,9 +495,18 @@
543	495	ioat->issued);
544	496	spin_unlock_bh(&ioat->ring_lock);
545	497
546		- /* do direct reclaim in the allocation failure case */
547		- ioat2_cleanup(ioat);
548		-
	498	+ /* progress reclaim in the allocation failure case we
	499	+ * may be called under bh_disabled so we need to trigger
	500	+ * the timer event directly
	501	+ */
	502	+ spin_lock_bh(&chan->cleanup_lock);
	503	+ if (jiffies > chan->timer.expires &&
	504	+ timer_pending(&chan->timer)) {
	505	+ mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
	506	+ spin_unlock_bh(&chan->cleanup_lock);
	507	+ ioat2_timer_event((unsigned long) ioat);
	508	+ } else
	509	+ spin_unlock_bh(&chan->cleanup_lock);
549	510	return -ENOMEM;
550	511	}
551	512
...	...	@@ -624,6 +585,7 @@
624	585	return;
625	586
626	587	tasklet_disable(&chan->cleanup_task);
	588	+ del_timer_sync(&chan->timer);
627	589	ioat2_cleanup(ioat);
628	590
629	591	/* Delay 100ms after reset to allow internal DMA logic to quiesce
...	...	@@ -663,10 +625,6 @@
663	625	chan->completion_dma = 0;
664	626	ioat->pending = 0;
665	627	ioat->dmacount = 0;
666		- chan->watchdog_completion = 0;
667		- chan->last_compl_desc_addr_hw = 0;
668		- chan->watchdog_tcp_cookie = 0;
669		- chan->watchdog_last_tcp_cookie = 0;
670	628	}
671	629
672	630	static enum dma_status
...	...	@@ -715,9 +673,6 @@
715	673	return err;
716	674	if (dca)
717	675	device->dca = ioat2_dca_init(pdev, device->reg_base);
718		-
719		- INIT_DELAYED_WORK(&device->work, ioat2_chan_watchdog);
720		- schedule_delayed_work(&device->work, WATCHDOG_DELAY);
721	676
722	677	return err;
723	678	}
...	...	@@ -127,6 +127,16 @@
127	127	return ioat->ring[idx & ioat2_ring_mask(ioat)];
128	128	}
129	129
	130	+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
	131	+{
	132	+ struct ioat_chan_common *chan = &ioat->base;
	133	+
	134	+ writel(addr & 0x00000000FFFFFFFF,
	135	+ chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
	136	+ writel(addr >> 32,
	137	+ chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
	138	+}
	139	+
130	140	int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
131	141	int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
132	142	struct dca_provider * __devinit ioat2_dca_init(struct pci_dev pdev, void __iomem iobase);
...	...	@@ -101,11 +101,11 @@
101	101	#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
102	102	#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
103	103	#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
104		-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x7ULL
105		-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
106		-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
107		-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
108		-#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
	104	+#define IOAT_CHANSTS_STATUS 0x7ULL
	105	+#define IOAT_CHANSTS_ACTIVE 0x0
	106	+#define IOAT_CHANSTS_DONE 0x1
	107	+#define IOAT_CHANSTS_SUSPENDED 0x2
	108	+#define IOAT_CHANSTS_HALTED 0x3
109	109
110	110
111	111
112	112
...	...	@@ -208,18 +208,18 @@
208	208	#define IOAT_CDAR_OFFSET_HIGH 0x24
209	209
210	210	#define IOAT_CHANERR_OFFSET 0x28 /* 32-bit Channel Error Register */
211		-#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR 0x0001
212		-#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
213		-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR 0x0004
214		-#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR 0x0008
	211	+#define IOAT_CHANERR_SRC_ADDR_ERR 0x0001
	212	+#define IOAT_CHANERR_DEST_ADDR_ERR 0x0002
	213	+#define IOAT_CHANERR_NEXT_ADDR_ERR 0x0004
	214	+#define IOAT_CHANERR_NEXT_DESC_ALIGN_ERR 0x0008
215	215	#define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR 0x0010
216	216	#define IOAT_CHANERR_CHANCMD_ERR 0x0020
217	217	#define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0040
218	218	#define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0080
219	219	#define IOAT_CHANERR_READ_DATA_ERR 0x0100
220	220	#define IOAT_CHANERR_WRITE_DATA_ERR 0x0200
221		-#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR 0x0400
222		-#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR 0x0800
	221	+#define IOAT_CHANERR_CONTROL_ERR 0x0400
	222	+#define IOAT_CHANERR_LENGTH_ERR 0x0800
223	223	#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
224	224	#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
225	225	#define IOAT_CHANERR_SOFT_ERR 0x4000
...	...	@@ -126,9 +126,9 @@
126	126	udelay(10);
127	127
128	128	sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
129		- IOAT_CHANSTS_DMA_TRANSFER_STATUS;
	129	+ IOAT_CHANSTS_STATUS;
130	130
131		- if (sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE)
	131	+ if (sts != IOAT_CHANSTS_ACTIVE)
132	132	break;
133	133
134	134	}
135	135
...	...	@@ -160,9 +160,9 @@
160	160	udelay(1000);
161	161
162	162	chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
163		- IOAT_CHANSTS_DMA_TRANSFER_STATUS;
	163	+ IOAT_CHANSTS_STATUS;
164	164
165		- if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE) {
	165	+ if (chan_sts != IOAT_CHANSTS_DONE) {
166	166	/* Not complete, reset the channel */
167	167	writeb(IOAT_CHANCMD_RESET,
168	168	ioat_chanbase + IOAT1_CHANCMD_OFFSET);
169	169
...	...	@@ -288,9 +288,9 @@
288	288	ioat_chanbase + IOAT1_CHANCMD_OFFSET);
289	289
290	290	chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
291		- IOAT_CHANSTS_DMA_TRANSFER_STATUS;
	291	+ IOAT_CHANSTS_STATUS;
292	292
293		- if (chan_sts != IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
	293	+ if (chan_sts != IOAT_CHANSTS_ACTIVE) {
294	294	writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET);
295	295	break;
296	296	}
297	297
...	...	@@ -298,14 +298,14 @@
298	298	}
299	299
300	300	chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) &
301		- IOAT_CHANSTS_DMA_TRANSFER_STATUS;
	301	+ IOAT_CHANSTS_STATUS;
302	302
303	303	/*
304	304	* We tried to reset multiple times. If IO A/T channel is still active
305	305	* flag an error and return without cleanup. Memory leak is better
306	306	* than random corruption in that extreme error situation.
307	307	*/
308		- if (chan_sts == IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE) {
	308	+ if (chan_sts == IOAT_CHANSTS_ACTIVE) {
309	309	printk(KERN_ERR I7300_PRINT "Unable to stop IO A/T channels."
310	310	" Not freeing resources\n");
311	311	return;