Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 058276303dbc4ed089c1f7dad0871810b1f5ddf1

Authored by Linus Walleij 2010-05-18 07:30:42 +0800

Committed by Dan Williams 2010-05-18 07:30:42 +0800

Exists in master and in 7 other branches

DMAENGINE: extend the control command to include an arg

This adds an argument to the DMAengine control function, so that
we can later provide control commands that need some external data
passed in through an argument akin to the ioctl() operation
prototype.

[dan.j.williams@intel.com: fix up some missed conversions]
Signed-off-by: Linus Walleij <linus.walleij@stericsson.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>

Showing 14 changed files with 32 additions and 21 deletions Inline Diff

drivers/dma/at_hdmac.c
drivers/dma/coh901318.c
drivers/dma/dw_dmac.c
drivers/dma/fsldma.c
drivers/dma/ipu/ipu_idmac.c
drivers/dma/shdma.c
drivers/dma/ste_dma40.c
drivers/dma/timb_dma.c
drivers/dma/txx9dmac.c
drivers/mmc/host/atmel-mci.c
drivers/serial/sh-sci.c
drivers/video/mx3fb.c
include/linux/dmaengine.h
sound/soc/txx9/txx9aclc.c

drivers/dma/at_hdmac.c

Diff comments View file @ 0582763

 /*
  * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
  *
  * Copyright (C) 2008 Atmel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  *
  * This supports the Atmel AHB DMA Controller,
  *
  * The driver has currently been tested with the Atmel AT91SAM9RL
  * and AT91SAM9G45 series.
  */
 #include <linux/clk.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
 #include <linux/dmapool.h>
 #include <linux/interrupt.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include "at_hdmac_regs.h"
 /*
  * Glossary
  * --------
  *
  * at_hdmac		: Name of the ATmel AHB DMA Controller
  * at_dma_ / atdma	: ATmel DMA controller entity related
  * atc_	/ atchan	: ATmel DMA Channel entity related
  */
 #define	ATC_DEFAULT_CFG		(ATC_FIFOCFG_HALFFIFO)
 #define	ATC_DEFAULT_CTRLA	(0)
 #define	ATC_DEFAULT_CTRLB	(ATC_SIF(0)	\
 				|ATC_DIF(1))
 /*
  * Initial number of descriptors to allocate for each channel. This could
  * be increased during dma usage.
  */
 static unsigned int init_nr_desc_per_channel = 64;
 module_param(init_nr_desc_per_channel, uint, 0644);
 MODULE_PARM_DESC(init_nr_desc_per_channel,
 		 "initial descriptors per channel (default: 64)");
 /* prototypes */
 static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
 /*----------------------------------------------------------------------*/
 static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
 {
 	return list_first_entry(&atchan->active_list,
 				struct at_desc, desc_node);
 }
 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
 {
 	return list_first_entry(&atchan->queue,
 				struct at_desc, desc_node);
 }
 /**
  * atc_alloc_descriptor - allocate and return an initilized descriptor
  * @chan: the channel to allocate descriptors for
  * @gfp_flags: GFP allocation flags
  *
  * Note: The ack-bit is positioned in the descriptor flag at creation time
  *       to make initial allocation more convenient. This bit will be cleared
  *       and control will be given to client at usage time (during
  *       preparation functions).
  */
 static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
 					    gfp_t gfp_flags)
 {
 	struct at_desc	*desc = NULL;
 	struct at_dma	*atdma = to_at_dma(chan->device);
 	dma_addr_t phys;
 	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
 	if (desc) {
 		memset(desc, 0, sizeof(struct at_desc));
 		INIT_LIST_HEAD(&desc->tx_list);
 		dma_async_tx_descriptor_init(&desc->txd, chan);
 		/* txd.flags will be overwritten in prep functions */
 		desc->txd.flags = DMA_CTRL_ACK;
 		desc->txd.tx_submit = atc_tx_submit;
 		desc->txd.phys = phys;
 	}
 	return desc;
 }
 /**
  * atc_desc_get - get an unused descriptor from free_list
  * @atchan: channel we want a new descriptor for
  */
 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
 {
 	struct at_desc *desc, *_desc;
 	struct at_desc *ret = NULL;
 	unsigned int i = 0;
 	LIST_HEAD(tmp_list);
 	spin_lock_bh(&atchan->lock);
 	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
 		i++;
 		if (async_tx_test_ack(&desc->txd)) {
 			list_del(&desc->desc_node);
 			ret = desc;
 			break;
 		}
 		dev_dbg(chan2dev(&atchan->chan_common),
 				"desc %p not ACKed\n", desc);
 	}
 	spin_unlock_bh(&atchan->lock);
 	dev_vdbg(chan2dev(&atchan->chan_common),
 		"scanned %u descriptors on freelist\n", i);
 	/* no more descriptor available in initial pool: create one more */
 	if (!ret) {
 		ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
 		if (ret) {
 			spin_lock_bh(&atchan->lock);
 			atchan->descs_allocated++;
 			spin_unlock_bh(&atchan->lock);
 		} else {
 			dev_err(chan2dev(&atchan->chan_common),
 					"not enough descriptors available\n");
 		}
 	}
 	return ret;
 }
 /**
  * atc_desc_put - move a descriptor, including any children, to the free list
  * @atchan: channel we work on
  * @desc: descriptor, at the head of a chain, to move to free list
  */
 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
 {
 	if (desc) {
 		struct at_desc *child;
 		spin_lock_bh(&atchan->lock);
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			dev_vdbg(chan2dev(&atchan->chan_common),
 					"moving child desc %p to freelist\n",
 					child);
 		list_splice_init(&desc->tx_list, &atchan->free_list);
 		dev_vdbg(chan2dev(&atchan->chan_common),
 			 "moving desc %p to freelist\n", desc);
 		list_add(&desc->desc_node, &atchan->free_list);
 		spin_unlock_bh(&atchan->lock);
 	}
 }
 /**
  * atc_assign_cookie - compute and assign new cookie
  * @atchan: channel we work on
  * @desc: descriptor to asign cookie for
  *
  * Called with atchan->lock held and bh disabled
  */
 static dma_cookie_t
 atc_assign_cookie(struct at_dma_chan *atchan, struct at_desc *desc)
 {
 	dma_cookie_t cookie = atchan->chan_common.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	atchan->chan_common.cookie = cookie;
 	desc->txd.cookie = cookie;
 	return cookie;
 }
 /**
  * atc_dostart - starts the DMA engine for real
  * @atchan: the channel we want to start
  * @first: first descriptor in the list we want to begin with
  *
  * Called with atchan->lock held and bh disabled
  */
 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
 {
 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);
 	/* ASSERT:  channel is idle */
 	if (atc_chan_is_enabled(atchan)) {
 		dev_err(chan2dev(&atchan->chan_common),
 			"BUG: Attempted to start non-idle channel\n");
 		dev_err(chan2dev(&atchan->chan_common),
 			"  channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
 			channel_readl(atchan, SADDR),
 			channel_readl(atchan, DADDR),
 			channel_readl(atchan, CTRLA),
 			channel_readl(atchan, CTRLB),
 			channel_readl(atchan, DSCR));
 		/* The tasklet will hopefully advance the queue... */
 		return;
 	}
 	vdbg_dump_regs(atchan);
 	/* clear any pending interrupt */
 	while (dma_readl(atdma, EBCISR))
 		cpu_relax();
 	channel_writel(atchan, SADDR, 0);
 	channel_writel(atchan, DADDR, 0);
 	channel_writel(atchan, CTRLA, 0);
 	channel_writel(atchan, CTRLB, 0);
 	channel_writel(atchan, DSCR, first->txd.phys);
 	dma_writel(atdma, CHER, atchan->mask);
 	vdbg_dump_regs(atchan);
 }
 /**
  * atc_chain_complete - finish work for one transaction chain
  * @atchan: channel we work on
  * @desc: descriptor at the head of the chain we want do complete
  *
  * Called with atchan->lock held and bh disabled */
 static void
 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
 {
 	dma_async_tx_callback		callback;
 	void				*param;
 	struct dma_async_tx_descriptor	*txd = &desc->txd;
 	dev_vdbg(chan2dev(&atchan->chan_common),
 		"descriptor %u complete\n", txd->cookie);
 	atchan->completed_cookie = txd->cookie;
 	callback = txd->callback;
 	param = txd->callback_param;
 	/* move children to free_list */
 	list_splice_init(&desc->tx_list, &atchan->free_list);
 	/* move myself to free_list */
 	list_move(&desc->desc_node, &atchan->free_list);
 	/* unmap dma addresses */
 	if (!atchan->chan_common.private) {
 		struct device *parent = chan2parent(&atchan->chan_common);
 		if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 			if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 				dma_unmap_single(parent,
 						desc->lli.daddr,
 						desc->len, DMA_FROM_DEVICE);
 			else
 				dma_unmap_page(parent,
 						desc->lli.daddr,
 						desc->len, DMA_FROM_DEVICE);
 		}
 		if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 			if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 				dma_unmap_single(parent,
 						desc->lli.saddr,
 						desc->len, DMA_TO_DEVICE);
 			else
 				dma_unmap_page(parent,
 						desc->lli.saddr,
 						desc->len, DMA_TO_DEVICE);
 		}
 	}
 	/*
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
 	if (callback)
 		callback(param);
 	dma_run_dependencies(txd);
 }
 /**
  * atc_complete_all - finish work for all transactions
  * @atchan: channel to complete transactions for
  *
  * Eventually submit queued descriptors if any
  *
  * Assume channel is idle while calling this function
  * Called with atchan->lock held and bh disabled
  */
 static void atc_complete_all(struct at_dma_chan *atchan)
 {
 	struct at_desc *desc, *_desc;
 	LIST_HEAD(list);
 	dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
 	BUG_ON(atc_chan_is_enabled(atchan));
 	/*
 	 * Submit queued descriptors ASAP, i.e. before we go through
 	 * the completed ones.
 	 */
 	if (!list_empty(&atchan->queue))
 		atc_dostart(atchan, atc_first_queued(atchan));
 	/* empty active_list now it is completed */
 	list_splice_init(&atchan->active_list, &list);
 	/* empty queue list by moving descriptors (if any) to active_list */
 	list_splice_init(&atchan->queue, &atchan->active_list);
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		atc_chain_complete(atchan, desc);
 }
 /**
  * atc_cleanup_descriptors - cleanup up finished descriptors in active_list
  * @atchan: channel to be cleaned up
  *
  * Called with atchan->lock held and bh disabled
  */
 static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
 {
 	struct at_desc	*desc, *_desc;
 	struct at_desc	*child;
 	dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n");
 	list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
 		if (!(desc->lli.ctrla & ATC_DONE))
 			/* This one is currently in progress */
 			return;
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			if (!(child->lli.ctrla & ATC_DONE))
 				/* Currently in progress */
 				return;
 		/*
 		 * No descriptors so far seem to be in progress, i.e.
 		 * this chain must be done.
 		 */
 		atc_chain_complete(atchan, desc);
 	}
 }
 /**
  * atc_advance_work - at the end of a transaction, move forward
  * @atchan: channel where the transaction ended
  *
  * Called with atchan->lock held and bh disabled
  */
 static void atc_advance_work(struct at_dma_chan *atchan)
 {
 	dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
 	if (list_empty(&atchan->active_list) ||
 	    list_is_singular(&atchan->active_list)) {
 		atc_complete_all(atchan);
 	} else {
 		atc_chain_complete(atchan, atc_first_active(atchan));
 		/* advance work */
 		atc_dostart(atchan, atc_first_active(atchan));
 	}
 }
 /**
  * atc_handle_error - handle errors reported by DMA controller
  * @atchan: channel where error occurs
  *
  * Called with atchan->lock held and bh disabled
  */
 static void atc_handle_error(struct at_dma_chan *atchan)
 {
 	struct at_desc *bad_desc;
 	struct at_desc *child;
 	/*
 	 * The descriptor currently at the head of the active list is
 	 * broked. Since we don't have any way to report errors, we'll
 	 * just have to scream loudly and try to carry on.
 	 */
 	bad_desc = atc_first_active(atchan);
 	list_del_init(&bad_desc->desc_node);
 	/* As we are stopped, take advantage to push queued descriptors
 	 * in active_list */
 	list_splice_init(&atchan->queue, atchan->active_list.prev);
 	/* Try to restart the controller */
 	if (!list_empty(&atchan->active_list))
 		atc_dostart(atchan, atc_first_active(atchan));
 	/*
 	 * KERN_CRITICAL may seem harsh, but since this only happens
 	 * when someone submits a bad physical address in a
 	 * descriptor, we should consider ourselves lucky that the
 	 * controller flagged an error instead of scribbling over
 	 * random memory locations.
 	 */
 	dev_crit(chan2dev(&atchan->chan_common),
 			"Bad descriptor submitted for DMA!\n");
 	dev_crit(chan2dev(&atchan->chan_common),
 			"  cookie: %d\n", bad_desc->txd.cookie);
 	atc_dump_lli(atchan, &bad_desc->lli);
 	list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 		atc_dump_lli(atchan, &child->lli);
 	/* Pretend the descriptor completed successfully */
 	atc_chain_complete(atchan, bad_desc);
 }
 /*--  IRQ & Tasklet  ---------------------------------------------------*/
 static void atc_tasklet(unsigned long data)
 {
 	struct at_dma_chan *atchan = (struct at_dma_chan *)data;
 	/* Channel cannot be enabled here */
 	if (atc_chan_is_enabled(atchan)) {
 		dev_err(chan2dev(&atchan->chan_common),
 			"BUG: channel enabled in tasklet\n");
 		return;
 	}
 	spin_lock(&atchan->lock);
 	if (test_and_clear_bit(0, &atchan->error_status))
 		atc_handle_error(atchan);
 	else
 		atc_advance_work(atchan);
 	spin_unlock(&atchan->lock);
 }
 static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
 {
 	struct at_dma		*atdma = (struct at_dma *)dev_id;
 	struct at_dma_chan	*atchan;
 	int			i;
 	u32			status, pending, imr;
 	int			ret = IRQ_NONE;
 	do {
 		imr = dma_readl(atdma, EBCIMR);
 		status = dma_readl(atdma, EBCISR);
 		pending = status & imr;
 		if (!pending)
 			break;
 		dev_vdbg(atdma->dma_common.dev,
 			"interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
 			 status, imr, pending);
 		for (i = 0; i < atdma->dma_common.chancnt; i++) {
 			atchan = &atdma->chan[i];
 			if (pending & (AT_DMA_CBTC(i) | AT_DMA_ERR(i))) {
 				if (pending & AT_DMA_ERR(i)) {
 					/* Disable channel on AHB error */
 					dma_writel(atdma, CHDR, atchan->mask);
 					/* Give information to tasklet */
 					set_bit(0, &atchan->error_status);
 				}
 				tasklet_schedule(&atchan->tasklet);
 				ret = IRQ_HANDLED;
 			}
 		}
 	} while (pending);
 	return ret;
 }
 /*--  DMA Engine API  --------------------------------------------------*/
 /**
  * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
  * @desc: descriptor at the head of the transaction chain
  *
  * Queue chain if DMA engine is working already
  *
  * Cookie increment and adding to active_list or queue must be atomic
  */
 static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct at_desc		*desc = txd_to_at_desc(tx);
 	struct at_dma_chan	*atchan = to_at_dma_chan(tx->chan);
 	dma_cookie_t		cookie;
 	spin_lock_bh(&atchan->lock);
 	cookie = atc_assign_cookie(atchan, desc);
 	if (list_empty(&atchan->active_list)) {
 		dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
 				desc->txd.cookie);
 		atc_dostart(atchan, desc);
 		list_add_tail(&desc->desc_node, &atchan->active_list);
 	} else {
 		dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
 				desc->txd.cookie);
 		list_add_tail(&desc->desc_node, &atchan->queue);
 	}
 	spin_unlock_bh(&atchan->lock);
 	return cookie;
 }
 /**
  * atc_prep_dma_memcpy - prepare a memcpy operation
  * @chan: the channel to prepare operation on
  * @dest: operation virtual destination address
  * @src: operation virtual source address
  * @len: operation length
  * @flags: tx descriptor status flags
  */
 static struct dma_async_tx_descriptor *
 atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		size_t len, unsigned long flags)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_desc		*desc = NULL;
 	struct at_desc		*first = NULL;
 	struct at_desc		*prev = NULL;
 	size_t			xfer_count;
 	size_t			offset;
 	unsigned int		src_width;
 	unsigned int		dst_width;
 	u32			ctrla;
 	u32			ctrlb;
 	dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d0x%x s0x%x l0x%zx f0x%lx\n",
 			dest, src, len, flags);
 	if (unlikely(!len)) {
 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 	ctrla =   ATC_DEFAULT_CTRLA;
 	ctrlb =   ATC_DEFAULT_CTRLB
 		| ATC_SRC_ADDR_MODE_INCR
 		| ATC_DST_ADDR_MODE_INCR
 		| ATC_FC_MEM2MEM;
 	/*
 	 * We can be a lot more clever here, but this should take care
 	 * of the most common optimization.
 	 */
 	if (!((src | dest  | len) & 3)) {
 		ctrla |= ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD;
 		src_width = dst_width = 2;
 	} else if (!((src | dest | len) & 1)) {
 		ctrla |= ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD;
 		src_width = dst_width = 1;
 	} else {
 		ctrla |= ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE;
 		src_width = dst_width = 0;
 	}
 	for (offset = 0; offset < len; offset += xfer_count << src_width) {
 		xfer_count = min_t(size_t, (len - offset) >> src_width,
 				ATC_BTSIZE_MAX);
 		desc = atc_desc_get(atchan);
 		if (!desc)
 			goto err_desc_get;
 		desc->lli.saddr = src + offset;
 		desc->lli.daddr = dest + offset;
 		desc->lli.ctrla = ctrla | xfer_count;
 		desc->lli.ctrlb = ctrlb;
 		desc->txd.cookie = 0;
 		async_tx_ack(&desc->txd);
 		if (!first) {
 			first = desc;
 		} else {
 			/* inform the HW lli about chaining */
 			prev->lli.dscr = desc->txd.phys;
 			/* insert the link descriptor to the LD ring */
 			list_add_tail(&desc->desc_node,
 					&first->tx_list);
 		}
 		prev = desc;
 	}
 	/* First descriptor of the chain embedds additional information */
 	first->txd.cookie = -EBUSY;
 	first->len = len;
 	/* set end-of-link to the last link descriptor of list*/
 	set_desc_eol(desc);
 	desc->txd.flags = flags; /* client is in control of this ack */
 	return &first->txd;
 err_desc_get:
 	atc_desc_put(atchan, first);
 	return NULL;
 }
 /**
  * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
  * @chan: DMA channel
  * @sgl: scatterlist to transfer to/from
  * @sg_len: number of entries in @scatterlist
  * @direction: DMA direction
  * @flags: tx descriptor status flags
  */
 static struct dma_async_tx_descriptor *
 atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma_slave	*atslave = chan->private;
 	struct at_desc		*first = NULL;
 	struct at_desc		*prev = NULL;
 	u32			ctrla;
 	u32			ctrlb;
 	dma_addr_t		reg;
 	unsigned int		reg_width;
 	unsigned int		mem_width;
 	unsigned int		i;
 	struct scatterlist	*sg;
 	size_t			total_len = 0;
 	dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n",
 			direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE",
 			flags);
 	if (unlikely(!atslave || !sg_len)) {
 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 	reg_width = atslave->reg_width;
 	ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
 	ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
 	switch (direction) {
 	case DMA_TO_DEVICE:
 		ctrla |=  ATC_DST_WIDTH(reg_width);
 		ctrlb |=  ATC_DST_ADDR_MODE_FIXED
 			| ATC_SRC_ADDR_MODE_INCR
 			| ATC_FC_MEM2PER;
 		reg = atslave->tx_reg;
 		for_each_sg(sgl, sg, sg_len, i) {
 			struct at_desc	*desc;
 			u32		len;
 			u32		mem;
 			desc = atc_desc_get(atchan);
 			if (!desc)
 				goto err_desc_get;
 			mem = sg_phys(sg);
 			len = sg_dma_len(sg);
 			mem_width = 2;
 			if (unlikely(mem & 3 || len & 3))
 				mem_width = 0;
 			desc->lli.saddr = mem;
 			desc->lli.daddr = reg;
 			desc->lli.ctrla = ctrla
 					| ATC_SRC_WIDTH(mem_width)
 					| len >> mem_width;
 			desc->lli.ctrlb = ctrlb;
 			if (!first) {
 				first = desc;
 			} else {
 				/* inform the HW lli about chaining */
 				prev->lli.dscr = desc->txd.phys;
 				/* insert the link descriptor to the LD ring */
 				list_add_tail(&desc->desc_node,
 						&first->tx_list);
 			}
 			prev = desc;
 			total_len += len;
 		}
 		break;
 	case DMA_FROM_DEVICE:
 		ctrla |=  ATC_SRC_WIDTH(reg_width);
 		ctrlb |=  ATC_DST_ADDR_MODE_INCR
 			| ATC_SRC_ADDR_MODE_FIXED
 			| ATC_FC_PER2MEM;
 		reg = atslave->rx_reg;
 		for_each_sg(sgl, sg, sg_len, i) {
 			struct at_desc	*desc;
 			u32		len;
 			u32		mem;
 			desc = atc_desc_get(atchan);
 			if (!desc)
 				goto err_desc_get;
 			mem = sg_phys(sg);
 			len = sg_dma_len(sg);
 			mem_width = 2;
 			if (unlikely(mem & 3 || len & 3))
 				mem_width = 0;
 			desc->lli.saddr = reg;
 			desc->lli.daddr = mem;
 			desc->lli.ctrla = ctrla
 					| ATC_DST_WIDTH(mem_width)
 					| len >> mem_width;
 			desc->lli.ctrlb = ctrlb;
 			if (!first) {
 				first = desc;
 			} else {
 				/* inform the HW lli about chaining */
 				prev->lli.dscr = desc->txd.phys;
 				/* insert the link descriptor to the LD ring */
 				list_add_tail(&desc->desc_node,
 						&first->tx_list);
 			}
 			prev = desc;
 			total_len += len;
 		}
 		break;
 	default:
 		return NULL;
 	}
 	/* set end-of-link to the last link descriptor of list*/
 	set_desc_eol(prev);
 	/* First descriptor of the chain embedds additional information */
 	first->txd.cookie = -EBUSY;
 	first->len = total_len;
 	/* last link descriptor of list is responsible of flags */
 	prev->txd.flags = flags; /* client is in control of this ack */
 	return &first->txd;
 err_desc_get:
 	dev_err(chan2dev(chan), "not enough descriptors available\n");
 	atc_desc_put(atchan, first);
 	return NULL;
 }
-static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		       unsigned long arg)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc, *_desc;
 	LIST_HEAD(list);
 	/* Only supports DMA_TERMINATE_ALL */
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	/*
 	 * This is only called when something went wrong elsewhere, so
 	 * we don't really care about the data. Just disable the
 	 * channel. We still have to poll the channel enable bit due
 	 * to AHB/HSB limitations.
 	 */
 	spin_lock_bh(&atchan->lock);
 	dma_writel(atdma, CHDR, atchan->mask);
 	/* confirm that this channel is disabled */
 	while (dma_readl(atdma, CHSR) & atchan->mask)
 		cpu_relax();
 	/* active_list entries will end up before queued entries */
 	list_splice_init(&atchan->queue, &list);
 	list_splice_init(&atchan->active_list, &list);
 	spin_unlock_bh(&atchan->lock);
 	/* Flush all pending and queued descriptors */
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		atc_chain_complete(atchan, desc);
 	return 0;
 }
 /**
  * atc_tx_status - poll for transaction completion
  * @chan: DMA channel
  * @cookie: transaction identifier to check status of
  * @txstate: if not %NULL updated with transaction state
  *
  * If @txstate is passed in, upon return it reflect the driver
  * internal state and can be used with dma_async_is_complete() to check
  * the status of multiple cookies without re-checking hardware state.
  */
 static enum dma_status
 atc_tx_status(struct dma_chan *chan,
 		dma_cookie_t cookie,
 		struct dma_tx_state *txstate)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	dma_cookie_t		last_used;
 	dma_cookie_t		last_complete;
 	enum dma_status		ret;
 	spin_lock_bh(&atchan->lock);
 	last_complete = atchan->completed_cookie;
 	last_used = chan->cookie;
 	ret = dma_async_is_complete(cookie, last_complete, last_used);
 	if (ret != DMA_SUCCESS) {
 		atc_cleanup_descriptors(atchan);
 		last_complete = atchan->completed_cookie;
 		last_used = chan->cookie;
 		ret = dma_async_is_complete(cookie, last_complete, last_used);
 	}
 	spin_unlock_bh(&atchan->lock);
 	dma_set_tx_state(txstate, last_complete, last_used, 0);
 	dev_vdbg(chan2dev(chan), "tx_status: %d (d%d, u%d)\n",
 		 cookie, last_complete ? last_complete : 0,
 		 last_used ? last_used : 0);
 	return ret;
 }
 /**
  * atc_issue_pending - try to finish work
  * @chan: target DMA channel
  */
 static void atc_issue_pending(struct dma_chan *chan)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	dev_vdbg(chan2dev(chan), "issue_pending\n");
 	if (!atc_chan_is_enabled(atchan)) {
 		spin_lock_bh(&atchan->lock);
 		atc_advance_work(atchan);
 		spin_unlock_bh(&atchan->lock);
 	}
 }
 /**
  * atc_alloc_chan_resources - allocate resources for DMA channel
  * @chan: allocate descriptor resources for this channel
  * @client: current client requesting the channel be ready for requests
  *
  * return - the number of allocated descriptors
  */
 static int atc_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc;
 	struct at_dma_slave	*atslave;
 	int			i;
 	u32			cfg;
 	LIST_HEAD(tmp_list);
 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
 	/* ASSERT:  channel is idle */
 	if (atc_chan_is_enabled(atchan)) {
 		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
 		return -EIO;
 	}
 	cfg = ATC_DEFAULT_CFG;
 	atslave = chan->private;
 	if (atslave) {
 		/*
 		 * We need controller-specific data to set up slave
 		 * transfers.
 		 */
 		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
 		/* if cfg configuration specified take it instad of default */
 		if (atslave->cfg)
 			cfg = atslave->cfg;
 	}
 	/* have we already been set up?
 	 * reconfigure channel but no need to reallocate descriptors */
 	if (!list_empty(&atchan->free_list))
 		return atchan->descs_allocated;
 	/* Allocate initial pool of descriptors */
 	for (i = 0; i < init_nr_desc_per_channel; i++) {
 		desc = atc_alloc_descriptor(chan, GFP_KERNEL);
 		if (!desc) {
 			dev_err(atdma->dma_common.dev,
 				"Only %d initial descriptors\n", i);
 			break;
 		}
 		list_add_tail(&desc->desc_node, &tmp_list);
 	}
 	spin_lock_bh(&atchan->lock);
 	atchan->descs_allocated = i;
 	list_splice(&tmp_list, &atchan->free_list);
 	atchan->completed_cookie = chan->cookie = 1;
 	spin_unlock_bh(&atchan->lock);
 	/* channel parameters */
 	channel_writel(atchan, CFG, cfg);
 	dev_dbg(chan2dev(chan),
 		"alloc_chan_resources: allocated %d descriptors\n",
 		atchan->descs_allocated);
 	return atchan->descs_allocated;
 }
 /**
  * atc_free_chan_resources - free all channel resources
  * @chan: DMA channel
  */
 static void atc_free_chan_resources(struct dma_chan *chan)
 {
 	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 	struct at_dma		*atdma = to_at_dma(chan->device);
 	struct at_desc		*desc, *_desc;
 	LIST_HEAD(list);
 	dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
 		atchan->descs_allocated);
 	/* ASSERT:  channel is idle */
 	BUG_ON(!list_empty(&atchan->active_list));
 	BUG_ON(!list_empty(&atchan->queue));
 	BUG_ON(atc_chan_is_enabled(atchan));
 	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
 		list_del(&desc->desc_node);
 		/* free link descriptor */
 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
 	}
 	list_splice_init(&atchan->free_list, &list);
 	atchan->descs_allocated = 0;
 	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
 }
 /*--  Module Management  -----------------------------------------------*/
 /**
  * at_dma_off - disable DMA controller
  * @atdma: the Atmel HDAMC device
  */
 static void at_dma_off(struct at_dma *atdma)
 {
 	dma_writel(atdma, EN, 0);
 	/* disable all interrupts */
 	dma_writel(atdma, EBCIDR, -1L);
 	/* confirm that all channels are disabled */
 	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
 		cpu_relax();
 }
 static int __init at_dma_probe(struct platform_device *pdev)
 {
 	struct at_dma_platform_data *pdata;
 	struct resource		*io;
 	struct at_dma		*atdma;
 	size_t			size;
 	int			irq;
 	int			err;
 	int			i;
 	/* get DMA Controller parameters from platform */
 	pdata = pdev->dev.platform_data;
 	if (!pdata || pdata->nr_channels > AT_DMA_MAX_NR_CHANNELS)
 		return -EINVAL;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!io)
 		return -EINVAL;
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
 	size = sizeof(struct at_dma);
 	size += pdata->nr_channels * sizeof(struct at_dma_chan);
 	atdma = kzalloc(size, GFP_KERNEL);
 	if (!atdma)
 		return -ENOMEM;
 	/* discover transaction capabilites from the platform data */
 	atdma->dma_common.cap_mask = pdata->cap_mask;
 	atdma->all_chan_mask = (1 << pdata->nr_channels) - 1;
 	size = io->end - io->start + 1;
 	if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
 		err = -EBUSY;
 		goto err_kfree;
 	}
 	atdma->regs = ioremap(io->start, size);
 	if (!atdma->regs) {
 		err = -ENOMEM;
 		goto err_release_r;
 	}
 	atdma->clk = clk_get(&pdev->dev, "dma_clk");
 	if (IS_ERR(atdma->clk)) {
 		err = PTR_ERR(atdma->clk);
 		goto err_clk;
 	}
 	clk_enable(atdma->clk);
 	/* force dma off, just in case */
 	at_dma_off(atdma);
 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
 	if (err)
 		goto err_irq;
 	platform_set_drvdata(pdev, atdma);
 	/* create a pool of consistent memory blocks for hardware descriptors */
 	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
 			&pdev->dev, sizeof(struct at_desc),
 			4 /* word alignment */, 0);
 	if (!atdma->dma_desc_pool) {
 		dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
 		err = -ENOMEM;
 		goto err_pool_create;
 	}
 	/* clear any pending interrupt */
 	while (dma_readl(atdma, EBCISR))
 		cpu_relax();
 	/* initialize channels related values */
 	INIT_LIST_HEAD(&atdma->dma_common.channels);
 	for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) {
 		struct at_dma_chan	*atchan = &atdma->chan[i];
 		atchan->chan_common.device = &atdma->dma_common;
 		atchan->chan_common.cookie = atchan->completed_cookie = 1;
 		atchan->chan_common.chan_id = i;
 		list_add_tail(&atchan->chan_common.device_node,
 				&atdma->dma_common.channels);
 		atchan->ch_regs = atdma->regs + ch_regs(i);
 		spin_lock_init(&atchan->lock);
 		atchan->mask = 1 << i;
 		INIT_LIST_HEAD(&atchan->active_list);
 		INIT_LIST_HEAD(&atchan->queue);
 		INIT_LIST_HEAD(&atchan->free_list);
 		tasklet_init(&atchan->tasklet, atc_tasklet,
 				(unsigned long)atchan);
 		atc_enable_irq(atchan);
 	}
 	/* set base routines */
 	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
 	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
 	atdma->dma_common.device_tx_status = atc_tx_status;
 	atdma->dma_common.device_issue_pending = atc_issue_pending;
 	atdma->dma_common.dev = &pdev->dev;
 	/* set prep routines based on capability */
 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
 	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
 		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
 		atdma->dma_common.device_control = atc_control;
 	}
 	dma_writel(atdma, EN, AT_DMA_ENABLE);
 	dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n",
 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",
 	  atdma->dma_common.chancnt);
 	dma_async_device_register(&atdma->dma_common);
 	return 0;
 err_pool_create:
 	platform_set_drvdata(pdev, NULL);
 	free_irq(platform_get_irq(pdev, 0), atdma);
 err_irq:
 	clk_disable(atdma->clk);
 	clk_put(atdma->clk);
 err_clk:
 	iounmap(atdma->regs);
 	atdma->regs = NULL;
 err_release_r:
 	release_mem_region(io->start, size);
 err_kfree:
 	kfree(atdma);
 	return err;
 }
 static int __exit at_dma_remove(struct platform_device *pdev)
 {
 	struct at_dma		*atdma = platform_get_drvdata(pdev);
 	struct dma_chan		*chan, *_chan;
 	struct resource		*io;
 	at_dma_off(atdma);
 	dma_async_device_unregister(&atdma->dma_common);
 	dma_pool_destroy(atdma->dma_desc_pool);
 	platform_set_drvdata(pdev, NULL);
 	free_irq(platform_get_irq(pdev, 0), atdma);
 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
 			device_node) {
 		struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 		/* Disable interrupts */
 		atc_disable_irq(atchan);
 		tasklet_disable(&atchan->tasklet);
 		tasklet_kill(&atchan->tasklet);
 		list_del(&chan->device_node);
 	}
 	clk_disable(atdma->clk);
 	clk_put(atdma->clk);
 	iounmap(atdma->regs);
 	atdma->regs = NULL;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	release_mem_region(io->start, io->end - io->start + 1);
 	kfree(atdma);
 	return 0;
 }
 static void at_dma_shutdown(struct platform_device *pdev)
 {
 	struct at_dma	*atdma = platform_get_drvdata(pdev);
 	at_dma_off(platform_get_drvdata(pdev));
 	clk_disable(atdma->clk);
 }
 static int at_dma_suspend_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct at_dma *atdma = platform_get_drvdata(pdev);
 	at_dma_off(platform_get_drvdata(pdev));
 	clk_disable(atdma->clk);
 	return 0;
 }
 static int at_dma_resume_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct at_dma *atdma = platform_get_drvdata(pdev);
 	clk_enable(atdma->clk);
 	dma_writel(atdma, EN, AT_DMA_ENABLE);
 	return 0;
 }
 static const struct dev_pm_ops at_dma_dev_pm_ops = {
 	.suspend_noirq = at_dma_suspend_noirq,
 	.resume_noirq = at_dma_resume_noirq,
 };
 static struct platform_driver at_dma_driver = {
 	.remove		= __exit_p(at_dma_remove),
 	.shutdown	= at_dma_shutdown,
 	.driver = {
 		.name	= "at_hdmac",
 		.pm	= &at_dma_dev_pm_ops,
 	},
 };
 static int __init at_dma_init(void)
 {
 	return platform_driver_probe(&at_dma_driver, at_dma_probe);
 }
 module_init(at_dma_init);
 static void __exit at_dma_exit(void)
 {
 	platform_driver_unregister(&at_dma_driver);
 }
 module_exit(at_dma_exit);
 MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
 MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:at_hdmac");

drivers/dma/coh901318.c

Diff comments View file @ 0582763

 /*
  * driver/dma/coh901318.c
  *
  * Copyright (C) 2007-2009 ST-Ericsson
  * License terms: GNU General Public License (GPL) version 2
  * DMA driver for COH 901 318
  * Author: Per Friden <per.friden@stericsson.com>
  */
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/kernel.h> /* printk() */
 #include <linux/fs.h> /* everything... */
 #include <linux/slab.h> /* kmalloc() */
 #include <linux/dmaengine.h>
 #include <linux/platform_device.h>
 #include <linux/device.h>
 #include <linux/irqreturn.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/uaccess.h>
 #include <linux/debugfs.h>
 #include <mach/coh901318.h>
 #include "coh901318_lli.h"
 #define COHC_2_DEV(cohc) (&cohc->chan.dev->device)
 #ifdef VERBOSE_DEBUG
 #define COH_DBG(x) ({ if (1) x; 0; })
 #else
 #define COH_DBG(x) ({ if (0) x; 0; })
 #endif
 struct coh901318_desc {
 	struct dma_async_tx_descriptor desc;
 	struct list_head node;
 	struct scatterlist *sg;
 	unsigned int sg_len;
 	struct coh901318_lli *lli;
 	enum dma_data_direction dir;
 	unsigned long flags;
 };
 struct coh901318_base {
 	struct device *dev;
 	void __iomem *virtbase;
 	struct coh901318_pool pool;
 	struct powersave pm;
 	struct dma_device dma_slave;
 	struct dma_device dma_memcpy;
 	struct coh901318_chan *chans;
 	struct coh901318_platform *platform;
 };
 struct coh901318_chan {
 	spinlock_t lock;
 	int allocated;
 	int completed;
 	int id;
 	int stopped;
 	struct work_struct free_work;
 	struct dma_chan chan;
 	struct tasklet_struct tasklet;
 	struct list_head active;
 	struct list_head queue;
 	struct list_head free;
 	unsigned long nbr_active_done;
 	unsigned long busy;
 	struct coh901318_base *base;
 };
 static void coh901318_list_print(struct coh901318_chan *cohc,
 				 struct coh901318_lli *lli)
 {
 	struct coh901318_lli *l = lli;
 	int i = 0;
 	while (l) {
 		dev_vdbg(COHC_2_DEV(cohc), "i %d, lli %p, ctrl 0x%x, src 0x%x"
 			 ", dst 0x%x, link 0x%x virt_link_addr 0x%p\n",
 			 i, l, l->control, l->src_addr, l->dst_addr,
 			 l->link_addr, l->virt_link_addr);
 		i++;
 		l = l->virt_link_addr;
 	}
 }
 #ifdef CONFIG_DEBUG_FS
 #define COH901318_DEBUGFS_ASSIGN(x, y) (x = y)
 static struct coh901318_base *debugfs_dma_base;
 static struct dentry *dma_dentry;
 static int coh901318_debugfs_open(struct inode *inode, struct file *file)
 {
 	file->private_data = inode->i_private;
 	return 0;
 }
 static int coh901318_debugfs_read(struct file *file, char __user *buf,
 				  size_t count, loff_t *f_pos)
 {
 	u64 started_channels = debugfs_dma_base->pm.started_channels;
 	int pool_count = debugfs_dma_base->pool.debugfs_pool_counter;
 	int i;
 	int ret = 0;
 	char *dev_buf;
 	char *tmp;
 	int dev_size;
 	dev_buf = kmalloc(4*1024, GFP_KERNEL);
 	if (dev_buf == NULL)
 		goto err_kmalloc;
 	tmp = dev_buf;
 	tmp += sprintf(tmp, "DMA -- enabled dma channels\n");
 	for (i = 0; i < debugfs_dma_base->platform->max_channels; i++)
 		if (started_channels & (1 << i))
 			tmp += sprintf(tmp, "channel %d\n", i);
 	tmp += sprintf(tmp, "Pool alloc nbr %d\n", pool_count);
 	dev_size = tmp  - dev_buf;
 	/* No more to read if offset != 0 */
 	if (*f_pos > dev_size)
 		goto out;
 	if (count > dev_size - *f_pos)
 		count = dev_size - *f_pos;
 	if (copy_to_user(buf, dev_buf + *f_pos, count))
 		ret = -EINVAL;
 	ret = count;
 	*f_pos += count;
  out:
 	kfree(dev_buf);
 	return ret;
  err_kmalloc:
 	return 0;
 }
 static const struct file_operations coh901318_debugfs_status_operations = {
 	.owner		= THIS_MODULE,
 	.open		= coh901318_debugfs_open,
 	.read		= coh901318_debugfs_read,
 };
 static int __init init_coh901318_debugfs(void)
 {
 	dma_dentry = debugfs_create_dir("dma", NULL);
 	(void) debugfs_create_file("status",
 				   S_IFREG | S_IRUGO,
 				   dma_dentry, NULL,
 				   &coh901318_debugfs_status_operations);
 	return 0;
 }
 static void __exit exit_coh901318_debugfs(void)
 {
 	debugfs_remove_recursive(dma_dentry);
 }
 module_init(init_coh901318_debugfs);
 module_exit(exit_coh901318_debugfs);
 #else
 #define COH901318_DEBUGFS_ASSIGN(x, y)
 #endif /* CONFIG_DEBUG_FS */
 static inline struct coh901318_chan *to_coh901318_chan(struct dma_chan *chan)
 {
 	return container_of(chan, struct coh901318_chan, chan);
 }
 static inline dma_addr_t
 cohc_dev_addr(struct coh901318_chan *cohc)
 {
 	return cohc->base->platform->chan_conf[cohc->id].dev_addr;
 }
 static inline const struct coh901318_params *
 cohc_chan_param(struct coh901318_chan *cohc)
 {
 	return &cohc->base->platform->chan_conf[cohc->id].param;
 }
 static inline const struct coh_dma_channel *
 cohc_chan_conf(struct coh901318_chan *cohc)
 {
 	return &cohc->base->platform->chan_conf[cohc->id];
 }
 static void enable_powersave(struct coh901318_chan *cohc)
 {
 	unsigned long flags;
 	struct powersave *pm = &cohc->base->pm;
 	spin_lock_irqsave(&pm->lock, flags);
 	pm->started_channels &= ~(1ULL << cohc->id);
 	if (!pm->started_channels) {
 		/* DMA no longer intends to access memory */
 		cohc->base->platform->access_memory_state(cohc->base->dev,
 							  false);
 	}
 	spin_unlock_irqrestore(&pm->lock, flags);
 }
 static void disable_powersave(struct coh901318_chan *cohc)
 {
 	unsigned long flags;
 	struct powersave *pm = &cohc->base->pm;
 	spin_lock_irqsave(&pm->lock, flags);
 	if (!pm->started_channels) {
 		/* DMA intends to access memory */
 		cohc->base->platform->access_memory_state(cohc->base->dev,
 							  true);
 	}
 	pm->started_channels |= (1ULL << cohc->id);
 	spin_unlock_irqrestore(&pm->lock, flags);
 }
 static inline int coh901318_set_ctrl(struct coh901318_chan *cohc, u32 control)
 {
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	writel(control,
 	       virtbase + COH901318_CX_CTRL +
 	       COH901318_CX_CTRL_SPACING * channel);
 	return 0;
 }
 static inline int coh901318_set_conf(struct coh901318_chan *cohc, u32 conf)
 {
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	writel(conf,
 	       virtbase + COH901318_CX_CFG +
 	       COH901318_CX_CFG_SPACING*channel);
 	return 0;
 }
 static int coh901318_start(struct coh901318_chan *cohc)
 {
 	u32 val;
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	disable_powersave(cohc);
 	val = readl(virtbase + COH901318_CX_CFG +
 		    COH901318_CX_CFG_SPACING * channel);
 	/* Enable channel */
 	val |= COH901318_CX_CFG_CH_ENABLE;
 	writel(val, virtbase + COH901318_CX_CFG +
 	       COH901318_CX_CFG_SPACING * channel);
 	return 0;
 }
 static int coh901318_prep_linked_list(struct coh901318_chan *cohc,
 				      struct coh901318_lli *lli)
 {
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	BUG_ON(readl(virtbase + COH901318_CX_STAT +
 		     COH901318_CX_STAT_SPACING*channel) &
 	       COH901318_CX_STAT_ACTIVE);
 	writel(lli->src_addr,
 	       virtbase + COH901318_CX_SRC_ADDR +
 	       COH901318_CX_SRC_ADDR_SPACING * channel);
 	writel(lli->dst_addr, virtbase +
 	       COH901318_CX_DST_ADDR +
 	       COH901318_CX_DST_ADDR_SPACING * channel);
 	writel(lli->link_addr, virtbase + COH901318_CX_LNK_ADDR +
 	       COH901318_CX_LNK_ADDR_SPACING * channel);
 	writel(lli->control, virtbase + COH901318_CX_CTRL +
 	       COH901318_CX_CTRL_SPACING * channel);
 	return 0;
 }
 static dma_cookie_t
 coh901318_assign_cookie(struct coh901318_chan *cohc,
 			struct coh901318_desc *cohd)
 {
 	dma_cookie_t cookie = cohc->chan.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	cohc->chan.cookie = cookie;
 	cohd->desc.cookie = cookie;
 	return cookie;
 }
 static struct coh901318_desc *
 coh901318_desc_get(struct coh901318_chan *cohc)
 {
 	struct coh901318_desc *desc;
 	if (list_empty(&cohc->free)) {
 		/* alloc new desc because we're out of used ones
 		 * TODO: alloc a pile of descs instead of just one,
 		 * avoid many small allocations.
 		 */
 		desc = kzalloc(sizeof(struct coh901318_desc), GFP_NOWAIT);
 		if (desc == NULL)
 			goto out;
 		INIT_LIST_HEAD(&desc->node);
 		dma_async_tx_descriptor_init(&desc->desc, &cohc->chan);
 	} else {
 		/* Reuse an old desc. */
 		desc = list_first_entry(&cohc->free,
 					struct coh901318_desc,
 					node);
 		list_del(&desc->node);
 		/* Initialize it a bit so it's not insane */
 		desc->sg = NULL;
 		desc->sg_len = 0;
 		desc->desc.callback = NULL;
 		desc->desc.callback_param = NULL;
 	}
  out:
 	return desc;
 }
 static void
 coh901318_desc_free(struct coh901318_chan *cohc, struct coh901318_desc *cohd)
 {
 	list_add_tail(&cohd->node, &cohc->free);
 }
 /* call with irq lock held */
 static void
 coh901318_desc_submit(struct coh901318_chan *cohc, struct coh901318_desc *desc)
 {
 	list_add_tail(&desc->node, &cohc->active);
 }
 static struct coh901318_desc *
 coh901318_first_active_get(struct coh901318_chan *cohc)
 {
 	struct coh901318_desc *d;
 	if (list_empty(&cohc->active))
 		return NULL;
 	d = list_first_entry(&cohc->active,
 			     struct coh901318_desc,
 			     node);
 	return d;
 }
 static void
 coh901318_desc_remove(struct coh901318_desc *cohd)
 {
 	list_del(&cohd->node);
 }
 static void
 coh901318_desc_queue(struct coh901318_chan *cohc, struct coh901318_desc *desc)
 {
 	list_add_tail(&desc->node, &cohc->queue);
 }
 static struct coh901318_desc *
 coh901318_first_queued(struct coh901318_chan *cohc)
 {
 	struct coh901318_desc *d;
 	if (list_empty(&cohc->queue))
 		return NULL;
 	d = list_first_entry(&cohc->queue,
 			     struct coh901318_desc,
 			     node);
 	return d;
 }
 static inline u32 coh901318_get_bytes_in_lli(struct coh901318_lli *in_lli)
 {
 	struct coh901318_lli *lli = in_lli;
 	u32 bytes = 0;
 	while (lli) {
 		bytes += lli->control & COH901318_CX_CTRL_TC_VALUE_MASK;
 		lli = lli->virt_link_addr;
 	}
 	return bytes;
 }
 /*
  * Get the number of bytes left to transfer on this channel,
  * it is unwise to call this before stopping the channel for
  * absolute measures, but for a rough guess you can still call
  * it.
  */
 static u32 coh901318_get_bytes_left(struct dma_chan *chan)
 {
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	struct coh901318_desc *cohd;
 	struct list_head *pos;
 	unsigned long flags;
 	u32 left = 0;
 	int i = 0;
 	spin_lock_irqsave(&cohc->lock, flags);
 	/*
 	 * If there are many queued jobs, we iterate and add the
 	 * size of them all. We take a special look on the first
 	 * job though, since it is probably active.
 	 */
 	list_for_each(pos, &cohc->active) {
 		/*
 		 * The first job in the list will be working on the
 		 * hardware. The job can be stopped but still active,
 		 * so that the transfer counter is somewhere inside
 		 * the buffer.
 		 */
 		cohd = list_entry(pos, struct coh901318_desc, node);
 		if (i == 0) {
 			struct coh901318_lli *lli;
 			dma_addr_t ladd;
 			/* Read current transfer count value */
 			left = readl(cohc->base->virtbase +
 				     COH901318_CX_CTRL +
 				     COH901318_CX_CTRL_SPACING * cohc->id) &
 				COH901318_CX_CTRL_TC_VALUE_MASK;
 			/* See if the transfer is linked... */
 			ladd = readl(cohc->base->virtbase +
 				     COH901318_CX_LNK_ADDR +
 				     COH901318_CX_LNK_ADDR_SPACING *
 				     cohc->id) &
 				~COH901318_CX_LNK_LINK_IMMEDIATE;
 			/* Single transaction */
 			if (!ladd)
 				continue;
 			/*
 			 * Linked transaction, follow the lli, find the
 			 * currently processing lli, and proceed to the next
 			 */
 			lli = cohd->lli;
 			while (lli && lli->link_addr != ladd)
 				lli = lli->virt_link_addr;
 			if (lli)
 				lli = lli->virt_link_addr;
 			/*
 			 * Follow remaining lli links around to count the total
 			 * number of bytes left
 			 */
 			left += coh901318_get_bytes_in_lli(lli);
 		} else {
 			left += coh901318_get_bytes_in_lli(cohd->lli);
 		}
 		i++;
 	}
 	/* Also count bytes in the queued jobs */
 	list_for_each(pos, &cohc->queue) {
 		cohd = list_entry(pos, struct coh901318_desc, node);
 		left += coh901318_get_bytes_in_lli(cohd->lli);
 	}
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return left;
 }
 /*
  * Pauses a transfer without losing data. Enables power save.
  * Use this function in conjunction with coh901318_resume.
  */
 static void coh901318_pause(struct dma_chan *chan)
 {
 	u32 val;
 	unsigned long flags;
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	spin_lock_irqsave(&cohc->lock, flags);
 	/* Disable channel in HW */
 	val = readl(virtbase + COH901318_CX_CFG +
 		    COH901318_CX_CFG_SPACING * channel);
 	/* Stopping infinit transfer */
 	if ((val & COH901318_CX_CTRL_TC_ENABLE) == 0 &&
 	    (val & COH901318_CX_CFG_CH_ENABLE))
 		cohc->stopped = 1;
 	val &= ~COH901318_CX_CFG_CH_ENABLE;
 	/* Enable twice, HW bug work around */
 	writel(val, virtbase + COH901318_CX_CFG +
 	       COH901318_CX_CFG_SPACING * channel);
 	writel(val, virtbase + COH901318_CX_CFG +
 	       COH901318_CX_CFG_SPACING * channel);
 	/* Spin-wait for it to actually go inactive */
 	while (readl(virtbase + COH901318_CX_STAT+COH901318_CX_STAT_SPACING *
 		     channel) & COH901318_CX_STAT_ACTIVE)
 		cpu_relax();
 	/* Check if we stopped an active job */
 	if ((readl(virtbase + COH901318_CX_CTRL+COH901318_CX_CTRL_SPACING *
 		   channel) & COH901318_CX_CTRL_TC_VALUE_MASK) > 0)
 		cohc->stopped = 1;
 	enable_powersave(cohc);
 	spin_unlock_irqrestore(&cohc->lock, flags);
 }
 /* Resumes a transfer that has been stopped via 300_dma_stop(..).
    Power save is handled.
 */
 static void coh901318_resume(struct dma_chan *chan)
 {
 	u32 val;
 	unsigned long flags;
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	int channel = cohc->id;
 	spin_lock_irqsave(&cohc->lock, flags);
 	disable_powersave(cohc);
 	if (cohc->stopped) {
 		/* Enable channel in HW */
 		val = readl(cohc->base->virtbase + COH901318_CX_CFG +
 			    COH901318_CX_CFG_SPACING * channel);
 		val |= COH901318_CX_CFG_CH_ENABLE;
 		writel(val, cohc->base->virtbase + COH901318_CX_CFG +
 		       COH901318_CX_CFG_SPACING*channel);
 		cohc->stopped = 0;
 	}
 	spin_unlock_irqrestore(&cohc->lock, flags);
 }
 bool coh901318_filter_id(struct dma_chan *chan, void *chan_id)
 {
 	unsigned int ch_nr = (unsigned int) chan_id;
 	if (ch_nr == to_coh901318_chan(chan)->id)
 		return true;
 	return false;
 }
 EXPORT_SYMBOL(coh901318_filter_id);
 /*
  * DMA channel allocation
  */
 static int coh901318_config(struct coh901318_chan *cohc,
 			    struct coh901318_params *param)
 {
 	unsigned long flags;
 	const struct coh901318_params *p;
 	int channel = cohc->id;
 	void __iomem *virtbase = cohc->base->virtbase;
 	spin_lock_irqsave(&cohc->lock, flags);
 	if (param)
 		p = param;
 	else
 		p = &cohc->base->platform->chan_conf[channel].param;
 	/* Clear any pending BE or TC interrupt */
 	if (channel < 32) {
 		writel(1 << channel, virtbase + COH901318_BE_INT_CLEAR1);
 		writel(1 << channel, virtbase + COH901318_TC_INT_CLEAR1);
 	} else {
 		writel(1 << (channel - 32), virtbase +
 		       COH901318_BE_INT_CLEAR2);
 		writel(1 << (channel - 32), virtbase +
 		       COH901318_TC_INT_CLEAR2);
 	}
 	coh901318_set_conf(cohc, p->config);
 	coh901318_set_ctrl(cohc, p->ctrl_lli_last);
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return 0;
 }
 /* must lock when calling this function
  * start queued jobs, if any
  * TODO: start all queued jobs in one go
  *
  * Returns descriptor if queued job is started otherwise NULL.
  * If the queue is empty NULL is returned.
  */
 static struct coh901318_desc *coh901318_queue_start(struct coh901318_chan *cohc)
 {
 	struct coh901318_desc *cohd;
 	/*
 	 * start queued jobs, if any
 	 * TODO: transmit all queued jobs in one go
 	 */
 	cohd = coh901318_first_queued(cohc);
 	if (cohd != NULL) {
 		/* Remove from queue */
 		coh901318_desc_remove(cohd);
 		/* initiate DMA job */
 		cohc->busy = 1;
 		coh901318_desc_submit(cohc, cohd);
 		coh901318_prep_linked_list(cohc, cohd->lli);
 		/* start dma job on this channel */
 		coh901318_start(cohc);
 	}
 	return cohd;
 }
 /*
  * This tasklet is called from the interrupt handler to
  * handle each descriptor (DMA job) that is sent to a channel.
  */
 static void dma_tasklet(unsigned long data)
 {
 	struct coh901318_chan *cohc = (struct coh901318_chan *) data;
 	struct coh901318_desc *cohd_fin;
 	unsigned long flags;
 	dma_async_tx_callback callback;
 	void *callback_param;
 	dev_vdbg(COHC_2_DEV(cohc), "[%s] chan_id %d"
 		 " nbr_active_done %ld\n", __func__,
 		 cohc->id, cohc->nbr_active_done);
 	spin_lock_irqsave(&cohc->lock, flags);
 	/* get first active descriptor entry from list */
 	cohd_fin = coh901318_first_active_get(cohc);
 	if (cohd_fin == NULL)
 		goto err;
 	/* locate callback to client */
 	callback = cohd_fin->desc.callback;
 	callback_param = cohd_fin->desc.callback_param;
 	/* sign this job as completed on the channel */
 	cohc->completed = cohd_fin->desc.cookie;
 	/* release the lli allocation and remove the descriptor */
 	coh901318_lli_free(&cohc->base->pool, &cohd_fin->lli);
 	/* return desc to free-list */
 	coh901318_desc_remove(cohd_fin);
 	coh901318_desc_free(cohc, cohd_fin);
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	/* Call the callback when we're done */
 	if (callback)
 		callback(callback_param);
 	spin_lock_irqsave(&cohc->lock, flags);
 	/*
 	 * If another interrupt fired while the tasklet was scheduling,
 	 * we don't get called twice, so we have this number of active
 	 * counter that keep track of the number of IRQs expected to
 	 * be handled for this channel. If there happen to be more than
 	 * one IRQ to be ack:ed, we simply schedule this tasklet again.
 	 */
 	cohc->nbr_active_done--;
 	if (cohc->nbr_active_done) {
 		dev_dbg(COHC_2_DEV(cohc), "scheduling tasklet again, new IRQs "
 			"came in while we were scheduling this tasklet\n");
 		if (cohc_chan_conf(cohc)->priority_high)
 			tasklet_hi_schedule(&cohc->tasklet);
 		else
 			tasklet_schedule(&cohc->tasklet);
 	}
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return;
  err:
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	dev_err(COHC_2_DEV(cohc), "[%s] No active dma desc\n", __func__);
 }
 /* called from interrupt context */
 static void dma_tc_handle(struct coh901318_chan *cohc)
 {
 	/*
 	 * If the channel is not allocated, then we shouldn't have
 	 * any TC interrupts on it.
 	 */
 	if (!cohc->allocated) {
 		dev_err(COHC_2_DEV(cohc), "spurious interrupt from "
 			"unallocated channel\n");
 		return;
 	}
 	spin_lock(&cohc->lock);
 	/*
 	 * When we reach this point, at least one queue item
 	 * should have been moved over from cohc->queue to
 	 * cohc->active and run to completion, that is why we're
 	 * getting a terminal count interrupt is it not?
 	 * If you get this BUG() the most probable cause is that
 	 * the individual nodes in the lli chain have IRQ enabled,
 	 * so check your platform config for lli chain ctrl.
 	 */
 	BUG_ON(list_empty(&cohc->active));
 	cohc->nbr_active_done++;
 	/*
 	 * This attempt to take a job from cohc->queue, put it
 	 * into cohc->active and start it.
 	 */
 	if (coh901318_queue_start(cohc) == NULL)
 		cohc->busy = 0;
 	spin_unlock(&cohc->lock);
 	/*
 	 * This tasklet will remove items from cohc->active
 	 * and thus terminates them.
 	 */
 	if (cohc_chan_conf(cohc)->priority_high)
 		tasklet_hi_schedule(&cohc->tasklet);
 	else
 		tasklet_schedule(&cohc->tasklet);
 }
 static irqreturn_t dma_irq_handler(int irq, void *dev_id)
 {
 	u32 status1;
 	u32 status2;
 	int i;
 	int ch;
 	struct coh901318_base *base  = dev_id;
 	struct coh901318_chan *cohc;
 	void __iomem *virtbase = base->virtbase;
 	status1 = readl(virtbase + COH901318_INT_STATUS1);
 	status2 = readl(virtbase + COH901318_INT_STATUS2);
 	if (unlikely(status1 == 0 && status2 == 0)) {
 		dev_warn(base->dev, "spurious DMA IRQ from no channel!\n");
 		return IRQ_HANDLED;
 	}
 	/* TODO: consider handle IRQ in tasklet here to
 	 *       minimize interrupt latency */
 	/* Check the first 32 DMA channels for IRQ */
 	while (status1) {
 		/* Find first bit set, return as a number. */
 		i = ffs(status1) - 1;
 		ch = i;
 		cohc = &base->chans[ch];
 		spin_lock(&cohc->lock);
 		/* Mask off this bit */
 		status1 &= ~(1 << i);
 		/* Check the individual channel bits */
 		if (test_bit(i, virtbase + COH901318_BE_INT_STATUS1)) {
 			dev_crit(COHC_2_DEV(cohc),
 				 "DMA bus error on channel %d!\n", ch);
 			BUG_ON(1);
 			/* Clear BE interrupt */
 			__set_bit(i, virtbase + COH901318_BE_INT_CLEAR1);
 		} else {
 			/* Caused by TC, really? */
 			if (unlikely(!test_bit(i, virtbase +
 					       COH901318_TC_INT_STATUS1))) {
 				dev_warn(COHC_2_DEV(cohc),
 					 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
 				/* Clear TC interrupt */
 				BUG_ON(1);
 				__set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
 			} else {
 				/* Enable powersave if transfer has finished */
 				if (!(readl(virtbase + COH901318_CX_STAT +
 					    COH901318_CX_STAT_SPACING*ch) &
 				      COH901318_CX_STAT_ENABLED)) {
 					enable_powersave(cohc);
 				}
 				/* Must clear TC interrupt before calling
 				 * dma_tc_handle
 				 * in case tc_handle initate a new dma job
 				 */
 				__set_bit(i, virtbase + COH901318_TC_INT_CLEAR1);
 				dma_tc_handle(cohc);
 			}
 		}
 		spin_unlock(&cohc->lock);
 	}
 	/* Check the remaining 32 DMA channels for IRQ */
 	while (status2) {
 		/* Find first bit set, return as a number. */
 		i = ffs(status2) - 1;
 		ch = i + 32;
 		cohc = &base->chans[ch];
 		spin_lock(&cohc->lock);
 		/* Mask off this bit */
 		status2 &= ~(1 << i);
 		/* Check the individual channel bits */
 		if (test_bit(i, virtbase + COH901318_BE_INT_STATUS2)) {
 			dev_crit(COHC_2_DEV(cohc),
 				 "DMA bus error on channel %d!\n", ch);
 			/* Clear BE interrupt */
 			BUG_ON(1);
 			__set_bit(i, virtbase + COH901318_BE_INT_CLEAR2);
 		} else {
 			/* Caused by TC, really? */
 			if (unlikely(!test_bit(i, virtbase +
 					       COH901318_TC_INT_STATUS2))) {
 				dev_warn(COHC_2_DEV(cohc),
 					 "ignoring interrupt not caused by terminal count on channel %d\n", ch);
 				/* Clear TC interrupt */
 				__set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
 				BUG_ON(1);
 			} else {
 				/* Enable powersave if transfer has finished */
 				if (!(readl(virtbase + COH901318_CX_STAT +
 					    COH901318_CX_STAT_SPACING*ch) &
 				      COH901318_CX_STAT_ENABLED)) {
 					enable_powersave(cohc);
 				}
 				/* Must clear TC interrupt before calling
 				 * dma_tc_handle
 				 * in case tc_handle initate a new dma job
 				 */
 				__set_bit(i, virtbase + COH901318_TC_INT_CLEAR2);
 				dma_tc_handle(cohc);
 			}
 		}
 		spin_unlock(&cohc->lock);
 	}
 	return IRQ_HANDLED;
 }
 static int coh901318_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct coh901318_chan	*cohc = to_coh901318_chan(chan);
 	unsigned long flags;
 	dev_vdbg(COHC_2_DEV(cohc), "[%s] DMA channel %d\n",
 		 __func__, cohc->id);
 	if (chan->client_count > 1)
 		return -EBUSY;
 	spin_lock_irqsave(&cohc->lock, flags);
 	coh901318_config(cohc, NULL);
 	cohc->allocated = 1;
 	cohc->completed = chan->cookie = 1;
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return 1;
 }
 static void
 coh901318_free_chan_resources(struct dma_chan *chan)
 {
 	struct coh901318_chan	*cohc = to_coh901318_chan(chan);
 	int channel = cohc->id;
 	unsigned long flags;
 	spin_lock_irqsave(&cohc->lock, flags);
 	/* Disable HW */
 	writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CFG +
 	       COH901318_CX_CFG_SPACING*channel);
 	writel(0x00000000U, cohc->base->virtbase + COH901318_CX_CTRL +
 	       COH901318_CX_CTRL_SPACING*channel);
 	cohc->allocated = 0;
 	spin_unlock_irqrestore(&cohc->lock, flags);
-	chan->device->device_control(chan, DMA_TERMINATE_ALL);
+	chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
 }
 static dma_cookie_t
 coh901318_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct coh901318_desc *cohd = container_of(tx, struct coh901318_desc,
 						   desc);
 	struct coh901318_chan *cohc = to_coh901318_chan(tx->chan);
 	unsigned long flags;
 	spin_lock_irqsave(&cohc->lock, flags);
 	tx->cookie = coh901318_assign_cookie(cohc, cohd);
 	coh901318_desc_queue(cohc, cohd);
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return tx->cookie;
 }
 static struct dma_async_tx_descriptor *
 coh901318_prep_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		      size_t size, unsigned long flags)
 {
 	struct coh901318_lli *lli;
 	struct coh901318_desc *cohd;
 	unsigned long flg;
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	int lli_len;
 	u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
 	int ret;
 	spin_lock_irqsave(&cohc->lock, flg);
 	dev_vdbg(COHC_2_DEV(cohc),
 		 "[%s] channel %d src 0x%x dest 0x%x size %d\n",
 		 __func__, cohc->id, src, dest, size);
 	if (flags & DMA_PREP_INTERRUPT)
 		/* Trigger interrupt after last lli */
 		ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
 	lli_len = size >> MAX_DMA_PACKET_SIZE_SHIFT;
 	if ((lli_len << MAX_DMA_PACKET_SIZE_SHIFT) < size)
 		lli_len++;
 	lli = coh901318_lli_alloc(&cohc->base->pool, lli_len);
 	if (lli == NULL)
 		goto err;
 	ret = coh901318_lli_fill_memcpy(
 		&cohc->base->pool, lli, src, size, dest,
 		cohc_chan_param(cohc)->ctrl_lli_chained,
 		ctrl_last);
 	if (ret)
 		goto err;
 	COH_DBG(coh901318_list_print(cohc, lli));
 	/* Pick a descriptor to handle this transfer */
 	cohd = coh901318_desc_get(cohc);
 	cohd->lli = lli;
 	cohd->flags = flags;
 	cohd->desc.tx_submit = coh901318_tx_submit;
 	spin_unlock_irqrestore(&cohc->lock, flg);
 	return &cohd->desc;
  err:
 	spin_unlock_irqrestore(&cohc->lock, flg);
 	return NULL;
 }
 static struct dma_async_tx_descriptor *
 coh901318_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 			unsigned int sg_len, enum dma_data_direction direction,
 			unsigned long flags)
 {
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	struct coh901318_lli *lli;
 	struct coh901318_desc *cohd;
 	const struct coh901318_params *params;
 	struct scatterlist *sg;
 	int len = 0;
 	int size;
 	int i;
 	u32 ctrl_chained = cohc_chan_param(cohc)->ctrl_lli_chained;
 	u32 ctrl = cohc_chan_param(cohc)->ctrl_lli;
 	u32 ctrl_last = cohc_chan_param(cohc)->ctrl_lli_last;
 	u32 config;
 	unsigned long flg;
 	int ret;
 	if (!sgl)
 		goto out;
 	if (sgl->length == 0)
 		goto out;
 	spin_lock_irqsave(&cohc->lock, flg);
 	dev_vdbg(COHC_2_DEV(cohc), "[%s] sg_len %d dir %d\n",
 		 __func__, sg_len, direction);
 	if (flags & DMA_PREP_INTERRUPT)
 		/* Trigger interrupt after last lli */
 		ctrl_last |= COH901318_CX_CTRL_TC_IRQ_ENABLE;
 	params = cohc_chan_param(cohc);
 	config = params->config;
 	if (direction == DMA_TO_DEVICE) {
 		u32 tx_flags = COH901318_CX_CTRL_PRDD_SOURCE |
 			COH901318_CX_CTRL_SRC_ADDR_INC_ENABLE;
 		config |= COH901318_CX_CFG_RM_MEMORY_TO_PRIMARY;
 		ctrl_chained |= tx_flags;
 		ctrl_last |= tx_flags;
 		ctrl |= tx_flags;
 	} else if (direction == DMA_FROM_DEVICE) {
 		u32 rx_flags = COH901318_CX_CTRL_PRDD_DEST |
 			COH901318_CX_CTRL_DST_ADDR_INC_ENABLE;
 		config |= COH901318_CX_CFG_RM_PRIMARY_TO_MEMORY;
 		ctrl_chained |= rx_flags;
 		ctrl_last |= rx_flags;
 		ctrl |= rx_flags;
 	} else
 		goto err_direction;
 	coh901318_set_conf(cohc, config);
 	/* The dma only supports transmitting packages up to
 	 * MAX_DMA_PACKET_SIZE. Calculate to total number of
 	 * dma elemts required to send the entire sg list
 	 */
 	for_each_sg(sgl, sg, sg_len, i) {
 		unsigned int factor;
 		size = sg_dma_len(sg);
 		if (size <= MAX_DMA_PACKET_SIZE) {
 			len++;
 			continue;
 		}
 		factor = size >> MAX_DMA_PACKET_SIZE_SHIFT;
 		if ((factor << MAX_DMA_PACKET_SIZE_SHIFT) < size)
 			factor++;
 		len += factor;
 	}
 	pr_debug("Allocate %d lli:s for this transfer\n", len);
 	lli = coh901318_lli_alloc(&cohc->base->pool, len);
 	if (lli == NULL)
 		goto err_dma_alloc;
 	/* initiate allocated lli list */
 	ret = coh901318_lli_fill_sg(&cohc->base->pool, lli, sgl, sg_len,
 				    cohc_dev_addr(cohc),
 				    ctrl_chained,
 				    ctrl,
 				    ctrl_last,
 				    direction, COH901318_CX_CTRL_TC_IRQ_ENABLE);
 	if (ret)
 		goto err_lli_fill;
 	COH_DBG(coh901318_list_print(cohc, lli));
 	/* Pick a descriptor to handle this transfer */
 	cohd = coh901318_desc_get(cohc);
 	cohd->dir = direction;
 	cohd->flags = flags;
 	cohd->desc.tx_submit = coh901318_tx_submit;
 	cohd->lli = lli;
 	spin_unlock_irqrestore(&cohc->lock, flg);
 	return &cohd->desc;
  err_lli_fill:
  err_dma_alloc:
  err_direction:
 	spin_unlock_irqrestore(&cohc->lock, flg);
  out:
 	return NULL;
 }
 static enum dma_status
 coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 		 struct dma_tx_state *txstate)
 {
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	dma_cookie_t last_used;
 	dma_cookie_t last_complete;
 	int ret;
 	last_complete = cohc->completed;
 	last_used = chan->cookie;
 	ret = dma_async_is_complete(cookie, last_complete, last_used);
 	dma_set_tx_state(txstate, last_complete, last_used,
 			 coh901318_get_bytes_left(chan));
 	if (ret == DMA_IN_PROGRESS && cohc->stopped)
 		ret = DMA_PAUSED;
 	return ret;
 }
 static void
 coh901318_issue_pending(struct dma_chan *chan)
 {
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	unsigned long flags;
 	spin_lock_irqsave(&cohc->lock, flags);
 	/*
 	 * Busy means that pending jobs are already being processed,
 	 * and then there is no point in starting the queue: the
 	 * terminal count interrupt on the channel will take the next
 	 * job on the queue and execute it anyway.
 	 */
 	if (!cohc->busy)
 		coh901318_queue_start(cohc);
 	spin_unlock_irqrestore(&cohc->lock, flags);
 }
 static int
-coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+coh901318_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		  unsigned long arg)
 {
 	unsigned long flags;
 	struct coh901318_chan *cohc = to_coh901318_chan(chan);
 	struct coh901318_desc *cohd;
 	void __iomem *virtbase = cohc->base->virtbase;
 	if (cmd == DMA_PAUSE) {
 		coh901318_pause(chan);
 		return 0;
 	}
 	if (cmd == DMA_RESUME) {
 		coh901318_resume(chan);
 		return 0;
 	}
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	/* The remainder of this function terminates the transfer */
 	coh901318_pause(chan);
 	spin_lock_irqsave(&cohc->lock, flags);
 	/* Clear any pending BE or TC interrupt */
 	if (cohc->id < 32) {
 		writel(1 << cohc->id, virtbase + COH901318_BE_INT_CLEAR1);
 		writel(1 << cohc->id, virtbase + COH901318_TC_INT_CLEAR1);
 	} else {
 		writel(1 << (cohc->id - 32), virtbase +
 		       COH901318_BE_INT_CLEAR2);
 		writel(1 << (cohc->id - 32), virtbase +
 		       COH901318_TC_INT_CLEAR2);
 	}
 	enable_powersave(cohc);
 	while ((cohd = coh901318_first_active_get(cohc))) {
 		/* release the lli allocation*/
 		coh901318_lli_free(&cohc->base->pool, &cohd->lli);
 		/* return desc to free-list */
 		coh901318_desc_remove(cohd);
 		coh901318_desc_free(cohc, cohd);
 	}
 	while ((cohd = coh901318_first_queued(cohc))) {
 		/* release the lli allocation*/
 		coh901318_lli_free(&cohc->base->pool, &cohd->lli);
 		/* return desc to free-list */
 		coh901318_desc_remove(cohd);
 		coh901318_desc_free(cohc, cohd);
 	}
 	cohc->nbr_active_done = 0;
 	cohc->busy = 0;
 	spin_unlock_irqrestore(&cohc->lock, flags);
 	return 0;
 }
 void coh901318_base_init(struct dma_device *dma, const int *pick_chans,
 			 struct coh901318_base *base)
 {
 	int chans_i;
 	int i = 0;
 	struct coh901318_chan *cohc;
 	INIT_LIST_HEAD(&dma->channels);
 	for (chans_i = 0; pick_chans[chans_i] != -1; chans_i += 2) {
 		for (i = pick_chans[chans_i]; i <= pick_chans[chans_i+1]; i++) {
 			cohc = &base->chans[i];
 			cohc->base = base;
 			cohc->chan.device = dma;
 			cohc->id = i;
 			/* TODO: do we really need this lock if only one
 			 * client is connected to each channel?
 			 */
 			spin_lock_init(&cohc->lock);
 			cohc->nbr_active_done = 0;
 			cohc->busy = 0;
 			INIT_LIST_HEAD(&cohc->free);
 			INIT_LIST_HEAD(&cohc->active);
 			INIT_LIST_HEAD(&cohc->queue);
 			tasklet_init(&cohc->tasklet, dma_tasklet,
 				     (unsigned long) cohc);
 			list_add_tail(&cohc->chan.device_node,
 				      &dma->channels);
 		}
 	}
 }
 static int __init coh901318_probe(struct platform_device *pdev)
 {
 	int err = 0;
 	struct coh901318_platform *pdata;
 	struct coh901318_base *base;
 	int irq;
 	struct resource *io;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!io)
 		goto err_get_resource;
 	/* Map DMA controller registers to virtual memory */
 	if (request_mem_region(io->start,
 			       resource_size(io),
 			       pdev->dev.driver->name) == NULL) {
 		err = -EBUSY;
 		goto err_request_mem;
 	}
 	pdata = pdev->dev.platform_data;
 	if (!pdata)
 		goto err_no_platformdata;
 	base = kmalloc(ALIGN(sizeof(struct coh901318_base), 4) +
 		       pdata->max_channels *
 		       sizeof(struct coh901318_chan),
 		       GFP_KERNEL);
 	if (!base)
 		goto err_alloc_coh_dma_channels;
 	base->chans = ((void *)base) + ALIGN(sizeof(struct coh901318_base), 4);
 	base->virtbase = ioremap(io->start, resource_size(io));
 	if (!base->virtbase) {
 		err = -ENOMEM;
 		goto err_no_ioremap;
 	}
 	base->dev = &pdev->dev;
 	base->platform = pdata;
 	spin_lock_init(&base->pm.lock);
 	base->pm.started_channels = 0;
 	COH901318_DEBUGFS_ASSIGN(debugfs_dma_base, base);
 	platform_set_drvdata(pdev, base);
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		goto err_no_irq;
 	err = request_irq(irq, dma_irq_handler, IRQF_DISABLED,
 			  "coh901318", base);
 	if (err) {
 		dev_crit(&pdev->dev,
 			 "Cannot allocate IRQ for DMA controller!\n");
 		goto err_request_irq;
 	}
 	err = coh901318_pool_create(&base->pool, &pdev->dev,
 				    sizeof(struct coh901318_lli),
 				    32);
 	if (err)
 		goto err_pool_create;
 	/* init channels for device transfers */
 	coh901318_base_init(&base->dma_slave,  base->platform->chans_slave,
 			    base);
 	dma_cap_zero(base->dma_slave.cap_mask);
 	dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
 	base->dma_slave.device_alloc_chan_resources = coh901318_alloc_chan_resources;
 	base->dma_slave.device_free_chan_resources = coh901318_free_chan_resources;
 	base->dma_slave.device_prep_slave_sg = coh901318_prep_slave_sg;
 	base->dma_slave.device_tx_status = coh901318_tx_status;
 	base->dma_slave.device_issue_pending = coh901318_issue_pending;
 	base->dma_slave.device_control = coh901318_control;
 	base->dma_slave.dev = &pdev->dev;
 	err = dma_async_device_register(&base->dma_slave);
 	if (err)
 		goto err_register_slave;
 	/* init channels for memcpy */
 	coh901318_base_init(&base->dma_memcpy, base->platform->chans_memcpy,
 			    base);
 	dma_cap_zero(base->dma_memcpy.cap_mask);
 	dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
 	base->dma_memcpy.device_alloc_chan_resources = coh901318_alloc_chan_resources;
 	base->dma_memcpy.device_free_chan_resources = coh901318_free_chan_resources;
 	base->dma_memcpy.device_prep_dma_memcpy = coh901318_prep_memcpy;
 	base->dma_memcpy.device_tx_status = coh901318_tx_status;
 	base->dma_memcpy.device_issue_pending = coh901318_issue_pending;
 	base->dma_memcpy.device_control = coh901318_control;
 	base->dma_memcpy.dev = &pdev->dev;
 	/*
 	 * This controller can only access address at even 32bit boundaries,
 	 * i.e. 2^2
 	 */
 	base->dma_memcpy.copy_align = 2;
 	err = dma_async_device_register(&base->dma_memcpy);
 	if (err)
 		goto err_register_memcpy;
 	dev_info(&pdev->dev, "Initialized COH901318 DMA on virtual base 0x%08x\n",
 		(u32) base->virtbase);
 	return err;
  err_register_memcpy:
 	dma_async_device_unregister(&base->dma_slave);
  err_register_slave:
 	coh901318_pool_destroy(&base->pool);
  err_pool_create:
 	free_irq(platform_get_irq(pdev, 0), base);
  err_request_irq:
  err_no_irq:
 	iounmap(base->virtbase);
  err_no_ioremap:
 	kfree(base);
  err_alloc_coh_dma_channels:
  err_no_platformdata:
 	release_mem_region(pdev->resource->start,
 			   resource_size(pdev->resource));
  err_request_mem:
  err_get_resource:
 	return err;
 }
 static int __exit coh901318_remove(struct platform_device *pdev)
 {
 	struct coh901318_base *base = platform_get_drvdata(pdev);
 	dma_async_device_unregister(&base->dma_memcpy);
 	dma_async_device_unregister(&base->dma_slave);
 	coh901318_pool_destroy(&base->pool);
 	free_irq(platform_get_irq(pdev, 0), base);
 	iounmap(base->virtbase);
 	kfree(base);
 	release_mem_region(pdev->resource->start,
 			   resource_size(pdev->resource));
 	return 0;
 }
 static struct platform_driver coh901318_driver = {
 	.remove = __exit_p(coh901318_remove),
 	.driver = {
 		.name	= "coh901318",
 	},
 };
 int __init coh901318_init(void)
 {
 	return platform_driver_probe(&coh901318_driver, coh901318_probe);
 }
 subsys_initcall(coh901318_init);
 void __exit coh901318_exit(void)
 {
 	platform_driver_unregister(&coh901318_driver);
 }
 module_exit(coh901318_exit);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Per Friden");

drivers/dma/dw_dmac.c

Diff comments View file @ 0582763

 /*
  * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
  * AVR32 systems.)
  *
  * Copyright (C) 2007-2008 Atmel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include "dw_dmac_regs.h"
 /*
  * This supports the Synopsys "DesignWare AHB Central DMA Controller",
  * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
  * of which use ARM any more).  See the "Databook" from Synopsys for
  * information beyond what licensees probably provide.
  *
  * The driver has currently been tested only with the Atmel AT32AP7000,
  * which does not support descriptor writeback.
  */
 /* NOTE:  DMS+SMS is system-specific. We should get this information
  * from the platform code somehow.
  */
 #define DWC_DEFAULT_CTLLO	(DWC_CTLL_DST_MSIZE(0)		\
 				| DWC_CTLL_SRC_MSIZE(0)		\
 				| DWC_CTLL_DMS(0)		\
 				| DWC_CTLL_SMS(1)		\
 				| DWC_CTLL_LLP_D_EN		\
 				| DWC_CTLL_LLP_S_EN)
 /*
  * This is configuration-dependent and usually a funny size like 4095.
  * Let's round it down to the nearest power of two.
  *
  * Note that this is a transfer count, i.e. if we transfer 32-bit
  * words, we can do 8192 bytes per descriptor.
  *
  * This parameter is also system-specific.
  */
 #define DWC_MAX_COUNT	2048U
 /*
  * Number of descriptors to allocate for each channel. This should be
  * made configurable somehow; preferably, the clients (at least the
  * ones using slave transfers) should be able to give us a hint.
  */
 #define NR_DESCS_PER_CHANNEL	64
 /*----------------------------------------------------------------------*/
 /*
  * Because we're not relying on writeback from the controller (it may not
  * even be configured into the core!) we don't need to use dma_pool.  These
  * descriptors -- and associated data -- are cacheable.  We do need to make
  * sure their dcache entries are written back before handing them off to
  * the controller, though.
  */
 static struct device *chan2dev(struct dma_chan *chan)
 {
 	return &chan->dev->device;
 }
 static struct device *chan2parent(struct dma_chan *chan)
 {
 	return chan->dev->device.parent;
 }
 static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
 {
 	return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
 }
 static struct dw_desc *dwc_first_queued(struct dw_dma_chan *dwc)
 {
 	return list_entry(dwc->queue.next, struct dw_desc, desc_node);
 }
 static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
 {
 	struct dw_desc *desc, *_desc;
 	struct dw_desc *ret = NULL;
 	unsigned int i = 0;
 	spin_lock_bh(&dwc->lock);
 	list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
 		if (async_tx_test_ack(&desc->txd)) {
 			list_del(&desc->desc_node);
 			ret = desc;
 			break;
 		}
 		dev_dbg(chan2dev(&dwc->chan), "desc %p not ACKed\n", desc);
 		i++;
 	}
 	spin_unlock_bh(&dwc->lock);
 	dev_vdbg(chan2dev(&dwc->chan), "scanned %u descriptors on freelist\n", i);
 	return ret;
 }
 static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
 	struct dw_desc	*child;
 	list_for_each_entry(child, &desc->tx_list, desc_node)
 		dma_sync_single_for_cpu(chan2parent(&dwc->chan),
 				child->txd.phys, sizeof(child->lli),
 				DMA_TO_DEVICE);
 	dma_sync_single_for_cpu(chan2parent(&dwc->chan),
 			desc->txd.phys, sizeof(desc->lli),
 			DMA_TO_DEVICE);
 }
 /*
  * Move a descriptor, including any children, to the free list.
  * `desc' must not be on any lists.
  */
 static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
 	if (desc) {
 		struct dw_desc *child;
 		dwc_sync_desc_for_cpu(dwc, desc);
 		spin_lock_bh(&dwc->lock);
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			dev_vdbg(chan2dev(&dwc->chan),
 					"moving child desc %p to freelist\n",
 					child);
 		list_splice_init(&desc->tx_list, &dwc->free_list);
 		dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
 		list_add(&desc->desc_node, &dwc->free_list);
 		spin_unlock_bh(&dwc->lock);
 	}
 }
 /* Called with dwc->lock held and bh disabled */
 static dma_cookie_t
 dwc_assign_cookie(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
 	dma_cookie_t cookie = dwc->chan.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	dwc->chan.cookie = cookie;
 	desc->txd.cookie = cookie;
 	return cookie;
 }
 /*----------------------------------------------------------------------*/
 /* Called with dwc->lock held and bh disabled */
 static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first)
 {
 	struct dw_dma	*dw = to_dw_dma(dwc->chan.device);
 	/* ASSERT:  channel is idle */
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
 		dev_err(chan2dev(&dwc->chan),
 			"BUG: Attempted to start non-idle channel\n");
 		dev_err(chan2dev(&dwc->chan),
 			"  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 			channel_readl(dwc, SAR),
 			channel_readl(dwc, DAR),
 			channel_readl(dwc, LLP),
 			channel_readl(dwc, CTL_HI),
 			channel_readl(dwc, CTL_LO));
 		/* The tasklet will hopefully advance the queue... */
 		return;
 	}
 	channel_writel(dwc, LLP, first->txd.phys);
 	channel_writel(dwc, CTL_LO,
 			DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 	channel_writel(dwc, CTL_HI, 0);
 	channel_set_bit(dw, CH_EN, dwc->mask);
 }
 /*----------------------------------------------------------------------*/
 static void
 dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
 	dma_async_tx_callback		callback;
 	void				*param;
 	struct dma_async_tx_descriptor	*txd = &desc->txd;
 	dev_vdbg(chan2dev(&dwc->chan), "descriptor %u complete\n", txd->cookie);
 	dwc->completed = txd->cookie;
 	callback = txd->callback;
 	param = txd->callback_param;
 	dwc_sync_desc_for_cpu(dwc, desc);
 	list_splice_init(&desc->tx_list, &dwc->free_list);
 	list_move(&desc->desc_node, &dwc->free_list);
 	if (!dwc->chan.private) {
 		struct device *parent = chan2parent(&dwc->chan);
 		if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 			if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 				dma_unmap_single(parent, desc->lli.dar,
 						desc->len, DMA_FROM_DEVICE);
 			else
 				dma_unmap_page(parent, desc->lli.dar,
 						desc->len, DMA_FROM_DEVICE);
 		}
 		if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 			if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 				dma_unmap_single(parent, desc->lli.sar,
 						desc->len, DMA_TO_DEVICE);
 			else
 				dma_unmap_page(parent, desc->lli.sar,
 						desc->len, DMA_TO_DEVICE);
 		}
 	}
 	/*
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
 	if (callback)
 		callback(param);
 }
 static void dwc_complete_all(struct dw_dma *dw, struct dw_dma_chan *dwc)
 {
 	struct dw_desc *desc, *_desc;
 	LIST_HEAD(list);
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
 		dev_err(chan2dev(&dwc->chan),
 			"BUG: XFER bit set, but channel not idle!\n");
 		/* Try to continue after resetting the channel... */
 		channel_clear_bit(dw, CH_EN, dwc->mask);
 		while (dma_readl(dw, CH_EN) & dwc->mask)
 			cpu_relax();
 	}
 	/*
 	 * Submit queued descriptors ASAP, i.e. before we go through
 	 * the completed ones.
 	 */
 	if (!list_empty(&dwc->queue))
 		dwc_dostart(dwc, dwc_first_queued(dwc));
 	list_splice_init(&dwc->active_list, &list);
 	list_splice_init(&dwc->queue, &dwc->active_list);
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		dwc_descriptor_complete(dwc, desc);
 }
 static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
 {
 	dma_addr_t llp;
 	struct dw_desc *desc, *_desc;
 	struct dw_desc *child;
 	u32 status_xfer;
 	/*
 	 * Clear block interrupt flag before scanning so that we don't
 	 * miss any, and read LLP before RAW_XFER to ensure it is
 	 * valid if we decide to scan the list.
 	 */
 	dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 	llp = channel_readl(dwc, LLP);
 	status_xfer = dma_readl(dw, RAW.XFER);
 	if (status_xfer & dwc->mask) {
 		/* Everything we've submitted is done */
 		dma_writel(dw, CLEAR.XFER, dwc->mask);
 		dwc_complete_all(dw, dwc);
 		return;
 	}
 	dev_vdbg(chan2dev(&dwc->chan), "scan_descriptors: llp=0x%x\n", llp);
 	list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) {
 		if (desc->lli.llp == llp)
 			/* This one is currently in progress */
 			return;
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			if (child->lli.llp == llp)
 				/* Currently in progress */
 				return;
 		/*
 		 * No descriptors so far seem to be in progress, i.e.
 		 * this one must be done.
 		 */
 		dwc_descriptor_complete(dwc, desc);
 	}
 	dev_err(chan2dev(&dwc->chan),
 		"BUG: All descriptors done, but channel not idle!\n");
 	/* Try to continue after resetting the channel... */
 	channel_clear_bit(dw, CH_EN, dwc->mask);
 	while (dma_readl(dw, CH_EN) & dwc->mask)
 		cpu_relax();
 	if (!list_empty(&dwc->queue)) {
 		dwc_dostart(dwc, dwc_first_queued(dwc));
 		list_splice_init(&dwc->queue, &dwc->active_list);
 	}
 }
 static void dwc_dump_lli(struct dw_dma_chan *dwc, struct dw_lli *lli)
 {
 	dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
 			"  desc: s0x%x d0x%x l0x%x c0x%x:%x\n",
 			lli->sar, lli->dar, lli->llp,
 			lli->ctlhi, lli->ctllo);
 }
 static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
 {
 	struct dw_desc *bad_desc;
 	struct dw_desc *child;
 	dwc_scan_descriptors(dw, dwc);
 	/*
 	 * The descriptor currently at the head of the active list is
 	 * borked. Since we don't have any way to report errors, we'll
 	 * just have to scream loudly and try to carry on.
 	 */
 	bad_desc = dwc_first_active(dwc);
 	list_del_init(&bad_desc->desc_node);
 	list_splice_init(&dwc->queue, dwc->active_list.prev);
 	/* Clear the error flag and try to restart the controller */
 	dma_writel(dw, CLEAR.ERROR, dwc->mask);
 	if (!list_empty(&dwc->active_list))
 		dwc_dostart(dwc, dwc_first_active(dwc));
 	/*
 	 * KERN_CRITICAL may seem harsh, but since this only happens
 	 * when someone submits a bad physical address in a
 	 * descriptor, we should consider ourselves lucky that the
 	 * controller flagged an error instead of scribbling over
 	 * random memory locations.
 	 */
 	dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
 			"Bad descriptor submitted for DMA!\n");
 	dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
 			"  cookie: %d\n", bad_desc->txd.cookie);
 	dwc_dump_lli(dwc, &bad_desc->lli);
 	list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 		dwc_dump_lli(dwc, &child->lli);
 	/* Pretend the descriptor completed successfully */
 	dwc_descriptor_complete(dwc, bad_desc);
 }
 /* --------------------- Cyclic DMA API extensions -------------------- */
 inline dma_addr_t dw_dma_get_src_addr(struct dma_chan *chan)
 {
 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 	return channel_readl(dwc, SAR);
 }
 EXPORT_SYMBOL(dw_dma_get_src_addr);
 inline dma_addr_t dw_dma_get_dst_addr(struct dma_chan *chan)
 {
 	struct dw_dma_chan *dwc = to_dw_dma_chan(chan);
 	return channel_readl(dwc, DAR);
 }
 EXPORT_SYMBOL(dw_dma_get_dst_addr);
 /* called with dwc->lock held and all DMAC interrupts disabled */
 static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc,
 		u32 status_block, u32 status_err, u32 status_xfer)
 {
 	if (status_block & dwc->mask) {
 		void (*callback)(void *param);
 		void *callback_param;
 		dev_vdbg(chan2dev(&dwc->chan), "new cyclic period llp 0x%08x\n",
 				channel_readl(dwc, LLP));
 		dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 		callback = dwc->cdesc->period_callback;
 		callback_param = dwc->cdesc->period_callback_param;
 		if (callback) {
 			spin_unlock(&dwc->lock);
 			callback(callback_param);
 			spin_lock(&dwc->lock);
 		}
 	}
 	/*
 	 * Error and transfer complete are highly unlikely, and will most
 	 * likely be due to a configuration error by the user.
 	 */
 	if (unlikely(status_err & dwc->mask) ||
 			unlikely(status_xfer & dwc->mask)) {
 		int i;
 		dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s "
 				"interrupt, stopping DMA transfer\n",
 				status_xfer ? "xfer" : "error");
 		dev_err(chan2dev(&dwc->chan),
 			"  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 			channel_readl(dwc, SAR),
 			channel_readl(dwc, DAR),
 			channel_readl(dwc, LLP),
 			channel_readl(dwc, CTL_HI),
 			channel_readl(dwc, CTL_LO));
 		channel_clear_bit(dw, CH_EN, dwc->mask);
 		while (dma_readl(dw, CH_EN) & dwc->mask)
 			cpu_relax();
 		/* make sure DMA does not restart by loading a new list */
 		channel_writel(dwc, LLP, 0);
 		channel_writel(dwc, CTL_LO, 0);
 		channel_writel(dwc, CTL_HI, 0);
 		dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 		dma_writel(dw, CLEAR.ERROR, dwc->mask);
 		dma_writel(dw, CLEAR.XFER, dwc->mask);
 		for (i = 0; i < dwc->cdesc->periods; i++)
 			dwc_dump_lli(dwc, &dwc->cdesc->desc[i]->lli);
 	}
 }
 /* ------------------------------------------------------------------------- */
 static void dw_dma_tasklet(unsigned long data)
 {
 	struct dw_dma *dw = (struct dw_dma *)data;
 	struct dw_dma_chan *dwc;
 	u32 status_block;
 	u32 status_xfer;
 	u32 status_err;
 	int i;
 	status_block = dma_readl(dw, RAW.BLOCK);
 	status_xfer = dma_readl(dw, RAW.XFER);
 	status_err = dma_readl(dw, RAW.ERROR);
 	dev_vdbg(dw->dma.dev, "tasklet: status_block=%x status_err=%x\n",
 			status_block, status_err);
 	for (i = 0; i < dw->dma.chancnt; i++) {
 		dwc = &dw->chan[i];
 		spin_lock(&dwc->lock);
 		if (test_bit(DW_DMA_IS_CYCLIC, &dwc->flags))
 			dwc_handle_cyclic(dw, dwc, status_block, status_err,
 					status_xfer);
 		else if (status_err & (1 << i))
 			dwc_handle_error(dw, dwc);
 		else if ((status_block | status_xfer) & (1 << i))
 			dwc_scan_descriptors(dw, dwc);
 		spin_unlock(&dwc->lock);
 	}
 	/*
 	 * Re-enable interrupts. Block Complete interrupts are only
 	 * enabled if the INT_EN bit in the descriptor is set. This
 	 * will trigger a scan before the whole list is done.
 	 */
 	channel_set_bit(dw, MASK.XFER, dw->all_chan_mask);
 	channel_set_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_set_bit(dw, MASK.ERROR, dw->all_chan_mask);
 }
 static irqreturn_t dw_dma_interrupt(int irq, void *dev_id)
 {
 	struct dw_dma *dw = dev_id;
 	u32 status;
 	dev_vdbg(dw->dma.dev, "interrupt: status=0x%x\n",
 			dma_readl(dw, STATUS_INT));
 	/*
 	 * Just disable the interrupts. We'll turn them back on in the
 	 * softirq handler.
 	 */
 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 	status = dma_readl(dw, STATUS_INT);
 	if (status) {
 		dev_err(dw->dma.dev,
 			"BUG: Unexpected interrupts pending: 0x%x\n",
 			status);
 		/* Try to recover */
 		channel_clear_bit(dw, MASK.XFER, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.BLOCK, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.SRC_TRAN, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.DST_TRAN, (1 << 8) - 1);
 		channel_clear_bit(dw, MASK.ERROR, (1 << 8) - 1);
 	}
 	tasklet_schedule(&dw->tasklet);
 	return IRQ_HANDLED;
 }
 /*----------------------------------------------------------------------*/
 static dma_cookie_t dwc_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct dw_desc		*desc = txd_to_dw_desc(tx);
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(tx->chan);
 	dma_cookie_t		cookie;
 	spin_lock_bh(&dwc->lock);
 	cookie = dwc_assign_cookie(dwc, desc);
 	/*
 	 * REVISIT: We should attempt to chain as many descriptors as
 	 * possible, perhaps even appending to those already submitted
 	 * for DMA. But this is hard to do in a race-free manner.
 	 */
 	if (list_empty(&dwc->active_list)) {
 		dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
 				desc->txd.cookie);
 		dwc_dostart(dwc, desc);
 		list_add_tail(&desc->desc_node, &dwc->active_list);
 	} else {
 		dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
 				desc->txd.cookie);
 		list_add_tail(&desc->desc_node, &dwc->queue);
 	}
 	spin_unlock_bh(&dwc->lock);
 	return cookie;
 }
 static struct dma_async_tx_descriptor *
 dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		size_t len, unsigned long flags)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_desc		*desc;
 	struct dw_desc		*first;
 	struct dw_desc		*prev;
 	size_t			xfer_count;
 	size_t			offset;
 	unsigned int		src_width;
 	unsigned int		dst_width;
 	u32			ctllo;
 	dev_vdbg(chan2dev(chan), "prep_dma_memcpy d0x%x s0x%x l0x%zx f0x%lx\n",
 			dest, src, len, flags);
 	if (unlikely(!len)) {
 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 	/*
 	 * We can be a lot more clever here, but this should take care
 	 * of the most common optimization.
 	 */
 	if (!((src | dest  | len) & 3))
 		src_width = dst_width = 2;
 	else if (!((src | dest | len) & 1))
 		src_width = dst_width = 1;
 	else
 		src_width = dst_width = 0;
 	ctllo = DWC_DEFAULT_CTLLO
 			| DWC_CTLL_DST_WIDTH(dst_width)
 			| DWC_CTLL_SRC_WIDTH(src_width)
 			| DWC_CTLL_DST_INC
 			| DWC_CTLL_SRC_INC
 			| DWC_CTLL_FC_M2M;
 	prev = first = NULL;
 	for (offset = 0; offset < len; offset += xfer_count << src_width) {
 		xfer_count = min_t(size_t, (len - offset) >> src_width,
 				DWC_MAX_COUNT);
 		desc = dwc_desc_get(dwc);
 		if (!desc)
 			goto err_desc_get;
 		desc->lli.sar = src + offset;
 		desc->lli.dar = dest + offset;
 		desc->lli.ctllo = ctllo;
 		desc->lli.ctlhi = xfer_count;
 		if (!first) {
 			first = desc;
 		} else {
 			prev->lli.llp = desc->txd.phys;
 			dma_sync_single_for_device(chan2parent(chan),
 					prev->txd.phys, sizeof(prev->lli),
 					DMA_TO_DEVICE);
 			list_add_tail(&desc->desc_node,
 					&first->tx_list);
 		}
 		prev = desc;
 	}
 	if (flags & DMA_PREP_INTERRUPT)
 		/* Trigger interrupt after last block */
 		prev->lli.ctllo |= DWC_CTLL_INT_EN;
 	prev->lli.llp = 0;
 	dma_sync_single_for_device(chan2parent(chan),
 			prev->txd.phys, sizeof(prev->lli),
 			DMA_TO_DEVICE);
 	first->txd.flags = flags;
 	first->len = len;
 	return &first->txd;
 err_desc_get:
 	dwc_desc_put(dwc, first);
 	return NULL;
 }
 static struct dma_async_tx_descriptor *
 dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma_slave	*dws = chan->private;
 	struct dw_desc		*prev;
 	struct dw_desc		*first;
 	u32			ctllo;
 	dma_addr_t		reg;
 	unsigned int		reg_width;
 	unsigned int		mem_width;
 	unsigned int		i;
 	struct scatterlist	*sg;
 	size_t			total_len = 0;
 	dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
 	if (unlikely(!dws || !sg_len))
 		return NULL;
 	reg_width = dws->reg_width;
 	prev = first = NULL;
 	switch (direction) {
 	case DMA_TO_DEVICE:
 		ctllo = (DWC_DEFAULT_CTLLO
 				| DWC_CTLL_DST_WIDTH(reg_width)
 				| DWC_CTLL_DST_FIX
 				| DWC_CTLL_SRC_INC
 				| DWC_CTLL_FC_M2P);
 		reg = dws->tx_reg;
 		for_each_sg(sgl, sg, sg_len, i) {
 			struct dw_desc	*desc;
 			u32		len;
 			u32		mem;
 			desc = dwc_desc_get(dwc);
 			if (!desc) {
 				dev_err(chan2dev(chan),
 					"not enough descriptors available\n");
 				goto err_desc_get;
 			}
 			mem = sg_phys(sg);
 			len = sg_dma_len(sg);
 			mem_width = 2;
 			if (unlikely(mem & 3 || len & 3))
 				mem_width = 0;
 			desc->lli.sar = mem;
 			desc->lli.dar = reg;
 			desc->lli.ctllo = ctllo | DWC_CTLL_SRC_WIDTH(mem_width);
 			desc->lli.ctlhi = len >> mem_width;
 			if (!first) {
 				first = desc;
 			} else {
 				prev->lli.llp = desc->txd.phys;
 				dma_sync_single_for_device(chan2parent(chan),
 						prev->txd.phys,
 						sizeof(prev->lli),
 						DMA_TO_DEVICE);
 				list_add_tail(&desc->desc_node,
 						&first->tx_list);
 			}
 			prev = desc;
 			total_len += len;
 		}
 		break;
 	case DMA_FROM_DEVICE:
 		ctllo = (DWC_DEFAULT_CTLLO
 				| DWC_CTLL_SRC_WIDTH(reg_width)
 				| DWC_CTLL_DST_INC
 				| DWC_CTLL_SRC_FIX
 				| DWC_CTLL_FC_P2M);
 		reg = dws->rx_reg;
 		for_each_sg(sgl, sg, sg_len, i) {
 			struct dw_desc	*desc;
 			u32		len;
 			u32		mem;
 			desc = dwc_desc_get(dwc);
 			if (!desc) {
 				dev_err(chan2dev(chan),
 					"not enough descriptors available\n");
 				goto err_desc_get;
 			}
 			mem = sg_phys(sg);
 			len = sg_dma_len(sg);
 			mem_width = 2;
 			if (unlikely(mem & 3 || len & 3))
 				mem_width = 0;
 			desc->lli.sar = reg;
 			desc->lli.dar = mem;
 			desc->lli.ctllo = ctllo | DWC_CTLL_DST_WIDTH(mem_width);
 			desc->lli.ctlhi = len >> reg_width;
 			if (!first) {
 				first = desc;
 			} else {
 				prev->lli.llp = desc->txd.phys;
 				dma_sync_single_for_device(chan2parent(chan),
 						prev->txd.phys,
 						sizeof(prev->lli),
 						DMA_TO_DEVICE);
 				list_add_tail(&desc->desc_node,
 						&first->tx_list);
 			}
 			prev = desc;
 			total_len += len;
 		}
 		break;
 	default:
 		return NULL;
 	}
 	if (flags & DMA_PREP_INTERRUPT)
 		/* Trigger interrupt after last block */
 		prev->lli.ctllo |= DWC_CTLL_INT_EN;
 	prev->lli.llp = 0;
 	dma_sync_single_for_device(chan2parent(chan),
 			prev->txd.phys, sizeof(prev->lli),
 			DMA_TO_DEVICE);
 	first->len = total_len;
 	return &first->txd;
 err_desc_get:
 	dwc_desc_put(dwc, first);
 	return NULL;
 }
-static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int dwc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		       unsigned long arg)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(chan->device);
 	struct dw_desc		*desc, *_desc;
 	LIST_HEAD(list);
 	/* Only supports DMA_TERMINATE_ALL */
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	/*
 	 * This is only called when something went wrong elsewhere, so
 	 * we don't really care about the data. Just disable the
 	 * channel. We still have to poll the channel enable bit due
 	 * to AHB/HSB limitations.
 	 */
 	spin_lock_bh(&dwc->lock);
 	channel_clear_bit(dw, CH_EN, dwc->mask);
 	while (dma_readl(dw, CH_EN) & dwc->mask)
 		cpu_relax();
 	/* active_list entries will end up before queued entries */
 	list_splice_init(&dwc->queue, &list);
 	list_splice_init(&dwc->active_list, &list);
 	spin_unlock_bh(&dwc->lock);
 	/* Flush all pending and queued descriptors */
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		dwc_descriptor_complete(dwc, desc);
 	return 0;
 }
 static enum dma_status
 dwc_tx_status(struct dma_chan *chan,
 	      dma_cookie_t cookie,
 	      struct dma_tx_state *txstate)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	dma_cookie_t		last_used;
 	dma_cookie_t		last_complete;
 	int			ret;
 	last_complete = dwc->completed;
 	last_used = chan->cookie;
 	ret = dma_async_is_complete(cookie, last_complete, last_used);
 	if (ret != DMA_SUCCESS) {
 		dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
 		last_complete = dwc->completed;
 		last_used = chan->cookie;
 		ret = dma_async_is_complete(cookie, last_complete, last_used);
 	}
 	dma_set_tx_state(txstate, last_complete, last_used, 0);
 	return ret;
 }
 static void dwc_issue_pending(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	spin_lock_bh(&dwc->lock);
 	if (!list_empty(&dwc->queue))
 		dwc_scan_descriptors(to_dw_dma(chan->device), dwc);
 	spin_unlock_bh(&dwc->lock);
 }
 static int dwc_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(chan->device);
 	struct dw_desc		*desc;
 	struct dw_dma_slave	*dws;
 	int			i;
 	u32			cfghi;
 	u32			cfglo;
 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
 	/* ASSERT:  channel is idle */
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
 		dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
 		return -EIO;
 	}
 	dwc->completed = chan->cookie = 1;
 	cfghi = DWC_CFGH_FIFO_MODE;
 	cfglo = 0;
 	dws = chan->private;
 	if (dws) {
 		/*
 		 * We need controller-specific data to set up slave
 		 * transfers.
 		 */
 		BUG_ON(!dws->dma_dev || dws->dma_dev != dw->dma.dev);
 		cfghi = dws->cfg_hi;
 		cfglo = dws->cfg_lo;
 	}
 	channel_writel(dwc, CFG_LO, cfglo);
 	channel_writel(dwc, CFG_HI, cfghi);
 	/*
 	 * NOTE: some controllers may have additional features that we
 	 * need to initialize here, like "scatter-gather" (which
 	 * doesn't mean what you think it means), and status writeback.
 	 */
 	spin_lock_bh(&dwc->lock);
 	i = dwc->descs_allocated;
 	while (dwc->descs_allocated < NR_DESCS_PER_CHANNEL) {
 		spin_unlock_bh(&dwc->lock);
 		desc = kzalloc(sizeof(struct dw_desc), GFP_KERNEL);
 		if (!desc) {
 			dev_info(chan2dev(chan),
 				"only allocated %d descriptors\n", i);
 			spin_lock_bh(&dwc->lock);
 			break;
 		}
 		INIT_LIST_HEAD(&desc->tx_list);
 		dma_async_tx_descriptor_init(&desc->txd, chan);
 		desc->txd.tx_submit = dwc_tx_submit;
 		desc->txd.flags = DMA_CTRL_ACK;
 		desc->txd.phys = dma_map_single(chan2parent(chan), &desc->lli,
 				sizeof(desc->lli), DMA_TO_DEVICE);
 		dwc_desc_put(dwc, desc);
 		spin_lock_bh(&dwc->lock);
 		i = ++dwc->descs_allocated;
 	}
 	/* Enable interrupts */
 	channel_set_bit(dw, MASK.XFER, dwc->mask);
 	channel_set_bit(dw, MASK.BLOCK, dwc->mask);
 	channel_set_bit(dw, MASK.ERROR, dwc->mask);
 	spin_unlock_bh(&dwc->lock);
 	dev_dbg(chan2dev(chan),
 		"alloc_chan_resources allocated %d descriptors\n", i);
 	return i;
 }
 static void dwc_free_chan_resources(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(chan->device);
 	struct dw_desc		*desc, *_desc;
 	LIST_HEAD(list);
 	dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
 			dwc->descs_allocated);
 	/* ASSERT:  channel is idle */
 	BUG_ON(!list_empty(&dwc->active_list));
 	BUG_ON(!list_empty(&dwc->queue));
 	BUG_ON(dma_readl(to_dw_dma(chan->device), CH_EN) & dwc->mask);
 	spin_lock_bh(&dwc->lock);
 	list_splice_init(&dwc->free_list, &list);
 	dwc->descs_allocated = 0;
 	/* Disable interrupts */
 	channel_clear_bit(dw, MASK.XFER, dwc->mask);
 	channel_clear_bit(dw, MASK.BLOCK, dwc->mask);
 	channel_clear_bit(dw, MASK.ERROR, dwc->mask);
 	spin_unlock_bh(&dwc->lock);
 	list_for_each_entry_safe(desc, _desc, &list, desc_node) {
 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
 		dma_unmap_single(chan2parent(chan), desc->txd.phys,
 				sizeof(desc->lli), DMA_TO_DEVICE);
 		kfree(desc);
 	}
 	dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
 }
 /* --------------------- Cyclic DMA API extensions -------------------- */
 /**
  * dw_dma_cyclic_start - start the cyclic DMA transfer
  * @chan: the DMA channel to start
  *
  * Must be called with soft interrupts disabled. Returns zero on success or
  * -errno on failure.
  */
 int dw_dma_cyclic_start(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);
 	if (!test_bit(DW_DMA_IS_CYCLIC, &dwc->flags)) {
 		dev_err(chan2dev(&dwc->chan), "missing prep for cyclic DMA\n");
 		return -ENODEV;
 	}
 	spin_lock(&dwc->lock);
 	/* assert channel is idle */
 	if (dma_readl(dw, CH_EN) & dwc->mask) {
 		dev_err(chan2dev(&dwc->chan),
 			"BUG: Attempted to start non-idle channel\n");
 		dev_err(chan2dev(&dwc->chan),
 			"  SAR: 0x%x DAR: 0x%x LLP: 0x%x CTL: 0x%x:%08x\n",
 			channel_readl(dwc, SAR),
 			channel_readl(dwc, DAR),
 			channel_readl(dwc, LLP),
 			channel_readl(dwc, CTL_HI),
 			channel_readl(dwc, CTL_LO));
 		spin_unlock(&dwc->lock);
 		return -EBUSY;
 	}
 	dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 	dma_writel(dw, CLEAR.ERROR, dwc->mask);
 	dma_writel(dw, CLEAR.XFER, dwc->mask);
 	/* setup DMAC channel registers */
 	channel_writel(dwc, LLP, dwc->cdesc->desc[0]->txd.phys);
 	channel_writel(dwc, CTL_LO, DWC_CTLL_LLP_D_EN | DWC_CTLL_LLP_S_EN);
 	channel_writel(dwc, CTL_HI, 0);
 	channel_set_bit(dw, CH_EN, dwc->mask);
 	spin_unlock(&dwc->lock);
 	return 0;
 }
 EXPORT_SYMBOL(dw_dma_cyclic_start);
 /**
  * dw_dma_cyclic_stop - stop the cyclic DMA transfer
  * @chan: the DMA channel to stop
  *
  * Must be called with soft interrupts disabled.
  */
 void dw_dma_cyclic_stop(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);
 	spin_lock(&dwc->lock);
 	channel_clear_bit(dw, CH_EN, dwc->mask);
 	while (dma_readl(dw, CH_EN) & dwc->mask)
 		cpu_relax();
 	spin_unlock(&dwc->lock);
 }
 EXPORT_SYMBOL(dw_dma_cyclic_stop);
 /**
  * dw_dma_cyclic_prep - prepare the cyclic DMA transfer
  * @chan: the DMA channel to prepare
  * @buf_addr: physical DMA address where the buffer starts
  * @buf_len: total number of bytes for the entire buffer
  * @period_len: number of bytes for each period
  * @direction: transfer direction, to or from device
  *
  * Must be called before trying to start the transfer. Returns a valid struct
  * dw_cyclic_desc if successful or an ERR_PTR(-errno) if not successful.
  */
 struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan,
 		dma_addr_t buf_addr, size_t buf_len, size_t period_len,
 		enum dma_data_direction direction)
 {
 	struct dw_dma_chan		*dwc = to_dw_dma_chan(chan);
 	struct dw_cyclic_desc		*cdesc;
 	struct dw_cyclic_desc		*retval = NULL;
 	struct dw_desc			*desc;
 	struct dw_desc			*last = NULL;
 	struct dw_dma_slave		*dws = chan->private;
 	unsigned long			was_cyclic;
 	unsigned int			reg_width;
 	unsigned int			periods;
 	unsigned int			i;
 	spin_lock_bh(&dwc->lock);
 	if (!list_empty(&dwc->queue) || !list_empty(&dwc->active_list)) {
 		spin_unlock_bh(&dwc->lock);
 		dev_dbg(chan2dev(&dwc->chan),
 				"queue and/or active list are not empty\n");
 		return ERR_PTR(-EBUSY);
 	}
 	was_cyclic = test_and_set_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
 	spin_unlock_bh(&dwc->lock);
 	if (was_cyclic) {
 		dev_dbg(chan2dev(&dwc->chan),
 				"channel already prepared for cyclic DMA\n");
 		return ERR_PTR(-EBUSY);
 	}
 	retval = ERR_PTR(-EINVAL);
 	reg_width = dws->reg_width;
 	periods = buf_len / period_len;
 	/* Check for too big/unaligned periods and unaligned DMA buffer. */
 	if (period_len > (DWC_MAX_COUNT << reg_width))
 		goto out_err;
 	if (unlikely(period_len & ((1 << reg_width) - 1)))
 		goto out_err;
 	if (unlikely(buf_addr & ((1 << reg_width) - 1)))
 		goto out_err;
 	if (unlikely(!(direction & (DMA_TO_DEVICE | DMA_FROM_DEVICE))))
 		goto out_err;
 	retval = ERR_PTR(-ENOMEM);
 	if (periods > NR_DESCS_PER_CHANNEL)
 		goto out_err;
 	cdesc = kzalloc(sizeof(struct dw_cyclic_desc), GFP_KERNEL);
 	if (!cdesc)
 		goto out_err;
 	cdesc->desc = kzalloc(sizeof(struct dw_desc *) * periods, GFP_KERNEL);
 	if (!cdesc->desc)
 		goto out_err_alloc;
 	for (i = 0; i < periods; i++) {
 		desc = dwc_desc_get(dwc);
 		if (!desc)
 			goto out_err_desc_get;
 		switch (direction) {
 		case DMA_TO_DEVICE:
 			desc->lli.dar = dws->tx_reg;
 			desc->lli.sar = buf_addr + (period_len * i);
 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO
 					| DWC_CTLL_DST_WIDTH(reg_width)
 					| DWC_CTLL_SRC_WIDTH(reg_width)
 					| DWC_CTLL_DST_FIX
 					| DWC_CTLL_SRC_INC
 					| DWC_CTLL_FC_M2P
 					| DWC_CTLL_INT_EN);
 			break;
 		case DMA_FROM_DEVICE:
 			desc->lli.dar = buf_addr + (period_len * i);
 			desc->lli.sar = dws->rx_reg;
 			desc->lli.ctllo = (DWC_DEFAULT_CTLLO
 					| DWC_CTLL_SRC_WIDTH(reg_width)
 					| DWC_CTLL_DST_WIDTH(reg_width)
 					| DWC_CTLL_DST_INC
 					| DWC_CTLL_SRC_FIX
 					| DWC_CTLL_FC_P2M
 					| DWC_CTLL_INT_EN);
 			break;
 		default:
 			break;
 		}
 		desc->lli.ctlhi = (period_len >> reg_width);
 		cdesc->desc[i] = desc;
 		if (last) {
 			last->lli.llp = desc->txd.phys;
 			dma_sync_single_for_device(chan2parent(chan),
 					last->txd.phys, sizeof(last->lli),
 					DMA_TO_DEVICE);
 		}
 		last = desc;
 	}
 	/* lets make a cyclic list */
 	last->lli.llp = cdesc->desc[0]->txd.phys;
 	dma_sync_single_for_device(chan2parent(chan), last->txd.phys,
 			sizeof(last->lli), DMA_TO_DEVICE);
 	dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%08x len %zu "
 			"period %zu periods %d\n", buf_addr, buf_len,
 			period_len, periods);
 	cdesc->periods = periods;
 	dwc->cdesc = cdesc;
 	return cdesc;
 out_err_desc_get:
 	while (i--)
 		dwc_desc_put(dwc, cdesc->desc[i]);
 out_err_alloc:
 	kfree(cdesc);
 out_err:
 	clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
 	return (struct dw_cyclic_desc *)retval;
 }
 EXPORT_SYMBOL(dw_dma_cyclic_prep);
 /**
  * dw_dma_cyclic_free - free a prepared cyclic DMA transfer
  * @chan: the DMA channel to free
  */
 void dw_dma_cyclic_free(struct dma_chan *chan)
 {
 	struct dw_dma_chan	*dwc = to_dw_dma_chan(chan);
 	struct dw_dma		*dw = to_dw_dma(dwc->chan.device);
 	struct dw_cyclic_desc	*cdesc = dwc->cdesc;
 	int			i;
 	dev_dbg(chan2dev(&dwc->chan), "cyclic free\n");
 	if (!cdesc)
 		return;
 	spin_lock_bh(&dwc->lock);
 	channel_clear_bit(dw, CH_EN, dwc->mask);
 	while (dma_readl(dw, CH_EN) & dwc->mask)
 		cpu_relax();
 	dma_writel(dw, CLEAR.BLOCK, dwc->mask);
 	dma_writel(dw, CLEAR.ERROR, dwc->mask);
 	dma_writel(dw, CLEAR.XFER, dwc->mask);
 	spin_unlock_bh(&dwc->lock);
 	for (i = 0; i < cdesc->periods; i++)
 		dwc_desc_put(dwc, cdesc->desc[i]);
 	kfree(cdesc->desc);
 	kfree(cdesc);
 	clear_bit(DW_DMA_IS_CYCLIC, &dwc->flags);
 }
 EXPORT_SYMBOL(dw_dma_cyclic_free);
 /*----------------------------------------------------------------------*/
 static void dw_dma_off(struct dw_dma *dw)
 {
 	dma_writel(dw, CFG, 0);
 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 	while (dma_readl(dw, CFG) & DW_CFG_DMA_EN)
 		cpu_relax();
 }
 static int __init dw_probe(struct platform_device *pdev)
 {
 	struct dw_dma_platform_data *pdata;
 	struct resource		*io;
 	struct dw_dma		*dw;
 	size_t			size;
 	int			irq;
 	int			err;
 	int			i;
 	pdata = pdev->dev.platform_data;
 	if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS)
 		return -EINVAL;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!io)
 		return -EINVAL;
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
 	size = sizeof(struct dw_dma);
 	size += pdata->nr_channels * sizeof(struct dw_dma_chan);
 	dw = kzalloc(size, GFP_KERNEL);
 	if (!dw)
 		return -ENOMEM;
 	if (!request_mem_region(io->start, DW_REGLEN, pdev->dev.driver->name)) {
 		err = -EBUSY;
 		goto err_kfree;
 	}
 	dw->regs = ioremap(io->start, DW_REGLEN);
 	if (!dw->regs) {
 		err = -ENOMEM;
 		goto err_release_r;
 	}
 	dw->clk = clk_get(&pdev->dev, "hclk");
 	if (IS_ERR(dw->clk)) {
 		err = PTR_ERR(dw->clk);
 		goto err_clk;
 	}
 	clk_enable(dw->clk);
 	/* force dma off, just in case */
 	dw_dma_off(dw);
 	err = request_irq(irq, dw_dma_interrupt, 0, "dw_dmac", dw);
 	if (err)
 		goto err_irq;
 	platform_set_drvdata(pdev, dw);
 	tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw);
 	dw->all_chan_mask = (1 << pdata->nr_channels) - 1;
 	INIT_LIST_HEAD(&dw->dma.channels);
 	for (i = 0; i < pdata->nr_channels; i++, dw->dma.chancnt++) {
 		struct dw_dma_chan	*dwc = &dw->chan[i];
 		dwc->chan.device = &dw->dma;
 		dwc->chan.cookie = dwc->completed = 1;
 		dwc->chan.chan_id = i;
 		list_add_tail(&dwc->chan.device_node, &dw->dma.channels);
 		dwc->ch_regs = &__dw_regs(dw)->CHAN[i];
 		spin_lock_init(&dwc->lock);
 		dwc->mask = 1 << i;
 		INIT_LIST_HEAD(&dwc->active_list);
 		INIT_LIST_HEAD(&dwc->queue);
 		INIT_LIST_HEAD(&dwc->free_list);
 		channel_clear_bit(dw, CH_EN, dwc->mask);
 	}
 	/* Clear/disable all interrupts on all channels. */
 	dma_writel(dw, CLEAR.XFER, dw->all_chan_mask);
 	dma_writel(dw, CLEAR.BLOCK, dw->all_chan_mask);
 	dma_writel(dw, CLEAR.SRC_TRAN, dw->all_chan_mask);
 	dma_writel(dw, CLEAR.DST_TRAN, dw->all_chan_mask);
 	dma_writel(dw, CLEAR.ERROR, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.XFER, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.SRC_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.DST_TRAN, dw->all_chan_mask);
 	channel_clear_bit(dw, MASK.ERROR, dw->all_chan_mask);
 	dma_cap_set(DMA_MEMCPY, dw->dma.cap_mask);
 	dma_cap_set(DMA_SLAVE, dw->dma.cap_mask);
 	dw->dma.dev = &pdev->dev;
 	dw->dma.device_alloc_chan_resources = dwc_alloc_chan_resources;
 	dw->dma.device_free_chan_resources = dwc_free_chan_resources;
 	dw->dma.device_prep_dma_memcpy = dwc_prep_dma_memcpy;
 	dw->dma.device_prep_slave_sg = dwc_prep_slave_sg;
 	dw->dma.device_control = dwc_control;
 	dw->dma.device_tx_status = dwc_tx_status;
 	dw->dma.device_issue_pending = dwc_issue_pending;
 	dma_writel(dw, CFG, DW_CFG_DMA_EN);
 	printk(KERN_INFO "%s: DesignWare DMA Controller, %d channels\n",
 			dev_name(&pdev->dev), dw->dma.chancnt);
 	dma_async_device_register(&dw->dma);
 	return 0;
 err_irq:
 	clk_disable(dw->clk);
 	clk_put(dw->clk);
 err_clk:
 	iounmap(dw->regs);
 	dw->regs = NULL;
 err_release_r:
 	release_resource(io);
 err_kfree:
 	kfree(dw);
 	return err;
 }
 static int __exit dw_remove(struct platform_device *pdev)
 {
 	struct dw_dma		*dw = platform_get_drvdata(pdev);
 	struct dw_dma_chan	*dwc, *_dwc;
 	struct resource		*io;
 	dw_dma_off(dw);
 	dma_async_device_unregister(&dw->dma);
 	free_irq(platform_get_irq(pdev, 0), dw);
 	tasklet_kill(&dw->tasklet);
 	list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels,
 			chan.device_node) {
 		list_del(&dwc->chan.device_node);
 		channel_clear_bit(dw, CH_EN, dwc->mask);
 	}
 	clk_disable(dw->clk);
 	clk_put(dw->clk);
 	iounmap(dw->regs);
 	dw->regs = NULL;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	release_mem_region(io->start, DW_REGLEN);
 	kfree(dw);
 	return 0;
 }
 static void dw_shutdown(struct platform_device *pdev)
 {
 	struct dw_dma	*dw = platform_get_drvdata(pdev);
 	dw_dma_off(platform_get_drvdata(pdev));
 	clk_disable(dw->clk);
 }
 static int dw_suspend_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct dw_dma	*dw = platform_get_drvdata(pdev);
 	dw_dma_off(platform_get_drvdata(pdev));
 	clk_disable(dw->clk);
 	return 0;
 }
 static int dw_resume_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct dw_dma	*dw = platform_get_drvdata(pdev);
 	clk_enable(dw->clk);
 	dma_writel(dw, CFG, DW_CFG_DMA_EN);
 	return 0;
 }
 static const struct dev_pm_ops dw_dev_pm_ops = {
 	.suspend_noirq = dw_suspend_noirq,
 	.resume_noirq = dw_resume_noirq,
 };
 static struct platform_driver dw_driver = {
 	.remove		= __exit_p(dw_remove),
 	.shutdown	= dw_shutdown,
 	.driver = {
 		.name	= "dw_dmac",
 		.pm	= &dw_dev_pm_ops,
 	},
 };
 static int __init dw_init(void)
 {
 	return platform_driver_probe(&dw_driver, dw_probe);
 }
 module_init(dw_init);
 static void __exit dw_exit(void)
 {
 	platform_driver_unregister(&dw_driver);
 }
 module_exit(dw_exit);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Synopsys DesignWare DMA Controller driver");
 MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");

drivers/dma/fsldma.c

Diff comments View file @ 0582763

1	/*	1	/*
2	* Freescale MPC85xx, MPC83xx DMA Engine support	2	* Freescale MPC85xx, MPC83xx DMA Engine support
3	*	3	*
4	* Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.	4	* Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
5	*	5	*
6	* Author:	6	* Author:
7	* Zhang Wei <wei.zhang@freescale.com>, Jul 2007	7	* Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8	* Ebony Zhu <ebony.zhu@freescale.com>, May 2007	8	* Ebony Zhu <ebony.zhu@freescale.com>, May 2007
9	*	9	*
10	* Description:	10	* Description:
11	* DMA engine driver for Freescale MPC8540 DMA controller, which is	11	* DMA engine driver for Freescale MPC8540 DMA controller, which is
12	* also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.	12	* also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13	* The support for MPC8349 DMA contorller is also added.	13	* The support for MPC8349 DMA contorller is also added.
14	*	14	*
15	* This driver instructs the DMA controller to issue the PCI Read Multiple	15	* This driver instructs the DMA controller to issue the PCI Read Multiple
16	* command for PCI read operations, instead of using the default PCI Read Line	16	* command for PCI read operations, instead of using the default PCI Read Line
17	* command. Please be aware that this setting may result in read pre-fetching	17	* command. Please be aware that this setting may result in read pre-fetching
18	* on some platforms.	18	* on some platforms.
19	*	19	*
20	* This is free software; you can redistribute it and/or modify	20	* This is free software; you can redistribute it and/or modify
21	* it under the terms of the GNU General Public License as published by	21	* it under the terms of the GNU General Public License as published by
22	* the Free Software Foundation; either version 2 of the License, or	22	* the Free Software Foundation; either version 2 of the License, or
23	* (at your option) any later version.	23	* (at your option) any later version.
24	*	24	*
25	*/	25	*/
26		26
27	#include <linux/init.h>	27	#include <linux/init.h>
28	#include <linux/module.h>	28	#include <linux/module.h>
29	#include <linux/pci.h>	29	#include <linux/pci.h>
30	#include <linux/interrupt.h>	30	#include <linux/interrupt.h>
31	#include <linux/dmaengine.h>	31	#include <linux/dmaengine.h>
32	#include <linux/delay.h>	32	#include <linux/delay.h>
33	#include <linux/dma-mapping.h>	33	#include <linux/dma-mapping.h>
34	#include <linux/dmapool.h>	34	#include <linux/dmapool.h>
35	#include <linux/of_platform.h>	35	#include <linux/of_platform.h>
36		36
37	#include <asm/fsldma.h>	37	#include <asm/fsldma.h>
38	#include "fsldma.h"	38	#include "fsldma.h"
39		39
40	static void dma_init(struct fsldma_chan *chan)	40	static void dma_init(struct fsldma_chan *chan)
41	{	41	{
42	/* Reset the channel */	42	/* Reset the channel */
43	DMA_OUT(chan, &chan->regs->mr, 0, 32);	43	DMA_OUT(chan, &chan->regs->mr, 0, 32);
44		44
45	switch (chan->feature & FSL_DMA_IP_MASK) {	45	switch (chan->feature & FSL_DMA_IP_MASK) {
46	case FSL_DMA_IP_85XX:	46	case FSL_DMA_IP_85XX:
47	/* Set the channel to below modes:	47	/* Set the channel to below modes:
48	* EIE - Error interrupt enable	48	* EIE - Error interrupt enable
49	* EOSIE - End of segments interrupt enable (basic mode)	49	* EOSIE - End of segments interrupt enable (basic mode)
50	* EOLNIE - End of links interrupt enable	50	* EOLNIE - End of links interrupt enable
51	*/	51	*/
52	DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EIE	52	DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EIE
53	\| FSL_DMA_MR_EOLNIE \| FSL_DMA_MR_EOSIE, 32);	53	\| FSL_DMA_MR_EOLNIE \| FSL_DMA_MR_EOSIE, 32);
54	break;	54	break;
55	case FSL_DMA_IP_83XX:	55	case FSL_DMA_IP_83XX:
56	/* Set the channel to below modes:	56	/* Set the channel to below modes:
57	* EOTIE - End-of-transfer interrupt enable	57	* EOTIE - End-of-transfer interrupt enable
58	* PRC_RM - PCI read multiple	58	* PRC_RM - PCI read multiple
59	*/	59	*/
60	DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE	60	DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE
61	\| FSL_DMA_MR_PRC_RM, 32);	61	\| FSL_DMA_MR_PRC_RM, 32);
62	break;	62	break;
63	}	63	}
64	}	64	}
65		65
66	static void set_sr(struct fsldma_chan *chan, u32 val)	66	static void set_sr(struct fsldma_chan *chan, u32 val)
67	{	67	{
68	DMA_OUT(chan, &chan->regs->sr, val, 32);	68	DMA_OUT(chan, &chan->regs->sr, val, 32);
69	}	69	}
70		70
71	static u32 get_sr(struct fsldma_chan *chan)	71	static u32 get_sr(struct fsldma_chan *chan)
72	{	72	{
73	return DMA_IN(chan, &chan->regs->sr, 32);	73	return DMA_IN(chan, &chan->regs->sr, 32);
74	}	74	}
75		75
76	static void set_desc_cnt(struct fsldma_chan *chan,	76	static void set_desc_cnt(struct fsldma_chan *chan,
77	struct fsl_dma_ld_hw *hw, u32 count)	77	struct fsl_dma_ld_hw *hw, u32 count)
78	{	78	{
79	hw->count = CPU_TO_DMA(chan, count, 32);	79	hw->count = CPU_TO_DMA(chan, count, 32);
80	}	80	}
81		81
82	static void set_desc_src(struct fsldma_chan *chan,	82	static void set_desc_src(struct fsldma_chan *chan,
83	struct fsl_dma_ld_hw *hw, dma_addr_t src)	83	struct fsl_dma_ld_hw *hw, dma_addr_t src)
84	{	84	{
85	u64 snoop_bits;	85	u64 snoop_bits;
86		86
87	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)	87	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
88	? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;	88	? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
89	hw->src_addr = CPU_TO_DMA(chan, snoop_bits \| src, 64);	89	hw->src_addr = CPU_TO_DMA(chan, snoop_bits \| src, 64);
90	}	90	}
91		91
92	static void set_desc_dst(struct fsldma_chan *chan,	92	static void set_desc_dst(struct fsldma_chan *chan,
93	struct fsl_dma_ld_hw *hw, dma_addr_t dst)	93	struct fsl_dma_ld_hw *hw, dma_addr_t dst)
94	{	94	{
95	u64 snoop_bits;	95	u64 snoop_bits;
96		96
97	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)	97	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
98	? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;	98	? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
99	hw->dst_addr = CPU_TO_DMA(chan, snoop_bits \| dst, 64);	99	hw->dst_addr = CPU_TO_DMA(chan, snoop_bits \| dst, 64);
100	}	100	}
101		101
102	static void set_desc_next(struct fsldma_chan *chan,	102	static void set_desc_next(struct fsldma_chan *chan,
103	struct fsl_dma_ld_hw *hw, dma_addr_t next)	103	struct fsl_dma_ld_hw *hw, dma_addr_t next)
104	{	104	{
105	u64 snoop_bits;	105	u64 snoop_bits;
106		106
107	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)	107	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
108	? FSL_DMA_SNEN : 0;	108	? FSL_DMA_SNEN : 0;
109	hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits \| next, 64);	109	hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits \| next, 64);
110	}	110	}
111		111
112	static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)	112	static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
113	{	113	{
114	DMA_OUT(chan, &chan->regs->cdar, addr \| FSL_DMA_SNEN, 64);	114	DMA_OUT(chan, &chan->regs->cdar, addr \| FSL_DMA_SNEN, 64);
115	}	115	}
116		116
117	static dma_addr_t get_cdar(struct fsldma_chan *chan)	117	static dma_addr_t get_cdar(struct fsldma_chan *chan)
118	{	118	{
119	return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;	119	return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
120	}	120	}
121		121
122	static dma_addr_t get_ndar(struct fsldma_chan *chan)	122	static dma_addr_t get_ndar(struct fsldma_chan *chan)
123	{	123	{
124	return DMA_IN(chan, &chan->regs->ndar, 64);	124	return DMA_IN(chan, &chan->regs->ndar, 64);
125	}	125	}
126		126
127	static u32 get_bcr(struct fsldma_chan *chan)	127	static u32 get_bcr(struct fsldma_chan *chan)
128	{	128	{
129	return DMA_IN(chan, &chan->regs->bcr, 32);	129	return DMA_IN(chan, &chan->regs->bcr, 32);
130	}	130	}
131		131
132	static int dma_is_idle(struct fsldma_chan *chan)	132	static int dma_is_idle(struct fsldma_chan *chan)
133	{	133	{
134	u32 sr = get_sr(chan);	134	u32 sr = get_sr(chan);
135	return (!(sr & FSL_DMA_SR_CB)) \|\| (sr & FSL_DMA_SR_CH);	135	return (!(sr & FSL_DMA_SR_CB)) \|\| (sr & FSL_DMA_SR_CH);
136	}	136	}
137		137
138	static void dma_start(struct fsldma_chan *chan)	138	static void dma_start(struct fsldma_chan *chan)
139	{	139	{
140	u32 mode;	140	u32 mode;
141		141
142	mode = DMA_IN(chan, &chan->regs->mr, 32);	142	mode = DMA_IN(chan, &chan->regs->mr, 32);
143		143
144	if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {	144	if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
145	if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {	145	if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
146	DMA_OUT(chan, &chan->regs->bcr, 0, 32);	146	DMA_OUT(chan, &chan->regs->bcr, 0, 32);
147	mode \|= FSL_DMA_MR_EMP_EN;	147	mode \|= FSL_DMA_MR_EMP_EN;
148	} else {	148	} else {
149	mode &= ~FSL_DMA_MR_EMP_EN;	149	mode &= ~FSL_DMA_MR_EMP_EN;
150	}	150	}
151	}	151	}
152		152
153	if (chan->feature & FSL_DMA_CHAN_START_EXT)	153	if (chan->feature & FSL_DMA_CHAN_START_EXT)
154	mode \|= FSL_DMA_MR_EMS_EN;	154	mode \|= FSL_DMA_MR_EMS_EN;
155	else	155	else
156	mode \|= FSL_DMA_MR_CS;	156	mode \|= FSL_DMA_MR_CS;
157		157
158	DMA_OUT(chan, &chan->regs->mr, mode, 32);	158	DMA_OUT(chan, &chan->regs->mr, mode, 32);
159	}	159	}
160		160
161	static void dma_halt(struct fsldma_chan *chan)	161	static void dma_halt(struct fsldma_chan *chan)
162	{	162	{
163	u32 mode;	163	u32 mode;
164	int i;	164	int i;
165		165
166	mode = DMA_IN(chan, &chan->regs->mr, 32);	166	mode = DMA_IN(chan, &chan->regs->mr, 32);
167	mode \|= FSL_DMA_MR_CA;	167	mode \|= FSL_DMA_MR_CA;
168	DMA_OUT(chan, &chan->regs->mr, mode, 32);	168	DMA_OUT(chan, &chan->regs->mr, mode, 32);
169		169
170	mode &= ~(FSL_DMA_MR_CS \| FSL_DMA_MR_EMS_EN \| FSL_DMA_MR_CA);	170	mode &= ~(FSL_DMA_MR_CS \| FSL_DMA_MR_EMS_EN \| FSL_DMA_MR_CA);
171	DMA_OUT(chan, &chan->regs->mr, mode, 32);	171	DMA_OUT(chan, &chan->regs->mr, mode, 32);
172		172
173	for (i = 0; i < 100; i++) {	173	for (i = 0; i < 100; i++) {
174	if (dma_is_idle(chan))	174	if (dma_is_idle(chan))
175	return;	175	return;
176		176
177	udelay(10);	177	udelay(10);
178	}	178	}
179		179
180	if (!dma_is_idle(chan))	180	if (!dma_is_idle(chan))
181	dev_err(chan->dev, "DMA halt timeout!\n");	181	dev_err(chan->dev, "DMA halt timeout!\n");
182	}	182	}
183		183
184	static void set_ld_eol(struct fsldma_chan *chan,	184	static void set_ld_eol(struct fsldma_chan *chan,
185	struct fsl_desc_sw *desc)	185	struct fsl_desc_sw *desc)
186	{	186	{
187	u64 snoop_bits;	187	u64 snoop_bits;
188		188
189	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)	189	snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
190	? FSL_DMA_SNEN : 0;	190	? FSL_DMA_SNEN : 0;
191		191
192	desc->hw.next_ln_addr = CPU_TO_DMA(chan,	192	desc->hw.next_ln_addr = CPU_TO_DMA(chan,
193	DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) \| FSL_DMA_EOL	193	DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) \| FSL_DMA_EOL
194	\| snoop_bits, 64);	194	\| snoop_bits, 64);
195	}	195	}
196		196
197	/**	197	/**
198	* fsl_chan_set_src_loop_size - Set source address hold transfer size	198	* fsl_chan_set_src_loop_size - Set source address hold transfer size
199	* @chan : Freescale DMA channel	199	* @chan : Freescale DMA channel
200	* @size : Address loop size, 0 for disable loop	200	* @size : Address loop size, 0 for disable loop
201	*	201	*
202	* The set source address hold transfer size. The source	202	* The set source address hold transfer size. The source
203	* address hold or loop transfer size is when the DMA transfer	203	* address hold or loop transfer size is when the DMA transfer
204	* data from source address (SA), if the loop size is 4, the DMA will	204	* data from source address (SA), if the loop size is 4, the DMA will
205	* read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,	205	* read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
206	* SA + 1 ... and so on.	206	* SA + 1 ... and so on.
207	*/	207	*/
208	static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)	208	static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)
209	{	209	{
210	u32 mode;	210	u32 mode;
211		211
212	mode = DMA_IN(chan, &chan->regs->mr, 32);	212	mode = DMA_IN(chan, &chan->regs->mr, 32);
213		213
214	switch (size) {	214	switch (size) {
215	case 0:	215	case 0:
216	mode &= ~FSL_DMA_MR_SAHE;	216	mode &= ~FSL_DMA_MR_SAHE;
217	break;	217	break;
218	case 1:	218	case 1:
219	case 2:	219	case 2:
220	case 4:	220	case 4:
221	case 8:	221	case 8:
222	mode \|= FSL_DMA_MR_SAHE \| (__ilog2(size) << 14);	222	mode \|= FSL_DMA_MR_SAHE \| (__ilog2(size) << 14);
223	break;	223	break;
224	}	224	}
225		225
226	DMA_OUT(chan, &chan->regs->mr, mode, 32);	226	DMA_OUT(chan, &chan->regs->mr, mode, 32);
227	}	227	}
228		228
229	/**	229	/**
230	* fsl_chan_set_dst_loop_size - Set destination address hold transfer size	230	* fsl_chan_set_dst_loop_size - Set destination address hold transfer size
231	* @chan : Freescale DMA channel	231	* @chan : Freescale DMA channel
232	* @size : Address loop size, 0 for disable loop	232	* @size : Address loop size, 0 for disable loop
233	*	233	*
234	* The set destination address hold transfer size. The destination	234	* The set destination address hold transfer size. The destination
235	* address hold or loop transfer size is when the DMA transfer	235	* address hold or loop transfer size is when the DMA transfer
236	* data to destination address (TA), if the loop size is 4, the DMA will	236	* data to destination address (TA), if the loop size is 4, the DMA will
237	* write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,	237	* write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
238	* TA + 1 ... and so on.	238	* TA + 1 ... and so on.
239	*/	239	*/
240	static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)	240	static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)
241	{	241	{
242	u32 mode;	242	u32 mode;
243		243
244	mode = DMA_IN(chan, &chan->regs->mr, 32);	244	mode = DMA_IN(chan, &chan->regs->mr, 32);
245		245
246	switch (size) {	246	switch (size) {
247	case 0:	247	case 0:
248	mode &= ~FSL_DMA_MR_DAHE;	248	mode &= ~FSL_DMA_MR_DAHE;
249	break;	249	break;
250	case 1:	250	case 1:
251	case 2:	251	case 2:
252	case 4:	252	case 4:
253	case 8:	253	case 8:
254	mode \|= FSL_DMA_MR_DAHE \| (__ilog2(size) << 16);	254	mode \|= FSL_DMA_MR_DAHE \| (__ilog2(size) << 16);
255	break;	255	break;
256	}	256	}
257		257
258	DMA_OUT(chan, &chan->regs->mr, mode, 32);	258	DMA_OUT(chan, &chan->regs->mr, mode, 32);
259	}	259	}
260		260
261	/**	261	/**
262	* fsl_chan_set_request_count - Set DMA Request Count for external control	262	* fsl_chan_set_request_count - Set DMA Request Count for external control
263	* @chan : Freescale DMA channel	263	* @chan : Freescale DMA channel
264	* @size : Number of bytes to transfer in a single request	264	* @size : Number of bytes to transfer in a single request
265	*	265	*
266	* The Freescale DMA channel can be controlled by the external signal DREQ#.	266	* The Freescale DMA channel can be controlled by the external signal DREQ#.
267	* The DMA request count is how many bytes are allowed to transfer before	267	* The DMA request count is how many bytes are allowed to transfer before
268	* pausing the channel, after which a new assertion of DREQ# resumes channel	268	* pausing the channel, after which a new assertion of DREQ# resumes channel
269	* operation.	269	* operation.
270	*	270	*
271	* A size of 0 disables external pause control. The maximum size is 1024.	271	* A size of 0 disables external pause control. The maximum size is 1024.
272	*/	272	*/
273	static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)	273	static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)
274	{	274	{
275	u32 mode;	275	u32 mode;
276		276
277	BUG_ON(size > 1024);	277	BUG_ON(size > 1024);
278		278
279	mode = DMA_IN(chan, &chan->regs->mr, 32);	279	mode = DMA_IN(chan, &chan->regs->mr, 32);
280	mode \|= (__ilog2(size) << 24) & 0x0f000000;	280	mode \|= (__ilog2(size) << 24) & 0x0f000000;
281		281
282	DMA_OUT(chan, &chan->regs->mr, mode, 32);	282	DMA_OUT(chan, &chan->regs->mr, mode, 32);
283	}	283	}
284		284
285	/**	285	/**
286	* fsl_chan_toggle_ext_pause - Toggle channel external pause status	286	* fsl_chan_toggle_ext_pause - Toggle channel external pause status
287	* @chan : Freescale DMA channel	287	* @chan : Freescale DMA channel
288	* @enable : 0 is disabled, 1 is enabled.	288	* @enable : 0 is disabled, 1 is enabled.
289	*	289	*
290	* The Freescale DMA channel can be controlled by the external signal DREQ#.	290	* The Freescale DMA channel can be controlled by the external signal DREQ#.
291	* The DMA Request Count feature should be used in addition to this feature	291	* The DMA Request Count feature should be used in addition to this feature
292	* to set the number of bytes to transfer before pausing the channel.	292	* to set the number of bytes to transfer before pausing the channel.
293	*/	293	*/
294	static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)	294	static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)
295	{	295	{
296	if (enable)	296	if (enable)
297	chan->feature \|= FSL_DMA_CHAN_PAUSE_EXT;	297	chan->feature \|= FSL_DMA_CHAN_PAUSE_EXT;
298	else	298	else
299	chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;	299	chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
300	}	300	}
301		301
302	/**	302	/**
303	* fsl_chan_toggle_ext_start - Toggle channel external start status	303	* fsl_chan_toggle_ext_start - Toggle channel external start status
304	* @chan : Freescale DMA channel	304	* @chan : Freescale DMA channel
305	* @enable : 0 is disabled, 1 is enabled.	305	* @enable : 0 is disabled, 1 is enabled.
306	*	306	*
307	* If enable the external start, the channel can be started by an	307	* If enable the external start, the channel can be started by an
308	* external DMA start pin. So the dma_start() does not start the	308	* external DMA start pin. So the dma_start() does not start the
309	* transfer immediately. The DMA channel will wait for the	309	* transfer immediately. The DMA channel will wait for the
310	* control pin asserted.	310	* control pin asserted.
311	*/	311	*/
312	static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)	312	static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
313	{	313	{
314	if (enable)	314	if (enable)
315	chan->feature \|= FSL_DMA_CHAN_START_EXT;	315	chan->feature \|= FSL_DMA_CHAN_START_EXT;
316	else	316	else
317	chan->feature &= ~FSL_DMA_CHAN_START_EXT;	317	chan->feature &= ~FSL_DMA_CHAN_START_EXT;
318	}	318	}
319		319
320	static void append_ld_queue(struct fsldma_chan *chan,	320	static void append_ld_queue(struct fsldma_chan *chan,
321	struct fsl_desc_sw *desc)	321	struct fsl_desc_sw *desc)
322	{	322	{
323	struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);	323	struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);
324		324
325	if (list_empty(&chan->ld_pending))	325	if (list_empty(&chan->ld_pending))
326	goto out_splice;	326	goto out_splice;
327		327
328	/*	328	/*
329	* Add the hardware descriptor to the chain of hardware descriptors	329	* Add the hardware descriptor to the chain of hardware descriptors
330	* that already exists in memory.	330	* that already exists in memory.
331	*	331	*
332	* This will un-set the EOL bit of the existing transaction, and the	332	* This will un-set the EOL bit of the existing transaction, and the
333	* last link in this transaction will become the EOL descriptor.	333	* last link in this transaction will become the EOL descriptor.
334	*/	334	*/
335	set_desc_next(chan, &tail->hw, desc->async_tx.phys);	335	set_desc_next(chan, &tail->hw, desc->async_tx.phys);
336		336
337	/*	337	/*
338	* Add the software descriptor and all children to the list	338	* Add the software descriptor and all children to the list
339	* of pending transactions	339	* of pending transactions
340	*/	340	*/
341	out_splice:	341	out_splice:
342	list_splice_tail_init(&desc->tx_list, &chan->ld_pending);	342	list_splice_tail_init(&desc->tx_list, &chan->ld_pending);
343	}	343	}
344		344
345	static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)	345	static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
346	{	346	{
347	struct fsldma_chan *chan = to_fsl_chan(tx->chan);	347	struct fsldma_chan *chan = to_fsl_chan(tx->chan);
348	struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);	348	struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
349	struct fsl_desc_sw *child;	349	struct fsl_desc_sw *child;
350	unsigned long flags;	350	unsigned long flags;
351	dma_cookie_t cookie;	351	dma_cookie_t cookie;
352		352
353	spin_lock_irqsave(&chan->desc_lock, flags);	353	spin_lock_irqsave(&chan->desc_lock, flags);
354		354
355	/*	355	/*
356	* assign cookies to all of the software descriptors	356	* assign cookies to all of the software descriptors
357	* that make up this transaction	357	* that make up this transaction
358	*/	358	*/
359	cookie = chan->common.cookie;	359	cookie = chan->common.cookie;
360	list_for_each_entry(child, &desc->tx_list, node) {	360	list_for_each_entry(child, &desc->tx_list, node) {
361	cookie++;	361	cookie++;
362	if (cookie < 0)	362	if (cookie < 0)
363	cookie = 1;	363	cookie = 1;
364		364
365	child->async_tx.cookie = cookie;	365	child->async_tx.cookie = cookie;
366	}	366	}
367		367
368	chan->common.cookie = cookie;	368	chan->common.cookie = cookie;
369		369
370	/* put this transaction onto the tail of the pending queue */	370	/* put this transaction onto the tail of the pending queue */
371	append_ld_queue(chan, desc);	371	append_ld_queue(chan, desc);
372		372
373	spin_unlock_irqrestore(&chan->desc_lock, flags);	373	spin_unlock_irqrestore(&chan->desc_lock, flags);
374		374
375	return cookie;	375	return cookie;
376	}	376	}
377		377
378	/**	378	/**
379	* fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.	379	* fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
380	* @chan : Freescale DMA channel	380	* @chan : Freescale DMA channel
381	*	381	*
382	* Return - The descriptor allocated. NULL for failed.	382	* Return - The descriptor allocated. NULL for failed.
383	*/	383	*/
384	static struct fsl_desc_sw *fsl_dma_alloc_descriptor(	384	static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
385	struct fsldma_chan *chan)	385	struct fsldma_chan *chan)
386	{	386	{
387	struct fsl_desc_sw *desc;	387	struct fsl_desc_sw *desc;
388	dma_addr_t pdesc;	388	dma_addr_t pdesc;
389		389
390	desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);	390	desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
391	if (!desc) {	391	if (!desc) {
392	dev_dbg(chan->dev, "out of memory for link desc\n");	392	dev_dbg(chan->dev, "out of memory for link desc\n");
393	return NULL;	393	return NULL;
394	}	394	}
395		395
396	memset(desc, 0, sizeof(*desc));	396	memset(desc, 0, sizeof(*desc));
397	INIT_LIST_HEAD(&desc->tx_list);	397	INIT_LIST_HEAD(&desc->tx_list);
398	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);	398	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
399	desc->async_tx.tx_submit = fsl_dma_tx_submit;	399	desc->async_tx.tx_submit = fsl_dma_tx_submit;
400	desc->async_tx.phys = pdesc;	400	desc->async_tx.phys = pdesc;
401		401
402	return desc;	402	return desc;
403	}	403	}
404		404
405		405
406	/**	406	/**
407	* fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.	407	* fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
408	* @chan : Freescale DMA channel	408	* @chan : Freescale DMA channel
409	*	409	*
410	* This function will create a dma pool for descriptor allocation.	410	* This function will create a dma pool for descriptor allocation.
411	*	411	*
412	* Return - The number of descriptors allocated.	412	* Return - The number of descriptors allocated.
413	*/	413	*/
414	static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)	414	static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
415	{	415	{
416	struct fsldma_chan *chan = to_fsl_chan(dchan);	416	struct fsldma_chan *chan = to_fsl_chan(dchan);
417		417
418	/* Has this channel already been allocated? */	418	/* Has this channel already been allocated? */
419	if (chan->desc_pool)	419	if (chan->desc_pool)
420	return 1;	420	return 1;
421		421
422	/*	422	/*
423	* We need the descriptor to be aligned to 32bytes	423	* We need the descriptor to be aligned to 32bytes
424	* for meeting FSL DMA specification requirement.	424	* for meeting FSL DMA specification requirement.
425	*/	425	*/
426	chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool",	426	chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool",
427	chan->dev,	427	chan->dev,
428	sizeof(struct fsl_desc_sw),	428	sizeof(struct fsl_desc_sw),
429	__alignof__(struct fsl_desc_sw), 0);	429	__alignof__(struct fsl_desc_sw), 0);
430	if (!chan->desc_pool) {	430	if (!chan->desc_pool) {
431	dev_err(chan->dev, "unable to allocate channel %d "	431	dev_err(chan->dev, "unable to allocate channel %d "
432	"descriptor pool\n", chan->id);	432	"descriptor pool\n", chan->id);
433	return -ENOMEM;	433	return -ENOMEM;
434	}	434	}
435		435
436	/* there is at least one descriptor free to be allocated */	436	/* there is at least one descriptor free to be allocated */
437	return 1;	437	return 1;
438	}	438	}
439		439
440	/**	440	/**
441	* fsldma_free_desc_list - Free all descriptors in a queue	441	* fsldma_free_desc_list - Free all descriptors in a queue
442	* @chan: Freescae DMA channel	442	* @chan: Freescae DMA channel
443	* @list: the list to free	443	* @list: the list to free
444	*	444	*
445	* LOCKING: must hold chan->desc_lock	445	* LOCKING: must hold chan->desc_lock
446	*/	446	*/
447	static void fsldma_free_desc_list(struct fsldma_chan *chan,	447	static void fsldma_free_desc_list(struct fsldma_chan *chan,
448	struct list_head *list)	448	struct list_head *list)
449	{	449	{
450	struct fsl_desc_sw desc, _desc;	450	struct fsl_desc_sw desc, _desc;
451		451
452	list_for_each_entry_safe(desc, _desc, list, node) {	452	list_for_each_entry_safe(desc, _desc, list, node) {
453	list_del(&desc->node);	453	list_del(&desc->node);
454	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);	454	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
455	}	455	}
456	}	456	}
457		457
458	static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,	458	static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
459	struct list_head *list)	459	struct list_head *list)
460	{	460	{
461	struct fsl_desc_sw desc, _desc;	461	struct fsl_desc_sw desc, _desc;
462		462
463	list_for_each_entry_safe_reverse(desc, _desc, list, node) {	463	list_for_each_entry_safe_reverse(desc, _desc, list, node) {
464	list_del(&desc->node);	464	list_del(&desc->node);
465	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);	465	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
466	}	466	}
467	}	467	}
468		468
469	/**	469	/**
470	* fsl_dma_free_chan_resources - Free all resources of the channel.	470	* fsl_dma_free_chan_resources - Free all resources of the channel.
471	* @chan : Freescale DMA channel	471	* @chan : Freescale DMA channel
472	*/	472	*/
473	static void fsl_dma_free_chan_resources(struct dma_chan *dchan)	473	static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
474	{	474	{
475	struct fsldma_chan *chan = to_fsl_chan(dchan);	475	struct fsldma_chan *chan = to_fsl_chan(dchan);
476	unsigned long flags;	476	unsigned long flags;
477		477
478	dev_dbg(chan->dev, "Free all channel resources.\n");	478	dev_dbg(chan->dev, "Free all channel resources.\n");
479	spin_lock_irqsave(&chan->desc_lock, flags);	479	spin_lock_irqsave(&chan->desc_lock, flags);
480	fsldma_free_desc_list(chan, &chan->ld_pending);	480	fsldma_free_desc_list(chan, &chan->ld_pending);
481	fsldma_free_desc_list(chan, &chan->ld_running);	481	fsldma_free_desc_list(chan, &chan->ld_running);
482	spin_unlock_irqrestore(&chan->desc_lock, flags);	482	spin_unlock_irqrestore(&chan->desc_lock, flags);
483		483
484	dma_pool_destroy(chan->desc_pool);	484	dma_pool_destroy(chan->desc_pool);
485	chan->desc_pool = NULL;	485	chan->desc_pool = NULL;
486	}	486	}
487		487
488	static struct dma_async_tx_descriptor *	488	static struct dma_async_tx_descriptor *
489	fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)	489	fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags)
490	{	490	{
491	struct fsldma_chan *chan;	491	struct fsldma_chan *chan;
492	struct fsl_desc_sw *new;	492	struct fsl_desc_sw *new;
493		493
494	if (!dchan)	494	if (!dchan)
495	return NULL;	495	return NULL;
496		496
497	chan = to_fsl_chan(dchan);	497	chan = to_fsl_chan(dchan);
498		498
499	new = fsl_dma_alloc_descriptor(chan);	499	new = fsl_dma_alloc_descriptor(chan);
500	if (!new) {	500	if (!new) {
501	dev_err(chan->dev, "No free memory for link descriptor\n");	501	dev_err(chan->dev, "No free memory for link descriptor\n");
502	return NULL;	502	return NULL;
503	}	503	}
504		504
505	new->async_tx.cookie = -EBUSY;	505	new->async_tx.cookie = -EBUSY;
506	new->async_tx.flags = flags;	506	new->async_tx.flags = flags;
507		507
508	/* Insert the link descriptor to the LD ring */	508	/* Insert the link descriptor to the LD ring */
509	list_add_tail(&new->node, &new->tx_list);	509	list_add_tail(&new->node, &new->tx_list);
510		510
511	/* Set End-of-link to the last link descriptor of new list*/	511	/* Set End-of-link to the last link descriptor of new list*/
512	set_ld_eol(chan, new);	512	set_ld_eol(chan, new);
513		513
514	return &new->async_tx;	514	return &new->async_tx;
515	}	515	}
516		516
517	static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(	517	static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
518	struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src,	518	struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src,
519	size_t len, unsigned long flags)	519	size_t len, unsigned long flags)
520	{	520	{
521	struct fsldma_chan *chan;	521	struct fsldma_chan *chan;
522	struct fsl_desc_sw first = NULL, prev = NULL, *new;	522	struct fsl_desc_sw first = NULL, prev = NULL, *new;
523	size_t copy;	523	size_t copy;
524		524
525	if (!dchan)	525	if (!dchan)
526	return NULL;	526	return NULL;
527		527
528	if (!len)	528	if (!len)
529	return NULL;	529	return NULL;
530		530
531	chan = to_fsl_chan(dchan);	531	chan = to_fsl_chan(dchan);
532		532
533	do {	533	do {
534		534
535	/* Allocate the link descriptor from DMA pool */	535	/* Allocate the link descriptor from DMA pool */
536	new = fsl_dma_alloc_descriptor(chan);	536	new = fsl_dma_alloc_descriptor(chan);
537	if (!new) {	537	if (!new) {
538	dev_err(chan->dev,	538	dev_err(chan->dev,
539	"No free memory for link descriptor\n");	539	"No free memory for link descriptor\n");
540	goto fail;	540	goto fail;
541	}	541	}
542	#ifdef FSL_DMA_LD_DEBUG	542	#ifdef FSL_DMA_LD_DEBUG
543	dev_dbg(chan->dev, "new link desc alloc %p\n", new);	543	dev_dbg(chan->dev, "new link desc alloc %p\n", new);
544	#endif	544	#endif
545		545
546	copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);	546	copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
547		547
548	set_desc_cnt(chan, &new->hw, copy);	548	set_desc_cnt(chan, &new->hw, copy);
549	set_desc_src(chan, &new->hw, dma_src);	549	set_desc_src(chan, &new->hw, dma_src);
550	set_desc_dst(chan, &new->hw, dma_dst);	550	set_desc_dst(chan, &new->hw, dma_dst);
551		551
552	if (!first)	552	if (!first)
553	first = new;	553	first = new;
554	else	554	else
555	set_desc_next(chan, &prev->hw, new->async_tx.phys);	555	set_desc_next(chan, &prev->hw, new->async_tx.phys);
556		556
557	new->async_tx.cookie = 0;	557	new->async_tx.cookie = 0;
558	async_tx_ack(&new->async_tx);	558	async_tx_ack(&new->async_tx);
559		559
560	prev = new;	560	prev = new;
561	len -= copy;	561	len -= copy;
562	dma_src += copy;	562	dma_src += copy;
563	dma_dst += copy;	563	dma_dst += copy;
564		564
565	/* Insert the link descriptor to the LD ring */	565	/* Insert the link descriptor to the LD ring */
566	list_add_tail(&new->node, &first->tx_list);	566	list_add_tail(&new->node, &first->tx_list);
567	} while (len);	567	} while (len);
568		568
569	new->async_tx.flags = flags; /* client is in control of this ack */	569	new->async_tx.flags = flags; /* client is in control of this ack */
570	new->async_tx.cookie = -EBUSY;	570	new->async_tx.cookie = -EBUSY;
571		571
572	/* Set End-of-link to the last link descriptor of new list*/	572	/* Set End-of-link to the last link descriptor of new list*/
573	set_ld_eol(chan, new);	573	set_ld_eol(chan, new);
574		574
575	return &first->async_tx;	575	return &first->async_tx;
576		576
577	fail:	577	fail:
578	if (!first)	578	if (!first)
579	return NULL;	579	return NULL;
580		580
581	fsldma_free_desc_list_reverse(chan, &first->tx_list);	581	fsldma_free_desc_list_reverse(chan, &first->tx_list);
582	return NULL;	582	return NULL;
583	}	583	}
584		584
585	/**	585	/**
586	* fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction	586	* fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
587	* @chan: DMA channel	587	* @chan: DMA channel
588	* @sgl: scatterlist to transfer to/from	588	* @sgl: scatterlist to transfer to/from
589	* @sg_len: number of entries in @scatterlist	589	* @sg_len: number of entries in @scatterlist
590	* @direction: DMA direction	590	* @direction: DMA direction
591	* @flags: DMAEngine flags	591	* @flags: DMAEngine flags
592	*	592	*
593	* Prepare a set of descriptors for a DMA_SLAVE transaction. Following the	593	* Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
594	* DMA_SLAVE API, this gets the device-specific information from the	594	* DMA_SLAVE API, this gets the device-specific information from the
595	* chan->private variable.	595	* chan->private variable.
596	*/	596	*/
597	static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(	597	static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
598	struct dma_chan dchan, struct scatterlist sgl, unsigned int sg_len,	598	struct dma_chan dchan, struct scatterlist sgl, unsigned int sg_len,
599	enum dma_data_direction direction, unsigned long flags)	599	enum dma_data_direction direction, unsigned long flags)
600	{	600	{
601	struct fsldma_chan *chan;	601	struct fsldma_chan *chan;
602	struct fsl_desc_sw first = NULL, prev = NULL, *new = NULL;	602	struct fsl_desc_sw first = NULL, prev = NULL, *new = NULL;
603	struct fsl_dma_slave *slave;	603	struct fsl_dma_slave *slave;
604	size_t copy;	604	size_t copy;
605		605
606	int i;	606	int i;
607	struct scatterlist *sg;	607	struct scatterlist *sg;
608	size_t sg_used;	608	size_t sg_used;
609	size_t hw_used;	609	size_t hw_used;
610	struct fsl_dma_hw_addr *hw;	610	struct fsl_dma_hw_addr *hw;
611	dma_addr_t dma_dst, dma_src;	611	dma_addr_t dma_dst, dma_src;
612		612
613	if (!dchan)	613	if (!dchan)
614	return NULL;	614	return NULL;
615		615
616	if (!dchan->private)	616	if (!dchan->private)
617	return NULL;	617	return NULL;
618		618
619	chan = to_fsl_chan(dchan);	619	chan = to_fsl_chan(dchan);
620	slave = dchan->private;	620	slave = dchan->private;
621		621
622	if (list_empty(&slave->addresses))	622	if (list_empty(&slave->addresses))
623	return NULL;	623	return NULL;
624		624
625	hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);	625	hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
626	hw_used = 0;	626	hw_used = 0;
627		627
628	/*	628	/*
629	* Build the hardware transaction to copy from the scatterlist to	629	* Build the hardware transaction to copy from the scatterlist to
630	* the hardware, or from the hardware to the scatterlist	630	* the hardware, or from the hardware to the scatterlist
631	*	631	*
632	* If you are copying from the hardware to the scatterlist and it	632	* If you are copying from the hardware to the scatterlist and it
633	* takes two hardware entries to fill an entire page, then both	633	* takes two hardware entries to fill an entire page, then both
634	* hardware entries will be coalesced into the same page	634	* hardware entries will be coalesced into the same page
635	*	635	*
636	* If you are copying from the scatterlist to the hardware and a	636	* If you are copying from the scatterlist to the hardware and a
637	* single page can fill two hardware entries, then the data will	637	* single page can fill two hardware entries, then the data will
638	* be read out of the page into the first hardware entry, and so on	638	* be read out of the page into the first hardware entry, and so on
639	*/	639	*/
640	for_each_sg(sgl, sg, sg_len, i) {	640	for_each_sg(sgl, sg, sg_len, i) {
641	sg_used = 0;	641	sg_used = 0;
642		642
643	/* Loop until the entire scatterlist entry is used */	643	/* Loop until the entire scatterlist entry is used */
644	while (sg_used < sg_dma_len(sg)) {	644	while (sg_used < sg_dma_len(sg)) {
645		645
646	/*	646	/*
647	* If we've used up the current hardware address/length	647	* If we've used up the current hardware address/length
648	* pair, we need to load a new one	648	* pair, we need to load a new one
649	*	649	*
650	* This is done in a while loop so that descriptors with	650	* This is done in a while loop so that descriptors with
651	* length == 0 will be skipped	651	* length == 0 will be skipped
652	*/	652	*/
653	while (hw_used >= hw->length) {	653	while (hw_used >= hw->length) {
654		654
655	/*	655	/*
656	* If the current hardware entry is the last	656	* If the current hardware entry is the last
657	* entry in the list, we're finished	657	* entry in the list, we're finished
658	*/	658	*/
659	if (list_is_last(&hw->entry, &slave->addresses))	659	if (list_is_last(&hw->entry, &slave->addresses))
660	goto finished;	660	goto finished;
661		661
662	/* Get the next hardware address/length pair */	662	/* Get the next hardware address/length pair */
663	hw = list_entry(hw->entry.next,	663	hw = list_entry(hw->entry.next,
664	struct fsl_dma_hw_addr, entry);	664	struct fsl_dma_hw_addr, entry);
665	hw_used = 0;	665	hw_used = 0;
666	}	666	}
667		667
668	/* Allocate the link descriptor from DMA pool */	668	/* Allocate the link descriptor from DMA pool */
669	new = fsl_dma_alloc_descriptor(chan);	669	new = fsl_dma_alloc_descriptor(chan);
670	if (!new) {	670	if (!new) {
671	dev_err(chan->dev, "No free memory for "	671	dev_err(chan->dev, "No free memory for "
672	"link descriptor\n");	672	"link descriptor\n");
673	goto fail;	673	goto fail;
674	}	674	}
675	#ifdef FSL_DMA_LD_DEBUG	675	#ifdef FSL_DMA_LD_DEBUG
676	dev_dbg(chan->dev, "new link desc alloc %p\n", new);	676	dev_dbg(chan->dev, "new link desc alloc %p\n", new);
677	#endif	677	#endif
678		678
679	/*	679	/*
680	* Calculate the maximum number of bytes to transfer,	680	* Calculate the maximum number of bytes to transfer,
681	* making sure it is less than the DMA controller limit	681	* making sure it is less than the DMA controller limit
682	*/	682	*/
683	copy = min_t(size_t, sg_dma_len(sg) - sg_used,	683	copy = min_t(size_t, sg_dma_len(sg) - sg_used,
684	hw->length - hw_used);	684	hw->length - hw_used);
685	copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);	685	copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
686		686
687	/*	687	/*
688	* DMA_FROM_DEVICE	688	* DMA_FROM_DEVICE
689	* from the hardware to the scatterlist	689	* from the hardware to the scatterlist
690	*	690	*
691	* DMA_TO_DEVICE	691	* DMA_TO_DEVICE
692	* from the scatterlist to the hardware	692	* from the scatterlist to the hardware
693	*/	693	*/
694	if (direction == DMA_FROM_DEVICE) {	694	if (direction == DMA_FROM_DEVICE) {
695	dma_src = hw->address + hw_used;	695	dma_src = hw->address + hw_used;
696	dma_dst = sg_dma_address(sg) + sg_used;	696	dma_dst = sg_dma_address(sg) + sg_used;
697	} else {	697	} else {
698	dma_src = sg_dma_address(sg) + sg_used;	698	dma_src = sg_dma_address(sg) + sg_used;
699	dma_dst = hw->address + hw_used;	699	dma_dst = hw->address + hw_used;
700	}	700	}
701		701
702	/* Fill in the descriptor */	702	/* Fill in the descriptor */
703	set_desc_cnt(chan, &new->hw, copy);	703	set_desc_cnt(chan, &new->hw, copy);
704	set_desc_src(chan, &new->hw, dma_src);	704	set_desc_src(chan, &new->hw, dma_src);
705	set_desc_dst(chan, &new->hw, dma_dst);	705	set_desc_dst(chan, &new->hw, dma_dst);
706		706
707	/*	707	/*
708	* If this is not the first descriptor, chain the	708	* If this is not the first descriptor, chain the
709	* current descriptor after the previous descriptor	709	* current descriptor after the previous descriptor
710	*/	710	*/
711	if (!first) {	711	if (!first) {
712	first = new;	712	first = new;
713	} else {	713	} else {
714	set_desc_next(chan, &prev->hw,	714	set_desc_next(chan, &prev->hw,
715	new->async_tx.phys);	715	new->async_tx.phys);
716	}	716	}
717		717
718	new->async_tx.cookie = 0;	718	new->async_tx.cookie = 0;
719	async_tx_ack(&new->async_tx);	719	async_tx_ack(&new->async_tx);
720		720
721	prev = new;	721	prev = new;
722	sg_used += copy;	722	sg_used += copy;
723	hw_used += copy;	723	hw_used += copy;
724		724
725	/* Insert the link descriptor into the LD ring */	725	/* Insert the link descriptor into the LD ring */
726	list_add_tail(&new->node, &first->tx_list);	726	list_add_tail(&new->node, &first->tx_list);
727	}	727	}
728	}	728	}
729		729
730	finished:	730	finished:
731		731
732	/* All of the hardware address/length pairs had length == 0 */	732	/* All of the hardware address/length pairs had length == 0 */
733	if (!first \|\| !new)	733	if (!first \|\| !new)
734	return NULL;	734	return NULL;
735		735
736	new->async_tx.flags = flags;	736	new->async_tx.flags = flags;
737	new->async_tx.cookie = -EBUSY;	737	new->async_tx.cookie = -EBUSY;
738		738
739	/* Set End-of-link to the last link descriptor of new list */	739	/* Set End-of-link to the last link descriptor of new list */
740	set_ld_eol(chan, new);	740	set_ld_eol(chan, new);
741		741
742	/* Enable extra controller features */	742	/* Enable extra controller features */
743	if (chan->set_src_loop_size)	743	if (chan->set_src_loop_size)
744	chan->set_src_loop_size(chan, slave->src_loop_size);	744	chan->set_src_loop_size(chan, slave->src_loop_size);
745		745
746	if (chan->set_dst_loop_size)	746	if (chan->set_dst_loop_size)
747	chan->set_dst_loop_size(chan, slave->dst_loop_size);	747	chan->set_dst_loop_size(chan, slave->dst_loop_size);
748		748
749	if (chan->toggle_ext_start)	749	if (chan->toggle_ext_start)
750	chan->toggle_ext_start(chan, slave->external_start);	750	chan->toggle_ext_start(chan, slave->external_start);
751		751
752	if (chan->toggle_ext_pause)	752	if (chan->toggle_ext_pause)
753	chan->toggle_ext_pause(chan, slave->external_pause);	753	chan->toggle_ext_pause(chan, slave->external_pause);
754		754
755	if (chan->set_request_count)	755	if (chan->set_request_count)
756	chan->set_request_count(chan, slave->request_count);	756	chan->set_request_count(chan, slave->request_count);
757		757
758	return &first->async_tx;	758	return &first->async_tx;
759		759
760	fail:	760	fail:
761	/* If first was not set, then we failed to allocate the very first	761	/* If first was not set, then we failed to allocate the very first
762	* descriptor, and we're done */	762	* descriptor, and we're done */
763	if (!first)	763	if (!first)
764	return NULL;	764	return NULL;
765		765
766	/*	766	/*
767	* First is set, so all of the descriptors we allocated have been added	767	* First is set, so all of the descriptors we allocated have been added
768	* to first->tx_list, INCLUDING "first" itself. Therefore we	768	* to first->tx_list, INCLUDING "first" itself. Therefore we
769	* must traverse the list backwards freeing each descriptor in turn	769	* must traverse the list backwards freeing each descriptor in turn
770	*	770	*
771	* We're re-using variables for the loop, oh well	771	* We're re-using variables for the loop, oh well
772	*/	772	*/
773	fsldma_free_desc_list_reverse(chan, &first->tx_list);	773	fsldma_free_desc_list_reverse(chan, &first->tx_list);
774	return NULL;	774	return NULL;
775	}	775	}
776		776
777	static int fsl_dma_device_control(struct dma_chan *dchan,	777	static int fsl_dma_device_control(struct dma_chan *dchan,
778	enum dma_ctrl_cmd cmd)	778	enum dma_ctrl_cmd cmd, unsigned long arg)
779	{	779	{
780	struct fsldma_chan *chan;	780	struct fsldma_chan *chan;
781	unsigned long flags;	781	unsigned long flags;
782		782
783	/* Only supports DMA_TERMINATE_ALL */	783	/* Only supports DMA_TERMINATE_ALL */
784	if (cmd != DMA_TERMINATE_ALL)	784	if (cmd != DMA_TERMINATE_ALL)
785	return -ENXIO;	785	return -ENXIO;
786		786
787	if (!dchan)	787	if (!dchan)
788	return -EINVAL;	788	return -EINVAL;
789		789
790	chan = to_fsl_chan(dchan);	790	chan = to_fsl_chan(dchan);
791		791
792	/* Halt the DMA engine */	792	/* Halt the DMA engine */
793	dma_halt(chan);	793	dma_halt(chan);
794		794
795	spin_lock_irqsave(&chan->desc_lock, flags);	795	spin_lock_irqsave(&chan->desc_lock, flags);
796		796
797	/* Remove and free all of the descriptors in the LD queue */	797	/* Remove and free all of the descriptors in the LD queue */
798	fsldma_free_desc_list(chan, &chan->ld_pending);	798	fsldma_free_desc_list(chan, &chan->ld_pending);
799	fsldma_free_desc_list(chan, &chan->ld_running);	799	fsldma_free_desc_list(chan, &chan->ld_running);
800		800
801	spin_unlock_irqrestore(&chan->desc_lock, flags);	801	spin_unlock_irqrestore(&chan->desc_lock, flags);
802		802
803	return 0;	803	return 0;
804	}	804	}
805		805
806	/**	806	/**
807	* fsl_dma_update_completed_cookie - Update the completed cookie.	807	* fsl_dma_update_completed_cookie - Update the completed cookie.
808	* @chan : Freescale DMA channel	808	* @chan : Freescale DMA channel
809	*	809	*
810	* CONTEXT: hardirq	810	* CONTEXT: hardirq
811	*/	811	*/
812	static void fsl_dma_update_completed_cookie(struct fsldma_chan *chan)	812	static void fsl_dma_update_completed_cookie(struct fsldma_chan *chan)
813	{	813	{
814	struct fsl_desc_sw *desc;	814	struct fsl_desc_sw *desc;
815	unsigned long flags;	815	unsigned long flags;
816	dma_cookie_t cookie;	816	dma_cookie_t cookie;
817		817
818	spin_lock_irqsave(&chan->desc_lock, flags);	818	spin_lock_irqsave(&chan->desc_lock, flags);
819		819
820	if (list_empty(&chan->ld_running)) {	820	if (list_empty(&chan->ld_running)) {
821	dev_dbg(chan->dev, "no running descriptors\n");	821	dev_dbg(chan->dev, "no running descriptors\n");
822	goto out_unlock;	822	goto out_unlock;
823	}	823	}
824		824
825	/* Get the last descriptor, update the cookie to that */	825	/* Get the last descriptor, update the cookie to that */
826	desc = to_fsl_desc(chan->ld_running.prev);	826	desc = to_fsl_desc(chan->ld_running.prev);
827	if (dma_is_idle(chan))	827	if (dma_is_idle(chan))
828	cookie = desc->async_tx.cookie;	828	cookie = desc->async_tx.cookie;
829	else {	829	else {
830	cookie = desc->async_tx.cookie - 1;	830	cookie = desc->async_tx.cookie - 1;
831	if (unlikely(cookie < DMA_MIN_COOKIE))	831	if (unlikely(cookie < DMA_MIN_COOKIE))
832	cookie = DMA_MAX_COOKIE;	832	cookie = DMA_MAX_COOKIE;
833	}	833	}
834		834
835	chan->completed_cookie = cookie;	835	chan->completed_cookie = cookie;
836		836
837	out_unlock:	837	out_unlock:
838	spin_unlock_irqrestore(&chan->desc_lock, flags);	838	spin_unlock_irqrestore(&chan->desc_lock, flags);
839	}	839	}
840		840
841	/**	841	/**
842	* fsldma_desc_status - Check the status of a descriptor	842	* fsldma_desc_status - Check the status of a descriptor
843	* @chan: Freescale DMA channel	843	* @chan: Freescale DMA channel
844	* @desc: DMA SW descriptor	844	* @desc: DMA SW descriptor
845	*	845	*
846	* This function will return the status of the given descriptor	846	* This function will return the status of the given descriptor
847	*/	847	*/
848	static enum dma_status fsldma_desc_status(struct fsldma_chan *chan,	848	static enum dma_status fsldma_desc_status(struct fsldma_chan *chan,
849	struct fsl_desc_sw *desc)	849	struct fsl_desc_sw *desc)
850	{	850	{
851	return dma_async_is_complete(desc->async_tx.cookie,	851	return dma_async_is_complete(desc->async_tx.cookie,
852	chan->completed_cookie,	852	chan->completed_cookie,
853	chan->common.cookie);	853	chan->common.cookie);
854	}	854	}
855		855
856	/**	856	/**
857	* fsl_chan_ld_cleanup - Clean up link descriptors	857	* fsl_chan_ld_cleanup - Clean up link descriptors
858	* @chan : Freescale DMA channel	858	* @chan : Freescale DMA channel
859	*	859	*
860	* This function clean up the ld_queue of DMA channel.	860	* This function clean up the ld_queue of DMA channel.
861	*/	861	*/
862	static void fsl_chan_ld_cleanup(struct fsldma_chan *chan)	862	static void fsl_chan_ld_cleanup(struct fsldma_chan *chan)
863	{	863	{
864	struct fsl_desc_sw desc, _desc;	864	struct fsl_desc_sw desc, _desc;
865	unsigned long flags;	865	unsigned long flags;
866		866
867	spin_lock_irqsave(&chan->desc_lock, flags);	867	spin_lock_irqsave(&chan->desc_lock, flags);
868		868
869	dev_dbg(chan->dev, "chan completed_cookie = %d\n", chan->completed_cookie);	869	dev_dbg(chan->dev, "chan completed_cookie = %d\n", chan->completed_cookie);
870	list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {	870	list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {
871	dma_async_tx_callback callback;	871	dma_async_tx_callback callback;
872	void *callback_param;	872	void *callback_param;
873		873
874	if (fsldma_desc_status(chan, desc) == DMA_IN_PROGRESS)	874	if (fsldma_desc_status(chan, desc) == DMA_IN_PROGRESS)
875	break;	875	break;
876		876
877	/* Remove from the list of running transactions */	877	/* Remove from the list of running transactions */
878	list_del(&desc->node);	878	list_del(&desc->node);
879		879
880	/* Run the link descriptor callback function */	880	/* Run the link descriptor callback function */
881	callback = desc->async_tx.callback;	881	callback = desc->async_tx.callback;
882	callback_param = desc->async_tx.callback_param;	882	callback_param = desc->async_tx.callback_param;
883	if (callback) {	883	if (callback) {
884	spin_unlock_irqrestore(&chan->desc_lock, flags);	884	spin_unlock_irqrestore(&chan->desc_lock, flags);
885	dev_dbg(chan->dev, "LD %p callback\n", desc);	885	dev_dbg(chan->dev, "LD %p callback\n", desc);
886	callback(callback_param);	886	callback(callback_param);
887	spin_lock_irqsave(&chan->desc_lock, flags);	887	spin_lock_irqsave(&chan->desc_lock, flags);
888	}	888	}
889		889
890	/* Run any dependencies, then free the descriptor */	890	/* Run any dependencies, then free the descriptor */
891	dma_run_dependencies(&desc->async_tx);	891	dma_run_dependencies(&desc->async_tx);
892	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);	892	dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
893	}	893	}
894		894
895	spin_unlock_irqrestore(&chan->desc_lock, flags);	895	spin_unlock_irqrestore(&chan->desc_lock, flags);
896	}	896	}
897		897
898	/**	898	/**
899	* fsl_chan_xfer_ld_queue - transfer any pending transactions	899	* fsl_chan_xfer_ld_queue - transfer any pending transactions
900	* @chan : Freescale DMA channel	900	* @chan : Freescale DMA channel
901	*	901	*
902	* This will make sure that any pending transactions will be run.	902	* This will make sure that any pending transactions will be run.
903	* If the DMA controller is idle, it will be started. Otherwise,	903	* If the DMA controller is idle, it will be started. Otherwise,
904	* the DMA controller's interrupt handler will start any pending	904	* the DMA controller's interrupt handler will start any pending
905	* transactions when it becomes idle.	905	* transactions when it becomes idle.
906	*/	906	*/
907	static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)	907	static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
908	{	908	{
909	struct fsl_desc_sw *desc;	909	struct fsl_desc_sw *desc;
910	unsigned long flags;	910	unsigned long flags;
911		911
912	spin_lock_irqsave(&chan->desc_lock, flags);	912	spin_lock_irqsave(&chan->desc_lock, flags);
913		913
914	/*	914	/*
915	* If the list of pending descriptors is empty, then we	915	* If the list of pending descriptors is empty, then we
916	* don't need to do any work at all	916	* don't need to do any work at all
917	*/	917	*/
918	if (list_empty(&chan->ld_pending)) {	918	if (list_empty(&chan->ld_pending)) {
919	dev_dbg(chan->dev, "no pending LDs\n");	919	dev_dbg(chan->dev, "no pending LDs\n");
920	goto out_unlock;	920	goto out_unlock;
921	}	921	}
922		922
923	/*	923	/*
924	* The DMA controller is not idle, which means the interrupt	924	* The DMA controller is not idle, which means the interrupt
925	* handler will start any queued transactions when it runs	925	* handler will start any queued transactions when it runs
926	* at the end of the current transaction	926	* at the end of the current transaction
927	*/	927	*/
928	if (!dma_is_idle(chan)) {	928	if (!dma_is_idle(chan)) {
929	dev_dbg(chan->dev, "DMA controller still busy\n");	929	dev_dbg(chan->dev, "DMA controller still busy\n");
930	goto out_unlock;	930	goto out_unlock;
931	}	931	}
932		932
933	/*	933	/*
934	* TODO:	934	* TODO:
935	* make sure the dma_halt() function really un-wedges the	935	* make sure the dma_halt() function really un-wedges the
936	* controller as much as possible	936	* controller as much as possible
937	*/	937	*/
938	dma_halt(chan);	938	dma_halt(chan);
939		939
940	/*	940	/*
941	* If there are some link descriptors which have not been	941	* If there are some link descriptors which have not been
942	* transferred, we need to start the controller	942	* transferred, we need to start the controller
943	*/	943	*/
944		944
945	/*	945	/*
946	* Move all elements from the queue of pending transactions	946	* Move all elements from the queue of pending transactions
947	* onto the list of running transactions	947	* onto the list of running transactions
948	*/	948	*/
949	desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);	949	desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
950	list_splice_tail_init(&chan->ld_pending, &chan->ld_running);	950	list_splice_tail_init(&chan->ld_pending, &chan->ld_running);
951		951
952	/*	952	/*
953	* Program the descriptor's address into the DMA controller,	953	* Program the descriptor's address into the DMA controller,
954	* then start the DMA transaction	954	* then start the DMA transaction
955	*/	955	*/
956	set_cdar(chan, desc->async_tx.phys);	956	set_cdar(chan, desc->async_tx.phys);
957	dma_start(chan);	957	dma_start(chan);
958		958
959	out_unlock:	959	out_unlock:
960	spin_unlock_irqrestore(&chan->desc_lock, flags);	960	spin_unlock_irqrestore(&chan->desc_lock, flags);
961	}	961	}
962		962
963	/**	963	/**
964	* fsl_dma_memcpy_issue_pending - Issue the DMA start command	964	* fsl_dma_memcpy_issue_pending - Issue the DMA start command
965	* @chan : Freescale DMA channel	965	* @chan : Freescale DMA channel
966	*/	966	*/
967	static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)	967	static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
968	{	968	{
969	struct fsldma_chan *chan = to_fsl_chan(dchan);	969	struct fsldma_chan *chan = to_fsl_chan(dchan);
970	fsl_chan_xfer_ld_queue(chan);	970	fsl_chan_xfer_ld_queue(chan);
971	}	971	}
972		972
973	/**	973	/**
974	* fsl_tx_status - Determine the DMA status	974	* fsl_tx_status - Determine the DMA status
975	* @chan : Freescale DMA channel	975	* @chan : Freescale DMA channel
976	*/	976	*/
977	static enum dma_status fsl_tx_status(struct dma_chan *dchan,	977	static enum dma_status fsl_tx_status(struct dma_chan *dchan,
978	dma_cookie_t cookie,	978	dma_cookie_t cookie,
979	struct dma_tx_state *txstate)	979	struct dma_tx_state *txstate)
980	{	980	{
981	struct fsldma_chan *chan = to_fsl_chan(dchan);	981	struct fsldma_chan *chan = to_fsl_chan(dchan);
982	dma_cookie_t last_used;	982	dma_cookie_t last_used;
983	dma_cookie_t last_complete;	983	dma_cookie_t last_complete;
984		984
985	fsl_chan_ld_cleanup(chan);	985	fsl_chan_ld_cleanup(chan);
986		986
987	last_used = dchan->cookie;	987	last_used = dchan->cookie;
988	last_complete = chan->completed_cookie;	988	last_complete = chan->completed_cookie;
989		989
990	dma_set_tx_state(txstate, last_complete, last_used, 0);	990	dma_set_tx_state(txstate, last_complete, last_used, 0);
991		991
992	return dma_async_is_complete(cookie, last_complete, last_used);	992	return dma_async_is_complete(cookie, last_complete, last_used);
993	}	993	}
994		994
995	/----------------------------------------------------------------------------/	995	/----------------------------------------------------------------------------/
996	/* Interrupt Handling */	996	/* Interrupt Handling */
997	/----------------------------------------------------------------------------/	997	/----------------------------------------------------------------------------/
998		998
999	static irqreturn_t fsldma_chan_irq(int irq, void *data)	999	static irqreturn_t fsldma_chan_irq(int irq, void *data)
1000	{	1000	{
1001	struct fsldma_chan *chan = data;	1001	struct fsldma_chan *chan = data;
1002	int update_cookie = 0;	1002	int update_cookie = 0;
1003	int xfer_ld_q = 0;	1003	int xfer_ld_q = 0;
1004	u32 stat;	1004	u32 stat;
1005		1005
1006	/* save and clear the status register */	1006	/* save and clear the status register */
1007	stat = get_sr(chan);	1007	stat = get_sr(chan);
1008	set_sr(chan, stat);	1008	set_sr(chan, stat);
1009	dev_dbg(chan->dev, "irq: channel %d, stat = 0x%x\n", chan->id, stat);	1009	dev_dbg(chan->dev, "irq: channel %d, stat = 0x%x\n", chan->id, stat);
1010		1010
1011	stat &= ~(FSL_DMA_SR_CB \| FSL_DMA_SR_CH);	1011	stat &= ~(FSL_DMA_SR_CB \| FSL_DMA_SR_CH);
1012	if (!stat)	1012	if (!stat)
1013	return IRQ_NONE;	1013	return IRQ_NONE;
1014		1014
1015	if (stat & FSL_DMA_SR_TE)	1015	if (stat & FSL_DMA_SR_TE)
1016	dev_err(chan->dev, "Transfer Error!\n");	1016	dev_err(chan->dev, "Transfer Error!\n");
1017		1017
1018	/*	1018	/*
1019	* Programming Error	1019	* Programming Error
1020	* The DMA_INTERRUPT async_tx is a NULL transfer, which will	1020	* The DMA_INTERRUPT async_tx is a NULL transfer, which will
1021	* triger a PE interrupt.	1021	* triger a PE interrupt.
1022	*/	1022	*/
1023	if (stat & FSL_DMA_SR_PE) {	1023	if (stat & FSL_DMA_SR_PE) {
1024	dev_dbg(chan->dev, "irq: Programming Error INT\n");	1024	dev_dbg(chan->dev, "irq: Programming Error INT\n");
1025	if (get_bcr(chan) == 0) {	1025	if (get_bcr(chan) == 0) {
1026	/* BCR register is 0, this is a DMA_INTERRUPT async_tx.	1026	/* BCR register is 0, this is a DMA_INTERRUPT async_tx.
1027	* Now, update the completed cookie, and continue the	1027	* Now, update the completed cookie, and continue the
1028	* next uncompleted transfer.	1028	* next uncompleted transfer.
1029	*/	1029	*/
1030	update_cookie = 1;	1030	update_cookie = 1;
1031	xfer_ld_q = 1;	1031	xfer_ld_q = 1;
1032	}	1032	}
1033	stat &= ~FSL_DMA_SR_PE;	1033	stat &= ~FSL_DMA_SR_PE;
1034	}	1034	}
1035		1035
1036	/*	1036	/*
1037	* If the link descriptor segment transfer finishes,	1037	* If the link descriptor segment transfer finishes,
1038	* we will recycle the used descriptor.	1038	* we will recycle the used descriptor.
1039	*/	1039	*/
1040	if (stat & FSL_DMA_SR_EOSI) {	1040	if (stat & FSL_DMA_SR_EOSI) {
1041	dev_dbg(chan->dev, "irq: End-of-segments INT\n");	1041	dev_dbg(chan->dev, "irq: End-of-segments INT\n");
1042	dev_dbg(chan->dev, "irq: clndar 0x%llx, nlndar 0x%llx\n",	1042	dev_dbg(chan->dev, "irq: clndar 0x%llx, nlndar 0x%llx\n",
1043	(unsigned long long)get_cdar(chan),	1043	(unsigned long long)get_cdar(chan),
1044	(unsigned long long)get_ndar(chan));	1044	(unsigned long long)get_ndar(chan));
1045	stat &= ~FSL_DMA_SR_EOSI;	1045	stat &= ~FSL_DMA_SR_EOSI;
1046	update_cookie = 1;	1046	update_cookie = 1;
1047	}	1047	}
1048		1048
1049	/*	1049	/*
1050	* For MPC8349, EOCDI event need to update cookie	1050	* For MPC8349, EOCDI event need to update cookie
1051	* and start the next transfer if it exist.	1051	* and start the next transfer if it exist.
1052	*/	1052	*/
1053	if (stat & FSL_DMA_SR_EOCDI) {	1053	if (stat & FSL_DMA_SR_EOCDI) {
1054	dev_dbg(chan->dev, "irq: End-of-Chain link INT\n");	1054	dev_dbg(chan->dev, "irq: End-of-Chain link INT\n");
1055	stat &= ~FSL_DMA_SR_EOCDI;	1055	stat &= ~FSL_DMA_SR_EOCDI;
1056	update_cookie = 1;	1056	update_cookie = 1;
1057	xfer_ld_q = 1;	1057	xfer_ld_q = 1;
1058	}	1058	}
1059		1059
1060	/*	1060	/*
1061	* If it current transfer is the end-of-transfer,	1061	* If it current transfer is the end-of-transfer,
1062	* we should clear the Channel Start bit for	1062	* we should clear the Channel Start bit for
1063	* prepare next transfer.	1063	* prepare next transfer.
1064	*/	1064	*/
1065	if (stat & FSL_DMA_SR_EOLNI) {	1065	if (stat & FSL_DMA_SR_EOLNI) {
1066	dev_dbg(chan->dev, "irq: End-of-link INT\n");	1066	dev_dbg(chan->dev, "irq: End-of-link INT\n");
1067	stat &= ~FSL_DMA_SR_EOLNI;	1067	stat &= ~FSL_DMA_SR_EOLNI;
1068	xfer_ld_q = 1;	1068	xfer_ld_q = 1;
1069	}	1069	}
1070		1070
1071	if (update_cookie)	1071	if (update_cookie)
1072	fsl_dma_update_completed_cookie(chan);	1072	fsl_dma_update_completed_cookie(chan);
1073	if (xfer_ld_q)	1073	if (xfer_ld_q)
1074	fsl_chan_xfer_ld_queue(chan);	1074	fsl_chan_xfer_ld_queue(chan);
1075	if (stat)	1075	if (stat)
1076	dev_dbg(chan->dev, "irq: unhandled sr 0x%02x\n", stat);	1076	dev_dbg(chan->dev, "irq: unhandled sr 0x%02x\n", stat);
1077		1077
1078	dev_dbg(chan->dev, "irq: Exit\n");	1078	dev_dbg(chan->dev, "irq: Exit\n");
1079	tasklet_schedule(&chan->tasklet);	1079	tasklet_schedule(&chan->tasklet);
1080	return IRQ_HANDLED;	1080	return IRQ_HANDLED;
1081	}	1081	}
1082		1082
1083	static void dma_do_tasklet(unsigned long data)	1083	static void dma_do_tasklet(unsigned long data)
1084	{	1084	{
1085	struct fsldma_chan chan = (struct fsldma_chan )data;	1085	struct fsldma_chan chan = (struct fsldma_chan )data;
1086	fsl_chan_ld_cleanup(chan);	1086	fsl_chan_ld_cleanup(chan);
1087	}	1087	}
1088		1088
1089	static irqreturn_t fsldma_ctrl_irq(int irq, void *data)	1089	static irqreturn_t fsldma_ctrl_irq(int irq, void *data)
1090	{	1090	{
1091	struct fsldma_device *fdev = data;	1091	struct fsldma_device *fdev = data;
1092	struct fsldma_chan *chan;	1092	struct fsldma_chan *chan;
1093	unsigned int handled = 0;	1093	unsigned int handled = 0;
1094	u32 gsr, mask;	1094	u32 gsr, mask;
1095	int i;	1095	int i;
1096		1096
1097	gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)	1097	gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)
1098	: in_le32(fdev->regs);	1098	: in_le32(fdev->regs);
1099	mask = 0xff000000;	1099	mask = 0xff000000;
1100	dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);	1100	dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);
1101		1101
1102	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {	1102	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1103	chan = fdev->chan[i];	1103	chan = fdev->chan[i];
1104	if (!chan)	1104	if (!chan)
1105	continue;	1105	continue;
1106		1106
1107	if (gsr & mask) {	1107	if (gsr & mask) {
1108	dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);	1108	dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);
1109	fsldma_chan_irq(irq, chan);	1109	fsldma_chan_irq(irq, chan);
1110	handled++;	1110	handled++;
1111	}	1111	}
1112		1112
1113	gsr &= ~mask;	1113	gsr &= ~mask;
1114	mask >>= 8;	1114	mask >>= 8;
1115	}	1115	}
1116		1116
1117	return IRQ_RETVAL(handled);	1117	return IRQ_RETVAL(handled);
1118	}	1118	}
1119		1119
1120	static void fsldma_free_irqs(struct fsldma_device *fdev)	1120	static void fsldma_free_irqs(struct fsldma_device *fdev)
1121	{	1121	{
1122	struct fsldma_chan *chan;	1122	struct fsldma_chan *chan;
1123	int i;	1123	int i;
1124		1124
1125	if (fdev->irq != NO_IRQ) {	1125	if (fdev->irq != NO_IRQ) {
1126	dev_dbg(fdev->dev, "free per-controller IRQ\n");	1126	dev_dbg(fdev->dev, "free per-controller IRQ\n");
1127	free_irq(fdev->irq, fdev);	1127	free_irq(fdev->irq, fdev);
1128	return;	1128	return;
1129	}	1129	}
1130		1130
1131	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {	1131	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1132	chan = fdev->chan[i];	1132	chan = fdev->chan[i];
1133	if (chan && chan->irq != NO_IRQ) {	1133	if (chan && chan->irq != NO_IRQ) {
1134	dev_dbg(fdev->dev, "free channel %d IRQ\n", chan->id);	1134	dev_dbg(fdev->dev, "free channel %d IRQ\n", chan->id);
1135	free_irq(chan->irq, chan);	1135	free_irq(chan->irq, chan);
1136	}	1136	}
1137	}	1137	}
1138	}	1138	}
1139		1139
1140	static int fsldma_request_irqs(struct fsldma_device *fdev)	1140	static int fsldma_request_irqs(struct fsldma_device *fdev)
1141	{	1141	{
1142	struct fsldma_chan *chan;	1142	struct fsldma_chan *chan;
1143	int ret;	1143	int ret;
1144	int i;	1144	int i;
1145		1145
1146	/* if we have a per-controller IRQ, use that */	1146	/* if we have a per-controller IRQ, use that */
1147	if (fdev->irq != NO_IRQ) {	1147	if (fdev->irq != NO_IRQ) {
1148	dev_dbg(fdev->dev, "request per-controller IRQ\n");	1148	dev_dbg(fdev->dev, "request per-controller IRQ\n");
1149	ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,	1149	ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
1150	"fsldma-controller", fdev);	1150	"fsldma-controller", fdev);
1151	return ret;	1151	return ret;
1152	}	1152	}
1153		1153
1154	/* no per-controller IRQ, use the per-channel IRQs */	1154	/* no per-controller IRQ, use the per-channel IRQs */
1155	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {	1155	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1156	chan = fdev->chan[i];	1156	chan = fdev->chan[i];
1157	if (!chan)	1157	if (!chan)
1158	continue;	1158	continue;
1159		1159
1160	if (chan->irq == NO_IRQ) {	1160	if (chan->irq == NO_IRQ) {
1161	dev_err(fdev->dev, "no interrupts property defined for "	1161	dev_err(fdev->dev, "no interrupts property defined for "
1162	"DMA channel %d. Please fix your "	1162	"DMA channel %d. Please fix your "
1163	"device tree\n", chan->id);	1163	"device tree\n", chan->id);
1164	ret = -ENODEV;	1164	ret = -ENODEV;
1165	goto out_unwind;	1165	goto out_unwind;
1166	}	1166	}
1167		1167
1168	dev_dbg(fdev->dev, "request channel %d IRQ\n", chan->id);	1168	dev_dbg(fdev->dev, "request channel %d IRQ\n", chan->id);
1169	ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,	1169	ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
1170	"fsldma-chan", chan);	1170	"fsldma-chan", chan);
1171	if (ret) {	1171	if (ret) {
1172	dev_err(fdev->dev, "unable to request IRQ for DMA "	1172	dev_err(fdev->dev, "unable to request IRQ for DMA "
1173	"channel %d\n", chan->id);	1173	"channel %d\n", chan->id);
1174	goto out_unwind;	1174	goto out_unwind;
1175	}	1175	}
1176	}	1176	}
1177		1177
1178	return 0;	1178	return 0;
1179		1179
1180	out_unwind:	1180	out_unwind:
1181	for (/* none */; i >= 0; i--) {	1181	for (/* none */; i >= 0; i--) {
1182	chan = fdev->chan[i];	1182	chan = fdev->chan[i];
1183	if (!chan)	1183	if (!chan)
1184	continue;	1184	continue;
1185		1185
1186	if (chan->irq == NO_IRQ)	1186	if (chan->irq == NO_IRQ)
1187	continue;	1187	continue;
1188		1188
1189	free_irq(chan->irq, chan);	1189	free_irq(chan->irq, chan);
1190	}	1190	}
1191		1191
1192	return ret;	1192	return ret;
1193	}	1193	}
1194		1194
1195	/----------------------------------------------------------------------------/	1195	/----------------------------------------------------------------------------/
1196	/* OpenFirmware Subsystem */	1196	/* OpenFirmware Subsystem */
1197	/----------------------------------------------------------------------------/	1197	/----------------------------------------------------------------------------/
1198		1198
1199	static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,	1199	static int __devinit fsl_dma_chan_probe(struct fsldma_device *fdev,
1200	struct device_node node, u32 feature, const char compatible)	1200	struct device_node node, u32 feature, const char compatible)
1201	{	1201	{
1202	struct fsldma_chan *chan;	1202	struct fsldma_chan *chan;
1203	struct resource res;	1203	struct resource res;
1204	int err;	1204	int err;
1205		1205
1206	/* alloc channel */	1206	/* alloc channel */
1207	chan = kzalloc(sizeof(*chan), GFP_KERNEL);	1207	chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1208	if (!chan) {	1208	if (!chan) {
1209	dev_err(fdev->dev, "no free memory for DMA channels!\n");	1209	dev_err(fdev->dev, "no free memory for DMA channels!\n");
1210	err = -ENOMEM;	1210	err = -ENOMEM;
1211	goto out_return;	1211	goto out_return;
1212	}	1212	}
1213		1213
1214	/* ioremap registers for use */	1214	/* ioremap registers for use */
1215	chan->regs = of_iomap(node, 0);	1215	chan->regs = of_iomap(node, 0);
1216	if (!chan->regs) {	1216	if (!chan->regs) {
1217	dev_err(fdev->dev, "unable to ioremap registers\n");	1217	dev_err(fdev->dev, "unable to ioremap registers\n");
1218	err = -ENOMEM;	1218	err = -ENOMEM;
1219	goto out_free_chan;	1219	goto out_free_chan;
1220	}	1220	}
1221		1221
1222	err = of_address_to_resource(node, 0, &res);	1222	err = of_address_to_resource(node, 0, &res);
1223	if (err) {	1223	if (err) {
1224	dev_err(fdev->dev, "unable to find 'reg' property\n");	1224	dev_err(fdev->dev, "unable to find 'reg' property\n");
1225	goto out_iounmap_regs;	1225	goto out_iounmap_regs;
1226	}	1226	}
1227		1227
1228	chan->feature = feature;	1228	chan->feature = feature;
1229	if (!fdev->feature)	1229	if (!fdev->feature)
1230	fdev->feature = chan->feature;	1230	fdev->feature = chan->feature;
1231		1231
1232	/*	1232	/*
1233	* If the DMA device's feature is different than the feature	1233	* If the DMA device's feature is different than the feature
1234	* of its channels, report the bug	1234	* of its channels, report the bug
1235	*/	1235	*/
1236	WARN_ON(fdev->feature != chan->feature);	1236	WARN_ON(fdev->feature != chan->feature);
1237		1237
1238	chan->dev = fdev->dev;	1238	chan->dev = fdev->dev;
1239	chan->id = ((res.start - 0x100) & 0xfff) >> 7;	1239	chan->id = ((res.start - 0x100) & 0xfff) >> 7;
1240	if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {	1240	if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {
1241	dev_err(fdev->dev, "too many channels for device\n");	1241	dev_err(fdev->dev, "too many channels for device\n");
1242	err = -EINVAL;	1242	err = -EINVAL;
1243	goto out_iounmap_regs;	1243	goto out_iounmap_regs;
1244	}	1244	}
1245		1245
1246	fdev->chan[chan->id] = chan;	1246	fdev->chan[chan->id] = chan;
1247	tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);	1247	tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
1248		1248
1249	/* Initialize the channel */	1249	/* Initialize the channel */
1250	dma_init(chan);	1250	dma_init(chan);
1251		1251
1252	/* Clear cdar registers */	1252	/* Clear cdar registers */
1253	set_cdar(chan, 0);	1253	set_cdar(chan, 0);
1254		1254
1255	switch (chan->feature & FSL_DMA_IP_MASK) {	1255	switch (chan->feature & FSL_DMA_IP_MASK) {
1256	case FSL_DMA_IP_85XX:	1256	case FSL_DMA_IP_85XX:
1257	chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;	1257	chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
1258	case FSL_DMA_IP_83XX:	1258	case FSL_DMA_IP_83XX:
1259	chan->toggle_ext_start = fsl_chan_toggle_ext_start;	1259	chan->toggle_ext_start = fsl_chan_toggle_ext_start;
1260	chan->set_src_loop_size = fsl_chan_set_src_loop_size;	1260	chan->set_src_loop_size = fsl_chan_set_src_loop_size;
1261	chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;	1261	chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;
1262	chan->set_request_count = fsl_chan_set_request_count;	1262	chan->set_request_count = fsl_chan_set_request_count;
1263	}	1263	}
1264		1264
1265	spin_lock_init(&chan->desc_lock);	1265	spin_lock_init(&chan->desc_lock);
1266	INIT_LIST_HEAD(&chan->ld_pending);	1266	INIT_LIST_HEAD(&chan->ld_pending);
1267	INIT_LIST_HEAD(&chan->ld_running);	1267	INIT_LIST_HEAD(&chan->ld_running);
1268		1268
1269	chan->common.device = &fdev->common;	1269	chan->common.device = &fdev->common;
1270		1270
1271	/* find the IRQ line, if it exists in the device tree */	1271	/* find the IRQ line, if it exists in the device tree */
1272	chan->irq = irq_of_parse_and_map(node, 0);	1272	chan->irq = irq_of_parse_and_map(node, 0);
1273		1273
1274	/* Add the channel to DMA device channel list */	1274	/* Add the channel to DMA device channel list */
1275	list_add_tail(&chan->common.device_node, &fdev->common.channels);	1275	list_add_tail(&chan->common.device_node, &fdev->common.channels);
1276	fdev->common.chancnt++;	1276	fdev->common.chancnt++;
1277		1277
1278	dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,	1278	dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
1279	chan->irq != NO_IRQ ? chan->irq : fdev->irq);	1279	chan->irq != NO_IRQ ? chan->irq : fdev->irq);
1280		1280
1281	return 0;	1281	return 0;
1282		1282
1283	out_iounmap_regs:	1283	out_iounmap_regs:
1284	iounmap(chan->regs);	1284	iounmap(chan->regs);
1285	out_free_chan:	1285	out_free_chan:
1286	kfree(chan);	1286	kfree(chan);
1287	out_return:	1287	out_return:
1288	return err;	1288	return err;
1289	}	1289	}
1290		1290
1291	static void fsl_dma_chan_remove(struct fsldma_chan *chan)	1291	static void fsl_dma_chan_remove(struct fsldma_chan *chan)
1292	{	1292	{
1293	irq_dispose_mapping(chan->irq);	1293	irq_dispose_mapping(chan->irq);
1294	list_del(&chan->common.device_node);	1294	list_del(&chan->common.device_node);
1295	iounmap(chan->regs);	1295	iounmap(chan->regs);
1296	kfree(chan);	1296	kfree(chan);
1297	}	1297	}
1298		1298
1299	static int __devinit fsldma_of_probe(struct of_device *op,	1299	static int __devinit fsldma_of_probe(struct of_device *op,
1300	const struct of_device_id *match)	1300	const struct of_device_id *match)
1301	{	1301	{
1302	struct fsldma_device *fdev;	1302	struct fsldma_device *fdev;
1303	struct device_node *child;	1303	struct device_node *child;
1304	int err;	1304	int err;
1305		1305
1306	fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);	1306	fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);
1307	if (!fdev) {	1307	if (!fdev) {
1308	dev_err(&op->dev, "No enough memory for 'priv'\n");	1308	dev_err(&op->dev, "No enough memory for 'priv'\n");
1309	err = -ENOMEM;	1309	err = -ENOMEM;
1310	goto out_return;	1310	goto out_return;
1311	}	1311	}
1312		1312
1313	fdev->dev = &op->dev;	1313	fdev->dev = &op->dev;
1314	INIT_LIST_HEAD(&fdev->common.channels);	1314	INIT_LIST_HEAD(&fdev->common.channels);
1315		1315
1316	/* ioremap the registers for use */	1316	/* ioremap the registers for use */
1317	fdev->regs = of_iomap(op->node, 0);	1317	fdev->regs = of_iomap(op->node, 0);
1318	if (!fdev->regs) {	1318	if (!fdev->regs) {
1319	dev_err(&op->dev, "unable to ioremap registers\n");	1319	dev_err(&op->dev, "unable to ioremap registers\n");
1320	err = -ENOMEM;	1320	err = -ENOMEM;
1321	goto out_free_fdev;	1321	goto out_free_fdev;
1322	}	1322	}
1323		1323
1324	/* map the channel IRQ if it exists, but don't hookup the handler yet */	1324	/* map the channel IRQ if it exists, but don't hookup the handler yet */
1325	fdev->irq = irq_of_parse_and_map(op->node, 0);	1325	fdev->irq = irq_of_parse_and_map(op->node, 0);
1326		1326
1327	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);	1327	dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
1328	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);	1328	dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
1329	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);	1329	dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
1330	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;	1330	fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
1331	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;	1331	fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
1332	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;	1332	fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
1333	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;	1333	fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
1334	fdev->common.device_tx_status = fsl_tx_status;	1334	fdev->common.device_tx_status = fsl_tx_status;
1335	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;	1335	fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
1336	fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;	1336	fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
1337	fdev->common.device_control = fsl_dma_device_control;	1337	fdev->common.device_control = fsl_dma_device_control;
1338	fdev->common.dev = &op->dev;	1338	fdev->common.dev = &op->dev;
1339		1339
1340	dev_set_drvdata(&op->dev, fdev);	1340	dev_set_drvdata(&op->dev, fdev);
1341		1341
1342	/*	1342	/*
1343	* We cannot use of_platform_bus_probe() because there is no	1343	* We cannot use of_platform_bus_probe() because there is no
1344	* of_platform_bus_remove(). Instead, we manually instantiate every DMA	1344	* of_platform_bus_remove(). Instead, we manually instantiate every DMA
1345	* channel object.	1345	* channel object.
1346	*/	1346	*/
1347	for_each_child_of_node(op->node, child) {	1347	for_each_child_of_node(op->node, child) {
1348	if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {	1348	if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {
1349	fsl_dma_chan_probe(fdev, child,	1349	fsl_dma_chan_probe(fdev, child,
1350	FSL_DMA_IP_85XX \| FSL_DMA_BIG_ENDIAN,	1350	FSL_DMA_IP_85XX \| FSL_DMA_BIG_ENDIAN,
1351	"fsl,eloplus-dma-channel");	1351	"fsl,eloplus-dma-channel");
1352	}	1352	}
1353		1353
1354	if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {	1354	if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {
1355	fsl_dma_chan_probe(fdev, child,	1355	fsl_dma_chan_probe(fdev, child,
1356	FSL_DMA_IP_83XX \| FSL_DMA_LITTLE_ENDIAN,	1356	FSL_DMA_IP_83XX \| FSL_DMA_LITTLE_ENDIAN,
1357	"fsl,elo-dma-channel");	1357	"fsl,elo-dma-channel");
1358	}	1358	}
1359	}	1359	}
1360		1360
1361	/*	1361	/*
1362	* Hookup the IRQ handler(s)	1362	* Hookup the IRQ handler(s)
1363	*	1363	*
1364	* If we have a per-controller interrupt, we prefer that to the	1364	* If we have a per-controller interrupt, we prefer that to the
1365	* per-channel interrupts to reduce the number of shared interrupt	1365	* per-channel interrupts to reduce the number of shared interrupt
1366	* handlers on the same IRQ line	1366	* handlers on the same IRQ line
1367	*/	1367	*/
1368	err = fsldma_request_irqs(fdev);	1368	err = fsldma_request_irqs(fdev);
1369	if (err) {	1369	if (err) {
1370	dev_err(fdev->dev, "unable to request IRQs\n");	1370	dev_err(fdev->dev, "unable to request IRQs\n");
1371	goto out_free_fdev;	1371	goto out_free_fdev;
1372	}	1372	}
1373		1373
1374	dma_async_device_register(&fdev->common);	1374	dma_async_device_register(&fdev->common);
1375	return 0;	1375	return 0;
1376		1376
1377	out_free_fdev:	1377	out_free_fdev:
1378	irq_dispose_mapping(fdev->irq);	1378	irq_dispose_mapping(fdev->irq);
1379	kfree(fdev);	1379	kfree(fdev);
1380	out_return:	1380	out_return:
1381	return err;	1381	return err;
1382	}	1382	}
1383		1383
1384	static int fsldma_of_remove(struct of_device *op)	1384	static int fsldma_of_remove(struct of_device *op)
1385	{	1385	{
1386	struct fsldma_device *fdev;	1386	struct fsldma_device *fdev;
1387	unsigned int i;	1387	unsigned int i;
1388		1388
1389	fdev = dev_get_drvdata(&op->dev);	1389	fdev = dev_get_drvdata(&op->dev);
1390	dma_async_device_unregister(&fdev->common);	1390	dma_async_device_unregister(&fdev->common);
1391		1391
1392	fsldma_free_irqs(fdev);	1392	fsldma_free_irqs(fdev);
1393		1393
1394	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {	1394	for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1395	if (fdev->chan[i])	1395	if (fdev->chan[i])
1396	fsl_dma_chan_remove(fdev->chan[i]);	1396	fsl_dma_chan_remove(fdev->chan[i]);
1397	}	1397	}
1398		1398
1399	iounmap(fdev->regs);	1399	iounmap(fdev->regs);
1400	dev_set_drvdata(&op->dev, NULL);	1400	dev_set_drvdata(&op->dev, NULL);
1401	kfree(fdev);	1401	kfree(fdev);
1402		1402
1403	return 0;	1403	return 0;
1404	}	1404	}
1405		1405
1406	static const struct of_device_id fsldma_of_ids[] = {	1406	static const struct of_device_id fsldma_of_ids[] = {
1407	{ .compatible = "fsl,eloplus-dma", },	1407	{ .compatible = "fsl,eloplus-dma", },
1408	{ .compatible = "fsl,elo-dma", },	1408	{ .compatible = "fsl,elo-dma", },
1409	{}	1409	{}
1410	};	1410	};
1411		1411
1412	static struct of_platform_driver fsldma_of_driver = {	1412	static struct of_platform_driver fsldma_of_driver = {
1413	.name = "fsl-elo-dma",	1413	.name = "fsl-elo-dma",
1414	.match_table = fsldma_of_ids,	1414	.match_table = fsldma_of_ids,
1415	.probe = fsldma_of_probe,	1415	.probe = fsldma_of_probe,
1416	.remove = fsldma_of_remove,	1416	.remove = fsldma_of_remove,
1417	};	1417	};
1418		1418
1419	/----------------------------------------------------------------------------/	1419	/----------------------------------------------------------------------------/
1420	/* Module Init / Exit */	1420	/* Module Init / Exit */
1421	/----------------------------------------------------------------------------/	1421	/----------------------------------------------------------------------------/
1422		1422
1423	static __init int fsldma_init(void)	1423	static __init int fsldma_init(void)
1424	{	1424	{
1425	int ret;	1425	int ret;
1426		1426
1427	pr_info("Freescale Elo / Elo Plus DMA driver\n");	1427	pr_info("Freescale Elo / Elo Plus DMA driver\n");
1428		1428
1429	ret = of_register_platform_driver(&fsldma_of_driver);	1429	ret = of_register_platform_driver(&fsldma_of_driver);
1430	if (ret)	1430	if (ret)
1431	pr_err("fsldma: failed to register platform driver\n");	1431	pr_err("fsldma: failed to register platform driver\n");
1432		1432
1433	return ret;	1433	return ret;
1434	}	1434	}
1435		1435
1436	static void __exit fsldma_exit(void)	1436	static void __exit fsldma_exit(void)
1437	{	1437	{
1438	of_unregister_platform_driver(&fsldma_of_driver);	1438	of_unregister_platform_driver(&fsldma_of_driver);
1439	}	1439	}
1440		1440
1441	subsys_initcall(fsldma_init);	1441	subsys_initcall(fsldma_init);
1442	module_exit(fsldma_exit);	1442	module_exit(fsldma_exit);
1443		1443
1444	MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");	1444	MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver");
1445	MODULE_LICENSE("GPL");	1445	MODULE_LICENSE("GPL");
1446		1446

drivers/dma/ipu/ipu_idmac.c

Diff comments View file @ 0582763

 /*
  * Copyright (C) 2008
  * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
  *
  * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/err.h>
 #include <linux/spinlock.h>
 #include <linux/delay.h>
 #include <linux/list.h>
 #include <linux/clk.h>
 #include <linux/vmalloc.h>
 #include <linux/string.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <mach/ipu.h>
 #include "ipu_intern.h"
 #define FS_VF_IN_VALID	0x00000002
 #define FS_ENC_IN_VALID	0x00000001
 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
 			       bool wait_for_stop);
 /*
  * There can be only one, we could allocate it dynamically, but then we'd have
  * to add an extra parameter to some functions, and use something as ugly as
  *	struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
  * in the ISR
  */
 static struct ipu ipu_data;
 #define to_ipu(id) container_of(id, struct ipu, idmac)
 static u32 __idmac_read_icreg(struct ipu *ipu, unsigned long reg)
 {
 	return __raw_readl(ipu->reg_ic + reg);
 }
 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
 static void __idmac_write_icreg(struct ipu *ipu, u32 value, unsigned long reg)
 {
 	__raw_writel(value, ipu->reg_ic + reg);
 }
 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
 static u32 idmac_read_ipureg(struct ipu *ipu, unsigned long reg)
 {
 	return __raw_readl(ipu->reg_ipu + reg);
 }
 static void idmac_write_ipureg(struct ipu *ipu, u32 value, unsigned long reg)
 {
 	__raw_writel(value, ipu->reg_ipu + reg);
 }
 /*****************************************************************************
  * IPU / IC common functions
  */
 static void dump_idmac_reg(struct ipu *ipu)
 {
 	dev_dbg(ipu->dev, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
 		"IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
 		idmac_read_icreg(ipu, IDMAC_CONF),
 		idmac_read_icreg(ipu, IC_CONF),
 		idmac_read_icreg(ipu, IDMAC_CHA_EN),
 		idmac_read_icreg(ipu, IDMAC_CHA_PRI),
 		idmac_read_icreg(ipu, IDMAC_CHA_BUSY));
 	dev_dbg(ipu->dev, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
 		"DB_MODE 0x%x, TASKS_STAT 0x%x\n",
 		idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
 		idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
 		idmac_read_ipureg(ipu, IPU_CHA_CUR_BUF),
 		idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL),
 		idmac_read_ipureg(ipu, IPU_TASKS_STAT));
 }
 static uint32_t bytes_per_pixel(enum pixel_fmt fmt)
 {
 	switch (fmt) {
 	case IPU_PIX_FMT_GENERIC:	/* generic data */
 	case IPU_PIX_FMT_RGB332:
 	case IPU_PIX_FMT_YUV420P:
 	case IPU_PIX_FMT_YUV422P:
 	default:
 		return 1;
 	case IPU_PIX_FMT_RGB565:
 	case IPU_PIX_FMT_YUYV:
 	case IPU_PIX_FMT_UYVY:
 		return 2;
 	case IPU_PIX_FMT_BGR24:
 	case IPU_PIX_FMT_RGB24:
 		return 3;
 	case IPU_PIX_FMT_GENERIC_32:	/* generic data */
 	case IPU_PIX_FMT_BGR32:
 	case IPU_PIX_FMT_RGB32:
 	case IPU_PIX_FMT_ABGR32:
 		return 4;
 	}
 }
 /* Enable direct write to memory by the Camera Sensor Interface */
 static void ipu_ic_enable_task(struct ipu *ipu, enum ipu_channel channel)
 {
 	uint32_t ic_conf, mask;
 	switch (channel) {
 	case IDMAC_IC_0:
 		mask = IC_CONF_PRPENC_EN;
 		break;
 	case IDMAC_IC_7:
 		mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
 		break;
 	default:
 		return;
 	}
 	ic_conf = idmac_read_icreg(ipu, IC_CONF) | mask;
 	idmac_write_icreg(ipu, ic_conf, IC_CONF);
 }
 /* Called under spin_lock_irqsave(&ipu_data.lock) */
 static void ipu_ic_disable_task(struct ipu *ipu, enum ipu_channel channel)
 {
 	uint32_t ic_conf, mask;
 	switch (channel) {
 	case IDMAC_IC_0:
 		mask = IC_CONF_PRPENC_EN;
 		break;
 	case IDMAC_IC_7:
 		mask = IC_CONF_RWS_EN | IC_CONF_PRPENC_EN;
 		break;
 	default:
 		return;
 	}
 	ic_conf = idmac_read_icreg(ipu, IC_CONF) & ~mask;
 	idmac_write_icreg(ipu, ic_conf, IC_CONF);
 }
 static uint32_t ipu_channel_status(struct ipu *ipu, enum ipu_channel channel)
 {
 	uint32_t stat = TASK_STAT_IDLE;
 	uint32_t task_stat_reg = idmac_read_ipureg(ipu, IPU_TASKS_STAT);
 	switch (channel) {
 	case IDMAC_IC_7:
 		stat = (task_stat_reg & TSTAT_CSI2MEM_MASK) >>
 			TSTAT_CSI2MEM_OFFSET;
 		break;
 	case IDMAC_IC_0:
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 	default:
 		break;
 	}
 	return stat;
 }
 struct chan_param_mem_planar {
 	/* Word 0 */
 	u32	xv:10;
 	u32	yv:10;
 	u32	xb:12;
 	u32	yb:12;
 	u32	res1:2;
 	u32	nsb:1;
 	u32	lnpb:6;
 	u32	ubo_l:11;
 	u32	ubo_h:15;
 	u32	vbo_l:17;
 	u32	vbo_h:9;
 	u32	res2:3;
 	u32	fw:12;
 	u32	fh_l:8;
 	u32	fh_h:4;
 	u32	res3:28;
 	/* Word 1 */
 	u32	eba0;
 	u32	eba1;
 	u32	bpp:3;
 	u32	sl:14;
 	u32	pfs:3;
 	u32	bam:3;
 	u32	res4:2;
 	u32	npb:6;
 	u32	res5:1;
 	u32	sat:2;
 	u32	res6:30;
 } __attribute__ ((packed));
 struct chan_param_mem_interleaved {
 	/* Word 0 */
 	u32	xv:10;
 	u32	yv:10;
 	u32	xb:12;
 	u32	yb:12;
 	u32	sce:1;
 	u32	res1:1;
 	u32	nsb:1;
 	u32	lnpb:6;
 	u32	sx:10;
 	u32	sy_l:1;
 	u32	sy_h:9;
 	u32	ns:10;
 	u32	sm:10;
 	u32	sdx_l:3;
 	u32	sdx_h:2;
 	u32	sdy:5;
 	u32	sdrx:1;
 	u32	sdry:1;
 	u32	sdr1:1;
 	u32	res2:2;
 	u32	fw:12;
 	u32	fh_l:8;
 	u32	fh_h:4;
 	u32	res3:28;
 	/* Word 1 */
 	u32	eba0;
 	u32	eba1;
 	u32	bpp:3;
 	u32	sl:14;
 	u32	pfs:3;
 	u32	bam:3;
 	u32	res4:2;
 	u32	npb:6;
 	u32	res5:1;
 	u32	sat:2;
 	u32	scc:1;
 	u32	ofs0:5;
 	u32	ofs1:5;
 	u32	ofs2:5;
 	u32	ofs3:5;
 	u32	wid0:3;
 	u32	wid1:3;
 	u32	wid2:3;
 	u32	wid3:3;
 	u32	dec_sel:1;
 	u32	res6:28;
 } __attribute__ ((packed));
 union chan_param_mem {
 	struct chan_param_mem_planar		pp;
 	struct chan_param_mem_interleaved	ip;
 };
 static void ipu_ch_param_set_plane_offset(union chan_param_mem *params,
 					  u32 u_offset, u32 v_offset)
 {
 	params->pp.ubo_l = u_offset & 0x7ff;
 	params->pp.ubo_h = u_offset >> 11;
 	params->pp.vbo_l = v_offset & 0x1ffff;
 	params->pp.vbo_h = v_offset >> 17;
 }
 static void ipu_ch_param_set_size(union chan_param_mem *params,
 				  uint32_t pixel_fmt, uint16_t width,
 				  uint16_t height, uint16_t stride)
 {
 	u32 u_offset;
 	u32 v_offset;
 	params->pp.fw		= width - 1;
 	params->pp.fh_l		= height - 1;
 	params->pp.fh_h		= (height - 1) >> 8;
 	params->pp.sl		= stride - 1;
 	switch (pixel_fmt) {
 	case IPU_PIX_FMT_GENERIC:
 		/*Represents 8-bit Generic data */
 		params->pp.bpp	= 3;
 		params->pp.pfs	= 7;
 		params->pp.npb	= 31;
 		params->pp.sat	= 2;		/* SAT = use 32-bit access */
 		break;
 	case IPU_PIX_FMT_GENERIC_32:
 		/*Represents 32-bit Generic data */
 		params->pp.bpp	= 0;
 		params->pp.pfs	= 7;
 		params->pp.npb	= 7;
 		params->pp.sat	= 2;		/* SAT = use 32-bit access */
 		break;
 	case IPU_PIX_FMT_RGB565:
 		params->ip.bpp	= 2;
 		params->ip.pfs	= 4;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		params->ip.ofs0	= 0;		/* Red bit offset */
 		params->ip.ofs1	= 5;		/* Green bit offset */
 		params->ip.ofs2	= 11;		/* Blue bit offset */
 		params->ip.ofs3	= 16;		/* Alpha bit offset */
 		params->ip.wid0	= 4;		/* Red bit width - 1 */
 		params->ip.wid1	= 5;		/* Green bit width - 1 */
 		params->ip.wid2	= 4;		/* Blue bit width - 1 */
 		break;
 	case IPU_PIX_FMT_BGR24:
 		params->ip.bpp	= 1;		/* 24 BPP & RGB PFS */
 		params->ip.pfs	= 4;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		params->ip.ofs0	= 0;		/* Red bit offset */
 		params->ip.ofs1	= 8;		/* Green bit offset */
 		params->ip.ofs2	= 16;		/* Blue bit offset */
 		params->ip.ofs3	= 24;		/* Alpha bit offset */
 		params->ip.wid0	= 7;		/* Red bit width - 1 */
 		params->ip.wid1	= 7;		/* Green bit width - 1 */
 		params->ip.wid2	= 7;		/* Blue bit width - 1 */
 		break;
 	case IPU_PIX_FMT_RGB24:
 		params->ip.bpp	= 1;		/* 24 BPP & RGB PFS */
 		params->ip.pfs	= 4;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		params->ip.ofs0	= 16;		/* Red bit offset */
 		params->ip.ofs1	= 8;		/* Green bit offset */
 		params->ip.ofs2	= 0;		/* Blue bit offset */
 		params->ip.ofs3	= 24;		/* Alpha bit offset */
 		params->ip.wid0	= 7;		/* Red bit width - 1 */
 		params->ip.wid1	= 7;		/* Green bit width - 1 */
 		params->ip.wid2	= 7;		/* Blue bit width - 1 */
 		break;
 	case IPU_PIX_FMT_BGRA32:
 	case IPU_PIX_FMT_BGR32:
 	case IPU_PIX_FMT_ABGR32:
 		params->ip.bpp	= 0;
 		params->ip.pfs	= 4;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		params->ip.ofs0	= 8;		/* Red bit offset */
 		params->ip.ofs1	= 16;		/* Green bit offset */
 		params->ip.ofs2	= 24;		/* Blue bit offset */
 		params->ip.ofs3	= 0;		/* Alpha bit offset */
 		params->ip.wid0	= 7;		/* Red bit width - 1 */
 		params->ip.wid1	= 7;		/* Green bit width - 1 */
 		params->ip.wid2	= 7;		/* Blue bit width - 1 */
 		params->ip.wid3	= 7;		/* Alpha bit width - 1 */
 		break;
 	case IPU_PIX_FMT_RGBA32:
 	case IPU_PIX_FMT_RGB32:
 		params->ip.bpp	= 0;
 		params->ip.pfs	= 4;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		params->ip.ofs0	= 24;		/* Red bit offset */
 		params->ip.ofs1	= 16;		/* Green bit offset */
 		params->ip.ofs2	= 8;		/* Blue bit offset */
 		params->ip.ofs3	= 0;		/* Alpha bit offset */
 		params->ip.wid0	= 7;		/* Red bit width - 1 */
 		params->ip.wid1	= 7;		/* Green bit width - 1 */
 		params->ip.wid2	= 7;		/* Blue bit width - 1 */
 		params->ip.wid3	= 7;		/* Alpha bit width - 1 */
 		break;
 	case IPU_PIX_FMT_UYVY:
 		params->ip.bpp	= 2;
 		params->ip.pfs	= 6;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		break;
 	case IPU_PIX_FMT_YUV420P2:
 	case IPU_PIX_FMT_YUV420P:
 		params->ip.bpp	= 3;
 		params->ip.pfs	= 3;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		u_offset = stride * height;
 		v_offset = u_offset + u_offset / 4;
 		ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 		break;
 	case IPU_PIX_FMT_YVU422P:
 		params->ip.bpp	= 3;
 		params->ip.pfs	= 2;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		v_offset = stride * height;
 		u_offset = v_offset + v_offset / 2;
 		ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 		break;
 	case IPU_PIX_FMT_YUV422P:
 		params->ip.bpp	= 3;
 		params->ip.pfs	= 2;
 		params->ip.npb	= 7;
 		params->ip.sat	= 2;		/* SAT = 32-bit access */
 		u_offset = stride * height;
 		v_offset = u_offset + u_offset / 2;
 		ipu_ch_param_set_plane_offset(params, u_offset, v_offset);
 		break;
 	default:
 		dev_err(ipu_data.dev,
 			"mx3 ipu: unimplemented pixel format %d\n", pixel_fmt);
 		break;
 	}
 	params->pp.nsb = 1;
 }
 static void ipu_ch_param_set_burst_size(union chan_param_mem *params,
 					uint16_t burst_pixels)
 {
 	params->pp.npb = burst_pixels - 1;
 }
 static void ipu_ch_param_set_buffer(union chan_param_mem *params,
 				    dma_addr_t buf0, dma_addr_t buf1)
 {
 	params->pp.eba0 = buf0;
 	params->pp.eba1 = buf1;
 }
 static void ipu_ch_param_set_rotation(union chan_param_mem *params,
 				      enum ipu_rotate_mode rotate)
 {
 	params->pp.bam = rotate;
 }
 static void ipu_write_param_mem(uint32_t addr, uint32_t *data,
 				uint32_t num_words)
 {
 	for (; num_words > 0; num_words--) {
 		dev_dbg(ipu_data.dev,
 			"write param mem - addr = 0x%08X, data = 0x%08X\n",
 			addr, *data);
 		idmac_write_ipureg(&ipu_data, addr, IPU_IMA_ADDR);
 		idmac_write_ipureg(&ipu_data, *data++, IPU_IMA_DATA);
 		addr++;
 		if ((addr & 0x7) == 5) {
 			addr &= ~0x7;	/* set to word 0 */
 			addr += 8;	/* increment to next row */
 		}
 	}
 }
 static int calc_resize_coeffs(uint32_t in_size, uint32_t out_size,
 			      uint32_t *resize_coeff,
 			      uint32_t *downsize_coeff)
 {
 	uint32_t temp_size;
 	uint32_t temp_downsize;
 	*resize_coeff	= 1 << 13;
 	*downsize_coeff	= 1 << 13;
 	/* Cannot downsize more than 8:1 */
 	if (out_size << 3 < in_size)
 		return -EINVAL;
 	/* compute downsizing coefficient */
 	temp_downsize = 0;
 	temp_size = in_size;
 	while (temp_size >= out_size * 2 && temp_downsize < 2) {
 		temp_size >>= 1;
 		temp_downsize++;
 	}
 	*downsize_coeff = temp_downsize;
 	/*
 	 * compute resizing coefficient using the following formula:
 	 * resize_coeff = M*(SI -1)/(SO - 1)
 	 * where M = 2^13, SI - input size, SO - output size
 	 */
 	*resize_coeff = (8192L * (temp_size - 1)) / (out_size - 1);
 	if (*resize_coeff >= 16384L) {
 		dev_err(ipu_data.dev, "Warning! Overflow on resize coeff.\n");
 		*resize_coeff = 0x3FFF;
 	}
 	dev_dbg(ipu_data.dev, "resizing from %u -> %u pixels, "
 		"downsize=%u, resize=%u.%lu (reg=%u)\n", in_size, out_size,
 		*downsize_coeff, *resize_coeff >= 8192L ? 1 : 0,
 		((*resize_coeff & 0x1FFF) * 10000L) / 8192L, *resize_coeff);
 	return 0;
 }
 static enum ipu_color_space format_to_colorspace(enum pixel_fmt fmt)
 {
 	switch (fmt) {
 	case IPU_PIX_FMT_RGB565:
 	case IPU_PIX_FMT_BGR24:
 	case IPU_PIX_FMT_RGB24:
 	case IPU_PIX_FMT_BGR32:
 	case IPU_PIX_FMT_RGB32:
 		return IPU_COLORSPACE_RGB;
 	default:
 		return IPU_COLORSPACE_YCBCR;
 	}
 }
 static int ipu_ic_init_prpenc(struct ipu *ipu,
 			      union ipu_channel_param *params, bool src_is_csi)
 {
 	uint32_t reg, ic_conf;
 	uint32_t downsize_coeff, resize_coeff;
 	enum ipu_color_space in_fmt, out_fmt;
 	/* Setup vertical resizing */
 	calc_resize_coeffs(params->video.in_height,
 			    params->video.out_height,
 			    &resize_coeff, &downsize_coeff);
 	reg = (downsize_coeff << 30) | (resize_coeff << 16);
 	/* Setup horizontal resizing */
 	calc_resize_coeffs(params->video.in_width,
 			    params->video.out_width,
 			    &resize_coeff, &downsize_coeff);
 	reg |= (downsize_coeff << 14) | resize_coeff;
 	/* Setup color space conversion */
 	in_fmt = format_to_colorspace(params->video.in_pixel_fmt);
 	out_fmt = format_to_colorspace(params->video.out_pixel_fmt);
 	/*
 	 * Colourspace conversion unsupported yet - see _init_csc() in
 	 * Freescale sources
 	 */
 	if (in_fmt != out_fmt) {
 		dev_err(ipu->dev, "Colourspace conversion unsupported!\n");
 		return -EOPNOTSUPP;
 	}
 	idmac_write_icreg(ipu, reg, IC_PRP_ENC_RSC);
 	ic_conf = idmac_read_icreg(ipu, IC_CONF);
 	if (src_is_csi)
 		ic_conf &= ~IC_CONF_RWS_EN;
 	else
 		ic_conf |= IC_CONF_RWS_EN;
 	idmac_write_icreg(ipu, ic_conf, IC_CONF);
 	return 0;
 }
 static uint32_t dma_param_addr(uint32_t dma_ch)
 {
 	/* Channel Parameter Memory */
 	return 0x10000 | (dma_ch << 4);
 }
 static void ipu_channel_set_priority(struct ipu *ipu, enum ipu_channel channel,
 				     bool prio)
 {
 	u32 reg = idmac_read_icreg(ipu, IDMAC_CHA_PRI);
 	if (prio)
 		reg |= 1UL << channel;
 	else
 		reg &= ~(1UL << channel);
 	idmac_write_icreg(ipu, reg, IDMAC_CHA_PRI);
 	dump_idmac_reg(ipu);
 }
 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel)
 {
 	uint32_t mask;
 	switch (channel) {
 	case IDMAC_IC_0:
 	case IDMAC_IC_7:
 		mask = IPU_CONF_CSI_EN | IPU_CONF_IC_EN;
 		break;
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 		mask = IPU_CONF_SDC_EN | IPU_CONF_DI_EN;
 		break;
 	default:
 		mask = 0;
 		break;
 	}
 	return mask;
 }
 /**
  * ipu_enable_channel() - enable an IPU channel.
  * @idmac:	IPU DMAC context.
  * @ichan:	IDMAC channel.
  * @return:	0 on success or negative error code on failure.
  */
 static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)
 {
 	struct ipu *ipu = to_ipu(idmac);
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	uint32_t reg;
 	unsigned long flags;
 	spin_lock_irqsave(&ipu->lock, flags);
 	/* Reset to buffer 0 */
 	idmac_write_ipureg(ipu, 1UL << channel, IPU_CHA_CUR_BUF);
 	ichan->active_buffer = 0;
 	ichan->status = IPU_CHANNEL_ENABLED;
 	switch (channel) {
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 	case IDMAC_IC_7:
 		ipu_channel_set_priority(ipu, channel, true);
 	default:
 		break;
 	}
 	reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
 	idmac_write_icreg(ipu, reg | (1UL << channel), IDMAC_CHA_EN);
 	ipu_ic_enable_task(ipu, channel);
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	return 0;
 }
 /**
  * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
  * @ichan:	IDMAC channel.
  * @pixel_fmt:	pixel format of buffer. Pixel format is a FOURCC ASCII code.
  * @width:	width of buffer in pixels.
  * @height:	height of buffer in pixels.
  * @stride:	stride length of buffer in pixels.
  * @rot_mode:	rotation mode of buffer. A rotation setting other than
  *		IPU_ROTATE_VERT_FLIP should only be used for input buffers of
  *		rotation channels.
  * @phyaddr_0:	buffer 0 physical address.
  * @phyaddr_1:	buffer 1 physical address. Setting this to a value other than
  *		NULL enables double buffering mode.
  * @return:	0 on success or negative error code on failure.
  */
 static int ipu_init_channel_buffer(struct idmac_channel *ichan,
 				   enum pixel_fmt pixel_fmt,
 				   uint16_t width, uint16_t height,
 				   uint32_t stride,
 				   enum ipu_rotate_mode rot_mode,
 				   dma_addr_t phyaddr_0, dma_addr_t phyaddr_1)
 {
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);
 	struct ipu *ipu = to_ipu(idmac);
 	union chan_param_mem params = {};
 	unsigned long flags;
 	uint32_t reg;
 	uint32_t stride_bytes;
 	stride_bytes = stride * bytes_per_pixel(pixel_fmt);
 	if (stride_bytes % 4) {
 		dev_err(ipu->dev,
 			"Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
 			stride, stride_bytes);
 		return -EINVAL;
 	}
 	/* IC channel's stride must be a multiple of 8 pixels */
 	if ((channel <= IDMAC_IC_13) && (stride % 8)) {
 		dev_err(ipu->dev, "Stride must be 8 pixel multiple\n");
 		return -EINVAL;
 	}
 	/* Build parameter memory data for DMA channel */
 	ipu_ch_param_set_size(&params, pixel_fmt, width, height, stride_bytes);
 	ipu_ch_param_set_buffer(&params, phyaddr_0, phyaddr_1);
 	ipu_ch_param_set_rotation(&params, rot_mode);
 	/* Some channels (rotation) have restriction on burst length */
 	switch (channel) {
 	case IDMAC_IC_7:	/* Hangs with burst 8, 16, other values
 				   invalid - Table 44-30 */
 /*
 		ipu_ch_param_set_burst_size(&params, 8);
  */
 		break;
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 		/* In original code only IPU_PIX_FMT_RGB565 was setting burst */
 		ipu_ch_param_set_burst_size(&params, 16);
 		break;
 	case IDMAC_IC_0:
 	default:
 		break;
 	}
 	spin_lock_irqsave(&ipu->lock, flags);
 	ipu_write_param_mem(dma_param_addr(channel), (uint32_t *)&params, 10);
 	reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
 	if (phyaddr_1)
 		reg |= 1UL << channel;
 	else
 		reg &= ~(1UL << channel);
 	idmac_write_ipureg(ipu, reg, IPU_CHA_DB_MODE_SEL);
 	ichan->status = IPU_CHANNEL_READY;
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	return 0;
 }
 /**
  * ipu_select_buffer() - mark a channel's buffer as ready.
  * @channel:	channel ID.
  * @buffer_n:	buffer number to mark ready.
  */
 static void ipu_select_buffer(enum ipu_channel channel, int buffer_n)
 {
 	/* No locking - this is a write-one-to-set register, cleared by IPU */
 	if (buffer_n == 0)
 		/* Mark buffer 0 as ready. */
 		idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF0_RDY);
 	else
 		/* Mark buffer 1 as ready. */
 		idmac_write_ipureg(&ipu_data, 1UL << channel, IPU_CHA_BUF1_RDY);
 }
 /**
  * ipu_update_channel_buffer() - update physical address of a channel buffer.
  * @ichan:	IDMAC channel.
  * @buffer_n:	buffer number to update.
  *		0 or 1 are the only valid values.
  * @phyaddr:	buffer physical address.
  */
 /* Called under spin_lock(_irqsave)(&ichan->lock) */
 static void ipu_update_channel_buffer(struct idmac_channel *ichan,
 				      int buffer_n, dma_addr_t phyaddr)
 {
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	uint32_t reg;
 	unsigned long flags;
 	spin_lock_irqsave(&ipu_data.lock, flags);
 	if (buffer_n == 0) {
 		reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
 		if (reg & (1UL << channel)) {
 			ipu_ic_disable_task(&ipu_data, channel);
 			ichan->status = IPU_CHANNEL_READY;
 		}
 		/* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
 		idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
 				   0x0008UL, IPU_IMA_ADDR);
 		idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
 	} else {
 		reg = idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
 		if (reg & (1UL << channel)) {
 			ipu_ic_disable_task(&ipu_data, channel);
 			ichan->status = IPU_CHANNEL_READY;
 		}
 		/* Check if double-buffering is already enabled */
 		reg = idmac_read_ipureg(&ipu_data, IPU_CHA_DB_MODE_SEL);
 		if (!(reg & (1UL << channel)))
 			idmac_write_ipureg(&ipu_data, reg | (1UL << channel),
 					   IPU_CHA_DB_MODE_SEL);
 		/* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
 		idmac_write_ipureg(&ipu_data, dma_param_addr(channel) +
 				   0x0009UL, IPU_IMA_ADDR);
 		idmac_write_ipureg(&ipu_data, phyaddr, IPU_IMA_DATA);
 	}
 	spin_unlock_irqrestore(&ipu_data.lock, flags);
 }
 /* Called under spin_lock_irqsave(&ichan->lock) */
 static int ipu_submit_buffer(struct idmac_channel *ichan,
 	struct idmac_tx_desc *desc, struct scatterlist *sg, int buf_idx)
 {
 	unsigned int chan_id = ichan->dma_chan.chan_id;
 	struct device *dev = &ichan->dma_chan.dev->device;
 	if (async_tx_test_ack(&desc->txd))
 		return -EINTR;
 	/*
 	 * On first invocation this shouldn't be necessary, the call to
 	 * ipu_init_channel_buffer() above will set addresses for us, so we
 	 * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
 	 * doing it again shouldn't hurt either.
 	 */
 	ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));
 	ipu_select_buffer(chan_id, buf_idx);
 	dev_dbg(dev, "Updated sg %p on channel 0x%x buffer %d\n",
 		sg, chan_id, buf_idx);
 	return 0;
 }
 /* Called under spin_lock_irqsave(&ichan->lock) */
 static int ipu_submit_channel_buffers(struct idmac_channel *ichan,
 				      struct idmac_tx_desc *desc)
 {
 	struct scatterlist *sg;
 	int i, ret = 0;
 	for (i = 0, sg = desc->sg; i < 2 && sg; i++) {
 		if (!ichan->sg[i]) {
 			ichan->sg[i] = sg;
 			ret = ipu_submit_buffer(ichan, desc, sg, i);
 			if (ret < 0)
 				return ret;
 			sg = sg_next(sg);
 		}
 	}
 	return ret;
 }
 static dma_cookie_t idmac_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct idmac_tx_desc *desc = to_tx_desc(tx);
 	struct idmac_channel *ichan = to_idmac_chan(tx->chan);
 	struct idmac *idmac = to_idmac(tx->chan->device);
 	struct ipu *ipu = to_ipu(idmac);
 	struct device *dev = &ichan->dma_chan.dev->device;
 	dma_cookie_t cookie;
 	unsigned long flags;
 	int ret;
 	/* Sanity check */
 	if (!list_empty(&desc->list)) {
 		/* The descriptor doesn't belong to client */
 		dev_err(dev, "Descriptor %p not prepared!\n", tx);
 		return -EBUSY;
 	}
 	mutex_lock(&ichan->chan_mutex);
 	async_tx_clear_ack(tx);
 	if (ichan->status < IPU_CHANNEL_READY) {
 		struct idmac_video_param *video = &ichan->params.video;
 		/*
 		 * Initial buffer assignment - the first two sg-entries from
 		 * the descriptor will end up in the IDMAC buffers
 		 */
 		dma_addr_t dma_1 = sg_is_last(desc->sg) ? 0 :
 			sg_dma_address(&desc->sg[1]);
 		WARN_ON(ichan->sg[0] || ichan->sg[1]);
 		cookie = ipu_init_channel_buffer(ichan,
 						 video->out_pixel_fmt,
 						 video->out_width,
 						 video->out_height,
 						 video->out_stride,
 						 IPU_ROTATE_NONE,
 						 sg_dma_address(&desc->sg[0]),
 						 dma_1);
 		if (cookie < 0)
 			goto out;
 	}
 	dev_dbg(dev, "Submitting sg %p\n", &desc->sg[0]);
 	cookie = ichan->dma_chan.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	/* from dmaengine.h: "last cookie value returned to client" */
 	ichan->dma_chan.cookie = cookie;
 	tx->cookie = cookie;
 	/* ipu->lock can be taken under ichan->lock, but not v.v. */
 	spin_lock_irqsave(&ichan->lock, flags);
 	list_add_tail(&desc->list, &ichan->queue);
 	/* submit_buffers() atomically verifies and fills empty sg slots */
 	ret = ipu_submit_channel_buffers(ichan, desc);
 	spin_unlock_irqrestore(&ichan->lock, flags);
 	if (ret < 0) {
 		cookie = ret;
 		goto dequeue;
 	}
 	if (ichan->status < IPU_CHANNEL_ENABLED) {
 		ret = ipu_enable_channel(idmac, ichan);
 		if (ret < 0) {
 			cookie = ret;
 			goto dequeue;
 		}
 	}
 	dump_idmac_reg(ipu);
 dequeue:
 	if (cookie < 0) {
 		spin_lock_irqsave(&ichan->lock, flags);
 		list_del_init(&desc->list);
 		spin_unlock_irqrestore(&ichan->lock, flags);
 		tx->cookie = cookie;
 		ichan->dma_chan.cookie = cookie;
 	}
 out:
 	mutex_unlock(&ichan->chan_mutex);
 	return cookie;
 }
 /* Called with ichan->chan_mutex held */
 static int idmac_desc_alloc(struct idmac_channel *ichan, int n)
 {
 	struct idmac_tx_desc *desc = vmalloc(n * sizeof(struct idmac_tx_desc));
 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);
 	if (!desc)
 		return -ENOMEM;
 	/* No interrupts, just disable the tasklet for a moment */
 	tasklet_disable(&to_ipu(idmac)->tasklet);
 	ichan->n_tx_desc = n;
 	ichan->desc = desc;
 	INIT_LIST_HEAD(&ichan->queue);
 	INIT_LIST_HEAD(&ichan->free_list);
 	while (n--) {
 		struct dma_async_tx_descriptor *txd = &desc->txd;
 		memset(txd, 0, sizeof(*txd));
 		dma_async_tx_descriptor_init(txd, &ichan->dma_chan);
 		txd->tx_submit		= idmac_tx_submit;
 		list_add(&desc->list, &ichan->free_list);
 		desc++;
 	}
 	tasklet_enable(&to_ipu(idmac)->tasklet);
 	return 0;
 }
 /**
  * ipu_init_channel() - initialize an IPU channel.
  * @idmac:	IPU DMAC context.
  * @ichan:	pointer to the channel object.
  * @return      0 on success or negative error code on failure.
  */
 static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)
 {
 	union ipu_channel_param *params = &ichan->params;
 	uint32_t ipu_conf;
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	unsigned long flags;
 	uint32_t reg;
 	struct ipu *ipu = to_ipu(idmac);
 	int ret = 0, n_desc = 0;
 	dev_dbg(ipu->dev, "init channel = %d\n", channel);
 	if (channel != IDMAC_SDC_0 && channel != IDMAC_SDC_1 &&
 	    channel != IDMAC_IC_7)
 		return -EINVAL;
 	spin_lock_irqsave(&ipu->lock, flags);
 	switch (channel) {
 	case IDMAC_IC_7:
 		n_desc = 16;
 		reg = idmac_read_icreg(ipu, IC_CONF);
 		idmac_write_icreg(ipu, reg & ~IC_CONF_CSI_MEM_WR_EN, IC_CONF);
 		break;
 	case IDMAC_IC_0:
 		n_desc = 16;
 		reg = idmac_read_ipureg(ipu, IPU_FS_PROC_FLOW);
 		idmac_write_ipureg(ipu, reg & ~FS_ENC_IN_VALID, IPU_FS_PROC_FLOW);
 		ret = ipu_ic_init_prpenc(ipu, params, true);
 		break;
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 		n_desc = 4;
 	default:
 		break;
 	}
 	ipu->channel_init_mask |= 1L << channel;
 	/* Enable IPU sub module */
 	ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) |
 		ipu_channel_conf_mask(channel);
 	idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	if (n_desc && !ichan->desc)
 		ret = idmac_desc_alloc(ichan, n_desc);
 	dump_idmac_reg(ipu);
 	return ret;
 }
 /**
  * ipu_uninit_channel() - uninitialize an IPU channel.
  * @idmac:	IPU DMAC context.
  * @ichan:	pointer to the channel object.
  */
 static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)
 {
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	unsigned long flags;
 	uint32_t reg;
 	unsigned long chan_mask = 1UL << channel;
 	uint32_t ipu_conf;
 	struct ipu *ipu = to_ipu(idmac);
 	spin_lock_irqsave(&ipu->lock, flags);
 	if (!(ipu->channel_init_mask & chan_mask)) {
 		dev_err(ipu->dev, "Channel already uninitialized %d\n",
 			channel);
 		spin_unlock_irqrestore(&ipu->lock, flags);
 		return;
 	}
 	/* Reset the double buffer */
 	reg = idmac_read_ipureg(ipu, IPU_CHA_DB_MODE_SEL);
 	idmac_write_ipureg(ipu, reg & ~chan_mask, IPU_CHA_DB_MODE_SEL);
 	ichan->sec_chan_en = false;
 	switch (channel) {
 	case IDMAC_IC_7:
 		reg = idmac_read_icreg(ipu, IC_CONF);
 		idmac_write_icreg(ipu, reg & ~(IC_CONF_RWS_EN | IC_CONF_PRPENC_EN),
 			     IC_CONF);
 		break;
 	case IDMAC_IC_0:
 		reg = idmac_read_icreg(ipu, IC_CONF);
 		idmac_write_icreg(ipu, reg & ~(IC_CONF_PRPENC_EN | IC_CONF_PRPENC_CSC1),
 				  IC_CONF);
 		break;
 	case IDMAC_SDC_0:
 	case IDMAC_SDC_1:
 	default:
 		break;
 	}
 	ipu->channel_init_mask &= ~(1L << channel);
 	ipu_conf = idmac_read_ipureg(ipu, IPU_CONF) &
 		~ipu_channel_conf_mask(channel);
 	idmac_write_ipureg(ipu, ipu_conf, IPU_CONF);
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	ichan->n_tx_desc = 0;
 	vfree(ichan->desc);
 	ichan->desc = NULL;
 }
 /**
  * ipu_disable_channel() - disable an IPU channel.
  * @idmac:		IPU DMAC context.
  * @ichan:		channel object pointer.
  * @wait_for_stop:	flag to set whether to wait for channel end of frame or
  *			return immediately.
  * @return:		0 on success or negative error code on failure.
  */
 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
 			       bool wait_for_stop)
 {
 	enum ipu_channel channel = ichan->dma_chan.chan_id;
 	struct ipu *ipu = to_ipu(idmac);
 	uint32_t reg;
 	unsigned long flags;
 	unsigned long chan_mask = 1UL << channel;
 	unsigned int timeout;
 	if (wait_for_stop && channel != IDMAC_SDC_1 && channel != IDMAC_SDC_0) {
 		timeout = 40;
 		/* This waiting always fails. Related to spurious irq problem */
 		while ((idmac_read_icreg(ipu, IDMAC_CHA_BUSY) & chan_mask) ||
 		       (ipu_channel_status(ipu, channel) == TASK_STAT_ACTIVE)) {
 			timeout--;
 			msleep(10);
 			if (!timeout) {
 				dev_dbg(ipu->dev,
 					"Warning: timeout waiting for channel %u to "
 					"stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
 					"busy = 0x%08X, tstat = 0x%08X\n", channel,
 					idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY),
 					idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY),
 					idmac_read_icreg(ipu, IDMAC_CHA_BUSY),
 					idmac_read_ipureg(ipu, IPU_TASKS_STAT));
 				break;
 			}
 		}
 		dev_dbg(ipu->dev, "timeout = %d * 10ms\n", 40 - timeout);
 	}
 	/* SDC BG and FG must be disabled before DMA is disabled */
 	if (wait_for_stop && (channel == IDMAC_SDC_0 ||
 			      channel == IDMAC_SDC_1)) {
 		for (timeout = 5;
 		     timeout && !ipu_irq_status(ichan->eof_irq); timeout--)
 			msleep(5);
 	}
 	spin_lock_irqsave(&ipu->lock, flags);
 	/* Disable IC task */
 	ipu_ic_disable_task(ipu, channel);
 	/* Disable DMA channel(s) */
 	reg = idmac_read_icreg(ipu, IDMAC_CHA_EN);
 	idmac_write_icreg(ipu, reg & ~chan_mask, IDMAC_CHA_EN);
 	/*
 	 * Problem (observed with channel DMAIC_7): after enabling the channel
 	 * and initialising buffers, there comes an interrupt with current still
 	 * pointing at buffer 0, whereas it should use buffer 0 first and only
 	 * generate an interrupt when it is done, then current should already
 	 * point to buffer 1. This spurious interrupt also comes on channel
 	 * DMASDC_0. With DMAIC_7 normally, is we just leave the ISR after the
 	 * first interrupt, there comes the second with current correctly
 	 * pointing to buffer 1 this time. But sometimes this second interrupt
 	 * doesn't come and the channel hangs. Clearing BUFx_RDY when disabling
 	 * the channel seems to prevent the channel from hanging, but it doesn't
 	 * prevent the spurious interrupt. This might also be unsafe. Think
 	 * about the IDMAC controller trying to switch to a buffer, when we
 	 * clear the ready bit, and re-enable it a moment later.
 	 */
 	reg = idmac_read_ipureg(ipu, IPU_CHA_BUF0_RDY);
 	idmac_write_ipureg(ipu, 0, IPU_CHA_BUF0_RDY);
 	idmac_write_ipureg(ipu, reg & ~(1UL << channel), IPU_CHA_BUF0_RDY);
 	reg = idmac_read_ipureg(ipu, IPU_CHA_BUF1_RDY);
 	idmac_write_ipureg(ipu, 0, IPU_CHA_BUF1_RDY);
 	idmac_write_ipureg(ipu, reg & ~(1UL << channel), IPU_CHA_BUF1_RDY);
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	return 0;
 }
 static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,
 	struct idmac_tx_desc **desc, struct scatterlist *sg)
 {
 	struct scatterlist *sgnew = sg ? sg_next(sg) : NULL;
 	if (sgnew)
 		/* next sg-element in this list */
 		return sgnew;
 	if ((*desc)->list.next == &ichan->queue)
 		/* No more descriptors on the queue */
 		return NULL;
 	/* Fetch next descriptor */
 	*desc = list_entry((*desc)->list.next, struct idmac_tx_desc, list);
 	return (*desc)->sg;
 }
 /*
  * We have several possibilities here:
  * current BUF		next BUF
  *
  * not last sg		next not last sg
  * not last sg		next last sg
  * last sg		first sg from next descriptor
  * last sg		NULL
  *
  * Besides, the descriptor queue might be empty or not. We process all these
  * cases carefully.
  */
 static irqreturn_t idmac_interrupt(int irq, void *dev_id)
 {
 	struct idmac_channel *ichan = dev_id;
 	struct device *dev = &ichan->dma_chan.dev->device;
 	unsigned int chan_id = ichan->dma_chan.chan_id;
 	struct scatterlist **sg, *sgnext, *sgnew = NULL;
 	/* Next transfer descriptor */
 	struct idmac_tx_desc *desc, *descnew;
 	dma_async_tx_callback callback;
 	void *callback_param;
 	bool done = false;
 	u32 ready0, ready1, curbuf, err;
 	unsigned long flags;
 	/* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
 	dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);
 	spin_lock_irqsave(&ipu_data.lock, flags);
 	ready0	= idmac_read_ipureg(&ipu_data, IPU_CHA_BUF0_RDY);
 	ready1	= idmac_read_ipureg(&ipu_data, IPU_CHA_BUF1_RDY);
 	curbuf	= idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF);
 	err	= idmac_read_ipureg(&ipu_data, IPU_INT_STAT_4);
 	if (err & (1 << chan_id)) {
 		idmac_write_ipureg(&ipu_data, 1 << chan_id, IPU_INT_STAT_4);
 		spin_unlock_irqrestore(&ipu_data.lock, flags);
 		/*
 		 * Doing this
 		 * ichan->sg[0] = ichan->sg[1] = NULL;
 		 * you can force channel re-enable on the next tx_submit(), but
 		 * this is dirty - think about descriptors with multiple
 		 * sg elements.
 		 */
 		dev_warn(dev, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
 			 chan_id, ready0, ready1, curbuf);
 		return IRQ_HANDLED;
 	}
 	spin_unlock_irqrestore(&ipu_data.lock, flags);
 	/* Other interrupts do not interfere with this channel */
 	spin_lock(&ichan->lock);
 	if (unlikely(chan_id != IDMAC_SDC_0 && chan_id != IDMAC_SDC_1 &&
 		     ((curbuf >> chan_id) & 1) == ichan->active_buffer &&
 		     !list_is_last(ichan->queue.next, &ichan->queue))) {
 		int i = 100;
 		/* This doesn't help. See comment in ipu_disable_channel() */
 		while (--i) {
 			curbuf = idmac_read_ipureg(&ipu_data, IPU_CHA_CUR_BUF);
 			if (((curbuf >> chan_id) & 1) != ichan->active_buffer)
 				break;
 			cpu_relax();
 		}
 		if (!i) {
 			spin_unlock(&ichan->lock);
 			dev_dbg(dev,
 				"IRQ on active buffer on channel %x, active "
 				"%d, ready %x, %x, current %x!\n", chan_id,
 				ichan->active_buffer, ready0, ready1, curbuf);
 			return IRQ_NONE;
 		} else
 			dev_dbg(dev,
 				"Buffer deactivated on channel %x, active "
 				"%d, ready %x, %x, current %x, rest %d!\n", chan_id,
 				ichan->active_buffer, ready0, ready1, curbuf, i);
 	}
 	if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||
 		     (!ichan->active_buffer && (ready0 >> chan_id) & 1)
 		     )) {
 		spin_unlock(&ichan->lock);
 		dev_dbg(dev,
 			"IRQ with active buffer still ready on channel %x, "
 			"active %d, ready %x, %x!\n", chan_id,
 			ichan->active_buffer, ready0, ready1);
 		return IRQ_NONE;
 	}
 	if (unlikely(list_empty(&ichan->queue))) {
 		ichan->sg[ichan->active_buffer] = NULL;
 		spin_unlock(&ichan->lock);
 		dev_err(dev,
 			"IRQ without queued buffers on channel %x, active %d, "
 			"ready %x, %x!\n", chan_id,
 			ichan->active_buffer, ready0, ready1);
 		return IRQ_NONE;
 	}
 	/*
 	 * active_buffer is a software flag, it shows which buffer we are
 	 * currently expecting back from the hardware, IDMAC should be
 	 * processing the other buffer already
 	 */
 	sg = &ichan->sg[ichan->active_buffer];
 	sgnext = ichan->sg[!ichan->active_buffer];
 	if (!*sg) {
 		spin_unlock(&ichan->lock);
 		return IRQ_HANDLED;
 	}
 	desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);
 	descnew = desc;
 	dev_dbg(dev, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
 		irq, sg_dma_address(*sg), sgnext ? sg_dma_address(sgnext) : 0, ichan->active_buffer, curbuf);
 	/* Find the descriptor of sgnext */
 	sgnew = idmac_sg_next(ichan, &descnew, *sg);
 	if (sgnext != sgnew)
 		dev_err(dev, "Submitted buffer %p, next buffer %p\n", sgnext, sgnew);
 	/*
 	 * if sgnext == NULL sg must be the last element in a scatterlist and
 	 * queue must be empty
 	 */
 	if (unlikely(!sgnext)) {
 		if (!WARN_ON(sg_next(*sg)))
 			dev_dbg(dev, "Underrun on channel %x\n", chan_id);
 		ichan->sg[!ichan->active_buffer] = sgnew;
 		if (unlikely(sgnew)) {
 			ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);
 		} else {
 			spin_lock_irqsave(&ipu_data.lock, flags);
 			ipu_ic_disable_task(&ipu_data, chan_id);
 			spin_unlock_irqrestore(&ipu_data.lock, flags);
 			ichan->status = IPU_CHANNEL_READY;
 			/* Continue to check for complete descriptor */
 		}
 	}
 	/* Calculate and submit the next sg element */
 	sgnew = idmac_sg_next(ichan, &descnew, sgnew);
 	if (unlikely(!sg_next(*sg)) || !sgnext) {
 		/*
 		 * Last element in scatterlist done, remove from the queue,
 		 * _init for debugging
 		 */
 		list_del_init(&desc->list);
 		done = true;
 	}
 	*sg = sgnew;
 	if (likely(sgnew) &&
 	    ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {
 		callback = descnew->txd.callback;
 		callback_param = descnew->txd.callback_param;
 		spin_unlock(&ichan->lock);
 		if (callback)
 			callback(callback_param);
 		spin_lock(&ichan->lock);
 	}
 	/* Flip the active buffer - even if update above failed */
 	ichan->active_buffer = !ichan->active_buffer;
 	if (done)
 		ichan->completed = desc->txd.cookie;
 	callback = desc->txd.callback;
 	callback_param = desc->txd.callback_param;
 	spin_unlock(&ichan->lock);
 	if (done && (desc->txd.flags & DMA_PREP_INTERRUPT) && callback)
 		callback(callback_param);
 	return IRQ_HANDLED;
 }
 static void ipu_gc_tasklet(unsigned long arg)
 {
 	struct ipu *ipu = (struct ipu *)arg;
 	int i;
 	for (i = 0; i < IPU_CHANNELS_NUM; i++) {
 		struct idmac_channel *ichan = ipu->channel + i;
 		struct idmac_tx_desc *desc;
 		unsigned long flags;
 		struct scatterlist *sg;
 		int j, k;
 		for (j = 0; j < ichan->n_tx_desc; j++) {
 			desc = ichan->desc + j;
 			spin_lock_irqsave(&ichan->lock, flags);
 			if (async_tx_test_ack(&desc->txd)) {
 				list_move(&desc->list, &ichan->free_list);
 				for_each_sg(desc->sg, sg, desc->sg_len, k) {
 					if (ichan->sg[0] == sg)
 						ichan->sg[0] = NULL;
 					else if (ichan->sg[1] == sg)
 						ichan->sg[1] = NULL;
 				}
 				async_tx_clear_ack(&desc->txd);
 			}
 			spin_unlock_irqrestore(&ichan->lock, flags);
 		}
 	}
 }
 /* Allocate and initialise a transfer descriptor. */
 static struct dma_async_tx_descriptor *idmac_prep_slave_sg(struct dma_chan *chan,
 		struct scatterlist *sgl, unsigned int sg_len,
 		enum dma_data_direction direction, unsigned long tx_flags)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	struct idmac_tx_desc *desc = NULL;
 	struct dma_async_tx_descriptor *txd = NULL;
 	unsigned long flags;
 	/* We only can handle these three channels so far */
 	if (chan->chan_id != IDMAC_SDC_0 && chan->chan_id != IDMAC_SDC_1 &&
 	    chan->chan_id != IDMAC_IC_7)
 		return NULL;
 	if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) {
 		dev_err(chan->device->dev, "Invalid DMA direction %d!\n", direction);
 		return NULL;
 	}
 	mutex_lock(&ichan->chan_mutex);
 	spin_lock_irqsave(&ichan->lock, flags);
 	if (!list_empty(&ichan->free_list)) {
 		desc = list_entry(ichan->free_list.next,
 				  struct idmac_tx_desc, list);
 		list_del_init(&desc->list);
 		desc->sg_len	= sg_len;
 		desc->sg	= sgl;
 		txd		= &desc->txd;
 		txd->flags	= tx_flags;
 	}
 	spin_unlock_irqrestore(&ichan->lock, flags);
 	mutex_unlock(&ichan->chan_mutex);
 	tasklet_schedule(&to_ipu(to_idmac(chan->device))->tasklet);
 	return txd;
 }
 /* Re-select the current buffer and re-activate the channel */
 static void idmac_issue_pending(struct dma_chan *chan)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	struct idmac *idmac = to_idmac(chan->device);
 	struct ipu *ipu = to_ipu(idmac);
 	unsigned long flags;
 	/* This is not always needed, but doesn't hurt either */
 	spin_lock_irqsave(&ipu->lock, flags);
 	ipu_select_buffer(chan->chan_id, ichan->active_buffer);
 	spin_unlock_irqrestore(&ipu->lock, flags);
 	/*
 	 * Might need to perform some parts of initialisation from
 	 * ipu_enable_channel(), but not all, we do not want to reset to buffer
 	 * 0, don't need to set priority again either, but re-enabling the task
 	 * and the channel might be a good idea.
 	 */
 }
-static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int __idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+			   unsigned long arg)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	struct idmac *idmac = to_idmac(chan->device);
 	unsigned long flags;
 	int i;
 	/* Only supports DMA_TERMINATE_ALL */
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	ipu_disable_channel(idmac, ichan,
 			    ichan->status >= IPU_CHANNEL_ENABLED);
 	tasklet_disable(&to_ipu(idmac)->tasklet);
 	/* ichan->queue is modified in ISR, have to spinlock */
 	spin_lock_irqsave(&ichan->lock, flags);
 	list_splice_init(&ichan->queue, &ichan->free_list);
 	if (ichan->desc)
 		for (i = 0; i < ichan->n_tx_desc; i++) {
 			struct idmac_tx_desc *desc = ichan->desc + i;
 			if (list_empty(&desc->list))
 				/* Descriptor was prepared, but not submitted */
 				list_add(&desc->list, &ichan->free_list);
 			async_tx_clear_ack(&desc->txd);
 		}
 	ichan->sg[0] = NULL;
 	ichan->sg[1] = NULL;
 	spin_unlock_irqrestore(&ichan->lock, flags);
 	tasklet_enable(&to_ipu(idmac)->tasklet);
 	ichan->status = IPU_CHANNEL_INITIALIZED;
 	return 0;
 }
-static int idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int idmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+			 unsigned long arg)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	int ret;
 	mutex_lock(&ichan->chan_mutex);
-	ret = __idmac_control(chan, cmd);
+	ret = __idmac_control(chan, cmd, arg);
 	mutex_unlock(&ichan->chan_mutex);
 	return ret;
 }
 #ifdef DEBUG
 static irqreturn_t ic_sof_irq(int irq, void *dev_id)
 {
 	struct idmac_channel *ichan = dev_id;
 	printk(KERN_DEBUG "Got SOF IRQ %d on Channel %d\n",
 	       irq, ichan->dma_chan.chan_id);
 	disable_irq_nosync(irq);
 	return IRQ_HANDLED;
 }
 static irqreturn_t ic_eof_irq(int irq, void *dev_id)
 {
 	struct idmac_channel *ichan = dev_id;
 	printk(KERN_DEBUG "Got EOF IRQ %d on Channel %d\n",
 	       irq, ichan->dma_chan.chan_id);
 	disable_irq_nosync(irq);
 	return IRQ_HANDLED;
 }
 static int ic_sof = -EINVAL, ic_eof = -EINVAL;
 #endif
 static int idmac_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	struct idmac *idmac = to_idmac(chan->device);
 	int ret;
 	/* dmaengine.c now guarantees to only offer free channels */
 	BUG_ON(chan->client_count > 1);
 	WARN_ON(ichan->status != IPU_CHANNEL_FREE);
 	chan->cookie		= 1;
 	ichan->completed	= -ENXIO;
 	ret = ipu_irq_map(chan->chan_id);
 	if (ret < 0)
 		goto eimap;
 	ichan->eof_irq = ret;
 	/*
 	 * Important to first disable the channel, because maybe someone
 	 * used it before us, e.g., the bootloader
 	 */
 	ipu_disable_channel(idmac, ichan, true);
 	ret = ipu_init_channel(idmac, ichan);
 	if (ret < 0)
 		goto eichan;
 	ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,
 			  ichan->eof_name, ichan);
 	if (ret < 0)
 		goto erirq;
 #ifdef DEBUG
 	if (chan->chan_id == IDMAC_IC_7) {
 		ic_sof = ipu_irq_map(69);
 		if (ic_sof > 0)
 			request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);
 		ic_eof = ipu_irq_map(70);
 		if (ic_eof > 0)
 			request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);
 	}
 #endif
 	ichan->status = IPU_CHANNEL_INITIALIZED;
 	dev_dbg(&chan->dev->device, "Found channel 0x%x, irq %d\n",
 		chan->chan_id, ichan->eof_irq);
 	return ret;
 erirq:
 	ipu_uninit_channel(idmac, ichan);
 eichan:
 	ipu_irq_unmap(chan->chan_id);
 eimap:
 	return ret;
 }
 static void idmac_free_chan_resources(struct dma_chan *chan)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	struct idmac *idmac = to_idmac(chan->device);
 	mutex_lock(&ichan->chan_mutex);
-	__idmac_control(chan, DMA_TERMINATE_ALL);
+	__idmac_control(chan, DMA_TERMINATE_ALL, 0);
 	if (ichan->status > IPU_CHANNEL_FREE) {
 #ifdef DEBUG
 		if (chan->chan_id == IDMAC_IC_7) {
 			if (ic_sof > 0) {
 				free_irq(ic_sof, ichan);
 				ipu_irq_unmap(69);
 				ic_sof = -EINVAL;
 			}
 			if (ic_eof > 0) {
 				free_irq(ic_eof, ichan);
 				ipu_irq_unmap(70);
 				ic_eof = -EINVAL;
 			}
 		}
 #endif
 		free_irq(ichan->eof_irq, ichan);
 		ipu_irq_unmap(chan->chan_id);
 	}
 	ichan->status = IPU_CHANNEL_FREE;
 	ipu_uninit_channel(idmac, ichan);
 	mutex_unlock(&ichan->chan_mutex);
 	tasklet_schedule(&to_ipu(idmac)->tasklet);
 }
 static enum dma_status idmac_tx_status(struct dma_chan *chan,
 		       dma_cookie_t cookie, struct dma_tx_state *txstate)
 {
 	struct idmac_channel *ichan = to_idmac_chan(chan);
 	dma_set_tx_state(txstate, ichan->completed, chan->cookie, 0);
 	if (cookie != chan->cookie)
 		return DMA_ERROR;
 	return DMA_SUCCESS;
 }
 static int __init ipu_idmac_init(struct ipu *ipu)
 {
 	struct idmac *idmac = &ipu->idmac;
 	struct dma_device *dma = &idmac->dma;
 	int i;
 	dma_cap_set(DMA_SLAVE, dma->cap_mask);
 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
 	/* Compulsory common fields */
 	dma->dev				= ipu->dev;
 	dma->device_alloc_chan_resources	= idmac_alloc_chan_resources;
 	dma->device_free_chan_resources		= idmac_free_chan_resources;
 	dma->device_tx_status			= idmac_tx_status;
 	dma->device_issue_pending		= idmac_issue_pending;
 	/* Compulsory for DMA_SLAVE fields */
 	dma->device_prep_slave_sg		= idmac_prep_slave_sg;
 	dma->device_control			= idmac_control;
 	INIT_LIST_HEAD(&dma->channels);
 	for (i = 0; i < IPU_CHANNELS_NUM; i++) {
 		struct idmac_channel *ichan = ipu->channel + i;
 		struct dma_chan *dma_chan = &ichan->dma_chan;
 		spin_lock_init(&ichan->lock);
 		mutex_init(&ichan->chan_mutex);
 		ichan->status		= IPU_CHANNEL_FREE;
 		ichan->sec_chan_en	= false;
 		ichan->completed	= -ENXIO;
 		snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);
 		dma_chan->device	= &idmac->dma;
 		dma_chan->cookie	= 1;
 		dma_chan->chan_id	= i;
 		list_add_tail(&dma_chan->device_node, &dma->channels);
 	}
 	idmac_write_icreg(ipu, 0x00000070, IDMAC_CONF);
 	return dma_async_device_register(&idmac->dma);
 }
 static void __exit ipu_idmac_exit(struct ipu *ipu)
 {
 	int i;
 	struct idmac *idmac = &ipu->idmac;
 	for (i = 0; i < IPU_CHANNELS_NUM; i++) {
 		struct idmac_channel *ichan = ipu->channel + i;
-		idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL);
+		idmac_control(&ichan->dma_chan, DMA_TERMINATE_ALL, 0);
 		idmac_prep_slave_sg(&ichan->dma_chan, NULL, 0, DMA_NONE, 0);
 	}
 	dma_async_device_unregister(&idmac->dma);
 }
 /*****************************************************************************
  * IPU common probe / remove
  */
 static int __init ipu_probe(struct platform_device *pdev)
 {
 	struct ipu_platform_data *pdata = pdev->dev.platform_data;
 	struct resource *mem_ipu, *mem_ic;
 	int ret;
 	spin_lock_init(&ipu_data.lock);
 	mem_ipu	= platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	mem_ic	= platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	if (!pdata || !mem_ipu || !mem_ic)
 		return -EINVAL;
 	ipu_data.dev = &pdev->dev;
 	platform_set_drvdata(pdev, &ipu_data);
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0)
 		goto err_noirq;
 	ipu_data.irq_fn = ret;
 	ret = platform_get_irq(pdev, 1);
 	if (ret < 0)
 		goto err_noirq;
 	ipu_data.irq_err = ret;
 	ipu_data.irq_base = pdata->irq_base;
 	dev_dbg(&pdev->dev, "fn irq %u, err irq %u, irq-base %u\n",
 		ipu_data.irq_fn, ipu_data.irq_err, ipu_data.irq_base);
 	/* Remap IPU common registers */
 	ipu_data.reg_ipu = ioremap(mem_ipu->start,
 				   mem_ipu->end - mem_ipu->start + 1);
 	if (!ipu_data.reg_ipu) {
 		ret = -ENOMEM;
 		goto err_ioremap_ipu;
 	}
 	/* Remap Image Converter and Image DMA Controller registers */
 	ipu_data.reg_ic = ioremap(mem_ic->start,
 				  mem_ic->end - mem_ic->start + 1);
 	if (!ipu_data.reg_ic) {
 		ret = -ENOMEM;
 		goto err_ioremap_ic;
 	}
 	/* Get IPU clock */
 	ipu_data.ipu_clk = clk_get(&pdev->dev, NULL);
 	if (IS_ERR(ipu_data.ipu_clk)) {
 		ret = PTR_ERR(ipu_data.ipu_clk);
 		goto err_clk_get;
 	}
 	/* Make sure IPU HSP clock is running */
 	clk_enable(ipu_data.ipu_clk);
 	/* Disable all interrupts */
 	idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_1);
 	idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_2);
 	idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_3);
 	idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_4);
 	idmac_write_ipureg(&ipu_data, 0, IPU_INT_CTRL_5);
 	dev_dbg(&pdev->dev, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev->name,
 		(unsigned long)mem_ipu->start, ipu_data.irq_fn, ipu_data.irq_err);
 	ret = ipu_irq_attach_irq(&ipu_data, pdev);
 	if (ret < 0)
 		goto err_attach_irq;
 	/* Initialize DMA engine */
 	ret = ipu_idmac_init(&ipu_data);
 	if (ret < 0)
 		goto err_idmac_init;
 	tasklet_init(&ipu_data.tasklet, ipu_gc_tasklet, (unsigned long)&ipu_data);
 	ipu_data.dev = &pdev->dev;
 	dev_dbg(ipu_data.dev, "IPU initialized\n");
 	return 0;
 err_idmac_init:
 err_attach_irq:
 	ipu_irq_detach_irq(&ipu_data, pdev);
 	clk_disable(ipu_data.ipu_clk);
 	clk_put(ipu_data.ipu_clk);
 err_clk_get:
 	iounmap(ipu_data.reg_ic);
 err_ioremap_ic:
 	iounmap(ipu_data.reg_ipu);
 err_ioremap_ipu:
 err_noirq:
 	dev_err(&pdev->dev, "Failed to probe IPU: %d\n", ret);
 	return ret;
 }
 static int __exit ipu_remove(struct platform_device *pdev)
 {
 	struct ipu *ipu = platform_get_drvdata(pdev);
 	ipu_idmac_exit(ipu);
 	ipu_irq_detach_irq(ipu, pdev);
 	clk_disable(ipu->ipu_clk);
 	clk_put(ipu->ipu_clk);
 	iounmap(ipu->reg_ic);
 	iounmap(ipu->reg_ipu);
 	tasklet_kill(&ipu->tasklet);
 	platform_set_drvdata(pdev, NULL);
 	return 0;
 }
 /*
  * We need two MEM resources - with IPU-common and Image Converter registers,
  * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
  */
 static struct platform_driver ipu_platform_driver = {
 	.driver = {
 		.name	= "ipu-core",
 		.owner	= THIS_MODULE,
 	},
 	.remove		= __exit_p(ipu_remove),
 };
 static int __init ipu_init(void)
 {
 	return platform_driver_probe(&ipu_platform_driver, ipu_probe);
 }
 subsys_initcall(ipu_init);
 MODULE_DESCRIPTION("IPU core driver");
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
 MODULE_ALIAS("platform:ipu-core");

drivers/dma/shdma.c

Diff comments View file @ 0582763

 /*
  * Renesas SuperH DMA Engine support
  *
  * base is drivers/dma/flsdma.c
  *
  * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>
  * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved.
  * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
  *
  * This is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
  * - DMA of SuperH does not have Hardware DMA chain mode.
  * - MAX DMA size is 16MB.
  *
  */
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/dmaengine.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 #include <asm/dmaengine.h>
 #include "shdma.h"
 /* DMA descriptor control */
 enum sh_dmae_desc_status {
 	DESC_IDLE,
 	DESC_PREPARED,
 	DESC_SUBMITTED,
 	DESC_COMPLETED,	/* completed, have to call callback */
 	DESC_WAITING,	/* callback called, waiting for ack / re-submit */
 };
 #define NR_DESCS_PER_CHANNEL 32
 /* Default MEMCPY transfer size = 2^2 = 4 bytes */
 #define LOG2_DEFAULT_XFER_SIZE	2
 /* A bitmask with bits enough for enum sh_dmae_slave_chan_id */
 static unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SHDMA_SLAVE_NUMBER)];
 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all);
 static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg)
 {
 	__raw_writel(data, sh_dc->base + reg / sizeof(u32));
 }
 static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg)
 {
 	return __raw_readl(sh_dc->base + reg / sizeof(u32));
 }
 static u16 dmaor_read(struct sh_dmae_device *shdev)
 {
 	return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32));
 }
 static void dmaor_write(struct sh_dmae_device *shdev, u16 data)
 {
 	__raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32));
 }
 /*
  * Reset DMA controller
  *
  * SH7780 has two DMAOR register
  */
 static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev)
 {
 	unsigned short dmaor = dmaor_read(shdev);
 	dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME));
 }
 static int sh_dmae_rst(struct sh_dmae_device *shdev)
 {
 	unsigned short dmaor;
 	sh_dmae_ctl_stop(shdev);
 	dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init;
 	dmaor_write(shdev, dmaor);
 	if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) {
 		pr_warning("dma-sh: Can't initialize DMAOR.\n");
 		return -EINVAL;
 	}
 	return 0;
 }
 static bool dmae_is_busy(struct sh_dmae_chan *sh_chan)
 {
 	u32 chcr = sh_dmae_readl(sh_chan, CHCR);
 	if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE)
 		return true; /* working */
 	return false; /* waiting */
 }
 static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr)
 {
 	struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
 						struct sh_dmae_device, common);
 	struct sh_dmae_pdata *pdata = shdev->pdata;
 	int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) |
 		((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift);
 	if (cnt >= pdata->ts_shift_num)
 		cnt = 0;
 	return pdata->ts_shift[cnt];
 }
 static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size)
 {
 	struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
 						struct sh_dmae_device, common);
 	struct sh_dmae_pdata *pdata = shdev->pdata;
 	int i;
 	for (i = 0; i < pdata->ts_shift_num; i++)
 		if (pdata->ts_shift[i] == l2size)
 			break;
 	if (i == pdata->ts_shift_num)
 		i = 0;
 	return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) |
 		((i << pdata->ts_high_shift) & pdata->ts_high_mask);
 }
 static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw)
 {
 	sh_dmae_writel(sh_chan, hw->sar, SAR);
 	sh_dmae_writel(sh_chan, hw->dar, DAR);
 	sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR);
 }
 static void dmae_start(struct sh_dmae_chan *sh_chan)
 {
 	u32 chcr = sh_dmae_readl(sh_chan, CHCR);
 	chcr |= CHCR_DE | CHCR_IE;
 	sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR);
 }
 static void dmae_halt(struct sh_dmae_chan *sh_chan)
 {
 	u32 chcr = sh_dmae_readl(sh_chan, CHCR);
 	chcr &= ~(CHCR_DE | CHCR_TE | CHCR_IE);
 	sh_dmae_writel(sh_chan, chcr, CHCR);
 }
 static void dmae_init(struct sh_dmae_chan *sh_chan)
 {
 	/*
 	 * Default configuration for dual address memory-memory transfer.
 	 * 0x400 represents auto-request.
 	 */
 	u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan,
 						   LOG2_DEFAULT_XFER_SIZE);
 	sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr);
 	sh_dmae_writel(sh_chan, chcr, CHCR);
 }
 static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val)
 {
 	/* When DMA was working, can not set data to CHCR */
 	if (dmae_is_busy(sh_chan))
 		return -EBUSY;
 	sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val);
 	sh_dmae_writel(sh_chan, val, CHCR);
 	return 0;
 }
 static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val)
 {
 	struct sh_dmae_device *shdev = container_of(sh_chan->common.device,
 						struct sh_dmae_device, common);
 	struct sh_dmae_pdata *pdata = shdev->pdata;
 	struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id];
 	u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16);
 	int shift = chan_pdata->dmars_bit;
 	if (dmae_is_busy(sh_chan))
 		return -EBUSY;
 	__raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift),
 		     addr);
 	return 0;
 }
 static dma_cookie_t sh_dmae_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct sh_desc *desc = tx_to_sh_desc(tx), *chunk, *last = desc, *c;
 	struct sh_dmae_chan *sh_chan = to_sh_chan(tx->chan);
 	dma_async_tx_callback callback = tx->callback;
 	dma_cookie_t cookie;
 	spin_lock_bh(&sh_chan->desc_lock);
 	cookie = sh_chan->common.cookie;
 	cookie++;
 	if (cookie < 0)
 		cookie = 1;
 	sh_chan->common.cookie = cookie;
 	tx->cookie = cookie;
 	/* Mark all chunks of this descriptor as submitted, move to the queue */
 	list_for_each_entry_safe(chunk, c, desc->node.prev, node) {
 		/*
 		 * All chunks are on the global ld_free, so, we have to find
 		 * the end of the chain ourselves
 		 */
 		if (chunk != desc && (chunk->mark == DESC_IDLE ||
 				      chunk->async_tx.cookie > 0 ||
 				      chunk->async_tx.cookie == -EBUSY ||
 				      &chunk->node == &sh_chan->ld_free))
 			break;
 		chunk->mark = DESC_SUBMITTED;
 		/* Callback goes to the last chunk */
 		chunk->async_tx.callback = NULL;
 		chunk->cookie = cookie;
 		list_move_tail(&chunk->node, &sh_chan->ld_queue);
 		last = chunk;
 	}
 	last->async_tx.callback = callback;
 	last->async_tx.callback_param = tx->callback_param;
 	dev_dbg(sh_chan->dev, "submit #%d@%p on %d: %x[%d] -> %x\n",
 		tx->cookie, &last->async_tx, sh_chan->id,
 		desc->hw.sar, desc->hw.tcr, desc->hw.dar);
 	spin_unlock_bh(&sh_chan->desc_lock);
 	return cookie;
 }
 /* Called with desc_lock held */
 static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan)
 {
 	struct sh_desc *desc;
 	list_for_each_entry(desc, &sh_chan->ld_free, node)
 		if (desc->mark != DESC_PREPARED) {
 			BUG_ON(desc->mark != DESC_IDLE);
 			list_del(&desc->node);
 			return desc;
 		}
 	return NULL;
 }
 static struct sh_dmae_slave_config *sh_dmae_find_slave(
 	struct sh_dmae_chan *sh_chan, enum sh_dmae_slave_chan_id slave_id)
 {
 	struct dma_device *dma_dev = sh_chan->common.device;
 	struct sh_dmae_device *shdev = container_of(dma_dev,
 					struct sh_dmae_device, common);
 	struct sh_dmae_pdata *pdata = shdev->pdata;
 	int i;
 	if ((unsigned)slave_id >= SHDMA_SLAVE_NUMBER)
 		return NULL;
 	for (i = 0; i < pdata->slave_num; i++)
 		if (pdata->slave[i].slave_id == slave_id)
 			return pdata->slave + i;
 	return NULL;
 }
 static int sh_dmae_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
 	struct sh_desc *desc;
 	struct sh_dmae_slave *param = chan->private;
 	pm_runtime_get_sync(sh_chan->dev);
 	/*
 	 * This relies on the guarantee from dmaengine that alloc_chan_resources
 	 * never runs concurrently with itself or free_chan_resources.
 	 */
 	if (param) {
 		struct sh_dmae_slave_config *cfg;
 		cfg = sh_dmae_find_slave(sh_chan, param->slave_id);
 		if (!cfg)
 			return -EINVAL;
 		if (test_and_set_bit(param->slave_id, sh_dmae_slave_used))
 			return -EBUSY;
 		param->config = cfg;
 		dmae_set_dmars(sh_chan, cfg->mid_rid);
 		dmae_set_chcr(sh_chan, cfg->chcr);
 	} else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) {
 		dmae_init(sh_chan);
 	}
 	spin_lock_bh(&sh_chan->desc_lock);
 	while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) {
 		spin_unlock_bh(&sh_chan->desc_lock);
 		desc = kzalloc(sizeof(struct sh_desc), GFP_KERNEL);
 		if (!desc) {
 			spin_lock_bh(&sh_chan->desc_lock);
 			break;
 		}
 		dma_async_tx_descriptor_init(&desc->async_tx,
 					&sh_chan->common);
 		desc->async_tx.tx_submit = sh_dmae_tx_submit;
 		desc->mark = DESC_IDLE;
 		spin_lock_bh(&sh_chan->desc_lock);
 		list_add(&desc->node, &sh_chan->ld_free);
 		sh_chan->descs_allocated++;
 	}
 	spin_unlock_bh(&sh_chan->desc_lock);
 	if (!sh_chan->descs_allocated)
 		pm_runtime_put(sh_chan->dev);
 	return sh_chan->descs_allocated;
 }
 /*
  * sh_dma_free_chan_resources - Free all resources of the channel.
  */
 static void sh_dmae_free_chan_resources(struct dma_chan *chan)
 {
 	struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
 	struct sh_desc *desc, *_desc;
 	LIST_HEAD(list);
 	int descs = sh_chan->descs_allocated;
 	dmae_halt(sh_chan);
 	/* Prepared and not submitted descriptors can still be on the queue */
 	if (!list_empty(&sh_chan->ld_queue))
 		sh_dmae_chan_ld_cleanup(sh_chan, true);
 	if (chan->private) {
 		/* The caller is holding dma_list_mutex */
 		struct sh_dmae_slave *param = chan->private;
 		clear_bit(param->slave_id, sh_dmae_slave_used);
 	}
 	spin_lock_bh(&sh_chan->desc_lock);
 	list_splice_init(&sh_chan->ld_free, &list);
 	sh_chan->descs_allocated = 0;
 	spin_unlock_bh(&sh_chan->desc_lock);
 	if (descs > 0)
 		pm_runtime_put(sh_chan->dev);
 	list_for_each_entry_safe(desc, _desc, &list, node)
 		kfree(desc);
 }
 /**
  * sh_dmae_add_desc - get, set up and return one transfer descriptor
  * @sh_chan:	DMA channel
  * @flags:	DMA transfer flags
  * @dest:	destination DMA address, incremented when direction equals
  *		DMA_FROM_DEVICE or DMA_BIDIRECTIONAL
  * @src:	source DMA address, incremented when direction equals
  *		DMA_TO_DEVICE or DMA_BIDIRECTIONAL
  * @len:	DMA transfer length
  * @first:	if NULL, set to the current descriptor and cookie set to -EBUSY
  * @direction:	needed for slave DMA to decide which address to keep constant,
  *		equals DMA_BIDIRECTIONAL for MEMCPY
  * Returns 0 or an error
  * Locks: called with desc_lock held
  */
 static struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan,
 	unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len,
 	struct sh_desc **first, enum dma_data_direction direction)
 {
 	struct sh_desc *new;
 	size_t copy_size;
 	if (!*len)
 		return NULL;
 	/* Allocate the link descriptor from the free list */
 	new = sh_dmae_get_desc(sh_chan);
 	if (!new) {
 		dev_err(sh_chan->dev, "No free link descriptor available\n");
 		return NULL;
 	}
 	copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1);
 	new->hw.sar = *src;
 	new->hw.dar = *dest;
 	new->hw.tcr = copy_size;
 	if (!*first) {
 		/* First desc */
 		new->async_tx.cookie = -EBUSY;
 		*first = new;
 	} else {
 		/* Other desc - invisible to the user */
 		new->async_tx.cookie = -EINVAL;
 	}
 	dev_dbg(sh_chan->dev,
 		"chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n",
 		copy_size, *len, *src, *dest, &new->async_tx,
 		new->async_tx.cookie, sh_chan->xmit_shift);
 	new->mark = DESC_PREPARED;
 	new->async_tx.flags = flags;
 	new->direction = direction;
 	*len -= copy_size;
 	if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE)
 		*src += copy_size;
 	if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE)
 		*dest += copy_size;
 	return new;
 }
 /*
  * sh_dmae_prep_sg - prepare transfer descriptors from an SG list
  *
  * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also
  * converted to scatter-gather to guarantee consistent locking and a correct
  * list manipulation. For slave DMA direction carries the usual meaning, and,
  * logically, the SG list is RAM and the addr variable contains slave address,
  * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL
  * and the SG list contains only one element and points at the source buffer.
  */
 static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan,
 	struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr,
 	enum dma_data_direction direction, unsigned long flags)
 {
 	struct scatterlist *sg;
 	struct sh_desc *first = NULL, *new = NULL /* compiler... */;
 	LIST_HEAD(tx_list);
 	int chunks = 0;
 	int i;
 	if (!sg_len)
 		return NULL;
 	for_each_sg(sgl, sg, sg_len, i)
 		chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) /
 			(SH_DMA_TCR_MAX + 1);
 	/* Have to lock the whole loop to protect against concurrent release */
 	spin_lock_bh(&sh_chan->desc_lock);
 	/*
 	 * Chaining:
 	 * first descriptor is what user is dealing with in all API calls, its
 	 *	cookie is at first set to -EBUSY, at tx-submit to a positive
 	 *	number
 	 * if more than one chunk is needed further chunks have cookie = -EINVAL
 	 * the last chunk, if not equal to the first, has cookie = -ENOSPC
 	 * all chunks are linked onto the tx_list head with their .node heads
 	 *	only during this function, then they are immediately spliced
 	 *	back onto the free list in form of a chain
 	 */
 	for_each_sg(sgl, sg, sg_len, i) {
 		dma_addr_t sg_addr = sg_dma_address(sg);
 		size_t len = sg_dma_len(sg);
 		if (!len)
 			goto err_get_desc;
 		do {
 			dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n",
 				i, sg, len, (unsigned long long)sg_addr);
 			if (direction == DMA_FROM_DEVICE)
 				new = sh_dmae_add_desc(sh_chan, flags,
 						&sg_addr, addr, &len, &first,
 						direction);
 			else
 				new = sh_dmae_add_desc(sh_chan, flags,
 						addr, &sg_addr, &len, &first,
 						direction);
 			if (!new)
 				goto err_get_desc;
 			new->chunks = chunks--;
 			list_add_tail(&new->node, &tx_list);
 		} while (len);
 	}
 	if (new != first)
 		new->async_tx.cookie = -ENOSPC;
 	/* Put them back on the free list, so, they don't get lost */
 	list_splice_tail(&tx_list, &sh_chan->ld_free);
 	spin_unlock_bh(&sh_chan->desc_lock);
 	return &first->async_tx;
 err_get_desc:
 	list_for_each_entry(new, &tx_list, node)
 		new->mark = DESC_IDLE;
 	list_splice(&tx_list, &sh_chan->ld_free);
 	spin_unlock_bh(&sh_chan->desc_lock);
 	return NULL;
 }
 static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy(
 	struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src,
 	size_t len, unsigned long flags)
 {
 	struct sh_dmae_chan *sh_chan;
 	struct scatterlist sg;
 	if (!chan || !len)
 		return NULL;
 	chan->private = NULL;
 	sh_chan = to_sh_chan(chan);
 	sg_init_table(&sg, 1);
 	sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len,
 		    offset_in_page(dma_src));
 	sg_dma_address(&sg) = dma_src;
 	sg_dma_len(&sg) = len;
 	return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL,
 			       flags);
 }
 static struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg(
 	struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
 	enum dma_data_direction direction, unsigned long flags)
 {
 	struct sh_dmae_slave *param;
 	struct sh_dmae_chan *sh_chan;
 	if (!chan)
 		return NULL;
 	sh_chan = to_sh_chan(chan);
 	param = chan->private;
 	/* Someone calling slave DMA on a public channel? */
 	if (!param || !sg_len) {
 		dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n",
 			 __func__, param, sg_len, param ? param->slave_id : -1);
 		return NULL;
 	}
 	/*
 	 * if (param != NULL), this is a successfully requested slave channel,
 	 * therefore param->config != NULL too.
 	 */
 	return sh_dmae_prep_sg(sh_chan, sgl, sg_len, &param->config->addr,
 			       direction, flags);
 }
-static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int sh_dmae_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+			   unsigned long arg)
 {
 	struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
 	/* Only supports DMA_TERMINATE_ALL */
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	if (!chan)
 		return -EINVAL;
 	dmae_halt(sh_chan);
 	spin_lock_bh(&sh_chan->desc_lock);
 	if (!list_empty(&sh_chan->ld_queue)) {
 		/* Record partial transfer */
 		struct sh_desc *desc = list_entry(sh_chan->ld_queue.next,
 						  struct sh_desc, node);
 		desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) <<
 			sh_chan->xmit_shift;
 	}
 	spin_unlock_bh(&sh_chan->desc_lock);
 	sh_dmae_chan_ld_cleanup(sh_chan, true);
 	return 0;
 }
 static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
 {
 	struct sh_desc *desc, *_desc;
 	/* Is the "exposed" head of a chain acked? */
 	bool head_acked = false;
 	dma_cookie_t cookie = 0;
 	dma_async_tx_callback callback = NULL;
 	void *param = NULL;
 	spin_lock_bh(&sh_chan->desc_lock);
 	list_for_each_entry_safe(desc, _desc, &sh_chan->ld_queue, node) {
 		struct dma_async_tx_descriptor *tx = &desc->async_tx;
 		BUG_ON(tx->cookie > 0 && tx->cookie != desc->cookie);
 		BUG_ON(desc->mark != DESC_SUBMITTED &&
 		       desc->mark != DESC_COMPLETED &&
 		       desc->mark != DESC_WAITING);
 		/*
 		 * queue is ordered, and we use this loop to (1) clean up all
 		 * completed descriptors, and to (2) update descriptor flags of
 		 * any chunks in a (partially) completed chain
 		 */
 		if (!all && desc->mark == DESC_SUBMITTED &&
 		    desc->cookie != cookie)
 			break;
 		if (tx->cookie > 0)
 			cookie = tx->cookie;
 		if (desc->mark == DESC_COMPLETED && desc->chunks == 1) {
 			if (sh_chan->completed_cookie != desc->cookie - 1)
 				dev_dbg(sh_chan->dev,
 					"Completing cookie %d, expected %d\n",
 					desc->cookie,
 					sh_chan->completed_cookie + 1);
 			sh_chan->completed_cookie = desc->cookie;
 		}
 		/* Call callback on the last chunk */
 		if (desc->mark == DESC_COMPLETED && tx->callback) {
 			desc->mark = DESC_WAITING;
 			callback = tx->callback;
 			param = tx->callback_param;
 			dev_dbg(sh_chan->dev, "descriptor #%d@%p on %d callback\n",
 				tx->cookie, tx, sh_chan->id);
 			BUG_ON(desc->chunks != 1);
 			break;
 		}
 		if (tx->cookie > 0 || tx->cookie == -EBUSY) {
 			if (desc->mark == DESC_COMPLETED) {
 				BUG_ON(tx->cookie < 0);
 				desc->mark = DESC_WAITING;
 			}
 			head_acked = async_tx_test_ack(tx);
 		} else {
 			switch (desc->mark) {
 			case DESC_COMPLETED:
 				desc->mark = DESC_WAITING;
 				/* Fall through */
 			case DESC_WAITING:
 				if (head_acked)
 					async_tx_ack(&desc->async_tx);
 			}
 		}
 		dev_dbg(sh_chan->dev, "descriptor %p #%d completed.\n",
 			tx, tx->cookie);
 		if (((desc->mark == DESC_COMPLETED ||
 		      desc->mark == DESC_WAITING) &&
 		     async_tx_test_ack(&desc->async_tx)) || all) {
 			/* Remove from ld_queue list */
 			desc->mark = DESC_IDLE;
 			list_move(&desc->node, &sh_chan->ld_free);
 		}
 	}
 	spin_unlock_bh(&sh_chan->desc_lock);
 	if (callback)
 		callback(param);
 	return callback;
 }
 /*
  * sh_chan_ld_cleanup - Clean up link descriptors
  *
  * This function cleans up the ld_queue of DMA channel.
  */
 static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all)
 {
 	while (__ld_cleanup(sh_chan, all))
 		;
 }
 static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan)
 {
 	struct sh_desc *desc;
 	spin_lock_bh(&sh_chan->desc_lock);
 	/* DMA work check */
 	if (dmae_is_busy(sh_chan)) {
 		spin_unlock_bh(&sh_chan->desc_lock);
 		return;
 	}
 	/* Find the first not transferred desciptor */
 	list_for_each_entry(desc, &sh_chan->ld_queue, node)
 		if (desc->mark == DESC_SUBMITTED) {
 			dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n",
 				desc->async_tx.cookie, sh_chan->id,
 				desc->hw.tcr, desc->hw.sar, desc->hw.dar);
 			/* Get the ld start address from ld_queue */
 			dmae_set_reg(sh_chan, &desc->hw);
 			dmae_start(sh_chan);
 			break;
 		}
 	spin_unlock_bh(&sh_chan->desc_lock);
 }
 static void sh_dmae_memcpy_issue_pending(struct dma_chan *chan)
 {
 	struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
 	sh_chan_xfer_ld_queue(sh_chan);
 }
 static enum dma_status sh_dmae_tx_status(struct dma_chan *chan,
 					dma_cookie_t cookie,
 					struct dma_tx_state *txstate)
 {
 	struct sh_dmae_chan *sh_chan = to_sh_chan(chan);
 	dma_cookie_t last_used;
 	dma_cookie_t last_complete;
 	enum dma_status status;
 	sh_dmae_chan_ld_cleanup(sh_chan, false);
 	last_used = chan->cookie;
 	last_complete = sh_chan->completed_cookie;
 	BUG_ON(last_complete < 0);
 	dma_set_tx_state(txstate, last_complete, last_used, 0);
 	spin_lock_bh(&sh_chan->desc_lock);
 	status = dma_async_is_complete(cookie, last_complete, last_used);
 	/*
 	 * If we don't find cookie on the queue, it has been aborted and we have
 	 * to report error
 	 */
 	if (status != DMA_SUCCESS) {
 		struct sh_desc *desc;
 		status = DMA_ERROR;
 		list_for_each_entry(desc, &sh_chan->ld_queue, node)
 			if (desc->cookie == cookie) {
 				status = DMA_IN_PROGRESS;
 				break;
 			}
 	}
 	spin_unlock_bh(&sh_chan->desc_lock);
 	return status;
 }
 static irqreturn_t sh_dmae_interrupt(int irq, void *data)
 {
 	irqreturn_t ret = IRQ_NONE;
 	struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
 	u32 chcr = sh_dmae_readl(sh_chan, CHCR);
 	if (chcr & CHCR_TE) {
 		/* DMA stop */
 		dmae_halt(sh_chan);
 		ret = IRQ_HANDLED;
 		tasklet_schedule(&sh_chan->tasklet);
 	}
 	return ret;
 }
 #if defined(CONFIG_CPU_SH4)
 static irqreturn_t sh_dmae_err(int irq, void *data)
 {
 	struct sh_dmae_device *shdev = (struct sh_dmae_device *)data;
 	int i;
 	/* halt the dma controller */
 	sh_dmae_ctl_stop(shdev);
 	/* We cannot detect, which channel caused the error, have to reset all */
 	for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) {
 		struct sh_dmae_chan *sh_chan = shdev->chan[i];
 		if (sh_chan) {
 			struct sh_desc *desc;
 			/* Stop the channel */
 			dmae_halt(sh_chan);
 			/* Complete all  */
 			list_for_each_entry(desc, &sh_chan->ld_queue, node) {
 				struct dma_async_tx_descriptor *tx = &desc->async_tx;
 				desc->mark = DESC_IDLE;
 				if (tx->callback)
 					tx->callback(tx->callback_param);
 			}
 			list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free);
 		}
 	}
 	sh_dmae_rst(shdev);
 	return IRQ_HANDLED;
 }
 #endif
 static void dmae_do_tasklet(unsigned long data)
 {
 	struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data;
 	struct sh_desc *desc;
 	u32 sar_buf = sh_dmae_readl(sh_chan, SAR);
 	u32 dar_buf = sh_dmae_readl(sh_chan, DAR);
 	spin_lock(&sh_chan->desc_lock);
 	list_for_each_entry(desc, &sh_chan->ld_queue, node) {
 		if (desc->mark == DESC_SUBMITTED &&
 		    ((desc->direction == DMA_FROM_DEVICE &&
 		      (desc->hw.dar + desc->hw.tcr) == dar_buf) ||
 		     (desc->hw.sar + desc->hw.tcr) == sar_buf)) {
 			dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n",
 				desc->async_tx.cookie, &desc->async_tx,
 				desc->hw.dar);
 			desc->mark = DESC_COMPLETED;
 			break;
 		}
 	}
 	spin_unlock(&sh_chan->desc_lock);
 	/* Next desc */
 	sh_chan_xfer_ld_queue(sh_chan);
 	sh_dmae_chan_ld_cleanup(sh_chan, false);
 }
 static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id,
 					int irq, unsigned long flags)
 {
 	int err;
 	struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id];
 	struct platform_device *pdev = to_platform_device(shdev->common.dev);
 	struct sh_dmae_chan *new_sh_chan;
 	/* alloc channel */
 	new_sh_chan = kzalloc(sizeof(struct sh_dmae_chan), GFP_KERNEL);
 	if (!new_sh_chan) {
 		dev_err(shdev->common.dev,
 			"No free memory for allocating dma channels!\n");
 		return -ENOMEM;
 	}
 	/* copy struct dma_device */
 	new_sh_chan->common.device = &shdev->common;
 	new_sh_chan->dev = shdev->common.dev;
 	new_sh_chan->id = id;
 	new_sh_chan->irq = irq;
 	new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32);
 	/* Init DMA tasklet */
 	tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet,
 			(unsigned long)new_sh_chan);
 	/* Init the channel */
 	dmae_init(new_sh_chan);
 	spin_lock_init(&new_sh_chan->desc_lock);
 	/* Init descripter manage list */
 	INIT_LIST_HEAD(&new_sh_chan->ld_queue);
 	INIT_LIST_HEAD(&new_sh_chan->ld_free);
 	/* Add the channel to DMA device channel list */
 	list_add_tail(&new_sh_chan->common.device_node,
 			&shdev->common.channels);
 	shdev->common.chancnt++;
 	if (pdev->id >= 0)
 		snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
 			 "sh-dmae%d.%d", pdev->id, new_sh_chan->id);
 	else
 		snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id),
 			 "sh-dma%d", new_sh_chan->id);
 	/* set up channel irq */
 	err = request_irq(irq, &sh_dmae_interrupt, flags,
 			  new_sh_chan->dev_id, new_sh_chan);
 	if (err) {
 		dev_err(shdev->common.dev, "DMA channel %d request_irq error "
 			"with return %d\n", id, err);
 		goto err_no_irq;
 	}
 	shdev->chan[id] = new_sh_chan;
 	return 0;
 err_no_irq:
 	/* remove from dmaengine device node */
 	list_del(&new_sh_chan->common.device_node);
 	kfree(new_sh_chan);
 	return err;
 }
 static void sh_dmae_chan_remove(struct sh_dmae_device *shdev)
 {
 	int i;
 	for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) {
 		if (shdev->chan[i]) {
 			struct sh_dmae_chan *sh_chan = shdev->chan[i];
 			free_irq(sh_chan->irq, sh_chan);
 			list_del(&sh_chan->common.device_node);
 			kfree(sh_chan);
 			shdev->chan[i] = NULL;
 		}
 	}
 	shdev->common.chancnt = 0;
 }
 static int __init sh_dmae_probe(struct platform_device *pdev)
 {
 	struct sh_dmae_pdata *pdata = pdev->dev.platform_data;
 	unsigned long irqflags = IRQF_DISABLED,
 		chan_flag[SH_DMAC_MAX_CHANNELS] = {};
 	int errirq, chan_irq[SH_DMAC_MAX_CHANNELS];
 	int err, i, irq_cnt = 0, irqres = 0;
 	struct sh_dmae_device *shdev;
 	struct resource *chan, *dmars, *errirq_res, *chanirq_res;
 	/* get platform data */
 	if (!pdata || !pdata->channel_num)
 		return -ENODEV;
 	chan = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	/* DMARS area is optional, if absent, this controller cannot do slave DMA */
 	dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	/*
 	 * IRQ resources:
 	 * 1. there always must be at least one IRQ IO-resource. On SH4 it is
 	 *    the error IRQ, in which case it is the only IRQ in this resource:
 	 *    start == end. If it is the only IRQ resource, all channels also
 	 *    use the same IRQ.
 	 * 2. DMA channel IRQ resources can be specified one per resource or in
 	 *    ranges (start != end)
 	 * 3. iff all events (channels and, optionally, error) on this
 	 *    controller use the same IRQ, only one IRQ resource can be
 	 *    specified, otherwise there must be one IRQ per channel, even if
 	 *    some of them are equal
 	 * 4. if all IRQs on this controller are equal or if some specific IRQs
 	 *    specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be
 	 *    requested with the IRQF_SHARED flag
 	 */
 	errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 	if (!chan || !errirq_res)
 		return -ENODEV;
 	if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) {
 		dev_err(&pdev->dev, "DMAC register region already claimed\n");
 		return -EBUSY;
 	}
 	if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) {
 		dev_err(&pdev->dev, "DMAC DMARS region already claimed\n");
 		err = -EBUSY;
 		goto ermrdmars;
 	}
 	err = -ENOMEM;
 	shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL);
 	if (!shdev) {
 		dev_err(&pdev->dev, "Not enough memory\n");
 		goto ealloc;
 	}
 	shdev->chan_reg = ioremap(chan->start, resource_size(chan));
 	if (!shdev->chan_reg)
 		goto emapchan;
 	if (dmars) {
 		shdev->dmars = ioremap(dmars->start, resource_size(dmars));
 		if (!shdev->dmars)
 			goto emapdmars;
 	}
 	/* platform data */
 	shdev->pdata = pdata;
 	pm_runtime_enable(&pdev->dev);
 	pm_runtime_get_sync(&pdev->dev);
 	/* reset dma controller */
 	err = sh_dmae_rst(shdev);
 	if (err)
 		goto rst_err;
 	INIT_LIST_HEAD(&shdev->common.channels);
 	dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask);
 	if (dmars)
 		dma_cap_set(DMA_SLAVE, shdev->common.cap_mask);
 	shdev->common.device_alloc_chan_resources
 		= sh_dmae_alloc_chan_resources;
 	shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources;
 	shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy;
 	shdev->common.device_tx_status = sh_dmae_tx_status;
 	shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending;
 	/* Compulsory for DMA_SLAVE fields */
 	shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg;
 	shdev->common.device_control = sh_dmae_control;
 	shdev->common.dev = &pdev->dev;
 	/* Default transfer size of 32 bytes requires 32-byte alignment */
 	shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE;
 #if defined(CONFIG_CPU_SH4)
 	chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1);
 	if (!chanirq_res)
 		chanirq_res = errirq_res;
 	else
 		irqres++;
 	if (chanirq_res == errirq_res ||
 	    (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE)
 		irqflags = IRQF_SHARED;
 	errirq = errirq_res->start;
 	err = request_irq(errirq, sh_dmae_err, irqflags,
 			  "DMAC Address Error", shdev);
 	if (err) {
 		dev_err(&pdev->dev,
 			"DMA failed requesting irq #%d, error %d\n",
 			errirq, err);
 		goto eirq_err;
 	}
 #else
 	chanirq_res = errirq_res;
 #endif /* CONFIG_CPU_SH4 */
 	if (chanirq_res->start == chanirq_res->end &&
 	    !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) {
 		/* Special case - all multiplexed */
 		for (; irq_cnt < pdata->channel_num; irq_cnt++) {
 			chan_irq[irq_cnt] = chanirq_res->start;
 			chan_flag[irq_cnt] = IRQF_SHARED;
 		}
 	} else {
 		do {
 			for (i = chanirq_res->start; i <= chanirq_res->end; i++) {
 				if ((errirq_res->flags & IORESOURCE_BITS) ==
 				    IORESOURCE_IRQ_SHAREABLE)
 					chan_flag[irq_cnt] = IRQF_SHARED;
 				else
 					chan_flag[irq_cnt] = IRQF_DISABLED;
 				dev_dbg(&pdev->dev,
 					"Found IRQ %d for channel %d\n",
 					i, irq_cnt);
 				chan_irq[irq_cnt++] = i;
 			}
 			chanirq_res = platform_get_resource(pdev,
 						IORESOURCE_IRQ, ++irqres);
 		} while (irq_cnt < pdata->channel_num && chanirq_res);
 	}
 	if (irq_cnt < pdata->channel_num)
 		goto eirqres;
 	/* Create DMA Channel */
 	for (i = 0; i < pdata->channel_num; i++) {
 		err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]);
 		if (err)
 			goto chan_probe_err;
 	}
 	pm_runtime_put(&pdev->dev);
 	platform_set_drvdata(pdev, shdev);
 	dma_async_device_register(&shdev->common);
 	return err;
 chan_probe_err:
 	sh_dmae_chan_remove(shdev);
 eirqres:
 #if defined(CONFIG_CPU_SH4)
 	free_irq(errirq, shdev);
 eirq_err:
 #endif
 rst_err:
 	pm_runtime_put(&pdev->dev);
 	if (dmars)
 		iounmap(shdev->dmars);
 emapdmars:
 	iounmap(shdev->chan_reg);
 emapchan:
 	kfree(shdev);
 ealloc:
 	if (dmars)
 		release_mem_region(dmars->start, resource_size(dmars));
 ermrdmars:
 	release_mem_region(chan->start, resource_size(chan));
 	return err;
 }
 static int __exit sh_dmae_remove(struct platform_device *pdev)
 {
 	struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
 	struct resource *res;
 	int errirq = platform_get_irq(pdev, 0);
 	dma_async_device_unregister(&shdev->common);
 	if (errirq > 0)
 		free_irq(errirq, shdev);
 	/* channel data remove */
 	sh_dmae_chan_remove(shdev);
 	pm_runtime_disable(&pdev->dev);
 	if (shdev->dmars)
 		iounmap(shdev->dmars);
 	iounmap(shdev->chan_reg);
 	kfree(shdev);
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (res)
 		release_mem_region(res->start, resource_size(res));
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	if (res)
 		release_mem_region(res->start, resource_size(res));
 	return 0;
 }
 static void sh_dmae_shutdown(struct platform_device *pdev)
 {
 	struct sh_dmae_device *shdev = platform_get_drvdata(pdev);
 	sh_dmae_ctl_stop(shdev);
 }
 static struct platform_driver sh_dmae_driver = {
 	.remove		= __exit_p(sh_dmae_remove),
 	.shutdown	= sh_dmae_shutdown,
 	.driver = {
 		.name	= "sh-dma-engine",
 	},
 };
 static int __init sh_dmae_init(void)
 {
 	return platform_driver_probe(&sh_dmae_driver, sh_dmae_probe);
 }
 module_init(sh_dmae_init);
 static void __exit sh_dmae_exit(void)
 {
 	platform_driver_unregister(&sh_dmae_driver);
 }
 module_exit(sh_dmae_exit);
 MODULE_AUTHOR("Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com>");
 MODULE_DESCRIPTION("Renesas SH DMA Engine driver");
 MODULE_LICENSE("GPL");

drivers/dma/ste_dma40.c

Diff comments View file @ 0582763

 /*
  * driver/dma/ste_dma40.c
  *
  * Copyright (C) ST-Ericsson 2007-2010
  * License terms: GNU General Public License (GPL) version 2
  * Author: Per Friden <per.friden@stericsson.com>
  * Author: Jonas Aaberg <jonas.aberg@stericsson.com>
  *
  */
 #include <linux/kernel.h>
 #include <linux/slab.h>
 #include <linux/dmaengine.h>
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <plat/ste_dma40.h>
 #include "ste_dma40_ll.h"
 #define D40_NAME "dma40"
 #define D40_PHY_CHAN -1
 /* For masking out/in 2 bit channel positions */
 #define D40_CHAN_POS(chan)  (2 * (chan / 2))
 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
 /* Maximum iterations taken before giving up suspending a channel */
 #define D40_SUSPEND_MAX_IT 500
 #define D40_ALLOC_FREE		(1 << 31)
 #define D40_ALLOC_PHY		(1 << 30)
 #define D40_ALLOC_LOG_FREE	0
 /* The number of free d40_desc to keep in memory before starting
  * to kfree() them */
 #define D40_DESC_CACHE_SIZE 50
 /* Hardware designer of the block */
 #define D40_PERIPHID2_DESIGNER 0x8
 /**
  * enum 40_command - The different commands and/or statuses.
  *
  * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
  * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
  * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
  * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
  */
 enum d40_command {
 	D40_DMA_STOP		= 0,
 	D40_DMA_RUN		= 1,
 	D40_DMA_SUSPEND_REQ	= 2,
 	D40_DMA_SUSPENDED	= 3
 };
 /**
  * struct d40_lli_pool - Structure for keeping LLIs in memory
  *
  * @base: Pointer to memory area when the pre_alloc_lli's are not large
  * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
  * pre_alloc_lli is used.
  * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
  * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
  * one buffer to one buffer.
  */
 struct d40_lli_pool {
 	void	*base;
 	int	size;
 	/* Space for dst and src, plus an extra for padding */
 	u8	pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
 };
 /**
  * struct d40_desc - A descriptor is one DMA job.
  *
  * @lli_phy: LLI settings for physical channel. Both src and dst=
  * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
  * lli_len equals one.
  * @lli_log: Same as above but for logical channels.
  * @lli_pool: The pool with two entries pre-allocated.
  * @lli_len: Number of LLI's in lli_pool
  * @lli_tcount: Number of LLIs processed in the transfer. When equals lli_len
  * then this transfer job is done.
  * @txd: DMA engine struct. Used for among other things for communication
  * during a transfer.
  * @node: List entry.
  * @dir: The transfer direction of this job.
  * @is_in_client_list: true if the client owns this descriptor.
  *
  * This descriptor is used for both logical and physical transfers.
  */
 struct d40_desc {
 	/* LLI physical */
 	struct d40_phy_lli_bidir	 lli_phy;
 	/* LLI logical */
 	struct d40_log_lli_bidir	 lli_log;
 	struct d40_lli_pool		 lli_pool;
 	u32				 lli_len;
 	u32				 lli_tcount;
 	struct dma_async_tx_descriptor	 txd;
 	struct list_head		 node;
 	enum dma_data_direction		 dir;
 	bool				 is_in_client_list;
 };
 /**
  * struct d40_lcla_pool - LCLA pool settings and data.
  *
  * @base: The virtual address of LCLA.
  * @phy: Physical base address of LCLA.
  * @base_size: size of lcla.
  * @lock: Lock to protect the content in this struct.
  * @alloc_map: Mapping between physical channel and LCLA entries.
  * @num_blocks: The number of entries of alloc_map. Equals to the
  * number of physical channels.
  */
 struct d40_lcla_pool {
 	void		*base;
 	dma_addr_t	 phy;
 	resource_size_t  base_size;
 	spinlock_t	 lock;
 	u32		*alloc_map;
 	int		 num_blocks;
 };
 /**
  * struct d40_phy_res - struct for handling eventlines mapped to physical
  * channels.
  *
  * @lock: A lock protection this entity.
  * @num: The physical channel number of this entity.
  * @allocated_src: Bit mapped to show which src event line's are mapped to
  * this physical channel. Can also be free or physically allocated.
  * @allocated_dst: Same as for src but is dst.
  * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
  * event line number. Both allocated_src and allocated_dst can not be
  * allocated to a physical channel, since the interrupt handler has then
  * no way of figure out which one the interrupt belongs to.
  */
 struct d40_phy_res {
 	spinlock_t lock;
 	int	   num;
 	u32	   allocated_src;
 	u32	   allocated_dst;
 };
 struct d40_base;
 /**
  * struct d40_chan - Struct that describes a channel.
  *
  * @lock: A spinlock to protect this struct.
  * @log_num: The logical number, if any of this channel.
  * @completed: Starts with 1, after first interrupt it is set to dma engine's
  * current cookie.
  * @pending_tx: The number of pending transfers. Used between interrupt handler
  * and tasklet.
  * @busy: Set to true when transfer is ongoing on this channel.
  * @phy_chan: Pointer to physical channel which this instance runs on.
  * @chan: DMA engine handle.
  * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
  * transfer and call client callback.
  * @client: Cliented owned descriptor list.
  * @active: Active descriptor.
  * @queue: Queued jobs.
  * @free: List of free descripts, ready to be reused.
  * @free_len: Number of descriptors in the free list.
  * @dma_cfg: The client configuration of this dma channel.
  * @base: Pointer to the device instance struct.
  * @src_def_cfg: Default cfg register setting for src.
  * @dst_def_cfg: Default cfg register setting for dst.
  * @log_def: Default logical channel settings.
  * @lcla: Space for one dst src pair for logical channel transfers.
  * @lcpa: Pointer to dst and src lcpa settings.
  *
  * This struct can either "be" a logical or a physical channel.
  */
 struct d40_chan {
 	spinlock_t			 lock;
 	int				 log_num;
 	/* ID of the most recent completed transfer */
 	int				 completed;
 	int				 pending_tx;
 	bool				 busy;
 	struct d40_phy_res		*phy_chan;
 	struct dma_chan			 chan;
 	struct tasklet_struct		 tasklet;
 	struct list_head		 client;
 	struct list_head		 active;
 	struct list_head		 queue;
 	struct list_head		 free;
 	int				 free_len;
 	struct stedma40_chan_cfg	 dma_cfg;
 	struct d40_base			*base;
 	/* Default register configurations */
 	u32				 src_def_cfg;
 	u32				 dst_def_cfg;
 	struct d40_def_lcsp		 log_def;
 	struct d40_lcla_elem		 lcla;
 	struct d40_log_lli_full		*lcpa;
 };
 /**
  * struct d40_base - The big global struct, one for each probe'd instance.
  *
  * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
  * @execmd_lock: Lock for execute command usage since several channels share
  * the same physical register.
  * @dev: The device structure.
  * @virtbase: The virtual base address of the DMA's register.
  * @clk: Pointer to the DMA clock structure.
  * @phy_start: Physical memory start of the DMA registers.
  * @phy_size: Size of the DMA register map.
  * @irq: The IRQ number.
  * @num_phy_chans: The number of physical channels. Read from HW. This
  * is the number of available channels for this driver, not counting "Secure
  * mode" allocated physical channels.
  * @num_log_chans: The number of logical channels. Calculated from
  * num_phy_chans.
  * @dma_both: dma_device channels that can do both memcpy and slave transfers.
  * @dma_slave: dma_device channels that can do only do slave transfers.
  * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
  * @phy_chans: Room for all possible physical channels in system.
  * @log_chans: Room for all possible logical channels in system.
  * @lookup_log_chans: Used to map interrupt number to logical channel. Points
  * to log_chans entries.
  * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
  * to phy_chans entries.
  * @plat_data: Pointer to provided platform_data which is the driver
  * configuration.
  * @phy_res: Vector containing all physical channels.
  * @lcla_pool: lcla pool settings and data.
  * @lcpa_base: The virtual mapped address of LCPA.
  * @phy_lcpa: The physical address of the LCPA.
  * @lcpa_size: The size of the LCPA area.
  */
 struct d40_base {
 	spinlock_t			 interrupt_lock;
 	spinlock_t			 execmd_lock;
 	struct device			 *dev;
 	void __iomem			 *virtbase;
 	struct clk			 *clk;
 	phys_addr_t			  phy_start;
 	resource_size_t			  phy_size;
 	int				  irq;
 	int				  num_phy_chans;
 	int				  num_log_chans;
 	struct dma_device		  dma_both;
 	struct dma_device		  dma_slave;
 	struct dma_device		  dma_memcpy;
 	struct d40_chan			 *phy_chans;
 	struct d40_chan			 *log_chans;
 	struct d40_chan			**lookup_log_chans;
 	struct d40_chan			**lookup_phy_chans;
 	struct stedma40_platform_data	 *plat_data;
 	/* Physical half channels */
 	struct d40_phy_res		 *phy_res;
 	struct d40_lcla_pool		  lcla_pool;
 	void				 *lcpa_base;
 	dma_addr_t			  phy_lcpa;
 	resource_size_t			  lcpa_size;
 };
 /**
  * struct d40_interrupt_lookup - lookup table for interrupt handler
  *
  * @src: Interrupt mask register.
  * @clr: Interrupt clear register.
  * @is_error: true if this is an error interrupt.
  * @offset: start delta in the lookup_log_chans in d40_base. If equals to
  * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
  */
 struct d40_interrupt_lookup {
 	u32 src;
 	u32 clr;
 	bool is_error;
 	int offset;
 };
 /**
  * struct d40_reg_val - simple lookup struct
  *
  * @reg: The register.
  * @val: The value that belongs to the register in reg.
  */
 struct d40_reg_val {
 	unsigned int reg;
 	unsigned int val;
 };
 static int d40_pool_lli_alloc(struct d40_desc *d40d,
 			      int lli_len, bool is_log)
 {
 	u32 align;
 	void *base;
 	if (is_log)
 		align = sizeof(struct d40_log_lli);
 	else
 		align = sizeof(struct d40_phy_lli);
 	if (lli_len == 1) {
 		base = d40d->lli_pool.pre_alloc_lli;
 		d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
 		d40d->lli_pool.base = NULL;
 	} else {
 		d40d->lli_pool.size = ALIGN(lli_len * 2 * align, align);
 		base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
 		d40d->lli_pool.base = base;
 		if (d40d->lli_pool.base == NULL)
 			return -ENOMEM;
 	}
 	if (is_log) {
 		d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
 					      align);
 		d40d->lli_log.dst = PTR_ALIGN(d40d->lli_log.src + lli_len,
 					      align);
 	} else {
 		d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
 					      align);
 		d40d->lli_phy.dst = PTR_ALIGN(d40d->lli_phy.src + lli_len,
 					      align);
 		d40d->lli_phy.src_addr = virt_to_phys(d40d->lli_phy.src);
 		d40d->lli_phy.dst_addr = virt_to_phys(d40d->lli_phy.dst);
 	}
 	return 0;
 }
 static void d40_pool_lli_free(struct d40_desc *d40d)
 {
 	kfree(d40d->lli_pool.base);
 	d40d->lli_pool.base = NULL;
 	d40d->lli_pool.size = 0;
 	d40d->lli_log.src = NULL;
 	d40d->lli_log.dst = NULL;
 	d40d->lli_phy.src = NULL;
 	d40d->lli_phy.dst = NULL;
 	d40d->lli_phy.src_addr = 0;
 	d40d->lli_phy.dst_addr = 0;
 }
 static dma_cookie_t d40_assign_cookie(struct d40_chan *d40c,
 				      struct d40_desc *desc)
 {
 	dma_cookie_t cookie = d40c->chan.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	d40c->chan.cookie = cookie;
 	desc->txd.cookie = cookie;
 	return cookie;
 }
 static void d40_desc_reset(struct d40_desc *d40d)
 {
 	d40d->lli_tcount = 0;
 }
 static void d40_desc_remove(struct d40_desc *d40d)
 {
 	list_del(&d40d->node);
 }
 static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
 {
 	struct d40_desc *desc;
 	struct d40_desc *d;
 	struct d40_desc *_d;
 	if (!list_empty(&d40c->client)) {
 		list_for_each_entry_safe(d, _d, &d40c->client, node)
 			if (async_tx_test_ack(&d->txd)) {
 				d40_pool_lli_free(d);
 				d40_desc_remove(d);
 				desc = d;
 				goto out;
 			}
 	}
 	if (list_empty(&d40c->free)) {
 		/* Alloc new desc because we're out of used ones */
 		desc = kzalloc(sizeof(struct d40_desc), GFP_NOWAIT);
 		if (desc == NULL)
 			goto out;
 		INIT_LIST_HEAD(&desc->node);
 	} else {
 		/* Reuse an old desc. */
 		desc = list_first_entry(&d40c->free,
 					struct d40_desc,
 					node);
 		list_del(&desc->node);
 		d40c->free_len--;
 	}
 out:
 	return desc;
 }
 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
 {
 	if (d40c->free_len < D40_DESC_CACHE_SIZE) {
 		list_add_tail(&d40d->node, &d40c->free);
 		d40c->free_len++;
 	} else
 		kfree(d40d);
 }
 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
 {
 	list_add_tail(&desc->node, &d40c->active);
 }
 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
 {
 	struct d40_desc *d;
 	if (list_empty(&d40c->active))
 		return NULL;
 	d = list_first_entry(&d40c->active,
 			     struct d40_desc,
 			     node);
 	return d;
 }
 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
 {
 	list_add_tail(&desc->node, &d40c->queue);
 }
 static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
 {
 	struct d40_desc *d;
 	if (list_empty(&d40c->queue))
 		return NULL;
 	d = list_first_entry(&d40c->queue,
 			     struct d40_desc,
 			     node);
 	return d;
 }
 /* Support functions for logical channels */
 static int d40_lcla_id_get(struct d40_chan *d40c,
 			   struct d40_lcla_pool *pool)
 {
 	int src_id = 0;
 	int dst_id = 0;
 	struct d40_log_lli *lcla_lidx_base =
 		pool->base + d40c->phy_chan->num * 1024;
 	int i;
 	int lli_per_log = d40c->base->plat_data->llis_per_log;
 	if (d40c->lcla.src_id >= 0 && d40c->lcla.dst_id >= 0)
 		return 0;
 	if (pool->num_blocks > 32)
 		return -EINVAL;
 	spin_lock(&pool->lock);
 	for (i = 0; i < pool->num_blocks; i++) {
 		if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) {
 			pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i);
 			break;
 		}
 	}
 	src_id = i;
 	if (src_id >= pool->num_blocks)
 		goto err;
 	for (; i < pool->num_blocks; i++) {
 		if (!(pool->alloc_map[d40c->phy_chan->num] & (0x1 << i))) {
 			pool->alloc_map[d40c->phy_chan->num] |= (0x1 << i);
 			break;
 		}
 	}
 	dst_id = i;
 	if (dst_id == src_id)
 		goto err;
 	d40c->lcla.src_id = src_id;
 	d40c->lcla.dst_id = dst_id;
 	d40c->lcla.dst = lcla_lidx_base + dst_id * lli_per_log + 1;
 	d40c->lcla.src = lcla_lidx_base + src_id * lli_per_log + 1;
 	spin_unlock(&pool->lock);
 	return 0;
 err:
 	spin_unlock(&pool->lock);
 	return -EINVAL;
 }
 static void d40_lcla_id_put(struct d40_chan *d40c,
 			    struct d40_lcla_pool *pool,
 			    int id)
 {
 	if (id < 0)
 		return;
 	d40c->lcla.src_id = -1;
 	d40c->lcla.dst_id = -1;
 	spin_lock(&pool->lock);
 	pool->alloc_map[d40c->phy_chan->num] &= (~(0x1 << id));
 	spin_unlock(&pool->lock);
 }
 static int d40_channel_execute_command(struct d40_chan *d40c,
 				       enum d40_command command)
 {
 	int status, i;
 	void __iomem *active_reg;
 	int ret = 0;
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->base->execmd_lock, flags);
 	if (d40c->phy_chan->num % 2 == 0)
 		active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 	else
 		active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 	if (command == D40_DMA_SUSPEND_REQ) {
 		status = (readl(active_reg) &
 			  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 			D40_CHAN_POS(d40c->phy_chan->num);
 		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 			goto done;
 	}
 	writel(command << D40_CHAN_POS(d40c->phy_chan->num), active_reg);
 	if (command == D40_DMA_SUSPEND_REQ) {
 		for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
 			status = (readl(active_reg) &
 				  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 				D40_CHAN_POS(d40c->phy_chan->num);
 			cpu_relax();
 			/*
 			 * Reduce the number of bus accesses while
 			 * waiting for the DMA to suspend.
 			 */
 			udelay(3);
 			if (status == D40_DMA_STOP ||
 			    status == D40_DMA_SUSPENDED)
 				break;
 		}
 		if (i == D40_SUSPEND_MAX_IT) {
 			dev_err(&d40c->chan.dev->device,
 				"[%s]: unable to suspend the chl %d (log: %d) status %x\n",
 				__func__, d40c->phy_chan->num, d40c->log_num,
 				status);
 			dump_stack();
 			ret = -EBUSY;
 		}
 	}
 done:
 	spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
 	return ret;
 }
 static void d40_term_all(struct d40_chan *d40c)
 {
 	struct d40_desc *d40d;
 	struct d40_desc *d;
 	struct d40_desc *_d;
 	/* Release active descriptors */
 	while ((d40d = d40_first_active_get(d40c))) {
 		d40_desc_remove(d40d);
 		/* Return desc to free-list */
 		d40_desc_free(d40c, d40d);
 	}
 	/* Release queued descriptors waiting for transfer */
 	while ((d40d = d40_first_queued(d40c))) {
 		d40_desc_remove(d40d);
 		/* Return desc to free-list */
 		d40_desc_free(d40c, d40d);
 	}
 	/* Release client owned descriptors */
 	if (!list_empty(&d40c->client))
 		list_for_each_entry_safe(d, _d, &d40c->client, node) {
 			d40_pool_lli_free(d);
 			d40_desc_remove(d);
 			/* Return desc to free-list */
 			d40_desc_free(d40c, d40d);
 		}
 	d40_lcla_id_put(d40c, &d40c->base->lcla_pool,
 			d40c->lcla.src_id);
 	d40_lcla_id_put(d40c, &d40c->base->lcla_pool,
 			d40c->lcla.dst_id);
 	d40c->pending_tx = 0;
 	d40c->busy = false;
 }
 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
 {
 	u32 val;
 	unsigned long flags;
 	if (do_enable)
 		val = D40_ACTIVATE_EVENTLINE;
 	else
 		val = D40_DEACTIVATE_EVENTLINE;
 	spin_lock_irqsave(&d40c->phy_chan->lock, flags);
 	/* Enable event line connected to device (or memcpy) */
 	if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 	    (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
 		u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 		writel((val << D40_EVENTLINE_POS(event)) |
 		       ~D40_EVENTLINE_MASK(event),
 		       d40c->base->virtbase + D40_DREG_PCBASE +
 		       d40c->phy_chan->num * D40_DREG_PCDELTA +
 		       D40_CHAN_REG_SSLNK);
 	}
 	if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM) {
 		u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 		writel((val << D40_EVENTLINE_POS(event)) |
 		       ~D40_EVENTLINE_MASK(event),
 		       d40c->base->virtbase + D40_DREG_PCBASE +
 		       d40c->phy_chan->num * D40_DREG_PCDELTA +
 		       D40_CHAN_REG_SDLNK);
 	}
 	spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
 }
 static u32 d40_chan_has_events(struct d40_chan *d40c)
 {
 	u32 val = 0;
 	/* If SSLNK or SDLNK is zero all events are disabled */
 	if ((d40c->dma_cfg.dir ==  STEDMA40_PERIPH_TO_MEM) ||
 	    (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
 		val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 			    d40c->phy_chan->num * D40_DREG_PCDELTA +
 			    D40_CHAN_REG_SSLNK);
 	if (d40c->dma_cfg.dir !=  STEDMA40_PERIPH_TO_MEM)
 		val = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 			    d40c->phy_chan->num * D40_DREG_PCDELTA +
 			    D40_CHAN_REG_SDLNK);
 	return val;
 }
 static void d40_config_enable_lidx(struct d40_chan *d40c)
 {
 	/* Set LIDX for lcla */
 	writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 	       D40_SREG_ELEM_LOG_LIDX_MASK,
 	       d40c->base->virtbase + D40_DREG_PCBASE +
 	       d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SDELT);
 	writel((d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) &
 	       D40_SREG_ELEM_LOG_LIDX_MASK,
 	       d40c->base->virtbase + D40_DREG_PCBASE +
 	       d40c->phy_chan->num * D40_DREG_PCDELTA + D40_CHAN_REG_SSELT);
 }
 static int d40_config_write(struct d40_chan *d40c)
 {
 	u32 addr_base;
 	u32 var;
 	int res;
 	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 	if (res)
 		return res;
 	/* Odd addresses are even addresses + 4 */
 	addr_base = (d40c->phy_chan->num % 2) * 4;
 	/* Setup channel mode to logical or physical */
 	var = ((u32)(d40c->log_num != D40_PHY_CHAN) + 1) <<
 		D40_CHAN_POS(d40c->phy_chan->num);
 	writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
 	/* Setup operational mode option register */
 	var = ((d40c->dma_cfg.channel_type >> STEDMA40_INFO_CH_MODE_OPT_POS) &
 	       0x3) << D40_CHAN_POS(d40c->phy_chan->num);
 	writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
 	if (d40c->log_num != D40_PHY_CHAN) {
 		/* Set default config for CFG reg */
 		writel(d40c->src_def_cfg,
 		       d40c->base->virtbase + D40_DREG_PCBASE +
 		       d40c->phy_chan->num * D40_DREG_PCDELTA +
 		       D40_CHAN_REG_SSCFG);
 		writel(d40c->dst_def_cfg,
 		       d40c->base->virtbase + D40_DREG_PCBASE +
 		       d40c->phy_chan->num * D40_DREG_PCDELTA +
 		       D40_CHAN_REG_SDCFG);
 		d40_config_enable_lidx(d40c);
 	}
 	return res;
 }
 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
 {
 	if (d40d->lli_phy.dst && d40d->lli_phy.src) {
 		d40_phy_lli_write(d40c->base->virtbase,
 				  d40c->phy_chan->num,
 				  d40d->lli_phy.dst,
 				  d40d->lli_phy.src);
 		d40d->lli_tcount = d40d->lli_len;
 	} else if (d40d->lli_log.dst && d40d->lli_log.src) {
 		u32 lli_len;
 		struct d40_log_lli *src = d40d->lli_log.src;
 		struct d40_log_lli *dst = d40d->lli_log.dst;
 		src += d40d->lli_tcount;
 		dst += d40d->lli_tcount;
 		if (d40d->lli_len <= d40c->base->plat_data->llis_per_log)
 			lli_len = d40d->lli_len;
 		else
 			lli_len = d40c->base->plat_data->llis_per_log;
 		d40d->lli_tcount += lli_len;
 		d40_log_lli_write(d40c->lcpa, d40c->lcla.src,
 				  d40c->lcla.dst,
 				  dst, src,
 				  d40c->base->plat_data->llis_per_log);
 	}
 }
 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct d40_chan *d40c = container_of(tx->chan,
 					     struct d40_chan,
 					     chan);
 	struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	tx->cookie = d40_assign_cookie(d40c, d40d);
 	d40_desc_queue(d40c, d40d);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return tx->cookie;
 }
 static int d40_start(struct d40_chan *d40c)
 {
 	int err;
 	if (d40c->log_num != D40_PHY_CHAN) {
 		err = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 		if (err)
 			return err;
 		d40_config_set_event(d40c, true);
 	}
 	err = d40_channel_execute_command(d40c, D40_DMA_RUN);
 	return err;
 }
 static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
 {
 	struct d40_desc *d40d;
 	int err;
 	/* Start queued jobs, if any */
 	d40d = d40_first_queued(d40c);
 	if (d40d != NULL) {
 		d40c->busy = true;
 		/* Remove from queue */
 		d40_desc_remove(d40d);
 		/* Add to active queue */
 		d40_desc_submit(d40c, d40d);
 		/* Initiate DMA job */
 		d40_desc_load(d40c, d40d);
 		/* Start dma job */
 		err = d40_start(d40c);
 		if (err)
 			return NULL;
 	}
 	return d40d;
 }
 /* called from interrupt context */
 static void dma_tc_handle(struct d40_chan *d40c)
 {
 	struct d40_desc *d40d;
 	if (!d40c->phy_chan)
 		return;
 	/* Get first active entry from list */
 	d40d = d40_first_active_get(d40c);
 	if (d40d == NULL)
 		return;
 	if (d40d->lli_tcount < d40d->lli_len) {
 		d40_desc_load(d40c, d40d);
 		/* Start dma job */
 		(void) d40_start(d40c);
 		return;
 	}
 	if (d40_queue_start(d40c) == NULL)
 		d40c->busy = false;
 	d40c->pending_tx++;
 	tasklet_schedule(&d40c->tasklet);
 }
 static void dma_tasklet(unsigned long data)
 {
 	struct d40_chan *d40c = (struct d40_chan *) data;
 	struct d40_desc *d40d_fin;
 	unsigned long flags;
 	dma_async_tx_callback callback;
 	void *callback_param;
 	spin_lock_irqsave(&d40c->lock, flags);
 	/* Get first active entry from list */
 	d40d_fin = d40_first_active_get(d40c);
 	if (d40d_fin == NULL)
 		goto err;
 	d40c->completed = d40d_fin->txd.cookie;
 	/*
 	 * If terminating a channel pending_tx is set to zero.
 	 * This prevents any finished active jobs to return to the client.
 	 */
 	if (d40c->pending_tx == 0) {
 		spin_unlock_irqrestore(&d40c->lock, flags);
 		return;
 	}
 	/* Callback to client */
 	callback = d40d_fin->txd.callback;
 	callback_param = d40d_fin->txd.callback_param;
 	if (async_tx_test_ack(&d40d_fin->txd)) {
 		d40_pool_lli_free(d40d_fin);
 		d40_desc_remove(d40d_fin);
 		/* Return desc to free-list */
 		d40_desc_free(d40c, d40d_fin);
 	} else {
 		d40_desc_reset(d40d_fin);
 		if (!d40d_fin->is_in_client_list) {
 			d40_desc_remove(d40d_fin);
 			list_add_tail(&d40d_fin->node, &d40c->client);
 			d40d_fin->is_in_client_list = true;
 		}
 	}
 	d40c->pending_tx--;
 	if (d40c->pending_tx)
 		tasklet_schedule(&d40c->tasklet);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	if (callback)
 		callback(callback_param);
 	return;
  err:
 	/* Rescue manouver if receiving double interrupts */
 	if (d40c->pending_tx > 0)
 		d40c->pending_tx--;
 	spin_unlock_irqrestore(&d40c->lock, flags);
 }
 static irqreturn_t d40_handle_interrupt(int irq, void *data)
 {
 	static const struct d40_interrupt_lookup il[] = {
 		{D40_DREG_LCTIS0, D40_DREG_LCICR0, false,  0},
 		{D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
 		{D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
 		{D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
 		{D40_DREG_LCEIS0, D40_DREG_LCICR0, true,   0},
 		{D40_DREG_LCEIS1, D40_DREG_LCICR1, true,  32},
 		{D40_DREG_LCEIS2, D40_DREG_LCICR2, true,  64},
 		{D40_DREG_LCEIS3, D40_DREG_LCICR3, true,  96},
 		{D40_DREG_PCTIS,  D40_DREG_PCICR,  false, D40_PHY_CHAN},
 		{D40_DREG_PCEIS,  D40_DREG_PCICR,  true,  D40_PHY_CHAN},
 	};
 	int i;
 	u32 regs[ARRAY_SIZE(il)];
 	u32 tmp;
 	u32 idx;
 	u32 row;
 	long chan = -1;
 	struct d40_chan *d40c;
 	unsigned long flags;
 	struct d40_base *base = data;
 	spin_lock_irqsave(&base->interrupt_lock, flags);
 	/* Read interrupt status of both logical and physical channels */
 	for (i = 0; i < ARRAY_SIZE(il); i++)
 		regs[i] = readl(base->virtbase + il[i].src);
 	for (;;) {
 		chan = find_next_bit((unsigned long *)regs,
 				     BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
 		/* No more set bits found? */
 		if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
 			break;
 		row = chan / BITS_PER_LONG;
 		idx = chan & (BITS_PER_LONG - 1);
 		/* ACK interrupt */
 		tmp = readl(base->virtbase + il[row].clr);
 		tmp |= 1 << idx;
 		writel(tmp, base->virtbase + il[row].clr);
 		if (il[row].offset == D40_PHY_CHAN)
 			d40c = base->lookup_phy_chans[idx];
 		else
 			d40c = base->lookup_log_chans[il[row].offset + idx];
 		spin_lock(&d40c->lock);
 		if (!il[row].is_error)
 			dma_tc_handle(d40c);
 		else
 			dev_err(base->dev, "[%s] IRQ chan: %ld offset %d idx %d\n",
 				__func__, chan, il[row].offset, idx);
 		spin_unlock(&d40c->lock);
 	}
 	spin_unlock_irqrestore(&base->interrupt_lock, flags);
 	return IRQ_HANDLED;
 }
 static int d40_validate_conf(struct d40_chan *d40c,
 			     struct stedma40_chan_cfg *conf)
 {
 	int res = 0;
 	u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
 	u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
 	bool is_log = (conf->channel_type & STEDMA40_CHANNEL_IN_OPER_MODE)
 		== STEDMA40_CHANNEL_IN_LOG_MODE;
 	if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH &&
 	    dst_event_group == STEDMA40_DEV_DST_MEMORY) {
 		dev_err(&d40c->chan.dev->device, "[%s] Invalid dst\n",
 			__func__);
 		res = -EINVAL;
 	}
 	if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM &&
 	    src_event_group == STEDMA40_DEV_SRC_MEMORY) {
 		dev_err(&d40c->chan.dev->device, "[%s] Invalid src\n",
 			__func__);
 		res = -EINVAL;
 	}
 	if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
 	    dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] No event line\n", __func__);
 		res = -EINVAL;
 	}
 	if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
 	    (src_event_group != dst_event_group)) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Invalid event group\n", __func__);
 		res = -EINVAL;
 	}
 	if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
 		/*
 		 * DMAC HW supports it. Will be added to this driver,
 		 * in case any dma client requires it.
 		 */
 		dev_err(&d40c->chan.dev->device,
 			"[%s] periph to periph not supported\n",
 			__func__);
 		res = -EINVAL;
 	}
 	return res;
 }
 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
 			       int log_event_line, bool is_log)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&phy->lock, flags);
 	if (!is_log) {
 		/* Physical interrupts are masked per physical full channel */
 		if (phy->allocated_src == D40_ALLOC_FREE &&
 		    phy->allocated_dst == D40_ALLOC_FREE) {
 			phy->allocated_dst = D40_ALLOC_PHY;
 			phy->allocated_src = D40_ALLOC_PHY;
 			goto found;
 		} else
 			goto not_found;
 	}
 	/* Logical channel */
 	if (is_src) {
 		if (phy->allocated_src == D40_ALLOC_PHY)
 			goto not_found;
 		if (phy->allocated_src == D40_ALLOC_FREE)
 			phy->allocated_src = D40_ALLOC_LOG_FREE;
 		if (!(phy->allocated_src & (1 << log_event_line))) {
 			phy->allocated_src |= 1 << log_event_line;
 			goto found;
 		} else
 			goto not_found;
 	} else {
 		if (phy->allocated_dst == D40_ALLOC_PHY)
 			goto not_found;
 		if (phy->allocated_dst == D40_ALLOC_FREE)
 			phy->allocated_dst = D40_ALLOC_LOG_FREE;
 		if (!(phy->allocated_dst & (1 << log_event_line))) {
 			phy->allocated_dst |= 1 << log_event_line;
 			goto found;
 		} else
 			goto not_found;
 	}
 not_found:
 	spin_unlock_irqrestore(&phy->lock, flags);
 	return false;
 found:
 	spin_unlock_irqrestore(&phy->lock, flags);
 	return true;
 }
 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
 			       int log_event_line)
 {
 	unsigned long flags;
 	bool is_free = false;
 	spin_lock_irqsave(&phy->lock, flags);
 	if (!log_event_line) {
 		/* Physical interrupts are masked per physical full channel */
 		phy->allocated_dst = D40_ALLOC_FREE;
 		phy->allocated_src = D40_ALLOC_FREE;
 		is_free = true;
 		goto out;
 	}
 	/* Logical channel */
 	if (is_src) {
 		phy->allocated_src &= ~(1 << log_event_line);
 		if (phy->allocated_src == D40_ALLOC_LOG_FREE)
 			phy->allocated_src = D40_ALLOC_FREE;
 	} else {
 		phy->allocated_dst &= ~(1 << log_event_line);
 		if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
 			phy->allocated_dst = D40_ALLOC_FREE;
 	}
 	is_free = ((phy->allocated_src | phy->allocated_dst) ==
 		   D40_ALLOC_FREE);
 out:
 	spin_unlock_irqrestore(&phy->lock, flags);
 	return is_free;
 }
 static int d40_allocate_channel(struct d40_chan *d40c)
 {
 	int dev_type;
 	int event_group;
 	int event_line;
 	struct d40_phy_res *phys;
 	int i;
 	int j;
 	int log_num;
 	bool is_src;
 	bool is_log = (d40c->dma_cfg.channel_type & STEDMA40_CHANNEL_IN_OPER_MODE)
 		== STEDMA40_CHANNEL_IN_LOG_MODE;
 	phys = d40c->base->phy_res;
 	if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
 		dev_type = d40c->dma_cfg.src_dev_type;
 		log_num = 2 * dev_type;
 		is_src = true;
 	} else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
 		   d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
 		/* dst event lines are used for logical memcpy */
 		dev_type = d40c->dma_cfg.dst_dev_type;
 		log_num = 2 * dev_type + 1;
 		is_src = false;
 	} else
 		return -EINVAL;
 	event_group = D40_TYPE_TO_GROUP(dev_type);
 	event_line = D40_TYPE_TO_EVENT(dev_type);
 	if (!is_log) {
 		if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
 			/* Find physical half channel */
 			for (i = 0; i < d40c->base->num_phy_chans; i++) {
 				if (d40_alloc_mask_set(&phys[i], is_src,
 						       0, is_log))
 					goto found_phy;
 			}
 		} else
 			for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
 				int phy_num = j  + event_group * 2;
 				for (i = phy_num; i < phy_num + 2; i++) {
 					if (d40_alloc_mask_set(&phys[i], is_src,
 							       0, is_log))
 						goto found_phy;
 				}
 			}
 		return -EINVAL;
 found_phy:
 		d40c->phy_chan = &phys[i];
 		d40c->log_num = D40_PHY_CHAN;
 		goto out;
 	}
 	if (dev_type == -1)
 		return -EINVAL;
 	/* Find logical channel */
 	for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
 		int phy_num = j + event_group * 2;
 		/*
 		 * Spread logical channels across all available physical rather
 		 * than pack every logical channel at the first available phy
 		 * channels.
 		 */
 		if (is_src) {
 			for (i = phy_num; i < phy_num + 2; i++) {
 				if (d40_alloc_mask_set(&phys[i], is_src,
 						       event_line, is_log))
 					goto found_log;
 			}
 		} else {
 			for (i = phy_num + 1; i >= phy_num; i--) {
 				if (d40_alloc_mask_set(&phys[i], is_src,
 						       event_line, is_log))
 					goto found_log;
 			}
 		}
 	}
 	return -EINVAL;
 found_log:
 	d40c->phy_chan = &phys[i];
 	d40c->log_num = log_num;
 out:
 	if (is_log)
 		d40c->base->lookup_log_chans[d40c->log_num] = d40c;
 	else
 		d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
 	return 0;
 }
 static int d40_config_chan(struct d40_chan *d40c,
 			   struct stedma40_chan_cfg *info)
 {
 	/* Fill in basic CFG register values */
 	d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
 		    &d40c->dst_def_cfg, d40c->log_num != D40_PHY_CHAN);
 	if (d40c->log_num != D40_PHY_CHAN) {
 		d40_log_cfg(&d40c->dma_cfg,
 			    &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
 		if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
 			d40c->lcpa = d40c->base->lcpa_base +
 				d40c->dma_cfg.src_dev_type * 32;
 		else
 			d40c->lcpa = d40c->base->lcpa_base +
 				d40c->dma_cfg.dst_dev_type * 32 + 16;
 	}
 	/* Write channel configuration to the DMA */
 	return d40_config_write(d40c);
 }
 static int d40_config_memcpy(struct d40_chan *d40c)
 {
 	dma_cap_mask_t cap = d40c->chan.device->cap_mask;
 	if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
 		d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
 		d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
 		d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
 			memcpy[d40c->chan.chan_id];
 	} else if (dma_has_cap(DMA_MEMCPY, cap) &&
 		   dma_has_cap(DMA_SLAVE, cap)) {
 		d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
 	} else {
 		dev_err(&d40c->chan.dev->device, "[%s] No memcpy\n",
 			__func__);
 		return -EINVAL;
 	}
 	return 0;
 }
 static int d40_free_dma(struct d40_chan *d40c)
 {
 	int res = 0;
 	u32 event, dir;
 	struct d40_phy_res *phy = d40c->phy_chan;
 	bool is_src;
 	/* Terminate all queued and active transfers */
 	d40_term_all(d40c);
 	if (phy == NULL) {
 		dev_err(&d40c->chan.dev->device, "[%s] phy == null\n",
 			__func__);
 		return -EINVAL;
 	}
 	if (phy->allocated_src == D40_ALLOC_FREE &&
 	    phy->allocated_dst == D40_ALLOC_FREE) {
 		dev_err(&d40c->chan.dev->device, "[%s] channel already free\n",
 			__func__);
 		return -EINVAL;
 	}
 	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 	if (res) {
 		dev_err(&d40c->chan.dev->device, "[%s] suspend\n",
 			__func__);
 		return res;
 	}
 	if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
 	    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
 		event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 		dir = D40_CHAN_REG_SDLNK;
 		is_src = false;
 	} else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
 		event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 		dir = D40_CHAN_REG_SSLNK;
 		is_src = true;
 	} else {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Unknown direction\n", __func__);
 		return -EINVAL;
 	}
 	if (d40c->log_num != D40_PHY_CHAN) {
 		/*
 		 * Release logical channel, deactivate the event line during
 		 * the time physical res is suspended.
 		 */
 		writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) &
 		       D40_EVENTLINE_MASK(event),
 		       d40c->base->virtbase + D40_DREG_PCBASE +
 		       phy->num * D40_DREG_PCDELTA + dir);
 		d40c->base->lookup_log_chans[d40c->log_num] = NULL;
 		/*
 		 * Check if there are more logical allocation
 		 * on this phy channel.
 		 */
 		if (!d40_alloc_mask_free(phy, is_src, event)) {
 			/* Resume the other logical channels if any */
 			if (d40_chan_has_events(d40c)) {
 				res = d40_channel_execute_command(d40c,
 								  D40_DMA_RUN);
 				if (res) {
 					dev_err(&d40c->chan.dev->device,
 						"[%s] Executing RUN command\n",
 						__func__);
 					return res;
 				}
 			}
 			return 0;
 		}
 	} else
 		d40_alloc_mask_free(phy, is_src, 0);
 	/* Release physical channel */
 	res = d40_channel_execute_command(d40c, D40_DMA_STOP);
 	if (res) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Failed to stop channel\n", __func__);
 		return res;
 	}
 	d40c->phy_chan = NULL;
 	/* Invalidate channel type */
 	d40c->dma_cfg.channel_type = 0;
 	d40c->base->lookup_phy_chans[phy->num] = NULL;
 	return 0;
 }
 static int d40_pause(struct dma_chan *chan)
 {
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	int res;
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 	if (res == 0) {
 		if (d40c->log_num != D40_PHY_CHAN) {
 			d40_config_set_event(d40c, false);
 			/* Resume the other logical channels if any */
 			if (d40_chan_has_events(d40c))
 				res = d40_channel_execute_command(d40c,
 								  D40_DMA_RUN);
 		}
 	}
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return res;
 }
 static bool d40_is_paused(struct d40_chan *d40c)
 {
 	bool is_paused = false;
 	unsigned long flags;
 	void __iomem *active_reg;
 	u32 status;
 	u32 event;
 	int res;
 	spin_lock_irqsave(&d40c->lock, flags);
 	if (d40c->log_num == D40_PHY_CHAN) {
 		if (d40c->phy_chan->num % 2 == 0)
 			active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
 		else
 			active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
 		status = (readl(active_reg) &
 			  D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
 			D40_CHAN_POS(d40c->phy_chan->num);
 		if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
 			is_paused = true;
 		goto _exit;
 	}
 	res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 	if (res != 0)
 		goto _exit;
 	if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
 	    d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM)
 		event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
 	else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
 		event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
 	else {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Unknown direction\n", __func__);
 		goto _exit;
 	}
 	status = d40_chan_has_events(d40c);
 	status = (status & D40_EVENTLINE_MASK(event)) >>
 		D40_EVENTLINE_POS(event);
 	if (status != D40_DMA_RUN)
 		is_paused = true;
 	/* Resume the other logical channels if any */
 	if (d40_chan_has_events(d40c))
 		res = d40_channel_execute_command(d40c,
 						  D40_DMA_RUN);
 _exit:
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return is_paused;
 }
 static bool d40_tx_is_linked(struct d40_chan *d40c)
 {
 	bool is_link;
 	if (d40c->log_num != D40_PHY_CHAN)
 		is_link = readl(&d40c->lcpa->lcsp3) &  D40_MEM_LCSP3_DLOS_MASK;
 	else
 		is_link = readl(d40c->base->virtbase + D40_DREG_PCBASE +
 				d40c->phy_chan->num * D40_DREG_PCDELTA +
 				D40_CHAN_REG_SDLNK) &
 			D40_SREG_LNK_PHYS_LNK_MASK;
 	return is_link;
 }
 static u32 d40_residue(struct d40_chan *d40c)
 {
 	u32 num_elt;
 	if (d40c->log_num != D40_PHY_CHAN)
 		num_elt = (readl(&d40c->lcpa->lcsp2) &  D40_MEM_LCSP2_ECNT_MASK)
 			>> D40_MEM_LCSP2_ECNT_POS;
 	else
 		num_elt = (readl(d40c->base->virtbase + D40_DREG_PCBASE +
 				 d40c->phy_chan->num * D40_DREG_PCDELTA +
 				 D40_CHAN_REG_SDELT) &
 			   D40_SREG_ELEM_PHY_ECNT_MASK) >> D40_SREG_ELEM_PHY_ECNT_POS;
 	return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
 }
 static int d40_resume(struct dma_chan *chan)
 {
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	int res = 0;
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	if (d40c->log_num != D40_PHY_CHAN) {
 		res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
 		if (res)
 			goto out;
 		/* If bytes left to transfer or linked tx resume job */
 		if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
 			d40_config_set_event(d40c, true);
 			res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 		}
 	} else if (d40_residue(d40c) || d40_tx_is_linked(d40c))
 		res = d40_channel_execute_command(d40c, D40_DMA_RUN);
 out:
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return res;
 }
 static u32 stedma40_residue(struct dma_chan *chan)
 {
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	u32 bytes_left;
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	bytes_left = d40_residue(d40c);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return bytes_left;
 }
 /* Public DMA functions in addition to the DMA engine framework */
 int stedma40_set_psize(struct dma_chan *chan,
 		       int src_psize,
 		       int dst_psize)
 {
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	if (d40c->log_num != D40_PHY_CHAN) {
 		d40c->log_def.lcsp1 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
 		d40c->log_def.lcsp3 &= ~D40_MEM_LCSP1_SCFG_PSIZE_MASK;
 		d40c->log_def.lcsp1 |= src_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS;
 		d40c->log_def.lcsp3 |= dst_psize << D40_MEM_LCSP1_SCFG_PSIZE_POS;
 		goto out;
 	}
 	if (src_psize == STEDMA40_PSIZE_PHY_1)
 		d40c->src_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
 	else {
 		d40c->src_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
 		d40c->src_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
 				       D40_SREG_CFG_PSIZE_POS);
 		d40c->src_def_cfg |= src_psize << D40_SREG_CFG_PSIZE_POS;
 	}
 	if (dst_psize == STEDMA40_PSIZE_PHY_1)
 		d40c->dst_def_cfg &= ~(1 << D40_SREG_CFG_PHY_PEN_POS);
 	else {
 		d40c->dst_def_cfg |= 1 << D40_SREG_CFG_PHY_PEN_POS;
 		d40c->dst_def_cfg &= ~(STEDMA40_PSIZE_PHY_16 <<
 				       D40_SREG_CFG_PSIZE_POS);
 		d40c->dst_def_cfg |= dst_psize << D40_SREG_CFG_PSIZE_POS;
 	}
 out:
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return 0;
 }
 EXPORT_SYMBOL(stedma40_set_psize);
 struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
 						   struct scatterlist *sgl_dst,
 						   struct scatterlist *sgl_src,
 						   unsigned int sgl_len,
 						   unsigned long flags)
 {
 	int res;
 	struct d40_desc *d40d;
 	struct d40_chan *d40c = container_of(chan, struct d40_chan,
 					     chan);
 	unsigned long flg;
 	int lli_max = d40c->base->plat_data->llis_per_log;
 	spin_lock_irqsave(&d40c->lock, flg);
 	d40d = d40_desc_get(d40c);
 	if (d40d == NULL)
 		goto err;
 	memset(d40d, 0, sizeof(struct d40_desc));
 	d40d->lli_len = sgl_len;
 	d40d->txd.flags = flags;
 	if (d40c->log_num != D40_PHY_CHAN) {
 		if (sgl_len > 1)
 			/*
 			 * Check if there is space available in lcla. If not,
 			 * split list into 1-length and run only in lcpa
 			 * space.
 			 */
 			if (d40_lcla_id_get(d40c,
 					    &d40c->base->lcla_pool) != 0)
 				lli_max = 1;
 		if (d40_pool_lli_alloc(d40d, sgl_len, true) < 0) {
 			dev_err(&d40c->chan.dev->device,
 				"[%s] Out of memory\n", __func__);
 			goto err;
 		}
 		(void) d40_log_sg_to_lli(d40c->lcla.src_id,
 					 sgl_src,
 					 sgl_len,
 					 d40d->lli_log.src,
 					 d40c->log_def.lcsp1,
 					 d40c->dma_cfg.src_info.data_width,
 					 flags & DMA_PREP_INTERRUPT, lli_max,
 					 d40c->base->plat_data->llis_per_log);
 		(void) d40_log_sg_to_lli(d40c->lcla.dst_id,
 					 sgl_dst,
 					 sgl_len,
 					 d40d->lli_log.dst,
 					 d40c->log_def.lcsp3,
 					 d40c->dma_cfg.dst_info.data_width,
 					 flags & DMA_PREP_INTERRUPT, lli_max,
 					 d40c->base->plat_data->llis_per_log);
 	} else {
 		if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
 			dev_err(&d40c->chan.dev->device,
 				"[%s] Out of memory\n", __func__);
 			goto err;
 		}
 		res = d40_phy_sg_to_lli(sgl_src,
 					sgl_len,
 					0,
 					d40d->lli_phy.src,
 					d40d->lli_phy.src_addr,
 					d40c->src_def_cfg,
 					d40c->dma_cfg.src_info.data_width,
 					d40c->dma_cfg.src_info.psize,
 					true);
 		if (res < 0)
 			goto err;
 		res = d40_phy_sg_to_lli(sgl_dst,
 					sgl_len,
 					0,
 					d40d->lli_phy.dst,
 					d40d->lli_phy.dst_addr,
 					d40c->dst_def_cfg,
 					d40c->dma_cfg.dst_info.data_width,
 					d40c->dma_cfg.dst_info.psize,
 					true);
 		if (res < 0)
 			goto err;
 		(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
 				      d40d->lli_pool.size, DMA_TO_DEVICE);
 	}
 	dma_async_tx_descriptor_init(&d40d->txd, chan);
 	d40d->txd.tx_submit = d40_tx_submit;
 	spin_unlock_irqrestore(&d40c->lock, flg);
 	return &d40d->txd;
 err:
 	spin_unlock_irqrestore(&d40c->lock, flg);
 	return NULL;
 }
 EXPORT_SYMBOL(stedma40_memcpy_sg);
 bool stedma40_filter(struct dma_chan *chan, void *data)
 {
 	struct stedma40_chan_cfg *info = data;
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	int err;
 	if (data) {
 		err = d40_validate_conf(d40c, info);
 		if (!err)
 			d40c->dma_cfg = *info;
 	} else
 		err = d40_config_memcpy(d40c);
 	return err == 0;
 }
 EXPORT_SYMBOL(stedma40_filter);
 /* DMA ENGINE functions */
 static int d40_alloc_chan_resources(struct dma_chan *chan)
 {
 	int err;
 	unsigned long flags;
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	spin_lock_irqsave(&d40c->lock, flags);
 	d40c->completed = chan->cookie = 1;
 	/*
 	 * If no dma configuration is set (channel_type == 0)
 	 * use default configuration
 	 */
 	if (d40c->dma_cfg.channel_type == 0) {
 		err = d40_config_memcpy(d40c);
 		if (err)
 			goto err_alloc;
 	}
 	err = d40_allocate_channel(d40c);
 	if (err) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Failed to allocate channel\n", __func__);
 		goto err_alloc;
 	}
 	err = d40_config_chan(d40c, &d40c->dma_cfg);
 	if (err) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Failed to configure channel\n",
 			__func__);
 		goto err_config;
 	}
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	return 0;
  err_config:
 	(void) d40_free_dma(d40c);
  err_alloc:
 	spin_unlock_irqrestore(&d40c->lock, flags);
 	dev_err(&d40c->chan.dev->device,
 		"[%s] Channel allocation failed\n", __func__);
 	return -EINVAL;
 }
 static void d40_free_chan_resources(struct dma_chan *chan)
 {
 	struct d40_chan *d40c =
 		container_of(chan, struct d40_chan, chan);
 	int err;
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	err = d40_free_dma(d40c);
 	if (err)
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Failed to free channel\n", __func__);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 }
 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
 						       dma_addr_t dst,
 						       dma_addr_t src,
 						       size_t size,
 						       unsigned long flags)
 {
 	struct d40_desc *d40d;
 	struct d40_chan *d40c = container_of(chan, struct d40_chan,
 					     chan);
 	unsigned long flg;
 	int err = 0;
 	spin_lock_irqsave(&d40c->lock, flg);
 	d40d = d40_desc_get(d40c);
 	if (d40d == NULL) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Descriptor is NULL\n", __func__);
 		goto err;
 	}
 	memset(d40d, 0, sizeof(struct d40_desc));
 	d40d->txd.flags = flags;
 	dma_async_tx_descriptor_init(&d40d->txd, chan);
 	d40d->txd.tx_submit = d40_tx_submit;
 	if (d40c->log_num != D40_PHY_CHAN) {
 		if (d40_pool_lli_alloc(d40d, 1, true) < 0) {
 			dev_err(&d40c->chan.dev->device,
 				"[%s] Out of memory\n", __func__);
 			goto err;
 		}
 		d40d->lli_len = 1;
 		d40_log_fill_lli(d40d->lli_log.src,
 				 src,
 				 size,
 				 0,
 				 d40c->log_def.lcsp1,
 				 d40c->dma_cfg.src_info.data_width,
 				 true, true);
 		d40_log_fill_lli(d40d->lli_log.dst,
 				 dst,
 				 size,
 				 0,
 				 d40c->log_def.lcsp3,
 				 d40c->dma_cfg.dst_info.data_width,
 				 true, true);
 	} else {
 		if (d40_pool_lli_alloc(d40d, 1, false) < 0) {
 			dev_err(&d40c->chan.dev->device,
 				"[%s] Out of memory\n", __func__);
 			goto err;
 		}
 		err = d40_phy_fill_lli(d40d->lli_phy.src,
 				       src,
 				       size,
 				       d40c->dma_cfg.src_info.psize,
 				       0,
 				       d40c->src_def_cfg,
 				       true,
 				       d40c->dma_cfg.src_info.data_width,
 				       false);
 		if (err)
 			goto err_fill_lli;
 		err = d40_phy_fill_lli(d40d->lli_phy.dst,
 				       dst,
 				       size,
 				       d40c->dma_cfg.dst_info.psize,
 				       0,
 				       d40c->dst_def_cfg,
 				       true,
 				       d40c->dma_cfg.dst_info.data_width,
 				       false);
 		if (err)
 			goto err_fill_lli;
 		(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
 				      d40d->lli_pool.size, DMA_TO_DEVICE);
 	}
 	spin_unlock_irqrestore(&d40c->lock, flg);
 	return &d40d->txd;
 err_fill_lli:
 	dev_err(&d40c->chan.dev->device,
 		"[%s] Failed filling in PHY LLI\n", __func__);
 	d40_pool_lli_free(d40d);
 err:
 	spin_unlock_irqrestore(&d40c->lock, flg);
 	return NULL;
 }
 static int d40_prep_slave_sg_log(struct d40_desc *d40d,
 				 struct d40_chan *d40c,
 				 struct scatterlist *sgl,
 				 unsigned int sg_len,
 				 enum dma_data_direction direction,
 				 unsigned long flags)
 {
 	dma_addr_t dev_addr = 0;
 	int total_size;
 	int lli_max = d40c->base->plat_data->llis_per_log;
 	if (d40_pool_lli_alloc(d40d, sg_len, true) < 0) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Out of memory\n", __func__);
 		return -ENOMEM;
 	}
 	d40d->lli_len = sg_len;
 	d40d->lli_tcount = 0;
 	if (sg_len > 1)
 		/*
 		 * Check if there is space available in lcla.
 		 * If not, split list into 1-length and run only
 		 * in lcpa space.
 		 */
 		if (d40_lcla_id_get(d40c, &d40c->base->lcla_pool) != 0)
 			lli_max = 1;
 	if (direction == DMA_FROM_DEVICE) {
 		dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
 		total_size = d40_log_sg_to_dev(&d40c->lcla,
 					       sgl, sg_len,
 					       &d40d->lli_log,
 					       &d40c->log_def,
 					       d40c->dma_cfg.src_info.data_width,
 					       d40c->dma_cfg.dst_info.data_width,
 					       direction,
 					       flags & DMA_PREP_INTERRUPT,
 					       dev_addr, lli_max,
 					       d40c->base->plat_data->llis_per_log);
 	} else if (direction == DMA_TO_DEVICE) {
 		dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
 		total_size = d40_log_sg_to_dev(&d40c->lcla,
 					       sgl, sg_len,
 					       &d40d->lli_log,
 					       &d40c->log_def,
 					       d40c->dma_cfg.src_info.data_width,
 					       d40c->dma_cfg.dst_info.data_width,
 					       direction,
 					       flags & DMA_PREP_INTERRUPT,
 					       dev_addr, lli_max,
 					       d40c->base->plat_data->llis_per_log);
 	} else
 		return -EINVAL;
 	if (total_size < 0)
 		return -EINVAL;
 	return 0;
 }
 static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
 				 struct d40_chan *d40c,
 				 struct scatterlist *sgl,
 				 unsigned int sgl_len,
 				 enum dma_data_direction direction,
 				 unsigned long flags)
 {
 	dma_addr_t src_dev_addr;
 	dma_addr_t dst_dev_addr;
 	int res;
 	if (d40_pool_lli_alloc(d40d, sgl_len, false) < 0) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Out of memory\n", __func__);
 		return -ENOMEM;
 	}
 	d40d->lli_len = sgl_len;
 	d40d->lli_tcount = 0;
 	if (direction == DMA_FROM_DEVICE) {
 		dst_dev_addr = 0;
 		src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
 	} else if (direction == DMA_TO_DEVICE) {
 		dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
 		src_dev_addr = 0;
 	} else
 		return -EINVAL;
 	res = d40_phy_sg_to_lli(sgl,
 				sgl_len,
 				src_dev_addr,
 				d40d->lli_phy.src,
 				d40d->lli_phy.src_addr,
 				d40c->src_def_cfg,
 				d40c->dma_cfg.src_info.data_width,
 				d40c->dma_cfg.src_info.psize,
 				true);
 	if (res < 0)
 		return res;
 	res = d40_phy_sg_to_lli(sgl,
 				sgl_len,
 				dst_dev_addr,
 				d40d->lli_phy.dst,
 				d40d->lli_phy.dst_addr,
 				d40c->dst_def_cfg,
 				d40c->dma_cfg.dst_info.data_width,
 				d40c->dma_cfg.dst_info.psize,
 				 true);
 	if (res < 0)
 		return res;
 	(void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
 			      d40d->lli_pool.size, DMA_TO_DEVICE);
 	return 0;
 }
 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
 							 struct scatterlist *sgl,
 							 unsigned int sg_len,
 							 enum dma_data_direction direction,
 							 unsigned long flags)
 {
 	struct d40_desc *d40d;
 	struct d40_chan *d40c = container_of(chan, struct d40_chan,
 					     chan);
 	unsigned long flg;
 	int err;
 	if (d40c->dma_cfg.pre_transfer)
 		d40c->dma_cfg.pre_transfer(chan,
 					   d40c->dma_cfg.pre_transfer_data,
 					   sg_dma_len(sgl));
 	spin_lock_irqsave(&d40c->lock, flg);
 	d40d = d40_desc_get(d40c);
 	spin_unlock_irqrestore(&d40c->lock, flg);
 	if (d40d == NULL)
 		return NULL;
 	memset(d40d, 0, sizeof(struct d40_desc));
 	if (d40c->log_num != D40_PHY_CHAN)
 		err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
 					    direction, flags);
 	else
 		err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
 					    direction, flags);
 	if (err) {
 		dev_err(&d40c->chan.dev->device,
 			"[%s] Failed to prepare %s slave sg job: %d\n",
 			__func__,
 			d40c->log_num != D40_PHY_CHAN ? "log" : "phy", err);
 		return NULL;
 	}
 	d40d->txd.flags = flags;
 	dma_async_tx_descriptor_init(&d40d->txd, chan);
 	d40d->txd.tx_submit = d40_tx_submit;
 	return &d40d->txd;
 }
 static enum dma_status d40_tx_status(struct dma_chan *chan,
 				     dma_cookie_t cookie,
 				     struct dma_tx_state *txstate)
 {
 	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
 	dma_cookie_t last_used;
 	dma_cookie_t last_complete;
 	int ret;
 	last_complete = d40c->completed;
 	last_used = chan->cookie;
 	if (d40_is_paused(d40c))
 		ret = DMA_PAUSED;
 	else
 		ret = dma_async_is_complete(cookie, last_complete, last_used);
 	dma_set_tx_state(txstate, last_complete, last_used,
 			 stedma40_residue(chan));
 	return ret;
 }
 static void d40_issue_pending(struct dma_chan *chan)
 {
 	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
 	unsigned long flags;
 	spin_lock_irqsave(&d40c->lock, flags);
 	/* Busy means that pending jobs are already being processed */
 	if (!d40c->busy)
 		(void) d40_queue_start(d40c);
 	spin_unlock_irqrestore(&d40c->lock, flags);
 }
-static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		       unsigned long arg)
 {
 	unsigned long flags;
 	struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
 	switch (cmd) {
 	case DMA_TERMINATE_ALL:
 		spin_lock_irqsave(&d40c->lock, flags);
 		d40_term_all(d40c);
 		spin_unlock_irqrestore(&d40c->lock, flags);
 		return 0;
 	case DMA_PAUSE:
 		return d40_pause(chan);
 	case DMA_RESUME:
 		return d40_resume(chan);
 	}
 	/* Other commands are unimplemented */
 	return -ENXIO;
 }
 /* Initialization functions */
 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
 				 struct d40_chan *chans, int offset,
 				 int num_chans)
 {
 	int i = 0;
 	struct d40_chan *d40c;
 	INIT_LIST_HEAD(&dma->channels);
 	for (i = offset; i < offset + num_chans; i++) {
 		d40c = &chans[i];
 		d40c->base = base;
 		d40c->chan.device = dma;
 		/* Invalidate lcla element */
 		d40c->lcla.src_id = -1;
 		d40c->lcla.dst_id = -1;
 		spin_lock_init(&d40c->lock);
 		d40c->log_num = D40_PHY_CHAN;
 		INIT_LIST_HEAD(&d40c->free);
 		INIT_LIST_HEAD(&d40c->active);
 		INIT_LIST_HEAD(&d40c->queue);
 		INIT_LIST_HEAD(&d40c->client);
 		d40c->free_len = 0;
 		tasklet_init(&d40c->tasklet, dma_tasklet,
 			     (unsigned long) d40c);
 		list_add_tail(&d40c->chan.device_node,
 			      &dma->channels);
 	}
 }
 static int __init d40_dmaengine_init(struct d40_base *base,
 				     int num_reserved_chans)
 {
 	int err ;
 	d40_chan_init(base, &base->dma_slave, base->log_chans,
 		      0, base->num_log_chans);
 	dma_cap_zero(base->dma_slave.cap_mask);
 	dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
 	base->dma_slave.device_alloc_chan_resources = d40_alloc_chan_resources;
 	base->dma_slave.device_free_chan_resources = d40_free_chan_resources;
 	base->dma_slave.device_prep_dma_memcpy = d40_prep_memcpy;
 	base->dma_slave.device_prep_slave_sg = d40_prep_slave_sg;
 	base->dma_slave.device_tx_status = d40_tx_status;
 	base->dma_slave.device_issue_pending = d40_issue_pending;
 	base->dma_slave.device_control = d40_control;
 	base->dma_slave.dev = base->dev;
 	err = dma_async_device_register(&base->dma_slave);
 	if (err) {
 		dev_err(base->dev,
 			"[%s] Failed to register slave channels\n",
 			__func__);
 		goto failure1;
 	}
 	d40_chan_init(base, &base->dma_memcpy, base->log_chans,
 		      base->num_log_chans, base->plat_data->memcpy_len);
 	dma_cap_zero(base->dma_memcpy.cap_mask);
 	dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
 	base->dma_memcpy.device_alloc_chan_resources = d40_alloc_chan_resources;
 	base->dma_memcpy.device_free_chan_resources = d40_free_chan_resources;
 	base->dma_memcpy.device_prep_dma_memcpy = d40_prep_memcpy;
 	base->dma_memcpy.device_prep_slave_sg = d40_prep_slave_sg;
 	base->dma_memcpy.device_tx_status = d40_tx_status;
 	base->dma_memcpy.device_issue_pending = d40_issue_pending;
 	base->dma_memcpy.device_control = d40_control;
 	base->dma_memcpy.dev = base->dev;
 	/*
 	 * This controller can only access address at even
 	 * 32bit boundaries, i.e. 2^2
 	 */
 	base->dma_memcpy.copy_align = 2;
 	err = dma_async_device_register(&base->dma_memcpy);
 	if (err) {
 		dev_err(base->dev,
 			"[%s] Failed to regsiter memcpy only channels\n",
 			__func__);
 		goto failure2;
 	}
 	d40_chan_init(base, &base->dma_both, base->phy_chans,
 		      0, num_reserved_chans);
 	dma_cap_zero(base->dma_both.cap_mask);
 	dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
 	dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
 	base->dma_both.device_alloc_chan_resources = d40_alloc_chan_resources;
 	base->dma_both.device_free_chan_resources = d40_free_chan_resources;
 	base->dma_both.device_prep_dma_memcpy = d40_prep_memcpy;
 	base->dma_both.device_prep_slave_sg = d40_prep_slave_sg;
 	base->dma_both.device_tx_status = d40_tx_status;
 	base->dma_both.device_issue_pending = d40_issue_pending;
 	base->dma_both.device_control = d40_control;
 	base->dma_both.dev = base->dev;
 	base->dma_both.copy_align = 2;
 	err = dma_async_device_register(&base->dma_both);
 	if (err) {
 		dev_err(base->dev,
 			"[%s] Failed to register logical and physical capable channels\n",
 			__func__);
 		goto failure3;
 	}
 	return 0;
 failure3:
 	dma_async_device_unregister(&base->dma_memcpy);
 failure2:
 	dma_async_device_unregister(&base->dma_slave);
 failure1:
 	return err;
 }
 /* Initialization functions. */
 static int __init d40_phy_res_init(struct d40_base *base)
 {
 	int i;
 	int num_phy_chans_avail = 0;
 	u32 val[2];
 	int odd_even_bit = -2;
 	val[0] = readl(base->virtbase + D40_DREG_PRSME);
 	val[1] = readl(base->virtbase + D40_DREG_PRSMO);
 	for (i = 0; i < base->num_phy_chans; i++) {
 		base->phy_res[i].num = i;
 		odd_even_bit += 2 * ((i % 2) == 0);
 		if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
 			/* Mark security only channels as occupied */
 			base->phy_res[i].allocated_src = D40_ALLOC_PHY;
 			base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
 		} else {
 			base->phy_res[i].allocated_src = D40_ALLOC_FREE;
 			base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
 			num_phy_chans_avail++;
 		}
 		spin_lock_init(&base->phy_res[i].lock);
 	}
 	dev_info(base->dev, "%d of %d physical DMA channels available\n",
 		 num_phy_chans_avail, base->num_phy_chans);
 	/* Verify settings extended vs standard */
 	val[0] = readl(base->virtbase + D40_DREG_PRTYP);
 	for (i = 0; i < base->num_phy_chans; i++) {
 		if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
 		    (val[0] & 0x3) != 1)
 			dev_info(base->dev,
 				 "[%s] INFO: channel %d is misconfigured (%d)\n",
 				 __func__, i, val[0] & 0x3);
 		val[0] = val[0] >> 2;
 	}
 	return num_phy_chans_avail;
 }
 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
 {
 	static const struct d40_reg_val dma_id_regs[] = {
 		/* Peripheral Id */
 		{ .reg = D40_DREG_PERIPHID0, .val = 0x0040},
 		{ .reg = D40_DREG_PERIPHID1, .val = 0x0000},
 		/*
 		 * D40_DREG_PERIPHID2 Depends on HW revision:
 		 *  MOP500/HREF ED has 0x0008,
 		 *  ? has 0x0018,
 		 *  HREF V1 has 0x0028
 		 */
 		{ .reg = D40_DREG_PERIPHID3, .val = 0x0000},
 		/* PCell Id */
 		{ .reg = D40_DREG_CELLID0, .val = 0x000d},
 		{ .reg = D40_DREG_CELLID1, .val = 0x00f0},
 		{ .reg = D40_DREG_CELLID2, .val = 0x0005},
 		{ .reg = D40_DREG_CELLID3, .val = 0x00b1}
 	};
 	struct stedma40_platform_data *plat_data;
 	struct clk *clk = NULL;
 	void __iomem *virtbase = NULL;
 	struct resource *res = NULL;
 	struct d40_base *base = NULL;
 	int num_log_chans = 0;
 	int num_phy_chans;
 	int i;
 	clk = clk_get(&pdev->dev, NULL);
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "[%s] No matching clock found\n",
 			__func__);
 		goto failure;
 	}
 	clk_enable(clk);
 	/* Get IO for DMAC base address */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
 	if (!res)
 		goto failure;
 	if (request_mem_region(res->start, resource_size(res),
 			       D40_NAME " I/O base") == NULL)
 		goto failure;
 	virtbase = ioremap(res->start, resource_size(res));
 	if (!virtbase)
 		goto failure;
 	/* HW version check */
 	for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
 		if (dma_id_regs[i].val !=
 		    readl(virtbase + dma_id_regs[i].reg)) {
 			dev_err(&pdev->dev,
 				"[%s] Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
 				__func__,
 				dma_id_regs[i].val,
 				dma_id_regs[i].reg,
 				readl(virtbase + dma_id_regs[i].reg));
 			goto failure;
 		}
 	}
 	i = readl(virtbase + D40_DREG_PERIPHID2);
 	if ((i & 0xf) != D40_PERIPHID2_DESIGNER) {
 		dev_err(&pdev->dev,
 			"[%s] Unknown designer! Got %x wanted %x\n",
 			__func__, i & 0xf, D40_PERIPHID2_DESIGNER);
 		goto failure;
 	}
 	/* The number of physical channels on this HW */
 	num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
 	dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
 		 (i >> 4) & 0xf, res->start);
 	plat_data = pdev->dev.platform_data;
 	/* Count the number of logical channels in use */
 	for (i = 0; i < plat_data->dev_len; i++)
 		if (plat_data->dev_rx[i] != 0)
 			num_log_chans++;
 	for (i = 0; i < plat_data->dev_len; i++)
 		if (plat_data->dev_tx[i] != 0)
 			num_log_chans++;
 	base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
 		       (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
 		       sizeof(struct d40_chan), GFP_KERNEL);
 	if (base == NULL) {
 		dev_err(&pdev->dev, "[%s] Out of memory\n", __func__);
 		goto failure;
 	}
 	base->clk = clk;
 	base->num_phy_chans = num_phy_chans;
 	base->num_log_chans = num_log_chans;
 	base->phy_start = res->start;
 	base->phy_size = resource_size(res);
 	base->virtbase = virtbase;
 	base->plat_data = plat_data;
 	base->dev = &pdev->dev;
 	base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
 	base->log_chans = &base->phy_chans[num_phy_chans];
 	base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
 				GFP_KERNEL);
 	if (!base->phy_res)
 		goto failure;
 	base->lookup_phy_chans = kzalloc(num_phy_chans *
 					 sizeof(struct d40_chan *),
 					 GFP_KERNEL);
 	if (!base->lookup_phy_chans)
 		goto failure;
 	if (num_log_chans + plat_data->memcpy_len) {
 		/*
 		 * The max number of logical channels are event lines for all
 		 * src devices and dst devices
 		 */
 		base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
 						 sizeof(struct d40_chan *),
 						 GFP_KERNEL);
 		if (!base->lookup_log_chans)
 			goto failure;
 	}
 	base->lcla_pool.alloc_map = kzalloc(num_phy_chans * sizeof(u32),
 					    GFP_KERNEL);
 	if (!base->lcla_pool.alloc_map)
 		goto failure;
 	return base;
 failure:
 	if (clk) {
 		clk_disable(clk);
 		clk_put(clk);
 	}
 	if (virtbase)
 		iounmap(virtbase);
 	if (res)
 		release_mem_region(res->start,
 				   resource_size(res));
 	if (virtbase)
 		iounmap(virtbase);
 	if (base) {
 		kfree(base->lcla_pool.alloc_map);
 		kfree(base->lookup_log_chans);
 		kfree(base->lookup_phy_chans);
 		kfree(base->phy_res);
 		kfree(base);
 	}
 	return NULL;
 }
 static void __init d40_hw_init(struct d40_base *base)
 {
 	static const struct d40_reg_val dma_init_reg[] = {
 		/* Clock every part of the DMA block from start */
 		{ .reg = D40_DREG_GCC,    .val = 0x0000ff01},
 		/* Interrupts on all logical channels */
 		{ .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
 		{ .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
 	};
 	int i;
 	u32 prmseo[2] = {0, 0};
 	u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
 	u32 pcmis = 0;
 	u32 pcicr = 0;
 	for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
 		writel(dma_init_reg[i].val,
 		       base->virtbase + dma_init_reg[i].reg);
 	/* Configure all our dma channels to default settings */
 	for (i = 0; i < base->num_phy_chans; i++) {
 		activeo[i % 2] = activeo[i % 2] << 2;
 		if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
 		    == D40_ALLOC_PHY) {
 			activeo[i % 2] |= 3;
 			continue;
 		}
 		/* Enable interrupt # */
 		pcmis = (pcmis << 1) | 1;
 		/* Clear interrupt # */
 		pcicr = (pcicr << 1) | 1;
 		/* Set channel to physical mode */
 		prmseo[i % 2] = prmseo[i % 2] << 2;
 		prmseo[i % 2] |= 1;
 	}
 	writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
 	writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
 	writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
 	writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
 	/* Write which interrupt to enable */
 	writel(pcmis, base->virtbase + D40_DREG_PCMIS);
 	/* Write which interrupt to clear */
 	writel(pcicr, base->virtbase + D40_DREG_PCICR);
 }
 static int __init d40_probe(struct platform_device *pdev)
 {
 	int err;
 	int ret = -ENOENT;
 	struct d40_base *base;
 	struct resource *res = NULL;
 	int num_reserved_chans;
 	u32 val;
 	base = d40_hw_detect_init(pdev);
 	if (!base)
 		goto failure;
 	num_reserved_chans = d40_phy_res_init(base);
 	platform_set_drvdata(pdev, base);
 	spin_lock_init(&base->interrupt_lock);
 	spin_lock_init(&base->execmd_lock);
 	/* Get IO for logical channel parameter address */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
 	if (!res) {
 		ret = -ENOENT;
 		dev_err(&pdev->dev,
 			"[%s] No \"lcpa\" memory resource\n",
 			__func__);
 		goto failure;
 	}
 	base->lcpa_size = resource_size(res);
 	base->phy_lcpa = res->start;
 	if (request_mem_region(res->start, resource_size(res),
 			       D40_NAME " I/O lcpa") == NULL) {
 		ret = -EBUSY;
 		dev_err(&pdev->dev,
 			"[%s] Failed to request LCPA region 0x%x-0x%x\n",
 			__func__, res->start, res->end);
 		goto failure;
 	}
 	/* We make use of ESRAM memory for this. */
 	val = readl(base->virtbase + D40_DREG_LCPA);
 	if (res->start != val && val != 0) {
 		dev_warn(&pdev->dev,
 			 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
 			 __func__, val, res->start);
 	} else
 		writel(res->start, base->virtbase + D40_DREG_LCPA);
 	base->lcpa_base = ioremap(res->start, resource_size(res));
 	if (!base->lcpa_base) {
 		ret = -ENOMEM;
 		dev_err(&pdev->dev,
 			"[%s] Failed to ioremap LCPA region\n",
 			__func__);
 		goto failure;
 	}
 	/* Get IO for logical channel link address */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcla");
 	if (!res) {
 		ret = -ENOENT;
 		dev_err(&pdev->dev,
 			"[%s] No \"lcla\" resource defined\n",
 			__func__);
 		goto failure;
 	}
 	base->lcla_pool.base_size = resource_size(res);
 	base->lcla_pool.phy = res->start;
 	if (request_mem_region(res->start, resource_size(res),
 			       D40_NAME " I/O lcla") == NULL) {
 		ret = -EBUSY;
 		dev_err(&pdev->dev,
 			"[%s] Failed to request LCLA region 0x%x-0x%x\n",
 			__func__, res->start, res->end);
 		goto failure;
 	}
 	val = readl(base->virtbase + D40_DREG_LCLA);
 	if (res->start != val && val != 0) {
 		dev_warn(&pdev->dev,
 			 "[%s] Mismatch LCLA dma 0x%x, def 0x%x\n",
 			 __func__, val, res->start);
 	} else
 		writel(res->start, base->virtbase + D40_DREG_LCLA);
 	base->lcla_pool.base = ioremap(res->start, resource_size(res));
 	if (!base->lcla_pool.base) {
 		ret = -ENOMEM;
 		dev_err(&pdev->dev,
 			"[%s] Failed to ioremap LCLA 0x%x-0x%x\n",
 			__func__, res->start, res->end);
 		goto failure;
 	}
 	spin_lock_init(&base->lcla_pool.lock);
 	base->lcla_pool.num_blocks = base->num_phy_chans;
 	base->irq = platform_get_irq(pdev, 0);
 	ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
 	if (ret) {
 		dev_err(&pdev->dev, "[%s] No IRQ defined\n", __func__);
 		goto failure;
 	}
 	err = d40_dmaengine_init(base, num_reserved_chans);
 	if (err)
 		goto failure;
 	d40_hw_init(base);
 	dev_info(base->dev, "initialized\n");
 	return 0;
 failure:
 	if (base) {
 		if (base->virtbase)
 			iounmap(base->virtbase);
 		if (base->lcla_pool.phy)
 			release_mem_region(base->lcla_pool.phy,
 					   base->lcla_pool.base_size);
 		if (base->phy_lcpa)
 			release_mem_region(base->phy_lcpa,
 					   base->lcpa_size);
 		if (base->phy_start)
 			release_mem_region(base->phy_start,
 					   base->phy_size);
 		if (base->clk) {
 			clk_disable(base->clk);
 			clk_put(base->clk);
 		}
 		kfree(base->lcla_pool.alloc_map);
 		kfree(base->lookup_log_chans);
 		kfree(base->lookup_phy_chans);
 		kfree(base->phy_res);
 		kfree(base);
 	}
 	dev_err(&pdev->dev, "[%s] probe failed\n", __func__);
 	return ret;
 }
 static struct platform_driver d40_driver = {
 	.driver = {
 		.owner = THIS_MODULE,
 		.name  = D40_NAME,
 	},
 };
 int __init stedma40_init(void)
 {
 	return platform_driver_probe(&d40_driver, d40_probe);
 }
 arch_initcall(stedma40_init);

drivers/dma/timb_dma.c

Diff comments View file @ 0582763

 /*
  * timb_dma.c timberdale FPGA DMA driver
  * Copyright (c) 2010 Intel Corporation
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 /* Supports:
  * Timberdale FPGA DMA engine
  */
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/timb_dma.h>
 #define DRIVER_NAME "timb-dma"
 /* Global DMA registers */
 #define TIMBDMA_ACR		0x34
 #define TIMBDMA_32BIT_ADDR	0x01
 #define TIMBDMA_ISR		0x080000
 #define TIMBDMA_IPR		0x080004
 #define TIMBDMA_IER		0x080008
 /* Channel specific registers */
 /* RX instances base addresses are 0x00, 0x40, 0x80 ...
  * TX instances base addresses are 0x18, 0x58, 0x98 ...
  */
 #define TIMBDMA_INSTANCE_OFFSET		0x40
 #define TIMBDMA_INSTANCE_TX_OFFSET	0x18
 /* RX registers, relative the instance base */
 #define TIMBDMA_OFFS_RX_DHAR	0x00
 #define TIMBDMA_OFFS_RX_DLAR	0x04
 #define TIMBDMA_OFFS_RX_LR	0x0C
 #define TIMBDMA_OFFS_RX_BLR	0x10
 #define TIMBDMA_OFFS_RX_ER	0x14
 #define TIMBDMA_RX_EN		0x01
 /* bytes per Row, video specific register
  * which is placed after the TX registers...
  */
 #define TIMBDMA_OFFS_RX_BPRR	0x30
 /* TX registers, relative the instance base */
 #define TIMBDMA_OFFS_TX_DHAR	0x00
 #define TIMBDMA_OFFS_TX_DLAR	0x04
 #define TIMBDMA_OFFS_TX_BLR	0x0C
 #define TIMBDMA_OFFS_TX_LR	0x14
 #define TIMB_DMA_DESC_SIZE	8
 struct timb_dma_desc {
 	struct list_head		desc_node;
 	struct dma_async_tx_descriptor	txd;
 	u8				*desc_list;
 	unsigned int			desc_list_len;
 	bool				interrupt;
 };
 struct timb_dma_chan {
 	struct dma_chan		chan;
 	void __iomem		*membase;
 	spinlock_t		lock; /* Used to protect data structures,
 					especially the lists and descriptors,
 					from races between the tasklet and calls
 					from above */
 	dma_cookie_t		last_completed_cookie;
 	bool			ongoing;
 	struct list_head	active_list;
 	struct list_head	queue;
 	struct list_head	free_list;
 	unsigned int		bytes_per_line;
 	enum dma_data_direction	direction;
 	unsigned int		descs; /* Descriptors to allocate */
 	unsigned int		desc_elems; /* number of elems per descriptor */
 };
 struct timb_dma {
 	struct dma_device	dma;
 	void __iomem		*membase;
 	struct tasklet_struct	tasklet;
 	struct timb_dma_chan	channels[0];
 };
 static struct device *chan2dev(struct dma_chan *chan)
 {
 	return &chan->dev->device;
 }
 static struct device *chan2dmadev(struct dma_chan *chan)
 {
 	return chan2dev(chan)->parent->parent;
 }
 static struct timb_dma *tdchantotd(struct timb_dma_chan *td_chan)
 {
 	int id = td_chan->chan.chan_id;
 	return (struct timb_dma *)((u8 *)td_chan -
 		id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma));
 }
 /* Must be called with the spinlock held */
 static void __td_enable_chan_irq(struct timb_dma_chan *td_chan)
 {
 	int id = td_chan->chan.chan_id;
 	struct timb_dma *td = tdchantotd(td_chan);
 	u32 ier;
 	/* enable interrupt for this channel */
 	ier = ioread32(td->membase + TIMBDMA_IER);
 	ier |= 1 << id;
 	dev_dbg(chan2dev(&td_chan->chan), "Enabling irq: %d, IER: 0x%x\n", id,
 		ier);
 	iowrite32(ier, td->membase + TIMBDMA_IER);
 }
 /* Should be called with the spinlock held */
 static bool __td_dma_done_ack(struct timb_dma_chan *td_chan)
 {
 	int id = td_chan->chan.chan_id;
 	struct timb_dma *td = (struct timb_dma *)((u8 *)td_chan -
 		id * sizeof(struct timb_dma_chan) - sizeof(struct timb_dma));
 	u32 isr;
 	bool done = false;
 	dev_dbg(chan2dev(&td_chan->chan), "Checking irq: %d, td: %p\n", id, td);
 	isr = ioread32(td->membase + TIMBDMA_ISR) & (1 << id);
 	if (isr) {
 		iowrite32(isr, td->membase + TIMBDMA_ISR);
 		done = true;
 	}
 	return done;
 }
 static void __td_unmap_desc(struct timb_dma_chan *td_chan, const u8 *dma_desc,
 	bool single)
 {
 	dma_addr_t addr;
 	int len;
 	addr = (dma_desc[7] << 24) | (dma_desc[6] << 16) | (dma_desc[5] << 8) |
 		dma_desc[4];
 	len = (dma_desc[3] << 8) | dma_desc[2];
 	if (single)
 		dma_unmap_single(chan2dev(&td_chan->chan), addr, len,
 			td_chan->direction);
 	else
 		dma_unmap_page(chan2dev(&td_chan->chan), addr, len,
 			td_chan->direction);
 }
 static void __td_unmap_descs(struct timb_dma_desc *td_desc, bool single)
 {
 	struct timb_dma_chan *td_chan = container_of(td_desc->txd.chan,
 		struct timb_dma_chan, chan);
 	u8 *descs;
 	for (descs = td_desc->desc_list; ; descs += TIMB_DMA_DESC_SIZE) {
 		__td_unmap_desc(td_chan, descs, single);
 		if (descs[0] & 0x02)
 			break;
 	}
 }
 static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc,
 	struct scatterlist *sg, bool last)
 {
 	if (sg_dma_len(sg) > USHORT_MAX) {
 		dev_err(chan2dev(&td_chan->chan), "Too big sg element\n");
 		return -EINVAL;
 	}
 	/* length must be word aligned */
 	if (sg_dma_len(sg) % sizeof(u32)) {
 		dev_err(chan2dev(&td_chan->chan), "Incorrect length: %d\n",
 			sg_dma_len(sg));
 		return -EINVAL;
 	}
 	dev_dbg(chan2dev(&td_chan->chan), "desc: %p, addr: %p\n",
 		dma_desc, (void *)sg_dma_address(sg));
 	dma_desc[7] = (sg_dma_address(sg) >> 24) & 0xff;
 	dma_desc[6] = (sg_dma_address(sg) >> 16) & 0xff;
 	dma_desc[5] = (sg_dma_address(sg) >> 8) & 0xff;
 	dma_desc[4] = (sg_dma_address(sg) >> 0) & 0xff;
 	dma_desc[3] = (sg_dma_len(sg) >> 8) & 0xff;
 	dma_desc[2] = (sg_dma_len(sg) >> 0) & 0xff;
 	dma_desc[1] = 0x00;
 	dma_desc[0] = 0x21 | (last ? 0x02 : 0); /* tran, valid */
 	return 0;
 }
 /* Must be called with the spinlock held */
 static void __td_start_dma(struct timb_dma_chan *td_chan)
 {
 	struct timb_dma_desc *td_desc;
 	if (td_chan->ongoing) {
 		dev_err(chan2dev(&td_chan->chan),
 			"Transfer already ongoing\n");
 		return;
 	}
 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,
 		desc_node);
 	dev_dbg(chan2dev(&td_chan->chan),
 		"td_chan: %p, chan: %d, membase: %p\n",
 		td_chan, td_chan->chan.chan_id, td_chan->membase);
 	if (td_chan->direction == DMA_FROM_DEVICE) {
 		/* descriptor address */
 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_DHAR);
 		iowrite32(td_desc->txd.phys, td_chan->membase +
 			TIMBDMA_OFFS_RX_DLAR);
 		/* Bytes per line */
 		iowrite32(td_chan->bytes_per_line, td_chan->membase +
 			TIMBDMA_OFFS_RX_BPRR);
 		/* enable RX */
 		iowrite32(TIMBDMA_RX_EN, td_chan->membase + TIMBDMA_OFFS_RX_ER);
 	} else {
 		/* address high */
 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DHAR);
 		iowrite32(td_desc->txd.phys, td_chan->membase +
 			TIMBDMA_OFFS_TX_DLAR);
 	}
 	td_chan->ongoing = true;
 	if (td_desc->interrupt)
 		__td_enable_chan_irq(td_chan);
 }
 static void __td_finish(struct timb_dma_chan *td_chan)
 {
 	dma_async_tx_callback		callback;
 	void				*param;
 	struct dma_async_tx_descriptor	*txd;
 	struct timb_dma_desc		*td_desc;
 	/* can happen if the descriptor is canceled */
 	if (list_empty(&td_chan->active_list))
 		return;
 	td_desc = list_entry(td_chan->active_list.next, struct timb_dma_desc,
 		desc_node);
 	txd = &td_desc->txd;
 	dev_dbg(chan2dev(&td_chan->chan), "descriptor %u complete\n",
 		txd->cookie);
 	/* make sure to stop the transfer */
 	if (td_chan->direction == DMA_FROM_DEVICE)
 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_RX_ER);
 /* Currently no support for stopping DMA transfers
 	else
 		iowrite32(0, td_chan->membase + TIMBDMA_OFFS_TX_DLAR);
 */
 	td_chan->last_completed_cookie = txd->cookie;
 	td_chan->ongoing = false;
 	callback = txd->callback;
 	param = txd->callback_param;
 	list_move(&td_desc->desc_node, &td_chan->free_list);
 	if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP))
 		__td_unmap_descs(td_desc,
 			txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE);
 	/*
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
 	if (callback)
 		callback(param);
 }
 static u32 __td_ier_mask(struct timb_dma *td)
 {
 	int i;
 	u32 ret = 0;
 	for (i = 0; i < td->dma.chancnt; i++) {
 		struct timb_dma_chan *td_chan = td->channels + i;
 		if (td_chan->ongoing) {
 			struct timb_dma_desc *td_desc =
 				list_entry(td_chan->active_list.next,
 				struct timb_dma_desc, desc_node);
 			if (td_desc->interrupt)
 				ret |= 1 << i;
 		}
 	}
 	return ret;
 }
 static void __td_start_next(struct timb_dma_chan *td_chan)
 {
 	struct timb_dma_desc *td_desc;
 	BUG_ON(list_empty(&td_chan->queue));
 	BUG_ON(td_chan->ongoing);
 	td_desc = list_entry(td_chan->queue.next, struct timb_dma_desc,
 		desc_node);
 	dev_dbg(chan2dev(&td_chan->chan), "%s: started %u\n",
 		__func__, td_desc->txd.cookie);
 	list_move(&td_desc->desc_node, &td_chan->active_list);
 	__td_start_dma(td_chan);
 }
 static dma_cookie_t td_tx_submit(struct dma_async_tx_descriptor *txd)
 {
 	struct timb_dma_desc *td_desc = container_of(txd, struct timb_dma_desc,
 		txd);
 	struct timb_dma_chan *td_chan = container_of(txd->chan,
 		struct timb_dma_chan, chan);
 	dma_cookie_t cookie;
 	spin_lock_bh(&td_chan->lock);
 	cookie = txd->chan->cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	txd->chan->cookie = cookie;
 	txd->cookie = cookie;
 	if (list_empty(&td_chan->active_list)) {
 		dev_dbg(chan2dev(txd->chan), "%s: started %u\n", __func__,
 			txd->cookie);
 		list_add_tail(&td_desc->desc_node, &td_chan->active_list);
 		__td_start_dma(td_chan);
 	} else {
 		dev_dbg(chan2dev(txd->chan), "tx_submit: queued %u\n",
 			txd->cookie);
 		list_add_tail(&td_desc->desc_node, &td_chan->queue);
 	}
 	spin_unlock_bh(&td_chan->lock);
 	return cookie;
 }
 static struct timb_dma_desc *td_alloc_init_desc(struct timb_dma_chan *td_chan)
 {
 	struct dma_chan *chan = &td_chan->chan;
 	struct timb_dma_desc *td_desc;
 	int err;
 	td_desc = kzalloc(sizeof(struct timb_dma_desc), GFP_KERNEL);
 	if (!td_desc) {
 		dev_err(chan2dev(chan), "Failed to alloc descriptor\n");
 		goto err;
 	}
 	td_desc->desc_list_len = td_chan->desc_elems * TIMB_DMA_DESC_SIZE;
 	td_desc->desc_list = kzalloc(td_desc->desc_list_len, GFP_KERNEL);
 	if (!td_desc->desc_list) {
 		dev_err(chan2dev(chan), "Failed to alloc descriptor\n");
 		goto err;
 	}
 	dma_async_tx_descriptor_init(&td_desc->txd, chan);
 	td_desc->txd.tx_submit = td_tx_submit;
 	td_desc->txd.flags = DMA_CTRL_ACK;
 	td_desc->txd.phys = dma_map_single(chan2dmadev(chan),
 		td_desc->desc_list, td_desc->desc_list_len, DMA_TO_DEVICE);
 	err = dma_mapping_error(chan2dmadev(chan), td_desc->txd.phys);
 	if (err) {
 		dev_err(chan2dev(chan), "DMA mapping error: %d\n", err);
 		goto err;
 	}
 	return td_desc;
 err:
 	kfree(td_desc->desc_list);
 	kfree(td_desc);
 	return NULL;
 }
 static void td_free_desc(struct timb_dma_desc *td_desc)
 {
 	dev_dbg(chan2dev(td_desc->txd.chan), "Freeing desc: %p\n", td_desc);
 	dma_unmap_single(chan2dmadev(td_desc->txd.chan), td_desc->txd.phys,
 		td_desc->desc_list_len, DMA_TO_DEVICE);
 	kfree(td_desc->desc_list);
 	kfree(td_desc);
 }
 static void td_desc_put(struct timb_dma_chan *td_chan,
 	struct timb_dma_desc *td_desc)
 {
 	dev_dbg(chan2dev(&td_chan->chan), "Putting desc: %p\n", td_desc);
 	spin_lock_bh(&td_chan->lock);
 	list_add(&td_desc->desc_node, &td_chan->free_list);
 	spin_unlock_bh(&td_chan->lock);
 }
 static struct timb_dma_desc *td_desc_get(struct timb_dma_chan *td_chan)
 {
 	struct timb_dma_desc *td_desc, *_td_desc;
 	struct timb_dma_desc *ret = NULL;
 	spin_lock_bh(&td_chan->lock);
 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->free_list,
 		desc_node) {
 		if (async_tx_test_ack(&td_desc->txd)) {
 			list_del(&td_desc->desc_node);
 			ret = td_desc;
 			break;
 		}
 		dev_dbg(chan2dev(&td_chan->chan), "desc %p not ACKed\n",
 			td_desc);
 	}
 	spin_unlock_bh(&td_chan->lock);
 	return ret;
 }
 static int td_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	int i;
 	dev_dbg(chan2dev(chan), "%s: entry\n", __func__);
 	BUG_ON(!list_empty(&td_chan->free_list));
 	for (i = 0; i < td_chan->descs; i++) {
 		struct timb_dma_desc *td_desc = td_alloc_init_desc(td_chan);
 		if (!td_desc) {
 			if (i)
 				break;
 			else {
 				dev_err(chan2dev(chan),
 					"Couldnt allocate any descriptors\n");
 				return -ENOMEM;
 			}
 		}
 		td_desc_put(td_chan, td_desc);
 	}
 	spin_lock_bh(&td_chan->lock);
 	td_chan->last_completed_cookie = 1;
 	chan->cookie = 1;
 	spin_unlock_bh(&td_chan->lock);
 	return 0;
 }
 static void td_free_chan_resources(struct dma_chan *chan)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	struct timb_dma_desc *td_desc, *_td_desc;
 	LIST_HEAD(list);
 	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
 	/* check that all descriptors are free */
 	BUG_ON(!list_empty(&td_chan->active_list));
 	BUG_ON(!list_empty(&td_chan->queue));
 	spin_lock_bh(&td_chan->lock);
 	list_splice_init(&td_chan->free_list, &list);
 	spin_unlock_bh(&td_chan->lock);
 	list_for_each_entry_safe(td_desc, _td_desc, &list, desc_node) {
 		dev_dbg(chan2dev(chan), "%s: Freeing desc: %p\n", __func__,
 			td_desc);
 		td_free_desc(td_desc);
 	}
 }
 static enum dma_status td_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 				    struct dma_tx_state *txstate)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	dma_cookie_t		last_used;
 	dma_cookie_t		last_complete;
 	int			ret;
 	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
 	last_complete = td_chan->last_completed_cookie;
 	last_used = chan->cookie;
 	ret = dma_async_is_complete(cookie, last_complete, last_used);
 	dma_set_tx_state(txstate, last_complete, last_used, 0);
 	dev_dbg(chan2dev(chan),
 		"%s: exit, ret: %d, last_complete: %d, last_used: %d\n",
 		__func__, ret, last_complete, last_used);
 	return ret;
 }
 static void td_issue_pending(struct dma_chan *chan)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
 	spin_lock_bh(&td_chan->lock);
 	if (!list_empty(&td_chan->active_list))
 		/* transfer ongoing */
 		if (__td_dma_done_ack(td_chan))
 			__td_finish(td_chan);
 	if (list_empty(&td_chan->active_list) && !list_empty(&td_chan->queue))
 		__td_start_next(td_chan);
 	spin_unlock_bh(&td_chan->lock);
 }
 static struct dma_async_tx_descriptor *td_prep_slave_sg(struct dma_chan *chan,
 	struct scatterlist *sgl, unsigned int sg_len,
 	enum dma_data_direction direction, unsigned long flags)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	struct timb_dma_desc *td_desc;
 	struct scatterlist *sg;
 	unsigned int i;
 	unsigned int desc_usage = 0;
 	if (!sgl || !sg_len) {
 		dev_err(chan2dev(chan), "%s: No SG list\n", __func__);
 		return NULL;
 	}
 	/* even channels are for RX, odd for TX */
 	if (td_chan->direction != direction) {
 		dev_err(chan2dev(chan),
 			"Requesting channel in wrong direction\n");
 		return NULL;
 	}
 	td_desc = td_desc_get(td_chan);
 	if (!td_desc) {
 		dev_err(chan2dev(chan), "Not enough descriptors available\n");
 		return NULL;
 	}
 	td_desc->interrupt = (flags & DMA_PREP_INTERRUPT) != 0;
 	for_each_sg(sgl, sg, sg_len, i) {
 		int err;
 		if (desc_usage > td_desc->desc_list_len) {
 			dev_err(chan2dev(chan), "No descriptor space\n");
 			return NULL;
 		}
 		err = td_fill_desc(td_chan, td_desc->desc_list + desc_usage, sg,
 			i == (sg_len - 1));
 		if (err) {
 			dev_err(chan2dev(chan), "Failed to update desc: %d\n",
 				err);
 			td_desc_put(td_chan, td_desc);
 			return NULL;
 		}
 		desc_usage += TIMB_DMA_DESC_SIZE;
 	}
 	dma_sync_single_for_device(chan2dmadev(chan), td_desc->txd.phys,
 		td_desc->desc_list_len, DMA_TO_DEVICE);
 	return &td_desc->txd;
 }
-static int td_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int td_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		      unsigned long arg)
 {
 	struct timb_dma_chan *td_chan =
 		container_of(chan, struct timb_dma_chan, chan);
 	struct timb_dma_desc *td_desc, *_td_desc;
 	dev_dbg(chan2dev(chan), "%s: Entry\n", __func__);
 	if (cmd != DMA_TERMINATE_ALL)
 		return -ENXIO;
 	/* first the easy part, put the queue into the free list */
 	spin_lock_bh(&td_chan->lock);
 	list_for_each_entry_safe(td_desc, _td_desc, &td_chan->queue,
 		desc_node)
 		list_move(&td_desc->desc_node, &td_chan->free_list);
 	/* now tear down the runnning */
 	__td_finish(td_chan);
 	spin_unlock_bh(&td_chan->lock);
 	return 0;
 }
 static void td_tasklet(unsigned long data)
 {
 	struct timb_dma *td = (struct timb_dma *)data;
 	u32 isr;
 	u32 ipr;
 	u32 ier;
 	int i;
 	isr = ioread32(td->membase + TIMBDMA_ISR);
 	ipr = isr & __td_ier_mask(td);
 	/* ack the interrupts */
 	iowrite32(ipr, td->membase + TIMBDMA_ISR);
 	for (i = 0; i < td->dma.chancnt; i++)
 		if (ipr & (1 << i)) {
 			struct timb_dma_chan *td_chan = td->channels + i;
 			spin_lock(&td_chan->lock);
 			__td_finish(td_chan);
 			if (!list_empty(&td_chan->queue))
 				__td_start_next(td_chan);
 			spin_unlock(&td_chan->lock);
 		}
 	ier = __td_ier_mask(td);
 	iowrite32(ier, td->membase + TIMBDMA_IER);
 }
 static irqreturn_t td_irq(int irq, void *devid)
 {
 	struct timb_dma *td = devid;
 	u32 ipr = ioread32(td->membase + TIMBDMA_IPR);
 	if (ipr) {
 		/* disable interrupts, will be re-enabled in tasklet */
 		iowrite32(0, td->membase + TIMBDMA_IER);
 		tasklet_schedule(&td->tasklet);
 		return IRQ_HANDLED;
 	} else
 		return IRQ_NONE;
 }
 static int __devinit td_probe(struct platform_device *pdev)
 {
 	struct timb_dma_platform_data *pdata = pdev->dev.platform_data;
 	struct timb_dma *td;
 	struct resource *iomem;
 	int irq;
 	int err;
 	int i;
 	if (!pdata) {
 		dev_err(&pdev->dev, "No platform data\n");
 		return -EINVAL;
 	}
 	iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!iomem)
 		return -EINVAL;
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
 	if (!request_mem_region(iomem->start, resource_size(iomem),
 		DRIVER_NAME))
 		return -EBUSY;
 	td  = kzalloc(sizeof(struct timb_dma) +
 		sizeof(struct timb_dma_chan) * pdata->nr_channels, GFP_KERNEL);
 	if (!td) {
 		err = -ENOMEM;
 		goto err_release_region;
 	}
 	dev_dbg(&pdev->dev, "Allocated TD: %p\n", td);
 	td->membase = ioremap(iomem->start, resource_size(iomem));
 	if (!td->membase) {
 		dev_err(&pdev->dev, "Failed to remap I/O memory\n");
 		err = -ENOMEM;
 		goto err_free_mem;
 	}
 	/* 32bit addressing */
 	iowrite32(TIMBDMA_32BIT_ADDR, td->membase + TIMBDMA_ACR);
 	/* disable and clear any interrupts */
 	iowrite32(0x0, td->membase + TIMBDMA_IER);
 	iowrite32(0xFFFFFFFF, td->membase + TIMBDMA_ISR);
 	tasklet_init(&td->tasklet, td_tasklet, (unsigned long)td);
 	err = request_irq(irq, td_irq, IRQF_SHARED, DRIVER_NAME, td);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to request IRQ\n");
 		goto err_tasklet_kill;
 	}
 	td->dma.device_alloc_chan_resources	= td_alloc_chan_resources;
 	td->dma.device_free_chan_resources	= td_free_chan_resources;
 	td->dma.device_tx_status		= td_tx_status;
 	td->dma.device_issue_pending		= td_issue_pending;
 	dma_cap_set(DMA_SLAVE, td->dma.cap_mask);
 	dma_cap_set(DMA_PRIVATE, td->dma.cap_mask);
 	td->dma.device_prep_slave_sg = td_prep_slave_sg;
 	td->dma.device_control = td_control;
 	td->dma.dev = &pdev->dev;
 	INIT_LIST_HEAD(&td->dma.channels);
 	for (i = 0; i < pdata->nr_channels; i++, td->dma.chancnt++) {
 		struct timb_dma_chan *td_chan = &td->channels[i];
 		struct timb_dma_platform_data_channel *pchan =
 			pdata->channels + i;
 		/* even channels are RX, odd are TX */
 		if (((i % 2) && pchan->rx) || (!(i % 2) && !pchan->rx)) {
 			dev_err(&pdev->dev, "Wrong channel configuration\n");
 			err = -EINVAL;
 			goto err_tasklet_kill;
 		}
 		td_chan->chan.device = &td->dma;
 		td_chan->chan.cookie = 1;
 		td_chan->chan.chan_id = i;
 		spin_lock_init(&td_chan->lock);
 		INIT_LIST_HEAD(&td_chan->active_list);
 		INIT_LIST_HEAD(&td_chan->queue);
 		INIT_LIST_HEAD(&td_chan->free_list);
 		td_chan->descs = pchan->descriptors;
 		td_chan->desc_elems = pchan->descriptor_elements;
 		td_chan->bytes_per_line = pchan->bytes_per_line;
 		td_chan->direction = pchan->rx ? DMA_FROM_DEVICE :
 			DMA_TO_DEVICE;
 		td_chan->membase = td->membase +
 			(i / 2) * TIMBDMA_INSTANCE_OFFSET +
 			(pchan->rx ? 0 : TIMBDMA_INSTANCE_TX_OFFSET);
 		dev_dbg(&pdev->dev, "Chan: %d, membase: %p\n",
 			i, td_chan->membase);
 		list_add_tail(&td_chan->chan.device_node, &td->dma.channels);
 	}
 	err = dma_async_device_register(&td->dma);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to register async device\n");
 		goto err_free_irq;
 	}
 	platform_set_drvdata(pdev, td);
 	dev_dbg(&pdev->dev, "Probe result: %d\n", err);
 	return err;
 err_free_irq:
 	free_irq(irq, td);
 err_tasklet_kill:
 	tasklet_kill(&td->tasklet);
 	iounmap(td->membase);
 err_free_mem:
 	kfree(td);
 err_release_region:
 	release_mem_region(iomem->start, resource_size(iomem));
 	return err;
 }
 static int __devexit td_remove(struct platform_device *pdev)
 {
 	struct timb_dma *td = platform_get_drvdata(pdev);
 	struct resource *iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	int irq = platform_get_irq(pdev, 0);
 	dma_async_device_unregister(&td->dma);
 	free_irq(irq, td);
 	tasklet_kill(&td->tasklet);
 	iounmap(td->membase);
 	kfree(td);
 	release_mem_region(iomem->start, resource_size(iomem));
 	platform_set_drvdata(pdev, NULL);
 	dev_dbg(&pdev->dev, "Removed...\n");
 	return 0;
 }
 static struct platform_driver td_driver = {
 	.driver = {
 		.name	= DRIVER_NAME,
 		.owner  = THIS_MODULE,
 	},
 	.probe	= td_probe,
 	.remove	= __exit_p(td_remove),
 };
 static int __init td_init(void)
 {
 	return platform_driver_register(&td_driver);
 }
 module_init(td_init);
 static void __exit td_exit(void)
 {
 	platform_driver_unregister(&td_driver);
 }
 module_exit(td_exit);
 MODULE_LICENSE("GPL v2");
 MODULE_DESCRIPTION("Timberdale DMA controller driver");
 MODULE_AUTHOR("Pelagicore AB <info@pelagicore.com>");
 MODULE_ALIAS("platform:"DRIVER_NAME);

drivers/dma/txx9dmac.c

Diff comments View file @ 0582763

 /*
  * Driver for the TXx9 SoC DMA Controller
  *
  * Copyright (C) 2009 Atsushi Nemoto
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/dma-mapping.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/scatterlist.h>
 #include "txx9dmac.h"
 static struct txx9dmac_chan *to_txx9dmac_chan(struct dma_chan *chan)
 {
 	return container_of(chan, struct txx9dmac_chan, chan);
 }
 static struct txx9dmac_cregs __iomem *__dma_regs(const struct txx9dmac_chan *dc)
 {
 	return dc->ch_regs;
 }
 static struct txx9dmac_cregs32 __iomem *__dma_regs32(
 	const struct txx9dmac_chan *dc)
 {
 	return dc->ch_regs;
 }
 #define channel64_readq(dc, name) \
 	__raw_readq(&(__dma_regs(dc)->name))
 #define channel64_writeq(dc, name, val) \
 	__raw_writeq((val), &(__dma_regs(dc)->name))
 #define channel64_readl(dc, name) \
 	__raw_readl(&(__dma_regs(dc)->name))
 #define channel64_writel(dc, name, val) \
 	__raw_writel((val), &(__dma_regs(dc)->name))
 #define channel32_readl(dc, name) \
 	__raw_readl(&(__dma_regs32(dc)->name))
 #define channel32_writel(dc, name, val) \
 	__raw_writel((val), &(__dma_regs32(dc)->name))
 #define channel_readq(dc, name) channel64_readq(dc, name)
 #define channel_writeq(dc, name, val) channel64_writeq(dc, name, val)
 #define channel_readl(dc, name) \
 	(is_dmac64(dc) ? \
 	 channel64_readl(dc, name) : channel32_readl(dc, name))
 #define channel_writel(dc, name, val) \
 	(is_dmac64(dc) ? \
 	 channel64_writel(dc, name, val) : channel32_writel(dc, name, val))
 static dma_addr_t channel64_read_CHAR(const struct txx9dmac_chan *dc)
 {
 	if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64))
 		return channel64_readq(dc, CHAR);
 	else
 		return channel64_readl(dc, CHAR);
 }
 static void channel64_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val)
 {
 	if (sizeof(__dma_regs(dc)->CHAR) == sizeof(u64))
 		channel64_writeq(dc, CHAR, val);
 	else
 		channel64_writel(dc, CHAR, val);
 }
 static void channel64_clear_CHAR(const struct txx9dmac_chan *dc)
 {
 #if defined(CONFIG_32BIT) && !defined(CONFIG_64BIT_PHYS_ADDR)
 	channel64_writel(dc, CHAR, 0);
 	channel64_writel(dc, __pad_CHAR, 0);
 #else
 	channel64_writeq(dc, CHAR, 0);
 #endif
 }
 static dma_addr_t channel_read_CHAR(const struct txx9dmac_chan *dc)
 {
 	if (is_dmac64(dc))
 		return channel64_read_CHAR(dc);
 	else
 		return channel32_readl(dc, CHAR);
 }
 static void channel_write_CHAR(const struct txx9dmac_chan *dc, dma_addr_t val)
 {
 	if (is_dmac64(dc))
 		channel64_write_CHAR(dc, val);
 	else
 		channel32_writel(dc, CHAR, val);
 }
 static struct txx9dmac_regs __iomem *__txx9dmac_regs(
 	const struct txx9dmac_dev *ddev)
 {
 	return ddev->regs;
 }
 static struct txx9dmac_regs32 __iomem *__txx9dmac_regs32(
 	const struct txx9dmac_dev *ddev)
 {
 	return ddev->regs;
 }
 #define dma64_readl(ddev, name) \
 	__raw_readl(&(__txx9dmac_regs(ddev)->name))
 #define dma64_writel(ddev, name, val) \
 	__raw_writel((val), &(__txx9dmac_regs(ddev)->name))
 #define dma32_readl(ddev, name) \
 	__raw_readl(&(__txx9dmac_regs32(ddev)->name))
 #define dma32_writel(ddev, name, val) \
 	__raw_writel((val), &(__txx9dmac_regs32(ddev)->name))
 #define dma_readl(ddev, name) \
 	(__is_dmac64(ddev) ? \
 	dma64_readl(ddev, name) : dma32_readl(ddev, name))
 #define dma_writel(ddev, name, val) \
 	(__is_dmac64(ddev) ? \
 	dma64_writel(ddev, name, val) : dma32_writel(ddev, name, val))
 static struct device *chan2dev(struct dma_chan *chan)
 {
 	return &chan->dev->device;
 }
 static struct device *chan2parent(struct dma_chan *chan)
 {
 	return chan->dev->device.parent;
 }
 static struct txx9dmac_desc *
 txd_to_txx9dmac_desc(struct dma_async_tx_descriptor *txd)
 {
 	return container_of(txd, struct txx9dmac_desc, txd);
 }
 static dma_addr_t desc_read_CHAR(const struct txx9dmac_chan *dc,
 				 const struct txx9dmac_desc *desc)
 {
 	return is_dmac64(dc) ? desc->hwdesc.CHAR : desc->hwdesc32.CHAR;
 }
 static void desc_write_CHAR(const struct txx9dmac_chan *dc,
 			    struct txx9dmac_desc *desc, dma_addr_t val)
 {
 	if (is_dmac64(dc))
 		desc->hwdesc.CHAR = val;
 	else
 		desc->hwdesc32.CHAR = val;
 }
 #define TXX9_DMA_MAX_COUNT	0x04000000
 #define TXX9_DMA_INITIAL_DESC_COUNT	64
 static struct txx9dmac_desc *txx9dmac_first_active(struct txx9dmac_chan *dc)
 {
 	return list_entry(dc->active_list.next,
 			  struct txx9dmac_desc, desc_node);
 }
 static struct txx9dmac_desc *txx9dmac_last_active(struct txx9dmac_chan *dc)
 {
 	return list_entry(dc->active_list.prev,
 			  struct txx9dmac_desc, desc_node);
 }
 static struct txx9dmac_desc *txx9dmac_first_queued(struct txx9dmac_chan *dc)
 {
 	return list_entry(dc->queue.next, struct txx9dmac_desc, desc_node);
 }
 static struct txx9dmac_desc *txx9dmac_last_child(struct txx9dmac_desc *desc)
 {
 	if (!list_empty(&desc->tx_list))
 		desc = list_entry(desc->tx_list.prev, typeof(*desc), desc_node);
 	return desc;
 }
 static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx);
 static struct txx9dmac_desc *txx9dmac_desc_alloc(struct txx9dmac_chan *dc,
 						 gfp_t flags)
 {
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *desc;
 	desc = kzalloc(sizeof(*desc), flags);
 	if (!desc)
 		return NULL;
 	INIT_LIST_HEAD(&desc->tx_list);
 	dma_async_tx_descriptor_init(&desc->txd, &dc->chan);
 	desc->txd.tx_submit = txx9dmac_tx_submit;
 	/* txd.flags will be overwritten in prep funcs */
 	desc->txd.flags = DMA_CTRL_ACK;
 	desc->txd.phys = dma_map_single(chan2parent(&dc->chan), &desc->hwdesc,
 					ddev->descsize, DMA_TO_DEVICE);
 	return desc;
 }
 static struct txx9dmac_desc *txx9dmac_desc_get(struct txx9dmac_chan *dc)
 {
 	struct txx9dmac_desc *desc, *_desc;
 	struct txx9dmac_desc *ret = NULL;
 	unsigned int i = 0;
 	spin_lock_bh(&dc->lock);
 	list_for_each_entry_safe(desc, _desc, &dc->free_list, desc_node) {
 		if (async_tx_test_ack(&desc->txd)) {
 			list_del(&desc->desc_node);
 			ret = desc;
 			break;
 		}
 		dev_dbg(chan2dev(&dc->chan), "desc %p not ACKed\n", desc);
 		i++;
 	}
 	spin_unlock_bh(&dc->lock);
 	dev_vdbg(chan2dev(&dc->chan), "scanned %u descriptors on freelist\n",
 		 i);
 	if (!ret) {
 		ret = txx9dmac_desc_alloc(dc, GFP_ATOMIC);
 		if (ret) {
 			spin_lock_bh(&dc->lock);
 			dc->descs_allocated++;
 			spin_unlock_bh(&dc->lock);
 		} else
 			dev_err(chan2dev(&dc->chan),
 				"not enough descriptors available\n");
 	}
 	return ret;
 }
 static void txx9dmac_sync_desc_for_cpu(struct txx9dmac_chan *dc,
 				       struct txx9dmac_desc *desc)
 {
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *child;
 	list_for_each_entry(child, &desc->tx_list, desc_node)
 		dma_sync_single_for_cpu(chan2parent(&dc->chan),
 				child->txd.phys, ddev->descsize,
 				DMA_TO_DEVICE);
 	dma_sync_single_for_cpu(chan2parent(&dc->chan),
 			desc->txd.phys, ddev->descsize,
 			DMA_TO_DEVICE);
 }
 /*
  * Move a descriptor, including any children, to the free list.
  * `desc' must not be on any lists.
  */
 static void txx9dmac_desc_put(struct txx9dmac_chan *dc,
 			      struct txx9dmac_desc *desc)
 {
 	if (desc) {
 		struct txx9dmac_desc *child;
 		txx9dmac_sync_desc_for_cpu(dc, desc);
 		spin_lock_bh(&dc->lock);
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			dev_vdbg(chan2dev(&dc->chan),
 				 "moving child desc %p to freelist\n",
 				 child);
 		list_splice_init(&desc->tx_list, &dc->free_list);
 		dev_vdbg(chan2dev(&dc->chan), "moving desc %p to freelist\n",
 			 desc);
 		list_add(&desc->desc_node, &dc->free_list);
 		spin_unlock_bh(&dc->lock);
 	}
 }
 /* Called with dc->lock held and bh disabled */
 static dma_cookie_t
 txx9dmac_assign_cookie(struct txx9dmac_chan *dc, struct txx9dmac_desc *desc)
 {
 	dma_cookie_t cookie = dc->chan.cookie;
 	if (++cookie < 0)
 		cookie = 1;
 	dc->chan.cookie = cookie;
 	desc->txd.cookie = cookie;
 	return cookie;
 }
 /*----------------------------------------------------------------------*/
 static void txx9dmac_dump_regs(struct txx9dmac_chan *dc)
 {
 	if (is_dmac64(dc))
 		dev_err(chan2dev(&dc->chan),
 			"  CHAR: %#llx SAR: %#llx DAR: %#llx CNTR: %#x"
 			" SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
 			(u64)channel64_read_CHAR(dc),
 			channel64_readq(dc, SAR),
 			channel64_readq(dc, DAR),
 			channel64_readl(dc, CNTR),
 			channel64_readl(dc, SAIR),
 			channel64_readl(dc, DAIR),
 			channel64_readl(dc, CCR),
 			channel64_readl(dc, CSR));
 	else
 		dev_err(chan2dev(&dc->chan),
 			"  CHAR: %#x SAR: %#x DAR: %#x CNTR: %#x"
 			" SAIR: %#x DAIR: %#x CCR: %#x CSR: %#x\n",
 			channel32_readl(dc, CHAR),
 			channel32_readl(dc, SAR),
 			channel32_readl(dc, DAR),
 			channel32_readl(dc, CNTR),
 			channel32_readl(dc, SAIR),
 			channel32_readl(dc, DAIR),
 			channel32_readl(dc, CCR),
 			channel32_readl(dc, CSR));
 }
 static void txx9dmac_reset_chan(struct txx9dmac_chan *dc)
 {
 	channel_writel(dc, CCR, TXX9_DMA_CCR_CHRST);
 	if (is_dmac64(dc)) {
 		channel64_clear_CHAR(dc);
 		channel_writeq(dc, SAR, 0);
 		channel_writeq(dc, DAR, 0);
 	} else {
 		channel_writel(dc, CHAR, 0);
 		channel_writel(dc, SAR, 0);
 		channel_writel(dc, DAR, 0);
 	}
 	channel_writel(dc, CNTR, 0);
 	channel_writel(dc, SAIR, 0);
 	channel_writel(dc, DAIR, 0);
 	channel_writel(dc, CCR, 0);
 	mmiowb();
 }
 /* Called with dc->lock held and bh disabled */
 static void txx9dmac_dostart(struct txx9dmac_chan *dc,
 			     struct txx9dmac_desc *first)
 {
 	struct txx9dmac_slave *ds = dc->chan.private;
 	u32 sai, dai;
 	dev_vdbg(chan2dev(&dc->chan), "dostart %u %p\n",
 		 first->txd.cookie, first);
 	/* ASSERT:  channel is idle */
 	if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
 		dev_err(chan2dev(&dc->chan),
 			"BUG: Attempted to start non-idle channel\n");
 		txx9dmac_dump_regs(dc);
 		/* The tasklet will hopefully advance the queue... */
 		return;
 	}
 	if (is_dmac64(dc)) {
 		channel64_writel(dc, CNTR, 0);
 		channel64_writel(dc, CSR, 0xffffffff);
 		if (ds) {
 			if (ds->tx_reg) {
 				sai = ds->reg_width;
 				dai = 0;
 			} else {
 				sai = 0;
 				dai = ds->reg_width;
 			}
 		} else {
 			sai = 8;
 			dai = 8;
 		}
 		channel64_writel(dc, SAIR, sai);
 		channel64_writel(dc, DAIR, dai);
 		/* All 64-bit DMAC supports SMPCHN */
 		channel64_writel(dc, CCR, dc->ccr);
 		/* Writing a non zero value to CHAR will assert XFACT */
 		channel64_write_CHAR(dc, first->txd.phys);
 	} else {
 		channel32_writel(dc, CNTR, 0);
 		channel32_writel(dc, CSR, 0xffffffff);
 		if (ds) {
 			if (ds->tx_reg) {
 				sai = ds->reg_width;
 				dai = 0;
 			} else {
 				sai = 0;
 				dai = ds->reg_width;
 			}
 		} else {
 			sai = 4;
 			dai = 4;
 		}
 		channel32_writel(dc, SAIR, sai);
 		channel32_writel(dc, DAIR, dai);
 		if (txx9_dma_have_SMPCHN()) {
 			channel32_writel(dc, CCR, dc->ccr);
 			/* Writing a non zero value to CHAR will assert XFACT */
 			channel32_writel(dc, CHAR, first->txd.phys);
 		} else {
 			channel32_writel(dc, CHAR, first->txd.phys);
 			channel32_writel(dc, CCR, dc->ccr);
 		}
 	}
 }
 /*----------------------------------------------------------------------*/
 static void
 txx9dmac_descriptor_complete(struct txx9dmac_chan *dc,
 			     struct txx9dmac_desc *desc)
 {
 	dma_async_tx_callback callback;
 	void *param;
 	struct dma_async_tx_descriptor *txd = &desc->txd;
 	struct txx9dmac_slave *ds = dc->chan.private;
 	dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n",
 		 txd->cookie, desc);
 	dc->completed = txd->cookie;
 	callback = txd->callback;
 	param = txd->callback_param;
 	txx9dmac_sync_desc_for_cpu(dc, desc);
 	list_splice_init(&desc->tx_list, &dc->free_list);
 	list_move(&desc->desc_node, &dc->free_list);
 	if (!ds) {
 		dma_addr_t dmaaddr;
 		if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 			dmaaddr = is_dmac64(dc) ?
 				desc->hwdesc.DAR : desc->hwdesc32.DAR;
 			if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 				dma_unmap_single(chan2parent(&dc->chan),
 					dmaaddr, desc->len, DMA_FROM_DEVICE);
 			else
 				dma_unmap_page(chan2parent(&dc->chan),
 					dmaaddr, desc->len, DMA_FROM_DEVICE);
 		}
 		if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 			dmaaddr = is_dmac64(dc) ?
 				desc->hwdesc.SAR : desc->hwdesc32.SAR;
 			if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 				dma_unmap_single(chan2parent(&dc->chan),
 					dmaaddr, desc->len, DMA_TO_DEVICE);
 			else
 				dma_unmap_page(chan2parent(&dc->chan),
 					dmaaddr, desc->len, DMA_TO_DEVICE);
 		}
 	}
 	/*
 	 * The API requires that no submissions are done from a
 	 * callback, so we don't need to drop the lock here
 	 */
 	if (callback)
 		callback(param);
 	dma_run_dependencies(txd);
 }
 static void txx9dmac_dequeue(struct txx9dmac_chan *dc, struct list_head *list)
 {
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *desc;
 	struct txx9dmac_desc *prev = NULL;
 	BUG_ON(!list_empty(list));
 	do {
 		desc = txx9dmac_first_queued(dc);
 		if (prev) {
 			desc_write_CHAR(dc, prev, desc->txd.phys);
 			dma_sync_single_for_device(chan2parent(&dc->chan),
 				prev->txd.phys, ddev->descsize,
 				DMA_TO_DEVICE);
 		}
 		prev = txx9dmac_last_child(desc);
 		list_move_tail(&desc->desc_node, list);
 		/* Make chain-completion interrupt happen */
 		if ((desc->txd.flags & DMA_PREP_INTERRUPT) &&
 		    !txx9dmac_chan_INTENT(dc))
 			break;
 	} while (!list_empty(&dc->queue));
 }
 static void txx9dmac_complete_all(struct txx9dmac_chan *dc)
 {
 	struct txx9dmac_desc *desc, *_desc;
 	LIST_HEAD(list);
 	/*
 	 * Submit queued descriptors ASAP, i.e. before we go through
 	 * the completed ones.
 	 */
 	list_splice_init(&dc->active_list, &list);
 	if (!list_empty(&dc->queue)) {
 		txx9dmac_dequeue(dc, &dc->active_list);
 		txx9dmac_dostart(dc, txx9dmac_first_active(dc));
 	}
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		txx9dmac_descriptor_complete(dc, desc);
 }
 static void txx9dmac_dump_desc(struct txx9dmac_chan *dc,
 			       struct txx9dmac_hwdesc *desc)
 {
 	if (is_dmac64(dc)) {
 #ifdef TXX9_DMA_USE_SIMPLE_CHAIN
 		dev_crit(chan2dev(&dc->chan),
 			 "  desc: ch%#llx s%#llx d%#llx c%#x\n",
 			 (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR);
 #else
 		dev_crit(chan2dev(&dc->chan),
 			 "  desc: ch%#llx s%#llx d%#llx c%#x"
 			 " si%#x di%#x cc%#x cs%#x\n",
 			 (u64)desc->CHAR, desc->SAR, desc->DAR, desc->CNTR,
 			 desc->SAIR, desc->DAIR, desc->CCR, desc->CSR);
 #endif
 	} else {
 		struct txx9dmac_hwdesc32 *d = (struct txx9dmac_hwdesc32 *)desc;
 #ifdef TXX9_DMA_USE_SIMPLE_CHAIN
 		dev_crit(chan2dev(&dc->chan),
 			 "  desc: ch%#x s%#x d%#x c%#x\n",
 			 d->CHAR, d->SAR, d->DAR, d->CNTR);
 #else
 		dev_crit(chan2dev(&dc->chan),
 			 "  desc: ch%#x s%#x d%#x c%#x"
 			 " si%#x di%#x cc%#x cs%#x\n",
 			 d->CHAR, d->SAR, d->DAR, d->CNTR,
 			 d->SAIR, d->DAIR, d->CCR, d->CSR);
 #endif
 	}
 }
 static void txx9dmac_handle_error(struct txx9dmac_chan *dc, u32 csr)
 {
 	struct txx9dmac_desc *bad_desc;
 	struct txx9dmac_desc *child;
 	u32 errors;
 	/*
 	 * The descriptor currently at the head of the active list is
 	 * borked. Since we don't have any way to report errors, we'll
 	 * just have to scream loudly and try to carry on.
 	 */
 	dev_crit(chan2dev(&dc->chan), "Abnormal Chain Completion\n");
 	txx9dmac_dump_regs(dc);
 	bad_desc = txx9dmac_first_active(dc);
 	list_del_init(&bad_desc->desc_node);
 	/* Clear all error flags and try to restart the controller */
 	errors = csr & (TXX9_DMA_CSR_ABCHC |
 			TXX9_DMA_CSR_CFERR | TXX9_DMA_CSR_CHERR |
 			TXX9_DMA_CSR_DESERR | TXX9_DMA_CSR_SORERR);
 	channel_writel(dc, CSR, errors);
 	if (list_empty(&dc->active_list) && !list_empty(&dc->queue))
 		txx9dmac_dequeue(dc, &dc->active_list);
 	if (!list_empty(&dc->active_list))
 		txx9dmac_dostart(dc, txx9dmac_first_active(dc));
 	dev_crit(chan2dev(&dc->chan),
 		 "Bad descriptor submitted for DMA! (cookie: %d)\n",
 		 bad_desc->txd.cookie);
 	txx9dmac_dump_desc(dc, &bad_desc->hwdesc);
 	list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 		txx9dmac_dump_desc(dc, &child->hwdesc);
 	/* Pretend the descriptor completed successfully */
 	txx9dmac_descriptor_complete(dc, bad_desc);
 }
 static void txx9dmac_scan_descriptors(struct txx9dmac_chan *dc)
 {
 	dma_addr_t chain;
 	struct txx9dmac_desc *desc, *_desc;
 	struct txx9dmac_desc *child;
 	u32 csr;
 	if (is_dmac64(dc)) {
 		chain = channel64_read_CHAR(dc);
 		csr = channel64_readl(dc, CSR);
 		channel64_writel(dc, CSR, csr);
 	} else {
 		chain = channel32_readl(dc, CHAR);
 		csr = channel32_readl(dc, CSR);
 		channel32_writel(dc, CSR, csr);
 	}
 	/* For dynamic chain, we should look at XFACT instead of NCHNC */
 	if (!(csr & (TXX9_DMA_CSR_XFACT | TXX9_DMA_CSR_ABCHC))) {
 		/* Everything we've submitted is done */
 		txx9dmac_complete_all(dc);
 		return;
 	}
 	if (!(csr & TXX9_DMA_CSR_CHNEN))
 		chain = 0;	/* last descriptor of this chain */
 	dev_vdbg(chan2dev(&dc->chan), "scan_descriptors: char=%#llx\n",
 		 (u64)chain);
 	list_for_each_entry_safe(desc, _desc, &dc->active_list, desc_node) {
 		if (desc_read_CHAR(dc, desc) == chain) {
 			/* This one is currently in progress */
 			if (csr & TXX9_DMA_CSR_ABCHC)
 				goto scan_done;
 			return;
 		}
 		list_for_each_entry(child, &desc->tx_list, desc_node)
 			if (desc_read_CHAR(dc, child) == chain) {
 				/* Currently in progress */
 				if (csr & TXX9_DMA_CSR_ABCHC)
 					goto scan_done;
 				return;
 			}
 		/*
 		 * No descriptors so far seem to be in progress, i.e.
 		 * this one must be done.
 		 */
 		txx9dmac_descriptor_complete(dc, desc);
 	}
 scan_done:
 	if (csr & TXX9_DMA_CSR_ABCHC) {
 		txx9dmac_handle_error(dc, csr);
 		return;
 	}
 	dev_err(chan2dev(&dc->chan),
 		"BUG: All descriptors done, but channel not idle!\n");
 	/* Try to continue after resetting the channel... */
 	txx9dmac_reset_chan(dc);
 	if (!list_empty(&dc->queue)) {
 		txx9dmac_dequeue(dc, &dc->active_list);
 		txx9dmac_dostart(dc, txx9dmac_first_active(dc));
 	}
 }
 static void txx9dmac_chan_tasklet(unsigned long data)
 {
 	int irq;
 	u32 csr;
 	struct txx9dmac_chan *dc;
 	dc = (struct txx9dmac_chan *)data;
 	csr = channel_readl(dc, CSR);
 	dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n", csr);
 	spin_lock(&dc->lock);
 	if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
 		   TXX9_DMA_CSR_NTRNFC))
 		txx9dmac_scan_descriptors(dc);
 	spin_unlock(&dc->lock);
 	irq = dc->irq;
 	enable_irq(irq);
 }
 static irqreturn_t txx9dmac_chan_interrupt(int irq, void *dev_id)
 {
 	struct txx9dmac_chan *dc = dev_id;
 	dev_vdbg(chan2dev(&dc->chan), "interrupt: status=%#x\n",
 			channel_readl(dc, CSR));
 	tasklet_schedule(&dc->tasklet);
 	/*
 	 * Just disable the interrupts. We'll turn them back on in the
 	 * softirq handler.
 	 */
 	disable_irq_nosync(irq);
 	return IRQ_HANDLED;
 }
 static void txx9dmac_tasklet(unsigned long data)
 {
 	int irq;
 	u32 csr;
 	struct txx9dmac_chan *dc;
 	struct txx9dmac_dev *ddev = (struct txx9dmac_dev *)data;
 	u32 mcr;
 	int i;
 	mcr = dma_readl(ddev, MCR);
 	dev_vdbg(ddev->chan[0]->dma.dev, "tasklet: mcr=%x\n", mcr);
 	for (i = 0; i < TXX9_DMA_MAX_NR_CHANNELS; i++) {
 		if ((mcr >> (24 + i)) & 0x11) {
 			dc = ddev->chan[i];
 			csr = channel_readl(dc, CSR);
 			dev_vdbg(chan2dev(&dc->chan), "tasklet: status=%x\n",
 				 csr);
 			spin_lock(&dc->lock);
 			if (csr & (TXX9_DMA_CSR_ABCHC | TXX9_DMA_CSR_NCHNC |
 				   TXX9_DMA_CSR_NTRNFC))
 				txx9dmac_scan_descriptors(dc);
 			spin_unlock(&dc->lock);
 		}
 	}
 	irq = ddev->irq;
 	enable_irq(irq);
 }
 static irqreturn_t txx9dmac_interrupt(int irq, void *dev_id)
 {
 	struct txx9dmac_dev *ddev = dev_id;
 	dev_vdbg(ddev->chan[0]->dma.dev, "interrupt: status=%#x\n",
 			dma_readl(ddev, MCR));
 	tasklet_schedule(&ddev->tasklet);
 	/*
 	 * Just disable the interrupts. We'll turn them back on in the
 	 * softirq handler.
 	 */
 	disable_irq_nosync(irq);
 	return IRQ_HANDLED;
 }
 /*----------------------------------------------------------------------*/
 static dma_cookie_t txx9dmac_tx_submit(struct dma_async_tx_descriptor *tx)
 {
 	struct txx9dmac_desc *desc = txd_to_txx9dmac_desc(tx);
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(tx->chan);
 	dma_cookie_t cookie;
 	spin_lock_bh(&dc->lock);
 	cookie = txx9dmac_assign_cookie(dc, desc);
 	dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u %p\n",
 		 desc->txd.cookie, desc);
 	list_add_tail(&desc->desc_node, &dc->queue);
 	spin_unlock_bh(&dc->lock);
 	return cookie;
 }
 static struct dma_async_tx_descriptor *
 txx9dmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		size_t len, unsigned long flags)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *desc;
 	struct txx9dmac_desc *first;
 	struct txx9dmac_desc *prev;
 	size_t xfer_count;
 	size_t offset;
 	dev_vdbg(chan2dev(chan), "prep_dma_memcpy d%#llx s%#llx l%#zx f%#lx\n",
 		 (u64)dest, (u64)src, len, flags);
 	if (unlikely(!len)) {
 		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
 		return NULL;
 	}
 	prev = first = NULL;
 	for (offset = 0; offset < len; offset += xfer_count) {
 		xfer_count = min_t(size_t, len - offset, TXX9_DMA_MAX_COUNT);
 		/*
 		 * Workaround for ERT-TX49H2-033, ERT-TX49H3-020,
 		 * ERT-TX49H4-016 (slightly conservative)
 		 */
 		if (__is_dmac64(ddev)) {
 			if (xfer_count > 0x100 &&
 			    (xfer_count & 0xff) >= 0xfa &&
 			    (xfer_count & 0xff) <= 0xff)
 				xfer_count -= 0x20;
 		} else {
 			if (xfer_count > 0x80 &&
 			    (xfer_count & 0x7f) >= 0x7e &&
 			    (xfer_count & 0x7f) <= 0x7f)
 				xfer_count -= 0x20;
 		}
 		desc = txx9dmac_desc_get(dc);
 		if (!desc) {
 			txx9dmac_desc_put(dc, first);
 			return NULL;
 		}
 		if (__is_dmac64(ddev)) {
 			desc->hwdesc.SAR = src + offset;
 			desc->hwdesc.DAR = dest + offset;
 			desc->hwdesc.CNTR = xfer_count;
 			txx9dmac_desc_set_nosimple(ddev, desc, 8, 8,
 					dc->ccr | TXX9_DMA_CCR_XFACT);
 		} else {
 			desc->hwdesc32.SAR = src + offset;
 			desc->hwdesc32.DAR = dest + offset;
 			desc->hwdesc32.CNTR = xfer_count;
 			txx9dmac_desc_set_nosimple(ddev, desc, 4, 4,
 					dc->ccr | TXX9_DMA_CCR_XFACT);
 		}
 		/*
 		 * The descriptors on tx_list are not reachable from
 		 * the dc->queue list or dc->active_list after a
 		 * submit.  If we put all descriptors on active_list,
 		 * calling of callback on the completion will be more
 		 * complex.
 		 */
 		if (!first) {
 			first = desc;
 		} else {
 			desc_write_CHAR(dc, prev, desc->txd.phys);
 			dma_sync_single_for_device(chan2parent(&dc->chan),
 					prev->txd.phys, ddev->descsize,
 					DMA_TO_DEVICE);
 			list_add_tail(&desc->desc_node, &first->tx_list);
 		}
 		prev = desc;
 	}
 	/* Trigger interrupt after last block */
 	if (flags & DMA_PREP_INTERRUPT)
 		txx9dmac_desc_set_INTENT(ddev, prev);
 	desc_write_CHAR(dc, prev, 0);
 	dma_sync_single_for_device(chan2parent(&dc->chan),
 			prev->txd.phys, ddev->descsize,
 			DMA_TO_DEVICE);
 	first->txd.flags = flags;
 	first->len = len;
 	return &first->txd;
 }
 static struct dma_async_tx_descriptor *
 txx9dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_slave *ds = chan->private;
 	struct txx9dmac_desc *prev;
 	struct txx9dmac_desc *first;
 	unsigned int i;
 	struct scatterlist *sg;
 	dev_vdbg(chan2dev(chan), "prep_dma_slave\n");
 	BUG_ON(!ds || !ds->reg_width);
 	if (ds->tx_reg)
 		BUG_ON(direction != DMA_TO_DEVICE);
 	else
 		BUG_ON(direction != DMA_FROM_DEVICE);
 	if (unlikely(!sg_len))
 		return NULL;
 	prev = first = NULL;
 	for_each_sg(sgl, sg, sg_len, i) {
 		struct txx9dmac_desc *desc;
 		dma_addr_t mem;
 		u32 sai, dai;
 		desc = txx9dmac_desc_get(dc);
 		if (!desc) {
 			txx9dmac_desc_put(dc, first);
 			return NULL;
 		}
 		mem = sg_dma_address(sg);
 		if (__is_dmac64(ddev)) {
 			if (direction == DMA_TO_DEVICE) {
 				desc->hwdesc.SAR = mem;
 				desc->hwdesc.DAR = ds->tx_reg;
 			} else {
 				desc->hwdesc.SAR = ds->rx_reg;
 				desc->hwdesc.DAR = mem;
 			}
 			desc->hwdesc.CNTR = sg_dma_len(sg);
 		} else {
 			if (direction == DMA_TO_DEVICE) {
 				desc->hwdesc32.SAR = mem;
 				desc->hwdesc32.DAR = ds->tx_reg;
 			} else {
 				desc->hwdesc32.SAR = ds->rx_reg;
 				desc->hwdesc32.DAR = mem;
 			}
 			desc->hwdesc32.CNTR = sg_dma_len(sg);
 		}
 		if (direction == DMA_TO_DEVICE) {
 			sai = ds->reg_width;
 			dai = 0;
 		} else {
 			sai = 0;
 			dai = ds->reg_width;
 		}
 		txx9dmac_desc_set_nosimple(ddev, desc, sai, dai,
 					dc->ccr | TXX9_DMA_CCR_XFACT);
 		if (!first) {
 			first = desc;
 		} else {
 			desc_write_CHAR(dc, prev, desc->txd.phys);
 			dma_sync_single_for_device(chan2parent(&dc->chan),
 					prev->txd.phys,
 					ddev->descsize,
 					DMA_TO_DEVICE);
 			list_add_tail(&desc->desc_node, &first->tx_list);
 		}
 		prev = desc;
 	}
 	/* Trigger interrupt after last block */
 	if (flags & DMA_PREP_INTERRUPT)
 		txx9dmac_desc_set_INTENT(ddev, prev);
 	desc_write_CHAR(dc, prev, 0);
 	dma_sync_single_for_device(chan2parent(&dc->chan),
 			prev->txd.phys, ddev->descsize,
 			DMA_TO_DEVICE);
 	first->txd.flags = flags;
 	first->len = 0;
 	return &first->txd;
 }
-static int txx9dmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd)
+static int txx9dmac_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+			    unsigned long arg)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	struct txx9dmac_desc *desc, *_desc;
 	LIST_HEAD(list);
 	/* Only supports DMA_TERMINATE_ALL */
 	if (cmd != DMA_TERMINATE_ALL)
 		return -EINVAL;
 	dev_vdbg(chan2dev(chan), "terminate_all\n");
 	spin_lock_bh(&dc->lock);
 	txx9dmac_reset_chan(dc);
 	/* active_list entries will end up before queued entries */
 	list_splice_init(&dc->queue, &list);
 	list_splice_init(&dc->active_list, &list);
 	spin_unlock_bh(&dc->lock);
 	/* Flush all pending and queued descriptors */
 	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 		txx9dmac_descriptor_complete(dc, desc);
 	return 0;
 }
 static enum dma_status
 txx9dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
 		   struct dma_tx_state *txstate)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	dma_cookie_t last_used;
 	dma_cookie_t last_complete;
 	int ret;
 	last_complete = dc->completed;
 	last_used = chan->cookie;
 	ret = dma_async_is_complete(cookie, last_complete, last_used);
 	if (ret != DMA_SUCCESS) {
 		spin_lock_bh(&dc->lock);
 		txx9dmac_scan_descriptors(dc);
 		spin_unlock_bh(&dc->lock);
 		last_complete = dc->completed;
 		last_used = chan->cookie;
 		ret = dma_async_is_complete(cookie, last_complete, last_used);
 	}
 	dma_set_tx_state(txstate, last_complete, last_used, 0);
 	return ret;
 }
 static void txx9dmac_chain_dynamic(struct txx9dmac_chan *dc,
 				   struct txx9dmac_desc *prev)
 {
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *desc;
 	LIST_HEAD(list);
 	prev = txx9dmac_last_child(prev);
 	txx9dmac_dequeue(dc, &list);
 	desc = list_entry(list.next, struct txx9dmac_desc, desc_node);
 	desc_write_CHAR(dc, prev, desc->txd.phys);
 	dma_sync_single_for_device(chan2parent(&dc->chan),
 				   prev->txd.phys, ddev->descsize,
 				   DMA_TO_DEVICE);
 	mmiowb();
 	if (!(channel_readl(dc, CSR) & TXX9_DMA_CSR_CHNEN) &&
 	    channel_read_CHAR(dc) == prev->txd.phys)
 		/* Restart chain DMA */
 		channel_write_CHAR(dc, desc->txd.phys);
 	list_splice_tail(&list, &dc->active_list);
 }
 static void txx9dmac_issue_pending(struct dma_chan *chan)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	spin_lock_bh(&dc->lock);
 	if (!list_empty(&dc->active_list))
 		txx9dmac_scan_descriptors(dc);
 	if (!list_empty(&dc->queue)) {
 		if (list_empty(&dc->active_list)) {
 			txx9dmac_dequeue(dc, &dc->active_list);
 			txx9dmac_dostart(dc, txx9dmac_first_active(dc));
 		} else if (txx9_dma_have_SMPCHN()) {
 			struct txx9dmac_desc *prev = txx9dmac_last_active(dc);
 			if (!(prev->txd.flags & DMA_PREP_INTERRUPT) ||
 			    txx9dmac_chan_INTENT(dc))
 				txx9dmac_chain_dynamic(dc, prev);
 		}
 	}
 	spin_unlock_bh(&dc->lock);
 }
 static int txx9dmac_alloc_chan_resources(struct dma_chan *chan)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	struct txx9dmac_slave *ds = chan->private;
 	struct txx9dmac_desc *desc;
 	int i;
 	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
 	/* ASSERT:  channel is idle */
 	if (channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT) {
 		dev_dbg(chan2dev(chan), "DMA channel not idle?\n");
 		return -EIO;
 	}
 	dc->completed = chan->cookie = 1;
 	dc->ccr = TXX9_DMA_CCR_IMMCHN | TXX9_DMA_CCR_INTENE | CCR_LE;
 	txx9dmac_chan_set_SMPCHN(dc);
 	if (!txx9_dma_have_SMPCHN() || (dc->ccr & TXX9_DMA_CCR_SMPCHN))
 		dc->ccr |= TXX9_DMA_CCR_INTENC;
 	if (chan->device->device_prep_dma_memcpy) {
 		if (ds)
 			return -EINVAL;
 		dc->ccr |= TXX9_DMA_CCR_XFSZ_X8;
 	} else {
 		if (!ds ||
 		    (ds->tx_reg && ds->rx_reg) || (!ds->tx_reg && !ds->rx_reg))
 			return -EINVAL;
 		dc->ccr |= TXX9_DMA_CCR_EXTRQ |
 			TXX9_DMA_CCR_XFSZ(__ffs(ds->reg_width));
 		txx9dmac_chan_set_INTENT(dc);
 	}
 	spin_lock_bh(&dc->lock);
 	i = dc->descs_allocated;
 	while (dc->descs_allocated < TXX9_DMA_INITIAL_DESC_COUNT) {
 		spin_unlock_bh(&dc->lock);
 		desc = txx9dmac_desc_alloc(dc, GFP_KERNEL);
 		if (!desc) {
 			dev_info(chan2dev(chan),
 				"only allocated %d descriptors\n", i);
 			spin_lock_bh(&dc->lock);
 			break;
 		}
 		txx9dmac_desc_put(dc, desc);
 		spin_lock_bh(&dc->lock);
 		i = ++dc->descs_allocated;
 	}
 	spin_unlock_bh(&dc->lock);
 	dev_dbg(chan2dev(chan),
 		"alloc_chan_resources allocated %d descriptors\n", i);
 	return i;
 }
 static void txx9dmac_free_chan_resources(struct dma_chan *chan)
 {
 	struct txx9dmac_chan *dc = to_txx9dmac_chan(chan);
 	struct txx9dmac_dev *ddev = dc->ddev;
 	struct txx9dmac_desc *desc, *_desc;
 	LIST_HEAD(list);
 	dev_dbg(chan2dev(chan), "free_chan_resources (descs allocated=%u)\n",
 			dc->descs_allocated);
 	/* ASSERT:  channel is idle */
 	BUG_ON(!list_empty(&dc->active_list));
 	BUG_ON(!list_empty(&dc->queue));
 	BUG_ON(channel_readl(dc, CSR) & TXX9_DMA_CSR_XFACT);
 	spin_lock_bh(&dc->lock);
 	list_splice_init(&dc->free_list, &list);
 	dc->descs_allocated = 0;
 	spin_unlock_bh(&dc->lock);
 	list_for_each_entry_safe(desc, _desc, &list, desc_node) {
 		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
 		dma_unmap_single(chan2parent(chan), desc->txd.phys,
 				 ddev->descsize, DMA_TO_DEVICE);
 		kfree(desc);
 	}
 	dev_vdbg(chan2dev(chan), "free_chan_resources done\n");
 }
 /*----------------------------------------------------------------------*/
 static void txx9dmac_off(struct txx9dmac_dev *ddev)
 {
 	dma_writel(ddev, MCR, 0);
 	mmiowb();
 }
 static int __init txx9dmac_chan_probe(struct platform_device *pdev)
 {
 	struct txx9dmac_chan_platform_data *cpdata = pdev->dev.platform_data;
 	struct platform_device *dmac_dev = cpdata->dmac_dev;
 	struct txx9dmac_platform_data *pdata = dmac_dev->dev.platform_data;
 	struct txx9dmac_chan *dc;
 	int err;
 	int ch = pdev->id % TXX9_DMA_MAX_NR_CHANNELS;
 	int irq;
 	dc = devm_kzalloc(&pdev->dev, sizeof(*dc), GFP_KERNEL);
 	if (!dc)
 		return -ENOMEM;
 	dc->dma.dev = &pdev->dev;
 	dc->dma.device_alloc_chan_resources = txx9dmac_alloc_chan_resources;
 	dc->dma.device_free_chan_resources = txx9dmac_free_chan_resources;
 	dc->dma.device_control = txx9dmac_control;
 	dc->dma.device_tx_status = txx9dmac_tx_status;
 	dc->dma.device_issue_pending = txx9dmac_issue_pending;
 	if (pdata && pdata->memcpy_chan == ch) {
 		dc->dma.device_prep_dma_memcpy = txx9dmac_prep_dma_memcpy;
 		dma_cap_set(DMA_MEMCPY, dc->dma.cap_mask);
 	} else {
 		dc->dma.device_prep_slave_sg = txx9dmac_prep_slave_sg;
 		dma_cap_set(DMA_SLAVE, dc->dma.cap_mask);
 		dma_cap_set(DMA_PRIVATE, dc->dma.cap_mask);
 	}
 	INIT_LIST_HEAD(&dc->dma.channels);
 	dc->ddev = platform_get_drvdata(dmac_dev);
 	if (dc->ddev->irq < 0) {
 		irq = platform_get_irq(pdev, 0);
 		if (irq < 0)
 			return irq;
 		tasklet_init(&dc->tasklet, txx9dmac_chan_tasklet,
 				(unsigned long)dc);
 		dc->irq = irq;
 		err = devm_request_irq(&pdev->dev, dc->irq,
 			txx9dmac_chan_interrupt, 0, dev_name(&pdev->dev), dc);
 		if (err)
 			return err;
 	} else
 		dc->irq = -1;
 	dc->ddev->chan[ch] = dc;
 	dc->chan.device = &dc->dma;
 	list_add_tail(&dc->chan.device_node, &dc->chan.device->channels);
 	dc->chan.cookie = dc->completed = 1;
 	if (is_dmac64(dc))
 		dc->ch_regs = &__txx9dmac_regs(dc->ddev)->CHAN[ch];
 	else
 		dc->ch_regs = &__txx9dmac_regs32(dc->ddev)->CHAN[ch];
 	spin_lock_init(&dc->lock);
 	INIT_LIST_HEAD(&dc->active_list);
 	INIT_LIST_HEAD(&dc->queue);
 	INIT_LIST_HEAD(&dc->free_list);
 	txx9dmac_reset_chan(dc);
 	platform_set_drvdata(pdev, dc);
 	err = dma_async_device_register(&dc->dma);
 	if (err)
 		return err;
 	dev_dbg(&pdev->dev, "TXx9 DMA Channel (dma%d%s%s)\n",
 		dc->dma.dev_id,
 		dma_has_cap(DMA_MEMCPY, dc->dma.cap_mask) ? " memcpy" : "",
 		dma_has_cap(DMA_SLAVE, dc->dma.cap_mask) ? " slave" : "");
 	return 0;
 }
 static int __exit txx9dmac_chan_remove(struct platform_device *pdev)
 {
 	struct txx9dmac_chan *dc = platform_get_drvdata(pdev);
 	dma_async_device_unregister(&dc->dma);
 	if (dc->irq >= 0)
 		tasklet_kill(&dc->tasklet);
 	dc->ddev->chan[pdev->id % TXX9_DMA_MAX_NR_CHANNELS] = NULL;
 	return 0;
 }
 static int __init txx9dmac_probe(struct platform_device *pdev)
 {
 	struct txx9dmac_platform_data *pdata = pdev->dev.platform_data;
 	struct resource *io;
 	struct txx9dmac_dev *ddev;
 	u32 mcr;
 	int err;
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!io)
 		return -EINVAL;
 	ddev = devm_kzalloc(&pdev->dev, sizeof(*ddev), GFP_KERNEL);
 	if (!ddev)
 		return -ENOMEM;
 	if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io),
 				     dev_name(&pdev->dev)))
 		return -EBUSY;
 	ddev->regs = devm_ioremap(&pdev->dev, io->start, resource_size(io));
 	if (!ddev->regs)
 		return -ENOMEM;
 	ddev->have_64bit_regs = pdata->have_64bit_regs;
 	if (__is_dmac64(ddev))
 		ddev->descsize = sizeof(struct txx9dmac_hwdesc);
 	else
 		ddev->descsize = sizeof(struct txx9dmac_hwdesc32);
 	/* force dma off, just in case */
 	txx9dmac_off(ddev);
 	ddev->irq = platform_get_irq(pdev, 0);
 	if (ddev->irq >= 0) {
 		tasklet_init(&ddev->tasklet, txx9dmac_tasklet,
 				(unsigned long)ddev);
 		err = devm_request_irq(&pdev->dev, ddev->irq,
 			txx9dmac_interrupt, 0, dev_name(&pdev->dev), ddev);
 		if (err)
 			return err;
 	}
 	mcr = TXX9_DMA_MCR_MSTEN | MCR_LE;
 	if (pdata && pdata->memcpy_chan >= 0)
 		mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan);
 	dma_writel(ddev, MCR, mcr);
 	platform_set_drvdata(pdev, ddev);
 	return 0;
 }
 static int __exit txx9dmac_remove(struct platform_device *pdev)
 {
 	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
 	txx9dmac_off(ddev);
 	if (ddev->irq >= 0)
 		tasklet_kill(&ddev->tasklet);
 	return 0;
 }
 static void txx9dmac_shutdown(struct platform_device *pdev)
 {
 	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
 	txx9dmac_off(ddev);
 }
 static int txx9dmac_suspend_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
 	txx9dmac_off(ddev);
 	return 0;
 }
 static int txx9dmac_resume_noirq(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct txx9dmac_dev *ddev = platform_get_drvdata(pdev);
 	struct txx9dmac_platform_data *pdata = pdev->dev.platform_data;
 	u32 mcr;
 	mcr = TXX9_DMA_MCR_MSTEN | MCR_LE;
 	if (pdata && pdata->memcpy_chan >= 0)
 		mcr |= TXX9_DMA_MCR_FIFUM(pdata->memcpy_chan);
 	dma_writel(ddev, MCR, mcr);
 	return 0;
 }
 static const struct dev_pm_ops txx9dmac_dev_pm_ops = {
 	.suspend_noirq = txx9dmac_suspend_noirq,
 	.resume_noirq = txx9dmac_resume_noirq,
 };
 static struct platform_driver txx9dmac_chan_driver = {
 	.remove		= __exit_p(txx9dmac_chan_remove),
 	.driver = {
 		.name	= "txx9dmac-chan",
 	},
 };
 static struct platform_driver txx9dmac_driver = {
 	.remove		= __exit_p(txx9dmac_remove),
 	.shutdown	= txx9dmac_shutdown,
 	.driver = {
 		.name	= "txx9dmac",
 		.pm	= &txx9dmac_dev_pm_ops,
 	},
 };
 static int __init txx9dmac_init(void)
 {
 	int rc;
 	rc = platform_driver_probe(&txx9dmac_driver, txx9dmac_probe);
 	if (!rc) {
 		rc = platform_driver_probe(&txx9dmac_chan_driver,
 					   txx9dmac_chan_probe);
 		if (rc)
 			platform_driver_unregister(&txx9dmac_driver);
 	}
 	return rc;
 }
 module_init(txx9dmac_init);
 static void __exit txx9dmac_exit(void)
 {
 	platform_driver_unregister(&txx9dmac_chan_driver);
 	platform_driver_unregister(&txx9dmac_driver);
 }
 module_exit(txx9dmac_exit);
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("TXx9 DMA Controller driver");
 MODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>");

drivers/mmc/host/atmel-mci.c

Diff comments View file @ 0582763

1	/*	1	/*
2	* Atmel MultiMedia Card Interface driver	2	* Atmel MultiMedia Card Interface driver
3	*	3	*
4	* Copyright (C) 2004-2008 Atmel Corporation	4	* Copyright (C) 2004-2008 Atmel Corporation
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify	6	* This program is free software; you can redistribute it and/or modify
7	* it under the terms of the GNU General Public License version 2 as	7	* it under the terms of the GNU General Public License version 2 as
8	* published by the Free Software Foundation.	8	* published by the Free Software Foundation.
9	*/	9	*/
10	#include <linux/blkdev.h>	10	#include <linux/blkdev.h>
11	#include <linux/clk.h>	11	#include <linux/clk.h>
12	#include <linux/debugfs.h>	12	#include <linux/debugfs.h>
13	#include <linux/device.h>	13	#include <linux/device.h>
14	#include <linux/dmaengine.h>	14	#include <linux/dmaengine.h>
15	#include <linux/dma-mapping.h>	15	#include <linux/dma-mapping.h>
16	#include <linux/err.h>	16	#include <linux/err.h>
17	#include <linux/gpio.h>	17	#include <linux/gpio.h>
18	#include <linux/init.h>	18	#include <linux/init.h>
19	#include <linux/interrupt.h>	19	#include <linux/interrupt.h>
20	#include <linux/ioport.h>	20	#include <linux/ioport.h>
21	#include <linux/module.h>	21	#include <linux/module.h>
22	#include <linux/platform_device.h>	22	#include <linux/platform_device.h>
23	#include <linux/scatterlist.h>	23	#include <linux/scatterlist.h>
24	#include <linux/seq_file.h>	24	#include <linux/seq_file.h>
25	#include <linux/stat.h>	25	#include <linux/stat.h>
26		26
27	#include <linux/mmc/host.h>	27	#include <linux/mmc/host.h>
28		28
29	#include <mach/atmel-mci.h>	29	#include <mach/atmel-mci.h>
30	#include <linux/atmel-mci.h>	30	#include <linux/atmel-mci.h>
31		31
32	#include <asm/io.h>	32	#include <asm/io.h>
33	#include <asm/unaligned.h>	33	#include <asm/unaligned.h>
34		34
35	#include <mach/cpu.h>	35	#include <mach/cpu.h>
36	#include <mach/board.h>	36	#include <mach/board.h>
37		37
38	#include "atmel-mci-regs.h"	38	#include "atmel-mci-regs.h"
39		39
40	#define ATMCI_DATA_ERROR_FLAGS (MCI_DCRCE \| MCI_DTOE \| MCI_OVRE \| MCI_UNRE)	40	#define ATMCI_DATA_ERROR_FLAGS (MCI_DCRCE \| MCI_DTOE \| MCI_OVRE \| MCI_UNRE)
41	#define ATMCI_DMA_THRESHOLD 16	41	#define ATMCI_DMA_THRESHOLD 16
42		42
43	enum {	43	enum {
44	EVENT_CMD_COMPLETE = 0,	44	EVENT_CMD_COMPLETE = 0,
45	EVENT_XFER_COMPLETE,	45	EVENT_XFER_COMPLETE,
46	EVENT_DATA_COMPLETE,	46	EVENT_DATA_COMPLETE,
47	EVENT_DATA_ERROR,	47	EVENT_DATA_ERROR,
48	};	48	};
49		49
50	enum atmel_mci_state {	50	enum atmel_mci_state {
51	STATE_IDLE = 0,	51	STATE_IDLE = 0,
52	STATE_SENDING_CMD,	52	STATE_SENDING_CMD,
53	STATE_SENDING_DATA,	53	STATE_SENDING_DATA,
54	STATE_DATA_BUSY,	54	STATE_DATA_BUSY,
55	STATE_SENDING_STOP,	55	STATE_SENDING_STOP,
56	STATE_DATA_ERROR,	56	STATE_DATA_ERROR,
57	};	57	};
58		58
59	struct atmel_mci_dma {	59	struct atmel_mci_dma {
60	#ifdef CONFIG_MMC_ATMELMCI_DMA	60	#ifdef CONFIG_MMC_ATMELMCI_DMA
61	struct dma_chan *chan;	61	struct dma_chan *chan;
62	struct dma_async_tx_descriptor *data_desc;	62	struct dma_async_tx_descriptor *data_desc;
63	#endif	63	#endif
64	};	64	};
65		65
66	/**	66	/**
67	* struct atmel_mci - MMC controller state shared between all slots	67	* struct atmel_mci - MMC controller state shared between all slots
68	* @lock: Spinlock protecting the queue and associated data.	68	* @lock: Spinlock protecting the queue and associated data.
69	* @regs: Pointer to MMIO registers.	69	* @regs: Pointer to MMIO registers.
70	* @sg: Scatterlist entry currently being processed by PIO code, if any.	70	* @sg: Scatterlist entry currently being processed by PIO code, if any.
71	* @pio_offset: Offset into the current scatterlist entry.	71	* @pio_offset: Offset into the current scatterlist entry.
72	* @cur_slot: The slot which is currently using the controller.	72	* @cur_slot: The slot which is currently using the controller.
73	* @mrq: The request currently being processed on @cur_slot,	73	* @mrq: The request currently being processed on @cur_slot,
74	* or NULL if the controller is idle.	74	* or NULL if the controller is idle.
75	* @cmd: The command currently being sent to the card, or NULL.	75	* @cmd: The command currently being sent to the card, or NULL.
76	* @data: The data currently being transferred, or NULL if no data	76	* @data: The data currently being transferred, or NULL if no data
77	* transfer is in progress.	77	* transfer is in progress.
78	* @dma: DMA client state.	78	* @dma: DMA client state.
79	* @data_chan: DMA channel being used for the current data transfer.	79	* @data_chan: DMA channel being used for the current data transfer.
80	* @cmd_status: Snapshot of SR taken upon completion of the current	80	* @cmd_status: Snapshot of SR taken upon completion of the current
81	* command. Only valid when EVENT_CMD_COMPLETE is pending.	81	* command. Only valid when EVENT_CMD_COMPLETE is pending.
82	* @data_status: Snapshot of SR taken upon completion of the current	82	* @data_status: Snapshot of SR taken upon completion of the current
83	* data transfer. Only valid when EVENT_DATA_COMPLETE or	83	* data transfer. Only valid when EVENT_DATA_COMPLETE or
84	* EVENT_DATA_ERROR is pending.	84	* EVENT_DATA_ERROR is pending.
85	* @stop_cmdr: Value to be loaded into CMDR when the stop command is	85	* @stop_cmdr: Value to be loaded into CMDR when the stop command is
86	* to be sent.	86	* to be sent.
87	* @tasklet: Tasklet running the request state machine.	87	* @tasklet: Tasklet running the request state machine.
88	* @pending_events: Bitmask of events flagged by the interrupt handler	88	* @pending_events: Bitmask of events flagged by the interrupt handler
89	* to be processed by the tasklet.	89	* to be processed by the tasklet.
90	* @completed_events: Bitmask of events which the state machine has	90	* @completed_events: Bitmask of events which the state machine has
91	* processed.	91	* processed.
92	* @state: Tasklet state.	92	* @state: Tasklet state.
93	* @queue: List of slots waiting for access to the controller.	93	* @queue: List of slots waiting for access to the controller.
94	* @need_clock_update: Update the clock rate before the next request.	94	* @need_clock_update: Update the clock rate before the next request.
95	* @need_reset: Reset controller before next request.	95	* @need_reset: Reset controller before next request.
96	* @mode_reg: Value of the MR register.	96	* @mode_reg: Value of the MR register.
97	* @cfg_reg: Value of the CFG register.	97	* @cfg_reg: Value of the CFG register.
98	* @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus	98	* @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus
99	* rate and timeout calculations.	99	* rate and timeout calculations.
100	* @mapbase: Physical address of the MMIO registers.	100	* @mapbase: Physical address of the MMIO registers.
101	* @mck: The peripheral bus clock hooked up to the MMC controller.	101	* @mck: The peripheral bus clock hooked up to the MMC controller.
102	* @pdev: Platform device associated with the MMC controller.	102	* @pdev: Platform device associated with the MMC controller.
103	* @slot: Slots sharing this MMC controller.	103	* @slot: Slots sharing this MMC controller.
104	*	104	*
105	* Locking	105	* Locking
106	* =======	106	* =======
107	*	107	*
108	* @lock is a softirq-safe spinlock protecting @queue as well as	108	* @lock is a softirq-safe spinlock protecting @queue as well as
109	* @cur_slot, @mrq and @state. These must always be updated	109	* @cur_slot, @mrq and @state. These must always be updated
110	* at the same time while holding @lock.	110	* at the same time while holding @lock.
111	*	111	*
112	* @lock also protects mode_reg and need_clock_update since these are	112	* @lock also protects mode_reg and need_clock_update since these are
113	* used to synchronize mode register updates with the queue	113	* used to synchronize mode register updates with the queue
114	* processing.	114	* processing.
115	*	115	*
116	* The @mrq field of struct atmel_mci_slot is also protected by @lock,	116	* The @mrq field of struct atmel_mci_slot is also protected by @lock,
117	* and must always be written at the same time as the slot is added to	117	* and must always be written at the same time as the slot is added to
118	* @queue.	118	* @queue.
119	*	119	*
120	* @pending_events and @completed_events are accessed using atomic bit	120	* @pending_events and @completed_events are accessed using atomic bit
121	* operations, so they don't need any locking.	121	* operations, so they don't need any locking.
122	*	122	*
123	* None of the fields touched by the interrupt handler need any	123	* None of the fields touched by the interrupt handler need any
124	* locking. However, ordering is important: Before EVENT_DATA_ERROR or	124	* locking. However, ordering is important: Before EVENT_DATA_ERROR or
125	* EVENT_DATA_COMPLETE is set in @pending_events, all data-related	125	* EVENT_DATA_COMPLETE is set in @pending_events, all data-related
126	* interrupts must be disabled and @data_status updated with a	126	* interrupts must be disabled and @data_status updated with a
127	* snapshot of SR. Similarly, before EVENT_CMD_COMPLETE is set, the	127	* snapshot of SR. Similarly, before EVENT_CMD_COMPLETE is set, the
128	* CMDRDY interupt must be disabled and @cmd_status updated with a	128	* CMDRDY interupt must be disabled and @cmd_status updated with a
129	* snapshot of SR, and before EVENT_XFER_COMPLETE can be set, the	129	* snapshot of SR, and before EVENT_XFER_COMPLETE can be set, the
130	* bytes_xfered field of @data must be written. This is ensured by	130	* bytes_xfered field of @data must be written. This is ensured by
131	* using barriers.	131	* using barriers.
132	*/	132	*/
133	struct atmel_mci {	133	struct atmel_mci {
134	spinlock_t lock;	134	spinlock_t lock;
135	void __iomem *regs;	135	void __iomem *regs;
136		136
137	struct scatterlist *sg;	137	struct scatterlist *sg;
138	unsigned int pio_offset;	138	unsigned int pio_offset;
139		139
140	struct atmel_mci_slot *cur_slot;	140	struct atmel_mci_slot *cur_slot;
141	struct mmc_request *mrq;	141	struct mmc_request *mrq;
142	struct mmc_command *cmd;	142	struct mmc_command *cmd;
143	struct mmc_data *data;	143	struct mmc_data *data;
144		144
145	struct atmel_mci_dma dma;	145	struct atmel_mci_dma dma;
146	struct dma_chan *data_chan;	146	struct dma_chan *data_chan;
147		147
148	u32 cmd_status;	148	u32 cmd_status;
149	u32 data_status;	149	u32 data_status;
150	u32 stop_cmdr;	150	u32 stop_cmdr;
151		151
152	struct tasklet_struct tasklet;	152	struct tasklet_struct tasklet;
153	unsigned long pending_events;	153	unsigned long pending_events;
154	unsigned long completed_events;	154	unsigned long completed_events;
155	enum atmel_mci_state state;	155	enum atmel_mci_state state;
156	struct list_head queue;	156	struct list_head queue;
157		157
158	bool need_clock_update;	158	bool need_clock_update;
159	bool need_reset;	159	bool need_reset;
160	u32 mode_reg;	160	u32 mode_reg;
161	u32 cfg_reg;	161	u32 cfg_reg;
162	unsigned long bus_hz;	162	unsigned long bus_hz;
163	unsigned long mapbase;	163	unsigned long mapbase;
164	struct clk *mck;	164	struct clk *mck;
165	struct platform_device *pdev;	165	struct platform_device *pdev;
166		166
167	struct atmel_mci_slot *slot[ATMEL_MCI_MAX_NR_SLOTS];	167	struct atmel_mci_slot *slot[ATMEL_MCI_MAX_NR_SLOTS];
168	};	168	};
169		169
170	/**	170	/**
171	* struct atmel_mci_slot - MMC slot state	171	* struct atmel_mci_slot - MMC slot state
172	* @mmc: The mmc_host representing this slot.	172	* @mmc: The mmc_host representing this slot.
173	* @host: The MMC controller this slot is using.	173	* @host: The MMC controller this slot is using.
174	* @sdc_reg: Value of SDCR to be written before using this slot.	174	* @sdc_reg: Value of SDCR to be written before using this slot.
175	* @mrq: mmc_request currently being processed or waiting to be	175	* @mrq: mmc_request currently being processed or waiting to be
176	* processed, or NULL when the slot is idle.	176	* processed, or NULL when the slot is idle.
177	* @queue_node: List node for placing this node in the @queue list of	177	* @queue_node: List node for placing this node in the @queue list of
178	* &struct atmel_mci.	178	* &struct atmel_mci.
179	* @clock: Clock rate configured by set_ios(). Protected by host->lock.	179	* @clock: Clock rate configured by set_ios(). Protected by host->lock.
180	* @flags: Random state bits associated with the slot.	180	* @flags: Random state bits associated with the slot.
181	* @detect_pin: GPIO pin used for card detection, or negative if not	181	* @detect_pin: GPIO pin used for card detection, or negative if not
182	* available.	182	* available.
183	* @wp_pin: GPIO pin used for card write protect sending, or negative	183	* @wp_pin: GPIO pin used for card write protect sending, or negative
184	* if not available.	184	* if not available.
185	* @detect_is_active_high: The state of the detect pin when it is active.	185	* @detect_is_active_high: The state of the detect pin when it is active.
186	* @detect_timer: Timer used for debouncing @detect_pin interrupts.	186	* @detect_timer: Timer used for debouncing @detect_pin interrupts.
187	*/	187	*/
188	struct atmel_mci_slot {	188	struct atmel_mci_slot {
189	struct mmc_host *mmc;	189	struct mmc_host *mmc;
190	struct atmel_mci *host;	190	struct atmel_mci *host;
191		191
192	u32 sdc_reg;	192	u32 sdc_reg;
193		193
194	struct mmc_request *mrq;	194	struct mmc_request *mrq;
195	struct list_head queue_node;	195	struct list_head queue_node;
196		196
197	unsigned int clock;	197	unsigned int clock;
198	unsigned long flags;	198	unsigned long flags;
199	#define ATMCI_CARD_PRESENT 0	199	#define ATMCI_CARD_PRESENT 0
200	#define ATMCI_CARD_NEED_INIT 1	200	#define ATMCI_CARD_NEED_INIT 1
201	#define ATMCI_SHUTDOWN 2	201	#define ATMCI_SHUTDOWN 2
202		202
203	int detect_pin;	203	int detect_pin;
204	int wp_pin;	204	int wp_pin;
205	bool detect_is_active_high;	205	bool detect_is_active_high;
206		206
207	struct timer_list detect_timer;	207	struct timer_list detect_timer;
208	};	208	};
209		209
210	#define atmci_test_and_clear_pending(host, event) \	210	#define atmci_test_and_clear_pending(host, event) \
211	test_and_clear_bit(event, &host->pending_events)	211	test_and_clear_bit(event, &host->pending_events)
212	#define atmci_set_completed(host, event) \	212	#define atmci_set_completed(host, event) \
213	set_bit(event, &host->completed_events)	213	set_bit(event, &host->completed_events)
214	#define atmci_set_pending(host, event) \	214	#define atmci_set_pending(host, event) \
215	set_bit(event, &host->pending_events)	215	set_bit(event, &host->pending_events)
216		216
217	/*	217	/*
218	* Enable or disable features/registers based on	218	* Enable or disable features/registers based on
219	* whether the processor supports them	219	* whether the processor supports them
220	*/	220	*/
221	static bool mci_has_rwproof(void)	221	static bool mci_has_rwproof(void)
222	{	222	{
223	if (cpu_is_at91sam9261() \|\| cpu_is_at91rm9200())	223	if (cpu_is_at91sam9261() \|\| cpu_is_at91rm9200())
224	return false;	224	return false;
225	else	225	else
226	return true;	226	return true;
227	}	227	}
228		228
229	/*	229	/*
230	* The new MCI2 module isn't 100% compatible with the old MCI module,	230	* The new MCI2 module isn't 100% compatible with the old MCI module,
231	* and it has a few nice features which we want to use...	231	* and it has a few nice features which we want to use...
232	*/	232	*/
233	static inline bool atmci_is_mci2(void)	233	static inline bool atmci_is_mci2(void)
234	{	234	{
235	if (cpu_is_at91sam9g45())	235	if (cpu_is_at91sam9g45())
236	return true;	236	return true;
237		237
238	return false;	238	return false;
239	}	239	}
240		240
241		241
242	/*	242	/*
243	* The debugfs stuff below is mostly optimized away when	243	* The debugfs stuff below is mostly optimized away when
244	* CONFIG_DEBUG_FS is not set.	244	* CONFIG_DEBUG_FS is not set.
245	*/	245	*/
246	static int atmci_req_show(struct seq_file s, void v)	246	static int atmci_req_show(struct seq_file s, void v)
247	{	247	{
248	struct atmel_mci_slot *slot = s->private;	248	struct atmel_mci_slot *slot = s->private;
249	struct mmc_request *mrq;	249	struct mmc_request *mrq;
250	struct mmc_command *cmd;	250	struct mmc_command *cmd;
251	struct mmc_command *stop;	251	struct mmc_command *stop;
252	struct mmc_data *data;	252	struct mmc_data *data;
253		253
254	/* Make sure we get a consistent snapshot */	254	/* Make sure we get a consistent snapshot */
255	spin_lock_bh(&slot->host->lock);	255	spin_lock_bh(&slot->host->lock);
256	mrq = slot->mrq;	256	mrq = slot->mrq;
257		257
258	if (mrq) {	258	if (mrq) {
259	cmd = mrq->cmd;	259	cmd = mrq->cmd;
260	data = mrq->data;	260	data = mrq->data;
261	stop = mrq->stop;	261	stop = mrq->stop;
262		262
263	if (cmd)	263	if (cmd)
264	seq_printf(s,	264	seq_printf(s,
265	"CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",	265	"CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
266	cmd->opcode, cmd->arg, cmd->flags,	266	cmd->opcode, cmd->arg, cmd->flags,
267	cmd->resp[0], cmd->resp[1], cmd->resp[2],	267	cmd->resp[0], cmd->resp[1], cmd->resp[2],
268	cmd->resp[2], cmd->error);	268	cmd->resp[2], cmd->error);
269	if (data)	269	if (data)
270	seq_printf(s, "DATA %u / %u * %u flg %x err %d\n",	270	seq_printf(s, "DATA %u / %u * %u flg %x err %d\n",
271	data->bytes_xfered, data->blocks,	271	data->bytes_xfered, data->blocks,
272	data->blksz, data->flags, data->error);	272	data->blksz, data->flags, data->error);
273	if (stop)	273	if (stop)
274	seq_printf(s,	274	seq_printf(s,
275	"CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",	275	"CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
276	stop->opcode, stop->arg, stop->flags,	276	stop->opcode, stop->arg, stop->flags,
277	stop->resp[0], stop->resp[1], stop->resp[2],	277	stop->resp[0], stop->resp[1], stop->resp[2],
278	stop->resp[2], stop->error);	278	stop->resp[2], stop->error);
279	}	279	}
280		280
281	spin_unlock_bh(&slot->host->lock);	281	spin_unlock_bh(&slot->host->lock);
282		282
283	return 0;	283	return 0;
284	}	284	}
285		285
286	static int atmci_req_open(struct inode inode, struct file file)	286	static int atmci_req_open(struct inode inode, struct file file)
287	{	287	{
288	return single_open(file, atmci_req_show, inode->i_private);	288	return single_open(file, atmci_req_show, inode->i_private);
289	}	289	}
290		290
291	static const struct file_operations atmci_req_fops = {	291	static const struct file_operations atmci_req_fops = {
292	.owner = THIS_MODULE,	292	.owner = THIS_MODULE,
293	.open = atmci_req_open,	293	.open = atmci_req_open,
294	.read = seq_read,	294	.read = seq_read,
295	.llseek = seq_lseek,	295	.llseek = seq_lseek,
296	.release = single_release,	296	.release = single_release,
297	};	297	};
298		298
299	static void atmci_show_status_reg(struct seq_file *s,	299	static void atmci_show_status_reg(struct seq_file *s,
300	const char *regname, u32 value)	300	const char *regname, u32 value)
301	{	301	{
302	static const char *sr_bit[] = {	302	static const char *sr_bit[] = {
303	[0] = "CMDRDY",	303	[0] = "CMDRDY",
304	[1] = "RXRDY",	304	[1] = "RXRDY",
305	[2] = "TXRDY",	305	[2] = "TXRDY",
306	[3] = "BLKE",	306	[3] = "BLKE",
307	[4] = "DTIP",	307	[4] = "DTIP",
308	[5] = "NOTBUSY",	308	[5] = "NOTBUSY",
309	[6] = "ENDRX",	309	[6] = "ENDRX",
310	[7] = "ENDTX",	310	[7] = "ENDTX",
311	[8] = "SDIOIRQA",	311	[8] = "SDIOIRQA",
312	[9] = "SDIOIRQB",	312	[9] = "SDIOIRQB",
313	[12] = "SDIOWAIT",	313	[12] = "SDIOWAIT",
314	[14] = "RXBUFF",	314	[14] = "RXBUFF",
315	[15] = "TXBUFE",	315	[15] = "TXBUFE",
316	[16] = "RINDE",	316	[16] = "RINDE",
317	[17] = "RDIRE",	317	[17] = "RDIRE",
318	[18] = "RCRCE",	318	[18] = "RCRCE",
319	[19] = "RENDE",	319	[19] = "RENDE",
320	[20] = "RTOE",	320	[20] = "RTOE",
321	[21] = "DCRCE",	321	[21] = "DCRCE",
322	[22] = "DTOE",	322	[22] = "DTOE",
323	[23] = "CSTOE",	323	[23] = "CSTOE",
324	[24] = "BLKOVRE",	324	[24] = "BLKOVRE",
325	[25] = "DMADONE",	325	[25] = "DMADONE",
326	[26] = "FIFOEMPTY",	326	[26] = "FIFOEMPTY",
327	[27] = "XFRDONE",	327	[27] = "XFRDONE",
328	[30] = "OVRE",	328	[30] = "OVRE",
329	[31] = "UNRE",	329	[31] = "UNRE",
330	};	330	};
331	unsigned int i;	331	unsigned int i;
332		332
333	seq_printf(s, "%s:\t0x%08x", regname, value);	333	seq_printf(s, "%s:\t0x%08x", regname, value);
334	for (i = 0; i < ARRAY_SIZE(sr_bit); i++) {	334	for (i = 0; i < ARRAY_SIZE(sr_bit); i++) {
335	if (value & (1 << i)) {	335	if (value & (1 << i)) {
336	if (sr_bit[i])	336	if (sr_bit[i])
337	seq_printf(s, " %s", sr_bit[i]);	337	seq_printf(s, " %s", sr_bit[i]);
338	else	338	else
339	seq_puts(s, " UNKNOWN");	339	seq_puts(s, " UNKNOWN");
340	}	340	}
341	}	341	}
342	seq_putc(s, '\n');	342	seq_putc(s, '\n');
343	}	343	}
344		344
345	static int atmci_regs_show(struct seq_file s, void v)	345	static int atmci_regs_show(struct seq_file s, void v)
346	{	346	{
347	struct atmel_mci *host = s->private;	347	struct atmel_mci *host = s->private;
348	u32 *buf;	348	u32 *buf;
349		349
350	buf = kmalloc(MCI_REGS_SIZE, GFP_KERNEL);	350	buf = kmalloc(MCI_REGS_SIZE, GFP_KERNEL);
351	if (!buf)	351	if (!buf)
352	return -ENOMEM;	352	return -ENOMEM;
353		353
354	/*	354	/*
355	* Grab a more or less consistent snapshot. Note that we're	355	* Grab a more or less consistent snapshot. Note that we're
356	* not disabling interrupts, so IMR and SR may not be	356	* not disabling interrupts, so IMR and SR may not be
357	* consistent.	357	* consistent.
358	*/	358	*/
359	spin_lock_bh(&host->lock);	359	spin_lock_bh(&host->lock);
360	clk_enable(host->mck);	360	clk_enable(host->mck);
361	memcpy_fromio(buf, host->regs, MCI_REGS_SIZE);	361	memcpy_fromio(buf, host->regs, MCI_REGS_SIZE);
362	clk_disable(host->mck);	362	clk_disable(host->mck);
363	spin_unlock_bh(&host->lock);	363	spin_unlock_bh(&host->lock);
364		364
365	seq_printf(s, "MR:\t0x%08x%s%s CLKDIV=%u\n",	365	seq_printf(s, "MR:\t0x%08x%s%s CLKDIV=%u\n",
366	buf[MCI_MR / 4],	366	buf[MCI_MR / 4],
367	buf[MCI_MR / 4] & MCI_MR_RDPROOF ? " RDPROOF" : "",	367	buf[MCI_MR / 4] & MCI_MR_RDPROOF ? " RDPROOF" : "",
368	buf[MCI_MR / 4] & MCI_MR_WRPROOF ? " WRPROOF" : "",	368	buf[MCI_MR / 4] & MCI_MR_WRPROOF ? " WRPROOF" : "",
369	buf[MCI_MR / 4] & 0xff);	369	buf[MCI_MR / 4] & 0xff);
370	seq_printf(s, "DTOR:\t0x%08x\n", buf[MCI_DTOR / 4]);	370	seq_printf(s, "DTOR:\t0x%08x\n", buf[MCI_DTOR / 4]);
371	seq_printf(s, "SDCR:\t0x%08x\n", buf[MCI_SDCR / 4]);	371	seq_printf(s, "SDCR:\t0x%08x\n", buf[MCI_SDCR / 4]);
372	seq_printf(s, "ARGR:\t0x%08x\n", buf[MCI_ARGR / 4]);	372	seq_printf(s, "ARGR:\t0x%08x\n", buf[MCI_ARGR / 4]);
373	seq_printf(s, "BLKR:\t0x%08x BCNT=%u BLKLEN=%u\n",	373	seq_printf(s, "BLKR:\t0x%08x BCNT=%u BLKLEN=%u\n",
374	buf[MCI_BLKR / 4],	374	buf[MCI_BLKR / 4],
375	buf[MCI_BLKR / 4] & 0xffff,	375	buf[MCI_BLKR / 4] & 0xffff,
376	(buf[MCI_BLKR / 4] >> 16) & 0xffff);	376	(buf[MCI_BLKR / 4] >> 16) & 0xffff);
377	if (atmci_is_mci2())	377	if (atmci_is_mci2())
378	seq_printf(s, "CSTOR:\t0x%08x\n", buf[MCI_CSTOR / 4]);	378	seq_printf(s, "CSTOR:\t0x%08x\n", buf[MCI_CSTOR / 4]);
379		379
380	/* Don't read RSPR and RDR; it will consume the data there */	380	/* Don't read RSPR and RDR; it will consume the data there */
381		381
382	atmci_show_status_reg(s, "SR", buf[MCI_SR / 4]);	382	atmci_show_status_reg(s, "SR", buf[MCI_SR / 4]);
383	atmci_show_status_reg(s, "IMR", buf[MCI_IMR / 4]);	383	atmci_show_status_reg(s, "IMR", buf[MCI_IMR / 4]);
384		384
385	if (atmci_is_mci2()) {	385	if (atmci_is_mci2()) {
386	u32 val;	386	u32 val;
387		387
388	val = buf[MCI_DMA / 4];	388	val = buf[MCI_DMA / 4];
389	seq_printf(s, "DMA:\t0x%08x OFFSET=%u CHKSIZE=%u%s\n",	389	seq_printf(s, "DMA:\t0x%08x OFFSET=%u CHKSIZE=%u%s\n",
390	val, val & 3,	390	val, val & 3,
391	((val >> 4) & 3) ?	391	((val >> 4) & 3) ?
392	1 << (((val >> 4) & 3) + 1) : 1,	392	1 << (((val >> 4) & 3) + 1) : 1,
393	val & MCI_DMAEN ? " DMAEN" : "");	393	val & MCI_DMAEN ? " DMAEN" : "");
394		394
395	val = buf[MCI_CFG / 4];	395	val = buf[MCI_CFG / 4];
396	seq_printf(s, "CFG:\t0x%08x%s%s%s%s\n",	396	seq_printf(s, "CFG:\t0x%08x%s%s%s%s\n",
397	val,	397	val,
398	val & MCI_CFG_FIFOMODE_1DATA ? " FIFOMODE_ONE_DATA" : "",	398	val & MCI_CFG_FIFOMODE_1DATA ? " FIFOMODE_ONE_DATA" : "",
399	val & MCI_CFG_FERRCTRL_COR ? " FERRCTRL_CLEAR_ON_READ" : "",	399	val & MCI_CFG_FERRCTRL_COR ? " FERRCTRL_CLEAR_ON_READ" : "",
400	val & MCI_CFG_HSMODE ? " HSMODE" : "",	400	val & MCI_CFG_HSMODE ? " HSMODE" : "",
401	val & MCI_CFG_LSYNC ? " LSYNC" : "");	401	val & MCI_CFG_LSYNC ? " LSYNC" : "");
402	}	402	}
403		403
404	kfree(buf);	404	kfree(buf);
405		405
406	return 0;	406	return 0;
407	}	407	}
408		408
409	static int atmci_regs_open(struct inode inode, struct file file)	409	static int atmci_regs_open(struct inode inode, struct file file)
410	{	410	{
411	return single_open(file, atmci_regs_show, inode->i_private);	411	return single_open(file, atmci_regs_show, inode->i_private);
412	}	412	}
413		413
414	static const struct file_operations atmci_regs_fops = {	414	static const struct file_operations atmci_regs_fops = {
415	.owner = THIS_MODULE,	415	.owner = THIS_MODULE,
416	.open = atmci_regs_open,	416	.open = atmci_regs_open,
417	.read = seq_read,	417	.read = seq_read,
418	.llseek = seq_lseek,	418	.llseek = seq_lseek,
419	.release = single_release,	419	.release = single_release,
420	};	420	};
421		421
422	static void atmci_init_debugfs(struct atmel_mci_slot *slot)	422	static void atmci_init_debugfs(struct atmel_mci_slot *slot)
423	{	423	{
424	struct mmc_host *mmc = slot->mmc;	424	struct mmc_host *mmc = slot->mmc;
425	struct atmel_mci *host = slot->host;	425	struct atmel_mci *host = slot->host;
426	struct dentry *root;	426	struct dentry *root;
427	struct dentry *node;	427	struct dentry *node;
428		428
429	root = mmc->debugfs_root;	429	root = mmc->debugfs_root;
430	if (!root)	430	if (!root)
431	return;	431	return;
432		432
433	node = debugfs_create_file("regs", S_IRUSR, root, host,	433	node = debugfs_create_file("regs", S_IRUSR, root, host,
434	&atmci_regs_fops);	434	&atmci_regs_fops);
435	if (IS_ERR(node))	435	if (IS_ERR(node))
436	return;	436	return;
437	if (!node)	437	if (!node)
438	goto err;	438	goto err;
439		439
440	node = debugfs_create_file("req", S_IRUSR, root, slot, &atmci_req_fops);	440	node = debugfs_create_file("req", S_IRUSR, root, slot, &atmci_req_fops);
441	if (!node)	441	if (!node)
442	goto err;	442	goto err;
443		443
444	node = debugfs_create_u32("state", S_IRUSR, root, (u32 *)&host->state);	444	node = debugfs_create_u32("state", S_IRUSR, root, (u32 *)&host->state);
445	if (!node)	445	if (!node)
446	goto err;	446	goto err;
447		447
448	node = debugfs_create_x32("pending_events", S_IRUSR, root,	448	node = debugfs_create_x32("pending_events", S_IRUSR, root,
449	(u32 *)&host->pending_events);	449	(u32 *)&host->pending_events);
450	if (!node)	450	if (!node)
451	goto err;	451	goto err;
452		452
453	node = debugfs_create_x32("completed_events", S_IRUSR, root,	453	node = debugfs_create_x32("completed_events", S_IRUSR, root,
454	(u32 *)&host->completed_events);	454	(u32 *)&host->completed_events);
455	if (!node)	455	if (!node)
456	goto err;	456	goto err;
457		457
458	return;	458	return;
459		459
460	err:	460	err:
461	dev_err(&mmc->class_dev, "failed to initialize debugfs for slot\n");	461	dev_err(&mmc->class_dev, "failed to initialize debugfs for slot\n");
462	}	462	}
463		463
464	static inline unsigned int ns_to_clocks(struct atmel_mci *host,	464	static inline unsigned int ns_to_clocks(struct atmel_mci *host,
465	unsigned int ns)	465	unsigned int ns)
466	{	466	{
467	return (ns * (host->bus_hz / 1000000) + 999) / 1000;	467	return (ns * (host->bus_hz / 1000000) + 999) / 1000;
468	}	468	}
469		469
470	static void atmci_set_timeout(struct atmel_mci *host,	470	static void atmci_set_timeout(struct atmel_mci *host,
471	struct atmel_mci_slot slot, struct mmc_data data)	471	struct atmel_mci_slot slot, struct mmc_data data)
472	{	472	{
473	static unsigned dtomul_to_shift[] = {	473	static unsigned dtomul_to_shift[] = {
474	0, 4, 7, 8, 10, 12, 16, 20	474	0, 4, 7, 8, 10, 12, 16, 20
475	};	475	};
476	unsigned timeout;	476	unsigned timeout;
477	unsigned dtocyc;	477	unsigned dtocyc;
478	unsigned dtomul;	478	unsigned dtomul;
479		479
480	timeout = ns_to_clocks(host, data->timeout_ns) + data->timeout_clks;	480	timeout = ns_to_clocks(host, data->timeout_ns) + data->timeout_clks;
481		481
482	for (dtomul = 0; dtomul < 8; dtomul++) {	482	for (dtomul = 0; dtomul < 8; dtomul++) {
483	unsigned shift = dtomul_to_shift[dtomul];	483	unsigned shift = dtomul_to_shift[dtomul];
484	dtocyc = (timeout + (1 << shift) - 1) >> shift;	484	dtocyc = (timeout + (1 << shift) - 1) >> shift;
485	if (dtocyc < 15)	485	if (dtocyc < 15)
486	break;	486	break;
487	}	487	}
488		488
489	if (dtomul >= 8) {	489	if (dtomul >= 8) {
490	dtomul = 7;	490	dtomul = 7;
491	dtocyc = 15;	491	dtocyc = 15;
492	}	492	}
493		493
494	dev_vdbg(&slot->mmc->class_dev, "setting timeout to %u cycles\n",	494	dev_vdbg(&slot->mmc->class_dev, "setting timeout to %u cycles\n",
495	dtocyc << dtomul_to_shift[dtomul]);	495	dtocyc << dtomul_to_shift[dtomul]);
496	mci_writel(host, DTOR, (MCI_DTOMUL(dtomul) \| MCI_DTOCYC(dtocyc)));	496	mci_writel(host, DTOR, (MCI_DTOMUL(dtomul) \| MCI_DTOCYC(dtocyc)));
497	}	497	}
498		498
499	/*	499	/*
500	* Return mask with command flags to be enabled for this command.	500	* Return mask with command flags to be enabled for this command.
501	*/	501	*/
502	static u32 atmci_prepare_command(struct mmc_host *mmc,	502	static u32 atmci_prepare_command(struct mmc_host *mmc,
503	struct mmc_command *cmd)	503	struct mmc_command *cmd)
504	{	504	{
505	struct mmc_data *data;	505	struct mmc_data *data;
506	u32 cmdr;	506	u32 cmdr;
507		507
508	cmd->error = -EINPROGRESS;	508	cmd->error = -EINPROGRESS;
509		509
510	cmdr = MCI_CMDR_CMDNB(cmd->opcode);	510	cmdr = MCI_CMDR_CMDNB(cmd->opcode);
511		511
512	if (cmd->flags & MMC_RSP_PRESENT) {	512	if (cmd->flags & MMC_RSP_PRESENT) {
513	if (cmd->flags & MMC_RSP_136)	513	if (cmd->flags & MMC_RSP_136)
514	cmdr \|= MCI_CMDR_RSPTYP_136BIT;	514	cmdr \|= MCI_CMDR_RSPTYP_136BIT;
515	else	515	else
516	cmdr \|= MCI_CMDR_RSPTYP_48BIT;	516	cmdr \|= MCI_CMDR_RSPTYP_48BIT;
517	}	517	}
518		518
519	/*	519	/*
520	* This should really be MAXLAT_5 for CMD2 and ACMD41, but	520	* This should really be MAXLAT_5 for CMD2 and ACMD41, but
521	* it's too difficult to determine whether this is an ACMD or	521	* it's too difficult to determine whether this is an ACMD or
522	* not. Better make it 64.	522	* not. Better make it 64.
523	*/	523	*/
524	cmdr \|= MCI_CMDR_MAXLAT_64CYC;	524	cmdr \|= MCI_CMDR_MAXLAT_64CYC;
525		525
526	if (mmc->ios.bus_mode == MMC_BUSMODE_OPENDRAIN)	526	if (mmc->ios.bus_mode == MMC_BUSMODE_OPENDRAIN)
527	cmdr \|= MCI_CMDR_OPDCMD;	527	cmdr \|= MCI_CMDR_OPDCMD;
528		528
529	data = cmd->data;	529	data = cmd->data;
530	if (data) {	530	if (data) {
531	cmdr \|= MCI_CMDR_START_XFER;	531	cmdr \|= MCI_CMDR_START_XFER;
532	if (data->flags & MMC_DATA_STREAM)	532	if (data->flags & MMC_DATA_STREAM)
533	cmdr \|= MCI_CMDR_STREAM;	533	cmdr \|= MCI_CMDR_STREAM;
534	else if (data->blocks > 1)	534	else if (data->blocks > 1)
535	cmdr \|= MCI_CMDR_MULTI_BLOCK;	535	cmdr \|= MCI_CMDR_MULTI_BLOCK;
536	else	536	else
537	cmdr \|= MCI_CMDR_BLOCK;	537	cmdr \|= MCI_CMDR_BLOCK;
538		538
539	if (data->flags & MMC_DATA_READ)	539	if (data->flags & MMC_DATA_READ)
540	cmdr \|= MCI_CMDR_TRDIR_READ;	540	cmdr \|= MCI_CMDR_TRDIR_READ;
541	}	541	}
542		542
543	return cmdr;	543	return cmdr;
544	}	544	}
545		545
546	static void atmci_start_command(struct atmel_mci *host,	546	static void atmci_start_command(struct atmel_mci *host,
547	struct mmc_command *cmd, u32 cmd_flags)	547	struct mmc_command *cmd, u32 cmd_flags)
548	{	548	{
549	WARN_ON(host->cmd);	549	WARN_ON(host->cmd);
550	host->cmd = cmd;	550	host->cmd = cmd;
551		551
552	dev_vdbg(&host->pdev->dev,	552	dev_vdbg(&host->pdev->dev,
553	"start command: ARGR=0x%08x CMDR=0x%08x\n",	553	"start command: ARGR=0x%08x CMDR=0x%08x\n",
554	cmd->arg, cmd_flags);	554	cmd->arg, cmd_flags);
555		555
556	mci_writel(host, ARGR, cmd->arg);	556	mci_writel(host, ARGR, cmd->arg);
557	mci_writel(host, CMDR, cmd_flags);	557	mci_writel(host, CMDR, cmd_flags);
558	}	558	}
559		559
560	static void send_stop_cmd(struct atmel_mci host, struct mmc_data data)	560	static void send_stop_cmd(struct atmel_mci host, struct mmc_data data)
561	{	561	{
562	atmci_start_command(host, data->stop, host->stop_cmdr);	562	atmci_start_command(host, data->stop, host->stop_cmdr);
563	mci_writel(host, IER, MCI_CMDRDY);	563	mci_writel(host, IER, MCI_CMDRDY);
564	}	564	}
565		565
566	#ifdef CONFIG_MMC_ATMELMCI_DMA	566	#ifdef CONFIG_MMC_ATMELMCI_DMA
567	static void atmci_dma_cleanup(struct atmel_mci *host)	567	static void atmci_dma_cleanup(struct atmel_mci *host)
568	{	568	{
569	struct mmc_data *data = host->data;	569	struct mmc_data *data = host->data;
570		570
571	dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len,	571	dma_unmap_sg(&host->pdev->dev, data->sg, data->sg_len,
572	((data->flags & MMC_DATA_WRITE)	572	((data->flags & MMC_DATA_WRITE)
573	? DMA_TO_DEVICE : DMA_FROM_DEVICE));	573	? DMA_TO_DEVICE : DMA_FROM_DEVICE));
574	}	574	}
575		575
576	static void atmci_stop_dma(struct atmel_mci *host)	576	static void atmci_stop_dma(struct atmel_mci *host)
577	{	577	{
578	struct dma_chan *chan = host->data_chan;	578	struct dma_chan *chan = host->data_chan;
579		579
580	if (chan) {	580	if (chan) {
581	chan->device->device_control(chan, DMA_TERMINATE_ALL);	581	chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
582	atmci_dma_cleanup(host);	582	atmci_dma_cleanup(host);
583	} else {	583	} else {
584	/* Data transfer was stopped by the interrupt handler */	584	/* Data transfer was stopped by the interrupt handler */
585	atmci_set_pending(host, EVENT_XFER_COMPLETE);	585	atmci_set_pending(host, EVENT_XFER_COMPLETE);
586	mci_writel(host, IER, MCI_NOTBUSY);	586	mci_writel(host, IER, MCI_NOTBUSY);
587	}	587	}
588	}	588	}
589		589
590	/* This function is called by the DMA driver from tasklet context. */	590	/* This function is called by the DMA driver from tasklet context. */
591	static void atmci_dma_complete(void *arg)	591	static void atmci_dma_complete(void *arg)
592	{	592	{
593	struct atmel_mci *host = arg;	593	struct atmel_mci *host = arg;
594	struct mmc_data *data = host->data;	594	struct mmc_data *data = host->data;
595		595
596	dev_vdbg(&host->pdev->dev, "DMA complete\n");	596	dev_vdbg(&host->pdev->dev, "DMA complete\n");
597		597
598	if (atmci_is_mci2())	598	if (atmci_is_mci2())
599	/* Disable DMA hardware handshaking on MCI */	599	/* Disable DMA hardware handshaking on MCI */
600	mci_writel(host, DMA, mci_readl(host, DMA) & ~MCI_DMAEN);	600	mci_writel(host, DMA, mci_readl(host, DMA) & ~MCI_DMAEN);
601		601
602	atmci_dma_cleanup(host);	602	atmci_dma_cleanup(host);
603		603
604	/*	604	/*
605	* If the card was removed, data will be NULL. No point trying	605	* If the card was removed, data will be NULL. No point trying
606	* to send the stop command or waiting for NBUSY in this case.	606	* to send the stop command or waiting for NBUSY in this case.
607	*/	607	*/
608	if (data) {	608	if (data) {
609	atmci_set_pending(host, EVENT_XFER_COMPLETE);	609	atmci_set_pending(host, EVENT_XFER_COMPLETE);
610	tasklet_schedule(&host->tasklet);	610	tasklet_schedule(&host->tasklet);
611		611
612	/*	612	/*
613	* Regardless of what the documentation says, we have	613	* Regardless of what the documentation says, we have
614	* to wait for NOTBUSY even after block read	614	* to wait for NOTBUSY even after block read
615	* operations.	615	* operations.
616	*	616	*
617	* When the DMA transfer is complete, the controller	617	* When the DMA transfer is complete, the controller
618	* may still be reading the CRC from the card, i.e.	618	* may still be reading the CRC from the card, i.e.
619	* the data transfer is still in progress and we	619	* the data transfer is still in progress and we
620	* haven't seen all the potential error bits yet.	620	* haven't seen all the potential error bits yet.
621	*	621	*
622	* The interrupt handler will schedule a different	622	* The interrupt handler will schedule a different
623	* tasklet to finish things up when the data transfer	623	* tasklet to finish things up when the data transfer
624	* is completely done.	624	* is completely done.
625	*	625	*
626	* We may not complete the mmc request here anyway	626	* We may not complete the mmc request here anyway
627	* because the mmc layer may call back and cause us to	627	* because the mmc layer may call back and cause us to
628	* violate the "don't submit new operations from the	628	* violate the "don't submit new operations from the
629	* completion callback" rule of the dma engine	629	* completion callback" rule of the dma engine
630	* framework.	630	* framework.
631	*/	631	*/
632	mci_writel(host, IER, MCI_NOTBUSY);	632	mci_writel(host, IER, MCI_NOTBUSY);
633	}	633	}
634	}	634	}
635		635
636	static int	636	static int
637	atmci_prepare_data_dma(struct atmel_mci host, struct mmc_data data)	637	atmci_prepare_data_dma(struct atmel_mci host, struct mmc_data data)
638	{	638	{
639	struct dma_chan *chan;	639	struct dma_chan *chan;
640	struct dma_async_tx_descriptor *desc;	640	struct dma_async_tx_descriptor *desc;
641	struct scatterlist *sg;	641	struct scatterlist *sg;
642	unsigned int i;	642	unsigned int i;
643	enum dma_data_direction direction;	643	enum dma_data_direction direction;
644	unsigned int sglen;	644	unsigned int sglen;
645		645
646	/*	646	/*
647	* We don't do DMA on "complex" transfers, i.e. with	647	* We don't do DMA on "complex" transfers, i.e. with
648	* non-word-aligned buffers or lengths. Also, we don't bother	648	* non-word-aligned buffers or lengths. Also, we don't bother
649	* with all the DMA setup overhead for short transfers.	649	* with all the DMA setup overhead for short transfers.
650	*/	650	*/
651	if (data->blocks * data->blksz < ATMCI_DMA_THRESHOLD)	651	if (data->blocks * data->blksz < ATMCI_DMA_THRESHOLD)
652	return -EINVAL;	652	return -EINVAL;
653	if (data->blksz & 3)	653	if (data->blksz & 3)
654	return -EINVAL;	654	return -EINVAL;
655		655
656	for_each_sg(data->sg, sg, data->sg_len, i) {	656	for_each_sg(data->sg, sg, data->sg_len, i) {
657	if (sg->offset & 3 \|\| sg->length & 3)	657	if (sg->offset & 3 \|\| sg->length & 3)
658	return -EINVAL;	658	return -EINVAL;
659	}	659	}
660		660
661	/* If we don't have a channel, we can't do DMA */	661	/* If we don't have a channel, we can't do DMA */
662	chan = host->dma.chan;	662	chan = host->dma.chan;
663	if (chan)	663	if (chan)
664	host->data_chan = chan;	664	host->data_chan = chan;
665		665
666	if (!chan)	666	if (!chan)
667	return -ENODEV;	667	return -ENODEV;
668		668
669	if (atmci_is_mci2())	669	if (atmci_is_mci2())
670	mci_writel(host, DMA, MCI_DMA_CHKSIZE(3) \| MCI_DMAEN);	670	mci_writel(host, DMA, MCI_DMA_CHKSIZE(3) \| MCI_DMAEN);
671		671
672	if (data->flags & MMC_DATA_READ)	672	if (data->flags & MMC_DATA_READ)
673	direction = DMA_FROM_DEVICE;	673	direction = DMA_FROM_DEVICE;
674	else	674	else
675	direction = DMA_TO_DEVICE;	675	direction = DMA_TO_DEVICE;
676		676
677	sglen = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, direction);	677	sglen = dma_map_sg(&host->pdev->dev, data->sg, data->sg_len, direction);
678	if (sglen != data->sg_len)	678	if (sglen != data->sg_len)
679	goto unmap_exit;	679	goto unmap_exit;
680	desc = chan->device->device_prep_slave_sg(chan,	680	desc = chan->device->device_prep_slave_sg(chan,
681	data->sg, data->sg_len, direction,	681	data->sg, data->sg_len, direction,
682	DMA_PREP_INTERRUPT \| DMA_CTRL_ACK);	682	DMA_PREP_INTERRUPT \| DMA_CTRL_ACK);
683	if (!desc)	683	if (!desc)
684	goto unmap_exit;	684	goto unmap_exit;
685		685
686	host->dma.data_desc = desc;	686	host->dma.data_desc = desc;
687	desc->callback = atmci_dma_complete;	687	desc->callback = atmci_dma_complete;
688	desc->callback_param = host;	688	desc->callback_param = host;
689		689
690	return 0;	690	return 0;
691	unmap_exit:	691	unmap_exit:
692	dma_unmap_sg(&host->pdev->dev, data->sg, sglen, direction);	692	dma_unmap_sg(&host->pdev->dev, data->sg, sglen, direction);
693	return -ENOMEM;	693	return -ENOMEM;
694	}	694	}
695		695
696	static void atmci_submit_data(struct atmel_mci *host)	696	static void atmci_submit_data(struct atmel_mci *host)
697	{	697	{
698	struct dma_chan *chan = host->data_chan;	698	struct dma_chan *chan = host->data_chan;
699	struct dma_async_tx_descriptor *desc = host->dma.data_desc;	699	struct dma_async_tx_descriptor *desc = host->dma.data_desc;
700		700
701	if (chan) {	701	if (chan) {
702	desc->tx_submit(desc);	702	desc->tx_submit(desc);
703	chan->device->device_issue_pending(chan);	703	chan->device->device_issue_pending(chan);
704	}	704	}
705	}	705	}
706		706
707	#else /* CONFIG_MMC_ATMELMCI_DMA */	707	#else /* CONFIG_MMC_ATMELMCI_DMA */
708		708
709	static int atmci_prepare_data_dma(struct atmel_mci host, struct mmc_data data)	709	static int atmci_prepare_data_dma(struct atmel_mci host, struct mmc_data data)
710	{	710	{
711	return -ENOSYS;	711	return -ENOSYS;
712	}	712	}
713		713
714	static void atmci_submit_data(struct atmel_mci *host) {}	714	static void atmci_submit_data(struct atmel_mci *host) {}
715		715
716	static void atmci_stop_dma(struct atmel_mci *host)	716	static void atmci_stop_dma(struct atmel_mci *host)
717	{	717	{
718	/* Data transfer was stopped by the interrupt handler */	718	/* Data transfer was stopped by the interrupt handler */
719	atmci_set_pending(host, EVENT_XFER_COMPLETE);	719	atmci_set_pending(host, EVENT_XFER_COMPLETE);
720	mci_writel(host, IER, MCI_NOTBUSY);	720	mci_writel(host, IER, MCI_NOTBUSY);
721	}	721	}
722		722
723	#endif /* CONFIG_MMC_ATMELMCI_DMA */	723	#endif /* CONFIG_MMC_ATMELMCI_DMA */
724		724
725	/*	725	/*
726	* Returns a mask of interrupt flags to be enabled after the whole	726	* Returns a mask of interrupt flags to be enabled after the whole
727	* request has been prepared.	727	* request has been prepared.
728	*/	728	*/
729	static u32 atmci_prepare_data(struct atmel_mci host, struct mmc_data data)	729	static u32 atmci_prepare_data(struct atmel_mci host, struct mmc_data data)
730	{	730	{
731	u32 iflags;	731	u32 iflags;
732		732
733	data->error = -EINPROGRESS;	733	data->error = -EINPROGRESS;
734		734
735	WARN_ON(host->data);	735	WARN_ON(host->data);
736	host->sg = NULL;	736	host->sg = NULL;
737	host->data = data;	737	host->data = data;
738		738
739	iflags = ATMCI_DATA_ERROR_FLAGS;	739	iflags = ATMCI_DATA_ERROR_FLAGS;
740	if (atmci_prepare_data_dma(host, data)) {	740	if (atmci_prepare_data_dma(host, data)) {
741	host->data_chan = NULL;	741	host->data_chan = NULL;
742		742
743	/*	743	/*
744	* Errata: MMC data write operation with less than 12	744	* Errata: MMC data write operation with less than 12
745	* bytes is impossible.	745	* bytes is impossible.
746	*	746	*
747	* Errata: MCI Transmit Data Register (TDR) FIFO	747	* Errata: MCI Transmit Data Register (TDR) FIFO
748	* corruption when length is not multiple of 4.	748	* corruption when length is not multiple of 4.
749	*/	749	*/
750	if (data->blocks * data->blksz < 12	750	if (data->blocks * data->blksz < 12
751	\|\| (data->blocks * data->blksz) & 3)	751	\|\| (data->blocks * data->blksz) & 3)
752	host->need_reset = true;	752	host->need_reset = true;
753		753
754	host->sg = data->sg;	754	host->sg = data->sg;
755	host->pio_offset = 0;	755	host->pio_offset = 0;
756	if (data->flags & MMC_DATA_READ)	756	if (data->flags & MMC_DATA_READ)
757	iflags \|= MCI_RXRDY;	757	iflags \|= MCI_RXRDY;
758	else	758	else
759	iflags \|= MCI_TXRDY;	759	iflags \|= MCI_TXRDY;
760	}	760	}
761		761
762	return iflags;	762	return iflags;
763	}	763	}
764		764
765	static void atmci_start_request(struct atmel_mci *host,	765	static void atmci_start_request(struct atmel_mci *host,
766	struct atmel_mci_slot *slot)	766	struct atmel_mci_slot *slot)
767	{	767	{
768	struct mmc_request *mrq;	768	struct mmc_request *mrq;
769	struct mmc_command *cmd;	769	struct mmc_command *cmd;
770	struct mmc_data *data;	770	struct mmc_data *data;
771	u32 iflags;	771	u32 iflags;
772	u32 cmdflags;	772	u32 cmdflags;
773		773
774	mrq = slot->mrq;	774	mrq = slot->mrq;
775	host->cur_slot = slot;	775	host->cur_slot = slot;
776	host->mrq = mrq;	776	host->mrq = mrq;
777		777
778	host->pending_events = 0;	778	host->pending_events = 0;
779	host->completed_events = 0;	779	host->completed_events = 0;
780	host->data_status = 0;	780	host->data_status = 0;
781		781
782	if (host->need_reset) {	782	if (host->need_reset) {
783	mci_writel(host, CR, MCI_CR_SWRST);	783	mci_writel(host, CR, MCI_CR_SWRST);
784	mci_writel(host, CR, MCI_CR_MCIEN);	784	mci_writel(host, CR, MCI_CR_MCIEN);
785	mci_writel(host, MR, host->mode_reg);	785	mci_writel(host, MR, host->mode_reg);
786	if (atmci_is_mci2())	786	if (atmci_is_mci2())
787	mci_writel(host, CFG, host->cfg_reg);	787	mci_writel(host, CFG, host->cfg_reg);
788	host->need_reset = false;	788	host->need_reset = false;
789	}	789	}
790	mci_writel(host, SDCR, slot->sdc_reg);	790	mci_writel(host, SDCR, slot->sdc_reg);
791		791
792	iflags = mci_readl(host, IMR);	792	iflags = mci_readl(host, IMR);
793	if (iflags)	793	if (iflags)
794	dev_warn(&slot->mmc->class_dev, "WARNING: IMR=0x%08x\n",	794	dev_warn(&slot->mmc->class_dev, "WARNING: IMR=0x%08x\n",
795	iflags);	795	iflags);
796		796
797	if (unlikely(test_and_clear_bit(ATMCI_CARD_NEED_INIT, &slot->flags))) {	797	if (unlikely(test_and_clear_bit(ATMCI_CARD_NEED_INIT, &slot->flags))) {
798	/* Send init sequence (74 clock cycles) */	798	/* Send init sequence (74 clock cycles) */
799	mci_writel(host, CMDR, MCI_CMDR_SPCMD_INIT);	799	mci_writel(host, CMDR, MCI_CMDR_SPCMD_INIT);
800	while (!(mci_readl(host, SR) & MCI_CMDRDY))	800	while (!(mci_readl(host, SR) & MCI_CMDRDY))
801	cpu_relax();	801	cpu_relax();
802	}	802	}
803	iflags = 0;	803	iflags = 0;
804	data = mrq->data;	804	data = mrq->data;
805	if (data) {	805	if (data) {
806	atmci_set_timeout(host, slot, data);	806	atmci_set_timeout(host, slot, data);
807		807
808	/* Must set block count/size before sending command */	808	/* Must set block count/size before sending command */
809	mci_writel(host, BLKR, MCI_BCNT(data->blocks)	809	mci_writel(host, BLKR, MCI_BCNT(data->blocks)
810	\| MCI_BLKLEN(data->blksz));	810	\| MCI_BLKLEN(data->blksz));
811	dev_vdbg(&slot->mmc->class_dev, "BLKR=0x%08x\n",	811	dev_vdbg(&slot->mmc->class_dev, "BLKR=0x%08x\n",
812	MCI_BCNT(data->blocks) \| MCI_BLKLEN(data->blksz));	812	MCI_BCNT(data->blocks) \| MCI_BLKLEN(data->blksz));
813		813
814	iflags \|= atmci_prepare_data(host, data);	814	iflags \|= atmci_prepare_data(host, data);
815	}	815	}
816		816
817	iflags \|= MCI_CMDRDY;	817	iflags \|= MCI_CMDRDY;
818	cmd = mrq->cmd;	818	cmd = mrq->cmd;
819	cmdflags = atmci_prepare_command(slot->mmc, cmd);	819	cmdflags = atmci_prepare_command(slot->mmc, cmd);
820	atmci_start_command(host, cmd, cmdflags);	820	atmci_start_command(host, cmd, cmdflags);
821		821
822	if (data)	822	if (data)
823	atmci_submit_data(host);	823	atmci_submit_data(host);
824		824
825	if (mrq->stop) {	825	if (mrq->stop) {
826	host->stop_cmdr = atmci_prepare_command(slot->mmc, mrq->stop);	826	host->stop_cmdr = atmci_prepare_command(slot->mmc, mrq->stop);
827	host->stop_cmdr \|= MCI_CMDR_STOP_XFER;	827	host->stop_cmdr \|= MCI_CMDR_STOP_XFER;
828	if (!(data->flags & MMC_DATA_WRITE))	828	if (!(data->flags & MMC_DATA_WRITE))
829	host->stop_cmdr \|= MCI_CMDR_TRDIR_READ;	829	host->stop_cmdr \|= MCI_CMDR_TRDIR_READ;
830	if (data->flags & MMC_DATA_STREAM)	830	if (data->flags & MMC_DATA_STREAM)
831	host->stop_cmdr \|= MCI_CMDR_STREAM;	831	host->stop_cmdr \|= MCI_CMDR_STREAM;
832	else	832	else
833	host->stop_cmdr \|= MCI_CMDR_MULTI_BLOCK;	833	host->stop_cmdr \|= MCI_CMDR_MULTI_BLOCK;
834	}	834	}
835		835
836	/*	836	/*
837	* We could have enabled interrupts earlier, but I suspect	837	* We could have enabled interrupts earlier, but I suspect
838	* that would open up a nice can of interesting race	838	* that would open up a nice can of interesting race
839	* conditions (e.g. command and data complete, but stop not	839	* conditions (e.g. command and data complete, but stop not
840	* prepared yet.)	840	* prepared yet.)
841	*/	841	*/
842	mci_writel(host, IER, iflags);	842	mci_writel(host, IER, iflags);
843	}	843	}
844		844
845	static void atmci_queue_request(struct atmel_mci *host,	845	static void atmci_queue_request(struct atmel_mci *host,
846	struct atmel_mci_slot slot, struct mmc_request mrq)	846	struct atmel_mci_slot slot, struct mmc_request mrq)
847	{	847	{
848	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",	848	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
849	host->state);	849	host->state);
850		850
851	spin_lock_bh(&host->lock);	851	spin_lock_bh(&host->lock);
852	slot->mrq = mrq;	852	slot->mrq = mrq;
853	if (host->state == STATE_IDLE) {	853	if (host->state == STATE_IDLE) {
854	host->state = STATE_SENDING_CMD;	854	host->state = STATE_SENDING_CMD;
855	atmci_start_request(host, slot);	855	atmci_start_request(host, slot);
856	} else {	856	} else {
857	list_add_tail(&slot->queue_node, &host->queue);	857	list_add_tail(&slot->queue_node, &host->queue);
858	}	858	}
859	spin_unlock_bh(&host->lock);	859	spin_unlock_bh(&host->lock);
860	}	860	}
861		861
862	static void atmci_request(struct mmc_host mmc, struct mmc_request mrq)	862	static void atmci_request(struct mmc_host mmc, struct mmc_request mrq)
863	{	863	{
864	struct atmel_mci_slot *slot = mmc_priv(mmc);	864	struct atmel_mci_slot *slot = mmc_priv(mmc);
865	struct atmel_mci *host = slot->host;	865	struct atmel_mci *host = slot->host;
866	struct mmc_data *data;	866	struct mmc_data *data;
867		867
868	WARN_ON(slot->mrq);	868	WARN_ON(slot->mrq);
869		869
870	/*	870	/*
871	* We may "know" the card is gone even though there's still an	871	* We may "know" the card is gone even though there's still an
872	* electrical connection. If so, we really need to communicate	872	* electrical connection. If so, we really need to communicate
873	* this to the MMC core since there won't be any more	873	* this to the MMC core since there won't be any more
874	* interrupts as the card is completely removed. Otherwise,	874	* interrupts as the card is completely removed. Otherwise,
875	* the MMC core might believe the card is still there even	875	* the MMC core might believe the card is still there even
876	* though the card was just removed very slowly.	876	* though the card was just removed very slowly.
877	*/	877	*/
878	if (!test_bit(ATMCI_CARD_PRESENT, &slot->flags)) {	878	if (!test_bit(ATMCI_CARD_PRESENT, &slot->flags)) {
879	mrq->cmd->error = -ENOMEDIUM;	879	mrq->cmd->error = -ENOMEDIUM;
880	mmc_request_done(mmc, mrq);	880	mmc_request_done(mmc, mrq);
881	return;	881	return;
882	}	882	}
883		883
884	/* We don't support multiple blocks of weird lengths. */	884	/* We don't support multiple blocks of weird lengths. */
885	data = mrq->data;	885	data = mrq->data;
886	if (data && data->blocks > 1 && data->blksz & 3) {	886	if (data && data->blocks > 1 && data->blksz & 3) {
887	mrq->cmd->error = -EINVAL;	887	mrq->cmd->error = -EINVAL;
888	mmc_request_done(mmc, mrq);	888	mmc_request_done(mmc, mrq);
889	}	889	}
890		890
891	atmci_queue_request(host, slot, mrq);	891	atmci_queue_request(host, slot, mrq);
892	}	892	}
893		893
894	static void atmci_set_ios(struct mmc_host mmc, struct mmc_ios ios)	894	static void atmci_set_ios(struct mmc_host mmc, struct mmc_ios ios)
895	{	895	{
896	struct atmel_mci_slot *slot = mmc_priv(mmc);	896	struct atmel_mci_slot *slot = mmc_priv(mmc);
897	struct atmel_mci *host = slot->host;	897	struct atmel_mci *host = slot->host;
898	unsigned int i;	898	unsigned int i;
899		899
900	slot->sdc_reg &= ~MCI_SDCBUS_MASK;	900	slot->sdc_reg &= ~MCI_SDCBUS_MASK;
901	switch (ios->bus_width) {	901	switch (ios->bus_width) {
902	case MMC_BUS_WIDTH_1:	902	case MMC_BUS_WIDTH_1:
903	slot->sdc_reg \|= MCI_SDCBUS_1BIT;	903	slot->sdc_reg \|= MCI_SDCBUS_1BIT;
904	break;	904	break;
905	case MMC_BUS_WIDTH_4:	905	case MMC_BUS_WIDTH_4:
906	slot->sdc_reg \|= MCI_SDCBUS_4BIT;	906	slot->sdc_reg \|= MCI_SDCBUS_4BIT;
907	break;	907	break;
908	}	908	}
909		909
910	if (ios->clock) {	910	if (ios->clock) {
911	unsigned int clock_min = ~0U;	911	unsigned int clock_min = ~0U;
912	u32 clkdiv;	912	u32 clkdiv;
913		913
914	spin_lock_bh(&host->lock);	914	spin_lock_bh(&host->lock);
915	if (!host->mode_reg) {	915	if (!host->mode_reg) {
916	clk_enable(host->mck);	916	clk_enable(host->mck);
917	mci_writel(host, CR, MCI_CR_SWRST);	917	mci_writel(host, CR, MCI_CR_SWRST);
918	mci_writel(host, CR, MCI_CR_MCIEN);	918	mci_writel(host, CR, MCI_CR_MCIEN);
919	if (atmci_is_mci2())	919	if (atmci_is_mci2())
920	mci_writel(host, CFG, host->cfg_reg);	920	mci_writel(host, CFG, host->cfg_reg);
921	}	921	}
922		922
923	/*	923	/*
924	* Use mirror of ios->clock to prevent race with mmc	924	* Use mirror of ios->clock to prevent race with mmc
925	* core ios update when finding the minimum.	925	* core ios update when finding the minimum.
926	*/	926	*/
927	slot->clock = ios->clock;	927	slot->clock = ios->clock;
928	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {	928	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
929	if (host->slot[i] && host->slot[i]->clock	929	if (host->slot[i] && host->slot[i]->clock
930	&& host->slot[i]->clock < clock_min)	930	&& host->slot[i]->clock < clock_min)
931	clock_min = host->slot[i]->clock;	931	clock_min = host->slot[i]->clock;
932	}	932	}
933		933
934	/* Calculate clock divider */	934	/* Calculate clock divider */
935	clkdiv = DIV_ROUND_UP(host->bus_hz, 2 * clock_min) - 1;	935	clkdiv = DIV_ROUND_UP(host->bus_hz, 2 * clock_min) - 1;
936	if (clkdiv > 255) {	936	if (clkdiv > 255) {
937	dev_warn(&mmc->class_dev,	937	dev_warn(&mmc->class_dev,
938	"clock %u too slow; using %lu\n",	938	"clock %u too slow; using %lu\n",
939	clock_min, host->bus_hz / (2 * 256));	939	clock_min, host->bus_hz / (2 * 256));
940	clkdiv = 255;	940	clkdiv = 255;
941	}	941	}
942		942
943	host->mode_reg = MCI_MR_CLKDIV(clkdiv);	943	host->mode_reg = MCI_MR_CLKDIV(clkdiv);
944		944
945	/*	945	/*
946	* WRPROOF and RDPROOF prevent overruns/underruns by	946	* WRPROOF and RDPROOF prevent overruns/underruns by
947	* stopping the clock when the FIFO is full/empty.	947	* stopping the clock when the FIFO is full/empty.
948	* This state is not expected to last for long.	948	* This state is not expected to last for long.
949	*/	949	*/
950	if (mci_has_rwproof())	950	if (mci_has_rwproof())
951	host->mode_reg \|= (MCI_MR_WRPROOF \| MCI_MR_RDPROOF);	951	host->mode_reg \|= (MCI_MR_WRPROOF \| MCI_MR_RDPROOF);
952		952
953	if (list_empty(&host->queue))	953	if (list_empty(&host->queue))
954	mci_writel(host, MR, host->mode_reg);	954	mci_writel(host, MR, host->mode_reg);
955	else	955	else
956	host->need_clock_update = true;	956	host->need_clock_update = true;
957		957
958	spin_unlock_bh(&host->lock);	958	spin_unlock_bh(&host->lock);
959	} else {	959	} else {
960	bool any_slot_active = false;	960	bool any_slot_active = false;
961		961
962	spin_lock_bh(&host->lock);	962	spin_lock_bh(&host->lock);
963	slot->clock = 0;	963	slot->clock = 0;
964	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {	964	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
965	if (host->slot[i] && host->slot[i]->clock) {	965	if (host->slot[i] && host->slot[i]->clock) {
966	any_slot_active = true;	966	any_slot_active = true;
967	break;	967	break;
968	}	968	}
969	}	969	}
970	if (!any_slot_active) {	970	if (!any_slot_active) {
971	mci_writel(host, CR, MCI_CR_MCIDIS);	971	mci_writel(host, CR, MCI_CR_MCIDIS);
972	if (host->mode_reg) {	972	if (host->mode_reg) {
973	mci_readl(host, MR);	973	mci_readl(host, MR);
974	clk_disable(host->mck);	974	clk_disable(host->mck);
975	}	975	}
976	host->mode_reg = 0;	976	host->mode_reg = 0;
977	}	977	}
978	spin_unlock_bh(&host->lock);	978	spin_unlock_bh(&host->lock);
979	}	979	}
980		980
981	switch (ios->power_mode) {	981	switch (ios->power_mode) {
982	case MMC_POWER_UP:	982	case MMC_POWER_UP:
983	set_bit(ATMCI_CARD_NEED_INIT, &slot->flags);	983	set_bit(ATMCI_CARD_NEED_INIT, &slot->flags);
984	break;	984	break;
985	default:	985	default:
986	/*	986	/*
987	* TODO: None of the currently available AVR32-based	987	* TODO: None of the currently available AVR32-based
988	* boards allow MMC power to be turned off. Implement	988	* boards allow MMC power to be turned off. Implement
989	* power control when this can be tested properly.	989	* power control when this can be tested properly.
990	*	990	*
991	* We also need to hook this into the clock management	991	* We also need to hook this into the clock management
992	* somehow so that newly inserted cards aren't	992	* somehow so that newly inserted cards aren't
993	* subjected to a fast clock before we have a chance	993	* subjected to a fast clock before we have a chance
994	* to figure out what the maximum rate is. Currently,	994	* to figure out what the maximum rate is. Currently,
995	* there's no way to avoid this, and there never will	995	* there's no way to avoid this, and there never will
996	* be for boards that don't support power control.	996	* be for boards that don't support power control.
997	*/	997	*/
998	break;	998	break;
999	}	999	}
1000	}	1000	}
1001		1001
1002	static int atmci_get_ro(struct mmc_host *mmc)	1002	static int atmci_get_ro(struct mmc_host *mmc)
1003	{	1003	{
1004	int read_only = -ENOSYS;	1004	int read_only = -ENOSYS;
1005	struct atmel_mci_slot *slot = mmc_priv(mmc);	1005	struct atmel_mci_slot *slot = mmc_priv(mmc);
1006		1006
1007	if (gpio_is_valid(slot->wp_pin)) {	1007	if (gpio_is_valid(slot->wp_pin)) {
1008	read_only = gpio_get_value(slot->wp_pin);	1008	read_only = gpio_get_value(slot->wp_pin);
1009	dev_dbg(&mmc->class_dev, "card is %s\n",	1009	dev_dbg(&mmc->class_dev, "card is %s\n",
1010	read_only ? "read-only" : "read-write");	1010	read_only ? "read-only" : "read-write");
1011	}	1011	}
1012		1012
1013	return read_only;	1013	return read_only;
1014	}	1014	}
1015		1015
1016	static int atmci_get_cd(struct mmc_host *mmc)	1016	static int atmci_get_cd(struct mmc_host *mmc)
1017	{	1017	{
1018	int present = -ENOSYS;	1018	int present = -ENOSYS;
1019	struct atmel_mci_slot *slot = mmc_priv(mmc);	1019	struct atmel_mci_slot *slot = mmc_priv(mmc);
1020		1020
1021	if (gpio_is_valid(slot->detect_pin)) {	1021	if (gpio_is_valid(slot->detect_pin)) {
1022	present = !(gpio_get_value(slot->detect_pin) ^	1022	present = !(gpio_get_value(slot->detect_pin) ^
1023	slot->detect_is_active_high);	1023	slot->detect_is_active_high);
1024	dev_dbg(&mmc->class_dev, "card is %spresent\n",	1024	dev_dbg(&mmc->class_dev, "card is %spresent\n",
1025	present ? "" : "not ");	1025	present ? "" : "not ");
1026	}	1026	}
1027		1027
1028	return present;	1028	return present;
1029	}	1029	}
1030		1030
1031	static const struct mmc_host_ops atmci_ops = {	1031	static const struct mmc_host_ops atmci_ops = {
1032	.request = atmci_request,	1032	.request = atmci_request,
1033	.set_ios = atmci_set_ios,	1033	.set_ios = atmci_set_ios,
1034	.get_ro = atmci_get_ro,	1034	.get_ro = atmci_get_ro,
1035	.get_cd = atmci_get_cd,	1035	.get_cd = atmci_get_cd,
1036	};	1036	};
1037		1037
1038	/* Called with host->lock held */	1038	/* Called with host->lock held */
1039	static void atmci_request_end(struct atmel_mci host, struct mmc_request mrq)	1039	static void atmci_request_end(struct atmel_mci host, struct mmc_request mrq)
1040	__releases(&host->lock)	1040	__releases(&host->lock)
1041	__acquires(&host->lock)	1041	__acquires(&host->lock)
1042	{	1042	{
1043	struct atmel_mci_slot *slot = NULL;	1043	struct atmel_mci_slot *slot = NULL;
1044	struct mmc_host *prev_mmc = host->cur_slot->mmc;	1044	struct mmc_host *prev_mmc = host->cur_slot->mmc;
1045		1045
1046	WARN_ON(host->cmd \|\| host->data);	1046	WARN_ON(host->cmd \|\| host->data);
1047		1047
1048	/*	1048	/*
1049	* Update the MMC clock rate if necessary. This may be	1049	* Update the MMC clock rate if necessary. This may be
1050	* necessary if set_ios() is called when a different slot is	1050	* necessary if set_ios() is called when a different slot is
1051	* busy transfering data.	1051	* busy transfering data.
1052	*/	1052	*/
1053	if (host->need_clock_update)	1053	if (host->need_clock_update)
1054	mci_writel(host, MR, host->mode_reg);	1054	mci_writel(host, MR, host->mode_reg);
1055		1055
1056	host->cur_slot->mrq = NULL;	1056	host->cur_slot->mrq = NULL;
1057	host->mrq = NULL;	1057	host->mrq = NULL;
1058	if (!list_empty(&host->queue)) {	1058	if (!list_empty(&host->queue)) {
1059	slot = list_entry(host->queue.next,	1059	slot = list_entry(host->queue.next,
1060	struct atmel_mci_slot, queue_node);	1060	struct atmel_mci_slot, queue_node);
1061	list_del(&slot->queue_node);	1061	list_del(&slot->queue_node);
1062	dev_vdbg(&host->pdev->dev, "list not empty: %s is next\n",	1062	dev_vdbg(&host->pdev->dev, "list not empty: %s is next\n",
1063	mmc_hostname(slot->mmc));	1063	mmc_hostname(slot->mmc));
1064	host->state = STATE_SENDING_CMD;	1064	host->state = STATE_SENDING_CMD;
1065	atmci_start_request(host, slot);	1065	atmci_start_request(host, slot);
1066	} else {	1066	} else {
1067	dev_vdbg(&host->pdev->dev, "list empty\n");	1067	dev_vdbg(&host->pdev->dev, "list empty\n");
1068	host->state = STATE_IDLE;	1068	host->state = STATE_IDLE;
1069	}	1069	}
1070		1070
1071	spin_unlock(&host->lock);	1071	spin_unlock(&host->lock);
1072	mmc_request_done(prev_mmc, mrq);	1072	mmc_request_done(prev_mmc, mrq);
1073	spin_lock(&host->lock);	1073	spin_lock(&host->lock);
1074	}	1074	}
1075		1075
1076	static void atmci_command_complete(struct atmel_mci *host,	1076	static void atmci_command_complete(struct atmel_mci *host,
1077	struct mmc_command *cmd)	1077	struct mmc_command *cmd)
1078	{	1078	{
1079	u32 status = host->cmd_status;	1079	u32 status = host->cmd_status;
1080		1080
1081	/* Read the response from the card (up to 16 bytes) */	1081	/* Read the response from the card (up to 16 bytes) */
1082	cmd->resp[0] = mci_readl(host, RSPR);	1082	cmd->resp[0] = mci_readl(host, RSPR);
1083	cmd->resp[1] = mci_readl(host, RSPR);	1083	cmd->resp[1] = mci_readl(host, RSPR);
1084	cmd->resp[2] = mci_readl(host, RSPR);	1084	cmd->resp[2] = mci_readl(host, RSPR);
1085	cmd->resp[3] = mci_readl(host, RSPR);	1085	cmd->resp[3] = mci_readl(host, RSPR);
1086		1086
1087	if (status & MCI_RTOE)	1087	if (status & MCI_RTOE)
1088	cmd->error = -ETIMEDOUT;	1088	cmd->error = -ETIMEDOUT;
1089	else if ((cmd->flags & MMC_RSP_CRC) && (status & MCI_RCRCE))	1089	else if ((cmd->flags & MMC_RSP_CRC) && (status & MCI_RCRCE))
1090	cmd->error = -EILSEQ;	1090	cmd->error = -EILSEQ;
1091	else if (status & (MCI_RINDE \| MCI_RDIRE \| MCI_RENDE))	1091	else if (status & (MCI_RINDE \| MCI_RDIRE \| MCI_RENDE))
1092	cmd->error = -EIO;	1092	cmd->error = -EIO;
1093	else	1093	else
1094	cmd->error = 0;	1094	cmd->error = 0;
1095		1095
1096	if (cmd->error) {	1096	if (cmd->error) {
1097	dev_dbg(&host->pdev->dev,	1097	dev_dbg(&host->pdev->dev,
1098	"command error: status=0x%08x\n", status);	1098	"command error: status=0x%08x\n", status);
1099		1099
1100	if (cmd->data) {	1100	if (cmd->data) {
1101	host->data = NULL;	1101	host->data = NULL;
1102	atmci_stop_dma(host);	1102	atmci_stop_dma(host);
1103	mci_writel(host, IDR, MCI_NOTBUSY	1103	mci_writel(host, IDR, MCI_NOTBUSY
1104	\| MCI_TXRDY \| MCI_RXRDY	1104	\| MCI_TXRDY \| MCI_RXRDY
1105	\| ATMCI_DATA_ERROR_FLAGS);	1105	\| ATMCI_DATA_ERROR_FLAGS);
1106	}	1106	}
1107	}	1107	}
1108	}	1108	}
1109		1109
1110	static void atmci_detect_change(unsigned long data)	1110	static void atmci_detect_change(unsigned long data)
1111	{	1111	{
1112	struct atmel_mci_slot slot = (struct atmel_mci_slot )data;	1112	struct atmel_mci_slot slot = (struct atmel_mci_slot )data;
1113	bool present;	1113	bool present;
1114	bool present_old;	1114	bool present_old;
1115		1115
1116	/*	1116	/*
1117	* atmci_cleanup_slot() sets the ATMCI_SHUTDOWN flag before	1117	* atmci_cleanup_slot() sets the ATMCI_SHUTDOWN flag before
1118	* freeing the interrupt. We must not re-enable the interrupt	1118	* freeing the interrupt. We must not re-enable the interrupt
1119	* if it has been freed, and if we're shutting down, it	1119	* if it has been freed, and if we're shutting down, it
1120	* doesn't really matter whether the card is present or not.	1120	* doesn't really matter whether the card is present or not.
1121	*/	1121	*/
1122	smp_rmb();	1122	smp_rmb();
1123	if (test_bit(ATMCI_SHUTDOWN, &slot->flags))	1123	if (test_bit(ATMCI_SHUTDOWN, &slot->flags))
1124	return;	1124	return;
1125		1125
1126	enable_irq(gpio_to_irq(slot->detect_pin));	1126	enable_irq(gpio_to_irq(slot->detect_pin));
1127	present = !(gpio_get_value(slot->detect_pin) ^	1127	present = !(gpio_get_value(slot->detect_pin) ^
1128	slot->detect_is_active_high);	1128	slot->detect_is_active_high);
1129	present_old = test_bit(ATMCI_CARD_PRESENT, &slot->flags);	1129	present_old = test_bit(ATMCI_CARD_PRESENT, &slot->flags);
1130		1130
1131	dev_vdbg(&slot->mmc->class_dev, "detect change: %d (was %d)\n",	1131	dev_vdbg(&slot->mmc->class_dev, "detect change: %d (was %d)\n",
1132	present, present_old);	1132	present, present_old);
1133		1133
1134	if (present != present_old) {	1134	if (present != present_old) {
1135	struct atmel_mci *host = slot->host;	1135	struct atmel_mci *host = slot->host;
1136	struct mmc_request *mrq;	1136	struct mmc_request *mrq;
1137		1137
1138	dev_dbg(&slot->mmc->class_dev, "card %s\n",	1138	dev_dbg(&slot->mmc->class_dev, "card %s\n",
1139	present ? "inserted" : "removed");	1139	present ? "inserted" : "removed");
1140		1140
1141	spin_lock(&host->lock);	1141	spin_lock(&host->lock);
1142		1142
1143	if (!present)	1143	if (!present)
1144	clear_bit(ATMCI_CARD_PRESENT, &slot->flags);	1144	clear_bit(ATMCI_CARD_PRESENT, &slot->flags);
1145	else	1145	else
1146	set_bit(ATMCI_CARD_PRESENT, &slot->flags);	1146	set_bit(ATMCI_CARD_PRESENT, &slot->flags);
1147		1147
1148	/* Clean up queue if present */	1148	/* Clean up queue if present */
1149	mrq = slot->mrq;	1149	mrq = slot->mrq;
1150	if (mrq) {	1150	if (mrq) {
1151	if (mrq == host->mrq) {	1151	if (mrq == host->mrq) {
1152	/*	1152	/*
1153	* Reset controller to terminate any ongoing	1153	* Reset controller to terminate any ongoing
1154	* commands or data transfers.	1154	* commands or data transfers.
1155	*/	1155	*/
1156	mci_writel(host, CR, MCI_CR_SWRST);	1156	mci_writel(host, CR, MCI_CR_SWRST);
1157	mci_writel(host, CR, MCI_CR_MCIEN);	1157	mci_writel(host, CR, MCI_CR_MCIEN);
1158	mci_writel(host, MR, host->mode_reg);	1158	mci_writel(host, MR, host->mode_reg);
1159	if (atmci_is_mci2())	1159	if (atmci_is_mci2())
1160	mci_writel(host, CFG, host->cfg_reg);	1160	mci_writel(host, CFG, host->cfg_reg);
1161		1161
1162	host->data = NULL;	1162	host->data = NULL;
1163	host->cmd = NULL;	1163	host->cmd = NULL;
1164		1164
1165	switch (host->state) {	1165	switch (host->state) {
1166	case STATE_IDLE:	1166	case STATE_IDLE:
1167	break;	1167	break;
1168	case STATE_SENDING_CMD:	1168	case STATE_SENDING_CMD:
1169	mrq->cmd->error = -ENOMEDIUM;	1169	mrq->cmd->error = -ENOMEDIUM;
1170	if (!mrq->data)	1170	if (!mrq->data)
1171	break;	1171	break;
1172	/* fall through */	1172	/* fall through */
1173	case STATE_SENDING_DATA:	1173	case STATE_SENDING_DATA:
1174	mrq->data->error = -ENOMEDIUM;	1174	mrq->data->error = -ENOMEDIUM;
1175	atmci_stop_dma(host);	1175	atmci_stop_dma(host);
1176	break;	1176	break;
1177	case STATE_DATA_BUSY:	1177	case STATE_DATA_BUSY:
1178	case STATE_DATA_ERROR:	1178	case STATE_DATA_ERROR:
1179	if (mrq->data->error == -EINPROGRESS)	1179	if (mrq->data->error == -EINPROGRESS)
1180	mrq->data->error = -ENOMEDIUM;	1180	mrq->data->error = -ENOMEDIUM;
1181	if (!mrq->stop)	1181	if (!mrq->stop)
1182	break;	1182	break;
1183	/* fall through */	1183	/* fall through */
1184	case STATE_SENDING_STOP:	1184	case STATE_SENDING_STOP:
1185	mrq->stop->error = -ENOMEDIUM;	1185	mrq->stop->error = -ENOMEDIUM;
1186	break;	1186	break;
1187	}	1187	}
1188		1188
1189	atmci_request_end(host, mrq);	1189	atmci_request_end(host, mrq);
1190	} else {	1190	} else {
1191	list_del(&slot->queue_node);	1191	list_del(&slot->queue_node);
1192	mrq->cmd->error = -ENOMEDIUM;	1192	mrq->cmd->error = -ENOMEDIUM;
1193	if (mrq->data)	1193	if (mrq->data)
1194	mrq->data->error = -ENOMEDIUM;	1194	mrq->data->error = -ENOMEDIUM;
1195	if (mrq->stop)	1195	if (mrq->stop)
1196	mrq->stop->error = -ENOMEDIUM;	1196	mrq->stop->error = -ENOMEDIUM;
1197		1197
1198	spin_unlock(&host->lock);	1198	spin_unlock(&host->lock);
1199	mmc_request_done(slot->mmc, mrq);	1199	mmc_request_done(slot->mmc, mrq);
1200	spin_lock(&host->lock);	1200	spin_lock(&host->lock);
1201	}	1201	}
1202	}	1202	}
1203	spin_unlock(&host->lock);	1203	spin_unlock(&host->lock);
1204		1204
1205	mmc_detect_change(slot->mmc, 0);	1205	mmc_detect_change(slot->mmc, 0);
1206	}	1206	}
1207	}	1207	}
1208		1208
1209	static void atmci_tasklet_func(unsigned long priv)	1209	static void atmci_tasklet_func(unsigned long priv)
1210	{	1210	{
1211	struct atmel_mci host = (struct atmel_mci )priv;	1211	struct atmel_mci host = (struct atmel_mci )priv;
1212	struct mmc_request *mrq = host->mrq;	1212	struct mmc_request *mrq = host->mrq;
1213	struct mmc_data *data = host->data;	1213	struct mmc_data *data = host->data;
1214	struct mmc_command *cmd = host->cmd;	1214	struct mmc_command *cmd = host->cmd;
1215	enum atmel_mci_state state = host->state;	1215	enum atmel_mci_state state = host->state;
1216	enum atmel_mci_state prev_state;	1216	enum atmel_mci_state prev_state;
1217	u32 status;	1217	u32 status;
1218		1218
1219	spin_lock(&host->lock);	1219	spin_lock(&host->lock);
1220		1220
1221	state = host->state;	1221	state = host->state;
1222		1222
1223	dev_vdbg(&host->pdev->dev,	1223	dev_vdbg(&host->pdev->dev,
1224	"tasklet: state %u pending/completed/mask %lx/%lx/%x\n",	1224	"tasklet: state %u pending/completed/mask %lx/%lx/%x\n",
1225	state, host->pending_events, host->completed_events,	1225	state, host->pending_events, host->completed_events,
1226	mci_readl(host, IMR));	1226	mci_readl(host, IMR));
1227		1227
1228	do {	1228	do {
1229	prev_state = state;	1229	prev_state = state;
1230		1230
1231	switch (state) {	1231	switch (state) {
1232	case STATE_IDLE:	1232	case STATE_IDLE:
1233	break;	1233	break;
1234		1234
1235	case STATE_SENDING_CMD:	1235	case STATE_SENDING_CMD:
1236	if (!atmci_test_and_clear_pending(host,	1236	if (!atmci_test_and_clear_pending(host,
1237	EVENT_CMD_COMPLETE))	1237	EVENT_CMD_COMPLETE))
1238	break;	1238	break;
1239		1239
1240	host->cmd = NULL;	1240	host->cmd = NULL;
1241	atmci_set_completed(host, EVENT_CMD_COMPLETE);	1241	atmci_set_completed(host, EVENT_CMD_COMPLETE);
1242	atmci_command_complete(host, mrq->cmd);	1242	atmci_command_complete(host, mrq->cmd);
1243	if (!mrq->data \|\| cmd->error) {	1243	if (!mrq->data \|\| cmd->error) {
1244	atmci_request_end(host, host->mrq);	1244	atmci_request_end(host, host->mrq);
1245	goto unlock;	1245	goto unlock;
1246	}	1246	}
1247		1247
1248	prev_state = state = STATE_SENDING_DATA;	1248	prev_state = state = STATE_SENDING_DATA;
1249	/* fall through */	1249	/* fall through */
1250		1250
1251	case STATE_SENDING_DATA:	1251	case STATE_SENDING_DATA:
1252	if (atmci_test_and_clear_pending(host,	1252	if (atmci_test_and_clear_pending(host,
1253	EVENT_DATA_ERROR)) {	1253	EVENT_DATA_ERROR)) {
1254	atmci_stop_dma(host);	1254	atmci_stop_dma(host);
1255	if (data->stop)	1255	if (data->stop)
1256	send_stop_cmd(host, data);	1256	send_stop_cmd(host, data);
1257	state = STATE_DATA_ERROR;	1257	state = STATE_DATA_ERROR;
1258	break;	1258	break;
1259	}	1259	}
1260		1260
1261	if (!atmci_test_and_clear_pending(host,	1261	if (!atmci_test_and_clear_pending(host,
1262	EVENT_XFER_COMPLETE))	1262	EVENT_XFER_COMPLETE))
1263	break;	1263	break;
1264		1264
1265	atmci_set_completed(host, EVENT_XFER_COMPLETE);	1265	atmci_set_completed(host, EVENT_XFER_COMPLETE);
1266	prev_state = state = STATE_DATA_BUSY;	1266	prev_state = state = STATE_DATA_BUSY;
1267	/* fall through */	1267	/* fall through */
1268		1268
1269	case STATE_DATA_BUSY:	1269	case STATE_DATA_BUSY:
1270	if (!atmci_test_and_clear_pending(host,	1270	if (!atmci_test_and_clear_pending(host,
1271	EVENT_DATA_COMPLETE))	1271	EVENT_DATA_COMPLETE))
1272	break;	1272	break;
1273		1273
1274	host->data = NULL;	1274	host->data = NULL;
1275	atmci_set_completed(host, EVENT_DATA_COMPLETE);	1275	atmci_set_completed(host, EVENT_DATA_COMPLETE);
1276	status = host->data_status;	1276	status = host->data_status;
1277	if (unlikely(status & ATMCI_DATA_ERROR_FLAGS)) {	1277	if (unlikely(status & ATMCI_DATA_ERROR_FLAGS)) {
1278	if (status & MCI_DTOE) {	1278	if (status & MCI_DTOE) {
1279	dev_dbg(&host->pdev->dev,	1279	dev_dbg(&host->pdev->dev,
1280	"data timeout error\n");	1280	"data timeout error\n");
1281	data->error = -ETIMEDOUT;	1281	data->error = -ETIMEDOUT;
1282	} else if (status & MCI_DCRCE) {	1282	} else if (status & MCI_DCRCE) {
1283	dev_dbg(&host->pdev->dev,	1283	dev_dbg(&host->pdev->dev,
1284	"data CRC error\n");	1284	"data CRC error\n");
1285	data->error = -EILSEQ;	1285	data->error = -EILSEQ;
1286	} else {	1286	} else {
1287	dev_dbg(&host->pdev->dev,	1287	dev_dbg(&host->pdev->dev,
1288	"data FIFO error (status=%08x)\n",	1288	"data FIFO error (status=%08x)\n",
1289	status);	1289	status);
1290	data->error = -EIO;	1290	data->error = -EIO;
1291	}	1291	}
1292	} else {	1292	} else {
1293	data->bytes_xfered = data->blocks * data->blksz;	1293	data->bytes_xfered = data->blocks * data->blksz;
1294	data->error = 0;	1294	data->error = 0;
1295	}	1295	}
1296		1296
1297	if (!data->stop) {	1297	if (!data->stop) {
1298	atmci_request_end(host, host->mrq);	1298	atmci_request_end(host, host->mrq);
1299	goto unlock;	1299	goto unlock;
1300	}	1300	}
1301		1301
1302	prev_state = state = STATE_SENDING_STOP;	1302	prev_state = state = STATE_SENDING_STOP;
1303	if (!data->error)	1303	if (!data->error)
1304	send_stop_cmd(host, data);	1304	send_stop_cmd(host, data);
1305	/* fall through */	1305	/* fall through */
1306		1306
1307	case STATE_SENDING_STOP:	1307	case STATE_SENDING_STOP:
1308	if (!atmci_test_and_clear_pending(host,	1308	if (!atmci_test_and_clear_pending(host,
1309	EVENT_CMD_COMPLETE))	1309	EVENT_CMD_COMPLETE))
1310	break;	1310	break;
1311		1311
1312	host->cmd = NULL;	1312	host->cmd = NULL;
1313	atmci_command_complete(host, mrq->stop);	1313	atmci_command_complete(host, mrq->stop);
1314	atmci_request_end(host, host->mrq);	1314	atmci_request_end(host, host->mrq);
1315	goto unlock;	1315	goto unlock;
1316		1316
1317	case STATE_DATA_ERROR:	1317	case STATE_DATA_ERROR:
1318	if (!atmci_test_and_clear_pending(host,	1318	if (!atmci_test_and_clear_pending(host,
1319	EVENT_XFER_COMPLETE))	1319	EVENT_XFER_COMPLETE))
1320	break;	1320	break;
1321		1321
1322	state = STATE_DATA_BUSY;	1322	state = STATE_DATA_BUSY;
1323	break;	1323	break;
1324	}	1324	}
1325	} while (state != prev_state);	1325	} while (state != prev_state);
1326		1326
1327	host->state = state;	1327	host->state = state;
1328		1328
1329	unlock:	1329	unlock:
1330	spin_unlock(&host->lock);	1330	spin_unlock(&host->lock);
1331	}	1331	}
1332		1332
1333	static void atmci_read_data_pio(struct atmel_mci *host)	1333	static void atmci_read_data_pio(struct atmel_mci *host)
1334	{	1334	{
1335	struct scatterlist *sg = host->sg;	1335	struct scatterlist *sg = host->sg;
1336	void *buf = sg_virt(sg);	1336	void *buf = sg_virt(sg);
1337	unsigned int offset = host->pio_offset;	1337	unsigned int offset = host->pio_offset;
1338	struct mmc_data *data = host->data;	1338	struct mmc_data *data = host->data;
1339	u32 value;	1339	u32 value;
1340	u32 status;	1340	u32 status;
1341	unsigned int nbytes = 0;	1341	unsigned int nbytes = 0;
1342		1342
1343	do {	1343	do {
1344	value = mci_readl(host, RDR);	1344	value = mci_readl(host, RDR);
1345	if (likely(offset + 4 <= sg->length)) {	1345	if (likely(offset + 4 <= sg->length)) {
1346	put_unaligned(value, (u32 *)(buf + offset));	1346	put_unaligned(value, (u32 *)(buf + offset));
1347		1347
1348	offset += 4;	1348	offset += 4;
1349	nbytes += 4;	1349	nbytes += 4;
1350		1350
1351	if (offset == sg->length) {	1351	if (offset == sg->length) {
1352	flush_dcache_page(sg_page(sg));	1352	flush_dcache_page(sg_page(sg));
1353	host->sg = sg = sg_next(sg);	1353	host->sg = sg = sg_next(sg);
1354	if (!sg)	1354	if (!sg)
1355	goto done;	1355	goto done;
1356		1356
1357	offset = 0;	1357	offset = 0;
1358	buf = sg_virt(sg);	1358	buf = sg_virt(sg);
1359	}	1359	}
1360	} else {	1360	} else {
1361	unsigned int remaining = sg->length - offset;	1361	unsigned int remaining = sg->length - offset;
1362	memcpy(buf + offset, &value, remaining);	1362	memcpy(buf + offset, &value, remaining);
1363	nbytes += remaining;	1363	nbytes += remaining;
1364		1364
1365	flush_dcache_page(sg_page(sg));	1365	flush_dcache_page(sg_page(sg));
1366	host->sg = sg = sg_next(sg);	1366	host->sg = sg = sg_next(sg);
1367	if (!sg)	1367	if (!sg)
1368	goto done;	1368	goto done;
1369		1369
1370	offset = 4 - remaining;	1370	offset = 4 - remaining;
1371	buf = sg_virt(sg);	1371	buf = sg_virt(sg);
1372	memcpy(buf, (u8 *)&value + remaining, offset);	1372	memcpy(buf, (u8 *)&value + remaining, offset);
1373	nbytes += offset;	1373	nbytes += offset;
1374	}	1374	}
1375		1375
1376	status = mci_readl(host, SR);	1376	status = mci_readl(host, SR);
1377	if (status & ATMCI_DATA_ERROR_FLAGS) {	1377	if (status & ATMCI_DATA_ERROR_FLAGS) {
1378	mci_writel(host, IDR, (MCI_NOTBUSY \| MCI_RXRDY	1378	mci_writel(host, IDR, (MCI_NOTBUSY \| MCI_RXRDY
1379	\| ATMCI_DATA_ERROR_FLAGS));	1379	\| ATMCI_DATA_ERROR_FLAGS));
1380	host->data_status = status;	1380	host->data_status = status;
1381	data->bytes_xfered += nbytes;	1381	data->bytes_xfered += nbytes;
1382	smp_wmb();	1382	smp_wmb();
1383	atmci_set_pending(host, EVENT_DATA_ERROR);	1383	atmci_set_pending(host, EVENT_DATA_ERROR);
1384	tasklet_schedule(&host->tasklet);	1384	tasklet_schedule(&host->tasklet);
1385	return;	1385	return;
1386	}	1386	}
1387	} while (status & MCI_RXRDY);	1387	} while (status & MCI_RXRDY);
1388		1388
1389	host->pio_offset = offset;	1389	host->pio_offset = offset;
1390	data->bytes_xfered += nbytes;	1390	data->bytes_xfered += nbytes;
1391		1391
1392	return;	1392	return;
1393		1393
1394	done:	1394	done:
1395	mci_writel(host, IDR, MCI_RXRDY);	1395	mci_writel(host, IDR, MCI_RXRDY);
1396	mci_writel(host, IER, MCI_NOTBUSY);	1396	mci_writel(host, IER, MCI_NOTBUSY);
1397	data->bytes_xfered += nbytes;	1397	data->bytes_xfered += nbytes;
1398	smp_wmb();	1398	smp_wmb();
1399	atmci_set_pending(host, EVENT_XFER_COMPLETE);	1399	atmci_set_pending(host, EVENT_XFER_COMPLETE);
1400	}	1400	}
1401		1401
1402	static void atmci_write_data_pio(struct atmel_mci *host)	1402	static void atmci_write_data_pio(struct atmel_mci *host)
1403	{	1403	{
1404	struct scatterlist *sg = host->sg;	1404	struct scatterlist *sg = host->sg;
1405	void *buf = sg_virt(sg);	1405	void *buf = sg_virt(sg);
1406	unsigned int offset = host->pio_offset;	1406	unsigned int offset = host->pio_offset;
1407	struct mmc_data *data = host->data;	1407	struct mmc_data *data = host->data;
1408	u32 value;	1408	u32 value;
1409	u32 status;	1409	u32 status;
1410	unsigned int nbytes = 0;	1410	unsigned int nbytes = 0;
1411		1411
1412	do {	1412	do {
1413	if (likely(offset + 4 <= sg->length)) {	1413	if (likely(offset + 4 <= sg->length)) {
1414	value = get_unaligned((u32 *)(buf + offset));	1414	value = get_unaligned((u32 *)(buf + offset));
1415	mci_writel(host, TDR, value);	1415	mci_writel(host, TDR, value);
1416		1416
1417	offset += 4;	1417	offset += 4;
1418	nbytes += 4;	1418	nbytes += 4;
1419	if (offset == sg->length) {	1419	if (offset == sg->length) {
1420	host->sg = sg = sg_next(sg);	1420	host->sg = sg = sg_next(sg);
1421	if (!sg)	1421	if (!sg)
1422	goto done;	1422	goto done;
1423		1423
1424	offset = 0;	1424	offset = 0;
1425	buf = sg_virt(sg);	1425	buf = sg_virt(sg);
1426	}	1426	}
1427	} else {	1427	} else {
1428	unsigned int remaining = sg->length - offset;	1428	unsigned int remaining = sg->length - offset;
1429		1429
1430	value = 0;	1430	value = 0;
1431	memcpy(&value, buf + offset, remaining);	1431	memcpy(&value, buf + offset, remaining);
1432	nbytes += remaining;	1432	nbytes += remaining;
1433		1433
1434	host->sg = sg = sg_next(sg);	1434	host->sg = sg = sg_next(sg);
1435	if (!sg) {	1435	if (!sg) {
1436	mci_writel(host, TDR, value);	1436	mci_writel(host, TDR, value);
1437	goto done;	1437	goto done;
1438	}	1438	}
1439		1439
1440	offset = 4 - remaining;	1440	offset = 4 - remaining;
1441	buf = sg_virt(sg);	1441	buf = sg_virt(sg);
1442	memcpy((u8 *)&value + remaining, buf, offset);	1442	memcpy((u8 *)&value + remaining, buf, offset);
1443	mci_writel(host, TDR, value);	1443	mci_writel(host, TDR, value);
1444	nbytes += offset;	1444	nbytes += offset;
1445	}	1445	}
1446		1446
1447	status = mci_readl(host, SR);	1447	status = mci_readl(host, SR);
1448	if (status & ATMCI_DATA_ERROR_FLAGS) {	1448	if (status & ATMCI_DATA_ERROR_FLAGS) {
1449	mci_writel(host, IDR, (MCI_NOTBUSY \| MCI_TXRDY	1449	mci_writel(host, IDR, (MCI_NOTBUSY \| MCI_TXRDY
1450	\| ATMCI_DATA_ERROR_FLAGS));	1450	\| ATMCI_DATA_ERROR_FLAGS));
1451	host->data_status = status;	1451	host->data_status = status;
1452	data->bytes_xfered += nbytes;	1452	data->bytes_xfered += nbytes;
1453	smp_wmb();	1453	smp_wmb();
1454	atmci_set_pending(host, EVENT_DATA_ERROR);	1454	atmci_set_pending(host, EVENT_DATA_ERROR);
1455	tasklet_schedule(&host->tasklet);	1455	tasklet_schedule(&host->tasklet);
1456	return;	1456	return;
1457	}	1457	}
1458	} while (status & MCI_TXRDY);	1458	} while (status & MCI_TXRDY);
1459		1459
1460	host->pio_offset = offset;	1460	host->pio_offset = offset;
1461	data->bytes_xfered += nbytes;	1461	data->bytes_xfered += nbytes;
1462		1462
1463	return;	1463	return;
1464		1464
1465	done:	1465	done:
1466	mci_writel(host, IDR, MCI_TXRDY);	1466	mci_writel(host, IDR, MCI_TXRDY);
1467	mci_writel(host, IER, MCI_NOTBUSY);	1467	mci_writel(host, IER, MCI_NOTBUSY);
1468	data->bytes_xfered += nbytes;	1468	data->bytes_xfered += nbytes;
1469	smp_wmb();	1469	smp_wmb();
1470	atmci_set_pending(host, EVENT_XFER_COMPLETE);	1470	atmci_set_pending(host, EVENT_XFER_COMPLETE);
1471	}	1471	}
1472		1472
1473	static void atmci_cmd_interrupt(struct atmel_mci *host, u32 status)	1473	static void atmci_cmd_interrupt(struct atmel_mci *host, u32 status)
1474	{	1474	{
1475	mci_writel(host, IDR, MCI_CMDRDY);	1475	mci_writel(host, IDR, MCI_CMDRDY);
1476		1476
1477	host->cmd_status = status;	1477	host->cmd_status = status;
1478	smp_wmb();	1478	smp_wmb();
1479	atmci_set_pending(host, EVENT_CMD_COMPLETE);	1479	atmci_set_pending(host, EVENT_CMD_COMPLETE);
1480	tasklet_schedule(&host->tasklet);	1480	tasklet_schedule(&host->tasklet);
1481	}	1481	}
1482		1482
1483	static irqreturn_t atmci_interrupt(int irq, void *dev_id)	1483	static irqreturn_t atmci_interrupt(int irq, void *dev_id)
1484	{	1484	{
1485	struct atmel_mci *host = dev_id;	1485	struct atmel_mci *host = dev_id;
1486	u32 status, mask, pending;	1486	u32 status, mask, pending;
1487	unsigned int pass_count = 0;	1487	unsigned int pass_count = 0;
1488		1488
1489	do {	1489	do {
1490	status = mci_readl(host, SR);	1490	status = mci_readl(host, SR);
1491	mask = mci_readl(host, IMR);	1491	mask = mci_readl(host, IMR);
1492	pending = status & mask;	1492	pending = status & mask;
1493	if (!pending)	1493	if (!pending)
1494	break;	1494	break;
1495		1495
1496	if (pending & ATMCI_DATA_ERROR_FLAGS) {	1496	if (pending & ATMCI_DATA_ERROR_FLAGS) {
1497	mci_writel(host, IDR, ATMCI_DATA_ERROR_FLAGS	1497	mci_writel(host, IDR, ATMCI_DATA_ERROR_FLAGS
1498	\| MCI_RXRDY \| MCI_TXRDY);	1498	\| MCI_RXRDY \| MCI_TXRDY);
1499	pending &= mci_readl(host, IMR);	1499	pending &= mci_readl(host, IMR);
1500		1500
1501	host->data_status = status;	1501	host->data_status = status;
1502	smp_wmb();	1502	smp_wmb();
1503	atmci_set_pending(host, EVENT_DATA_ERROR);	1503	atmci_set_pending(host, EVENT_DATA_ERROR);
1504	tasklet_schedule(&host->tasklet);	1504	tasklet_schedule(&host->tasklet);
1505	}	1505	}
1506	if (pending & MCI_NOTBUSY) {	1506	if (pending & MCI_NOTBUSY) {
1507	mci_writel(host, IDR,	1507	mci_writel(host, IDR,
1508	ATMCI_DATA_ERROR_FLAGS \| MCI_NOTBUSY);	1508	ATMCI_DATA_ERROR_FLAGS \| MCI_NOTBUSY);
1509	if (!host->data_status)	1509	if (!host->data_status)
1510	host->data_status = status;	1510	host->data_status = status;
1511	smp_wmb();	1511	smp_wmb();
1512	atmci_set_pending(host, EVENT_DATA_COMPLETE);	1512	atmci_set_pending(host, EVENT_DATA_COMPLETE);
1513	tasklet_schedule(&host->tasklet);	1513	tasklet_schedule(&host->tasklet);
1514	}	1514	}
1515	if (pending & MCI_RXRDY)	1515	if (pending & MCI_RXRDY)
1516	atmci_read_data_pio(host);	1516	atmci_read_data_pio(host);
1517	if (pending & MCI_TXRDY)	1517	if (pending & MCI_TXRDY)
1518	atmci_write_data_pio(host);	1518	atmci_write_data_pio(host);
1519		1519
1520	if (pending & MCI_CMDRDY)	1520	if (pending & MCI_CMDRDY)
1521	atmci_cmd_interrupt(host, status);	1521	atmci_cmd_interrupt(host, status);
1522	} while (pass_count++ < 5);	1522	} while (pass_count++ < 5);
1523		1523
1524	return pass_count ? IRQ_HANDLED : IRQ_NONE;	1524	return pass_count ? IRQ_HANDLED : IRQ_NONE;
1525	}	1525	}
1526		1526
1527	static irqreturn_t atmci_detect_interrupt(int irq, void *dev_id)	1527	static irqreturn_t atmci_detect_interrupt(int irq, void *dev_id)
1528	{	1528	{
1529	struct atmel_mci_slot *slot = dev_id;	1529	struct atmel_mci_slot *slot = dev_id;
1530		1530
1531	/*	1531	/*
1532	* Disable interrupts until the pin has stabilized and check	1532	* Disable interrupts until the pin has stabilized and check
1533	* the state then. Use mod_timer() since we may be in the	1533	* the state then. Use mod_timer() since we may be in the
1534	* middle of the timer routine when this interrupt triggers.	1534	* middle of the timer routine when this interrupt triggers.
1535	*/	1535	*/
1536	disable_irq_nosync(irq);	1536	disable_irq_nosync(irq);
1537	mod_timer(&slot->detect_timer, jiffies + msecs_to_jiffies(20));	1537	mod_timer(&slot->detect_timer, jiffies + msecs_to_jiffies(20));
1538		1538
1539	return IRQ_HANDLED;	1539	return IRQ_HANDLED;
1540	}	1540	}
1541		1541
1542	static int __init atmci_init_slot(struct atmel_mci *host,	1542	static int __init atmci_init_slot(struct atmel_mci *host,
1543	struct mci_slot_pdata *slot_data, unsigned int id,	1543	struct mci_slot_pdata *slot_data, unsigned int id,
1544	u32 sdc_reg)	1544	u32 sdc_reg)
1545	{	1545	{
1546	struct mmc_host *mmc;	1546	struct mmc_host *mmc;
1547	struct atmel_mci_slot *slot;	1547	struct atmel_mci_slot *slot;
1548		1548
1549	mmc = mmc_alloc_host(sizeof(struct atmel_mci_slot), &host->pdev->dev);	1549	mmc = mmc_alloc_host(sizeof(struct atmel_mci_slot), &host->pdev->dev);
1550	if (!mmc)	1550	if (!mmc)
1551	return -ENOMEM;	1551	return -ENOMEM;
1552		1552
1553	slot = mmc_priv(mmc);	1553	slot = mmc_priv(mmc);
1554	slot->mmc = mmc;	1554	slot->mmc = mmc;
1555	slot->host = host;	1555	slot->host = host;
1556	slot->detect_pin = slot_data->detect_pin;	1556	slot->detect_pin = slot_data->detect_pin;
1557	slot->wp_pin = slot_data->wp_pin;	1557	slot->wp_pin = slot_data->wp_pin;
1558	slot->detect_is_active_high = slot_data->detect_is_active_high;	1558	slot->detect_is_active_high = slot_data->detect_is_active_high;
1559	slot->sdc_reg = sdc_reg;	1559	slot->sdc_reg = sdc_reg;
1560		1560
1561	mmc->ops = &atmci_ops;	1561	mmc->ops = &atmci_ops;
1562	mmc->f_min = DIV_ROUND_UP(host->bus_hz, 512);	1562	mmc->f_min = DIV_ROUND_UP(host->bus_hz, 512);
1563	mmc->f_max = host->bus_hz / 2;	1563	mmc->f_max = host->bus_hz / 2;
1564	mmc->ocr_avail = MMC_VDD_32_33 \| MMC_VDD_33_34;	1564	mmc->ocr_avail = MMC_VDD_32_33 \| MMC_VDD_33_34;
1565	if (slot_data->bus_width >= 4)	1565	if (slot_data->bus_width >= 4)
1566	mmc->caps \|= MMC_CAP_4_BIT_DATA;	1566	mmc->caps \|= MMC_CAP_4_BIT_DATA;
1567		1567
1568	mmc->max_hw_segs = 64;	1568	mmc->max_hw_segs = 64;
1569	mmc->max_phys_segs = 64;	1569	mmc->max_phys_segs = 64;
1570	mmc->max_req_size = 32768 * 512;	1570	mmc->max_req_size = 32768 * 512;
1571	mmc->max_blk_size = 32768;	1571	mmc->max_blk_size = 32768;
1572	mmc->max_blk_count = 512;	1572	mmc->max_blk_count = 512;
1573		1573
1574	/* Assume card is present initially */	1574	/* Assume card is present initially */
1575	set_bit(ATMCI_CARD_PRESENT, &slot->flags);	1575	set_bit(ATMCI_CARD_PRESENT, &slot->flags);
1576	if (gpio_is_valid(slot->detect_pin)) {	1576	if (gpio_is_valid(slot->detect_pin)) {
1577	if (gpio_request(slot->detect_pin, "mmc_detect")) {	1577	if (gpio_request(slot->detect_pin, "mmc_detect")) {
1578	dev_dbg(&mmc->class_dev, "no detect pin available\n");	1578	dev_dbg(&mmc->class_dev, "no detect pin available\n");
1579	slot->detect_pin = -EBUSY;	1579	slot->detect_pin = -EBUSY;
1580	} else if (gpio_get_value(slot->detect_pin) ^	1580	} else if (gpio_get_value(slot->detect_pin) ^
1581	slot->detect_is_active_high) {	1581	slot->detect_is_active_high) {
1582	clear_bit(ATMCI_CARD_PRESENT, &slot->flags);	1582	clear_bit(ATMCI_CARD_PRESENT, &slot->flags);
1583	}	1583	}
1584	}	1584	}
1585		1585
1586	if (!gpio_is_valid(slot->detect_pin))	1586	if (!gpio_is_valid(slot->detect_pin))
1587	mmc->caps \|= MMC_CAP_NEEDS_POLL;	1587	mmc->caps \|= MMC_CAP_NEEDS_POLL;
1588		1588
1589	if (gpio_is_valid(slot->wp_pin)) {	1589	if (gpio_is_valid(slot->wp_pin)) {
1590	if (gpio_request(slot->wp_pin, "mmc_wp")) {	1590	if (gpio_request(slot->wp_pin, "mmc_wp")) {
1591	dev_dbg(&mmc->class_dev, "no WP pin available\n");	1591	dev_dbg(&mmc->class_dev, "no WP pin available\n");
1592	slot->wp_pin = -EBUSY;	1592	slot->wp_pin = -EBUSY;
1593	}	1593	}
1594	}	1594	}
1595		1595
1596	host->slot[id] = slot;	1596	host->slot[id] = slot;
1597	mmc_add_host(mmc);	1597	mmc_add_host(mmc);
1598		1598
1599	if (gpio_is_valid(slot->detect_pin)) {	1599	if (gpio_is_valid(slot->detect_pin)) {
1600	int ret;	1600	int ret;
1601		1601
1602	setup_timer(&slot->detect_timer, atmci_detect_change,	1602	setup_timer(&slot->detect_timer, atmci_detect_change,
1603	(unsigned long)slot);	1603	(unsigned long)slot);
1604		1604
1605	ret = request_irq(gpio_to_irq(slot->detect_pin),	1605	ret = request_irq(gpio_to_irq(slot->detect_pin),
1606	atmci_detect_interrupt,	1606	atmci_detect_interrupt,
1607	IRQF_TRIGGER_FALLING \| IRQF_TRIGGER_RISING,	1607	IRQF_TRIGGER_FALLING \| IRQF_TRIGGER_RISING,
1608	"mmc-detect", slot);	1608	"mmc-detect", slot);
1609	if (ret) {	1609	if (ret) {
1610	dev_dbg(&mmc->class_dev,	1610	dev_dbg(&mmc->class_dev,
1611	"could not request IRQ %d for detect pin\n",	1611	"could not request IRQ %d for detect pin\n",
1612	gpio_to_irq(slot->detect_pin));	1612	gpio_to_irq(slot->detect_pin));
1613	gpio_free(slot->detect_pin);	1613	gpio_free(slot->detect_pin);
1614	slot->detect_pin = -EBUSY;	1614	slot->detect_pin = -EBUSY;
1615	}	1615	}
1616	}	1616	}
1617		1617
1618	atmci_init_debugfs(slot);	1618	atmci_init_debugfs(slot);
1619		1619
1620	return 0;	1620	return 0;
1621	}	1621	}
1622		1622
1623	static void __exit atmci_cleanup_slot(struct atmel_mci_slot *slot,	1623	static void __exit atmci_cleanup_slot(struct atmel_mci_slot *slot,
1624	unsigned int id)	1624	unsigned int id)
1625	{	1625	{
1626	/* Debugfs stuff is cleaned up by mmc core */	1626	/* Debugfs stuff is cleaned up by mmc core */
1627		1627
1628	set_bit(ATMCI_SHUTDOWN, &slot->flags);	1628	set_bit(ATMCI_SHUTDOWN, &slot->flags);
1629	smp_wmb();	1629	smp_wmb();
1630		1630
1631	mmc_remove_host(slot->mmc);	1631	mmc_remove_host(slot->mmc);
1632		1632
1633	if (gpio_is_valid(slot->detect_pin)) {	1633	if (gpio_is_valid(slot->detect_pin)) {
1634	int pin = slot->detect_pin;	1634	int pin = slot->detect_pin;
1635		1635
1636	free_irq(gpio_to_irq(pin), slot);	1636	free_irq(gpio_to_irq(pin), slot);
1637	del_timer_sync(&slot->detect_timer);	1637	del_timer_sync(&slot->detect_timer);
1638	gpio_free(pin);	1638	gpio_free(pin);
1639	}	1639	}
1640	if (gpio_is_valid(slot->wp_pin))	1640	if (gpio_is_valid(slot->wp_pin))
1641	gpio_free(slot->wp_pin);	1641	gpio_free(slot->wp_pin);
1642		1642
1643	slot->host->slot[id] = NULL;	1643	slot->host->slot[id] = NULL;
1644	mmc_free_host(slot->mmc);	1644	mmc_free_host(slot->mmc);
1645	}	1645	}
1646		1646
1647	#ifdef CONFIG_MMC_ATMELMCI_DMA	1647	#ifdef CONFIG_MMC_ATMELMCI_DMA
1648	static bool filter(struct dma_chan chan, void slave)	1648	static bool filter(struct dma_chan chan, void slave)
1649	{	1649	{
1650	struct mci_dma_data *sl = slave;	1650	struct mci_dma_data *sl = slave;
1651		1651
1652	if (sl && find_slave_dev(sl) == chan->device->dev) {	1652	if (sl && find_slave_dev(sl) == chan->device->dev) {
1653	chan->private = slave_data_ptr(sl);	1653	chan->private = slave_data_ptr(sl);
1654	return true;	1654	return true;
1655	} else {	1655	} else {
1656	return false;	1656	return false;
1657	}	1657	}
1658	}	1658	}
1659		1659
1660	static void atmci_configure_dma(struct atmel_mci *host)	1660	static void atmci_configure_dma(struct atmel_mci *host)
1661	{	1661	{
1662	struct mci_platform_data *pdata;	1662	struct mci_platform_data *pdata;
1663		1663
1664	if (host == NULL)	1664	if (host == NULL)
1665	return;	1665	return;
1666		1666
1667	pdata = host->pdev->dev.platform_data;	1667	pdata = host->pdev->dev.platform_data;
1668		1668
1669	if (pdata && find_slave_dev(pdata->dma_slave)) {	1669	if (pdata && find_slave_dev(pdata->dma_slave)) {
1670	dma_cap_mask_t mask;	1670	dma_cap_mask_t mask;
1671		1671
1672	setup_dma_addr(pdata->dma_slave,	1672	setup_dma_addr(pdata->dma_slave,
1673	host->mapbase + MCI_TDR,	1673	host->mapbase + MCI_TDR,
1674	host->mapbase + MCI_RDR);	1674	host->mapbase + MCI_RDR);
1675		1675
1676	/* Try to grab a DMA channel */	1676	/* Try to grab a DMA channel */
1677	dma_cap_zero(mask);	1677	dma_cap_zero(mask);
1678	dma_cap_set(DMA_SLAVE, mask);	1678	dma_cap_set(DMA_SLAVE, mask);
1679	host->dma.chan =	1679	host->dma.chan =
1680	dma_request_channel(mask, filter, pdata->dma_slave);	1680	dma_request_channel(mask, filter, pdata->dma_slave);
1681	}	1681	}
1682	if (!host->dma.chan)	1682	if (!host->dma.chan)
1683	dev_notice(&host->pdev->dev, "DMA not available, using PIO\n");	1683	dev_notice(&host->pdev->dev, "DMA not available, using PIO\n");
1684	else	1684	else
1685	dev_info(&host->pdev->dev,	1685	dev_info(&host->pdev->dev,
1686	"Using %s for DMA transfers\n",	1686	"Using %s for DMA transfers\n",
1687	dma_chan_name(host->dma.chan));	1687	dma_chan_name(host->dma.chan));
1688	}	1688	}
1689	#else	1689	#else
1690	static void atmci_configure_dma(struct atmel_mci *host) {}	1690	static void atmci_configure_dma(struct atmel_mci *host) {}
1691	#endif	1691	#endif
1692		1692
1693	static int __init atmci_probe(struct platform_device *pdev)	1693	static int __init atmci_probe(struct platform_device *pdev)
1694	{	1694	{
1695	struct mci_platform_data *pdata;	1695	struct mci_platform_data *pdata;
1696	struct atmel_mci *host;	1696	struct atmel_mci *host;
1697	struct resource *regs;	1697	struct resource *regs;
1698	unsigned int nr_slots;	1698	unsigned int nr_slots;
1699	int irq;	1699	int irq;
1700	int ret;	1700	int ret;
1701		1701
1702	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);	1702	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1703	if (!regs)	1703	if (!regs)
1704	return -ENXIO;	1704	return -ENXIO;
1705	pdata = pdev->dev.platform_data;	1705	pdata = pdev->dev.platform_data;
1706	if (!pdata)	1706	if (!pdata)
1707	return -ENXIO;	1707	return -ENXIO;
1708	irq = platform_get_irq(pdev, 0);	1708	irq = platform_get_irq(pdev, 0);
1709	if (irq < 0)	1709	if (irq < 0)
1710	return irq;	1710	return irq;
1711		1711
1712	host = kzalloc(sizeof(struct atmel_mci), GFP_KERNEL);	1712	host = kzalloc(sizeof(struct atmel_mci), GFP_KERNEL);
1713	if (!host)	1713	if (!host)
1714	return -ENOMEM;	1714	return -ENOMEM;
1715		1715
1716	host->pdev = pdev;	1716	host->pdev = pdev;
1717	spin_lock_init(&host->lock);	1717	spin_lock_init(&host->lock);
1718	INIT_LIST_HEAD(&host->queue);	1718	INIT_LIST_HEAD(&host->queue);
1719		1719
1720	host->mck = clk_get(&pdev->dev, "mci_clk");	1720	host->mck = clk_get(&pdev->dev, "mci_clk");
1721	if (IS_ERR(host->mck)) {	1721	if (IS_ERR(host->mck)) {
1722	ret = PTR_ERR(host->mck);	1722	ret = PTR_ERR(host->mck);
1723	goto err_clk_get;	1723	goto err_clk_get;
1724	}	1724	}
1725		1725
1726	ret = -ENOMEM;	1726	ret = -ENOMEM;
1727	host->regs = ioremap(regs->start, regs->end - regs->start + 1);	1727	host->regs = ioremap(regs->start, regs->end - regs->start + 1);
1728	if (!host->regs)	1728	if (!host->regs)
1729	goto err_ioremap;	1729	goto err_ioremap;
1730		1730
1731	clk_enable(host->mck);	1731	clk_enable(host->mck);
1732	mci_writel(host, CR, MCI_CR_SWRST);	1732	mci_writel(host, CR, MCI_CR_SWRST);
1733	host->bus_hz = clk_get_rate(host->mck);	1733	host->bus_hz = clk_get_rate(host->mck);
1734	clk_disable(host->mck);	1734	clk_disable(host->mck);
1735		1735
1736	host->mapbase = regs->start;	1736	host->mapbase = regs->start;
1737		1737
1738	tasklet_init(&host->tasklet, atmci_tasklet_func, (unsigned long)host);	1738	tasklet_init(&host->tasklet, atmci_tasklet_func, (unsigned long)host);
1739		1739
1740	ret = request_irq(irq, atmci_interrupt, 0, dev_name(&pdev->dev), host);	1740	ret = request_irq(irq, atmci_interrupt, 0, dev_name(&pdev->dev), host);
1741	if (ret)	1741	if (ret)
1742	goto err_request_irq;	1742	goto err_request_irq;
1743		1743
1744	atmci_configure_dma(host);	1744	atmci_configure_dma(host);
1745		1745
1746	platform_set_drvdata(pdev, host);	1746	platform_set_drvdata(pdev, host);
1747		1747
1748	/* We need at least one slot to succeed */	1748	/* We need at least one slot to succeed */
1749	nr_slots = 0;	1749	nr_slots = 0;
1750	ret = -ENODEV;	1750	ret = -ENODEV;
1751	if (pdata->slot[0].bus_width) {	1751	if (pdata->slot[0].bus_width) {
1752	ret = atmci_init_slot(host, &pdata->slot[0],	1752	ret = atmci_init_slot(host, &pdata->slot[0],
1753	MCI_SDCSEL_SLOT_A, 0);	1753	MCI_SDCSEL_SLOT_A, 0);
1754	if (!ret)	1754	if (!ret)
1755	nr_slots++;	1755	nr_slots++;
1756	}	1756	}
1757	if (pdata->slot[1].bus_width) {	1757	if (pdata->slot[1].bus_width) {
1758	ret = atmci_init_slot(host, &pdata->slot[1],	1758	ret = atmci_init_slot(host, &pdata->slot[1],
1759	MCI_SDCSEL_SLOT_B, 1);	1759	MCI_SDCSEL_SLOT_B, 1);
1760	if (!ret)	1760	if (!ret)
1761	nr_slots++;	1761	nr_slots++;
1762	}	1762	}
1763		1763
1764	if (!nr_slots) {	1764	if (!nr_slots) {
1765	dev_err(&pdev->dev, "init failed: no slot defined\n");	1765	dev_err(&pdev->dev, "init failed: no slot defined\n");
1766	goto err_init_slot;	1766	goto err_init_slot;
1767	}	1767	}
1768		1768
1769	dev_info(&pdev->dev,	1769	dev_info(&pdev->dev,
1770	"Atmel MCI controller at 0x%08lx irq %d, %u slots\n",	1770	"Atmel MCI controller at 0x%08lx irq %d, %u slots\n",
1771	host->mapbase, irq, nr_slots);	1771	host->mapbase, irq, nr_slots);
1772		1772
1773	return 0;	1773	return 0;
1774		1774
1775	err_init_slot:	1775	err_init_slot:
1776	#ifdef CONFIG_MMC_ATMELMCI_DMA	1776	#ifdef CONFIG_MMC_ATMELMCI_DMA
1777	if (host->dma.chan)	1777	if (host->dma.chan)
1778	dma_release_channel(host->dma.chan);	1778	dma_release_channel(host->dma.chan);
1779	#endif	1779	#endif
1780	free_irq(irq, host);	1780	free_irq(irq, host);
1781	err_request_irq:	1781	err_request_irq:
1782	iounmap(host->regs);	1782	iounmap(host->regs);
1783	err_ioremap:	1783	err_ioremap:
1784	clk_put(host->mck);	1784	clk_put(host->mck);
1785	err_clk_get:	1785	err_clk_get:
1786	kfree(host);	1786	kfree(host);
1787	return ret;	1787	return ret;
1788	}	1788	}
1789		1789
1790	static int __exit atmci_remove(struct platform_device *pdev)	1790	static int __exit atmci_remove(struct platform_device *pdev)
1791	{	1791	{
1792	struct atmel_mci *host = platform_get_drvdata(pdev);	1792	struct atmel_mci *host = platform_get_drvdata(pdev);
1793	unsigned int i;	1793	unsigned int i;
1794		1794
1795	platform_set_drvdata(pdev, NULL);	1795	platform_set_drvdata(pdev, NULL);
1796		1796
1797	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {	1797	for (i = 0; i < ATMEL_MCI_MAX_NR_SLOTS; i++) {
1798	if (host->slot[i])	1798	if (host->slot[i])
1799	atmci_cleanup_slot(host->slot[i], i);	1799	atmci_cleanup_slot(host->slot[i], i);
1800	}	1800	}
1801		1801
1802	clk_enable(host->mck);	1802	clk_enable(host->mck);
1803	mci_writel(host, IDR, ~0UL);	1803	mci_writel(host, IDR, ~0UL);
1804	mci_writel(host, CR, MCI_CR_MCIDIS);	1804	mci_writel(host, CR, MCI_CR_MCIDIS);
1805	mci_readl(host, SR);	1805	mci_readl(host, SR);
1806	clk_disable(host->mck);	1806	clk_disable(host->mck);
1807		1807
1808	#ifdef CONFIG_MMC_ATMELMCI_DMA	1808	#ifdef CONFIG_MMC_ATMELMCI_DMA
1809	if (host->dma.chan)	1809	if (host->dma.chan)
1810	dma_release_channel(host->dma.chan);	1810	dma_release_channel(host->dma.chan);
1811	#endif	1811	#endif
1812		1812
1813	free_irq(platform_get_irq(pdev, 0), host);	1813	free_irq(platform_get_irq(pdev, 0), host);
1814	iounmap(host->regs);	1814	iounmap(host->regs);
1815		1815
1816	clk_put(host->mck);	1816	clk_put(host->mck);
1817	kfree(host);	1817	kfree(host);
1818		1818
1819	return 0;	1819	return 0;
1820	}	1820	}
1821		1821
1822	static struct platform_driver atmci_driver = {	1822	static struct platform_driver atmci_driver = {
1823	.remove = __exit_p(atmci_remove),	1823	.remove = __exit_p(atmci_remove),
1824	.driver = {	1824	.driver = {
1825	.name = "atmel_mci",	1825	.name = "atmel_mci",
1826	},	1826	},
1827	};	1827	};
1828		1828
1829	static int __init atmci_init(void)	1829	static int __init atmci_init(void)
1830	{	1830	{
1831	return platform_driver_probe(&atmci_driver, atmci_probe);	1831	return platform_driver_probe(&atmci_driver, atmci_probe);
1832	}	1832	}
1833		1833
1834	static void __exit atmci_exit(void)	1834	static void __exit atmci_exit(void)
1835	{	1835	{
1836	platform_driver_unregister(&atmci_driver);	1836	platform_driver_unregister(&atmci_driver);
1837	}	1837	}
1838		1838
1839	late_initcall(atmci_init); /* try to load after dma driver when built-in */	1839	late_initcall(atmci_init); /* try to load after dma driver when built-in */
1840	module_exit(atmci_exit);	1840	module_exit(atmci_exit);
1841		1841
1842	MODULE_DESCRIPTION("Atmel Multimedia Card Interface driver");	1842	MODULE_DESCRIPTION("Atmel Multimedia Card Interface driver");
1843	MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");	1843	MODULE_AUTHOR("Haavard Skinnemoen <haavard.skinnemoen@atmel.com>");
1844	MODULE_LICENSE("GPL v2");	1844	MODULE_LICENSE("GPL v2");
1845		1845

drivers/serial/sh-sci.c

Diff comments View file @ 0582763

1	/*	1	/*
2	* drivers/serial/sh-sci.c	2	* drivers/serial/sh-sci.c
3	*	3	*
4	* SuperH on-chip serial module support. (SCI with no FIFO / with FIFO)	4	* SuperH on-chip serial module support. (SCI with no FIFO / with FIFO)
5	*	5	*
6	* Copyright (C) 2002 - 2008 Paul Mundt	6	* Copyright (C) 2002 - 2008 Paul Mundt
7	* Modified to support SH7720 SCIF. Markus Brunner, Mark Jonas (Jul 2007).	7	* Modified to support SH7720 SCIF. Markus Brunner, Mark Jonas (Jul 2007).
8	*	8	*
9	* based off of the old drivers/char/sh-sci.c by:	9	* based off of the old drivers/char/sh-sci.c by:
10	*	10	*
11	* Copyright (C) 1999, 2000 Niibe Yutaka	11	* Copyright (C) 1999, 2000 Niibe Yutaka
12	* Copyright (C) 2000 Sugioka Toshinobu	12	* Copyright (C) 2000 Sugioka Toshinobu
13	* Modified to support multiple serial ports. Stuart Menefy (May 2000).	13	* Modified to support multiple serial ports. Stuart Menefy (May 2000).
14	* Modified to support SecureEdge. David McCullough (2002)	14	* Modified to support SecureEdge. David McCullough (2002)
15	* Modified to support SH7300 SCIF. Takashi Kusuda (Jun 2003).	15	* Modified to support SH7300 SCIF. Takashi Kusuda (Jun 2003).
16	* Removed SH7300 support (Jul 2007).	16	* Removed SH7300 support (Jul 2007).
17	*	17	*
18	* This file is subject to the terms and conditions of the GNU General Public	18	* This file is subject to the terms and conditions of the GNU General Public
19	* License. See the file "COPYING" in the main directory of this archive	19	* License. See the file "COPYING" in the main directory of this archive
20	* for more details.	20	* for more details.
21	*/	21	*/
22	#if defined(CONFIG_SERIAL_SH_SCI_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)	22	#if defined(CONFIG_SERIAL_SH_SCI_CONSOLE) && defined(CONFIG_MAGIC_SYSRQ)
23	#define SUPPORT_SYSRQ	23	#define SUPPORT_SYSRQ
24	#endif	24	#endif
25		25
26	#undef DEBUG	26	#undef DEBUG
27		27
28	#include <linux/module.h>	28	#include <linux/module.h>
29	#include <linux/errno.h>	29	#include <linux/errno.h>
30	#include <linux/timer.h>	30	#include <linux/timer.h>
31	#include <linux/interrupt.h>	31	#include <linux/interrupt.h>
32	#include <linux/tty.h>	32	#include <linux/tty.h>
33	#include <linux/tty_flip.h>	33	#include <linux/tty_flip.h>
34	#include <linux/serial.h>	34	#include <linux/serial.h>
35	#include <linux/major.h>	35	#include <linux/major.h>
36	#include <linux/string.h>	36	#include <linux/string.h>
37	#include <linux/sysrq.h>	37	#include <linux/sysrq.h>
38	#include <linux/ioport.h>	38	#include <linux/ioport.h>
39	#include <linux/mm.h>	39	#include <linux/mm.h>
40	#include <linux/init.h>	40	#include <linux/init.h>
41	#include <linux/delay.h>	41	#include <linux/delay.h>
42	#include <linux/console.h>	42	#include <linux/console.h>
43	#include <linux/platform_device.h>	43	#include <linux/platform_device.h>
44	#include <linux/serial_sci.h>	44	#include <linux/serial_sci.h>
45	#include <linux/notifier.h>	45	#include <linux/notifier.h>
46	#include <linux/cpufreq.h>	46	#include <linux/cpufreq.h>
47	#include <linux/clk.h>	47	#include <linux/clk.h>
48	#include <linux/ctype.h>	48	#include <linux/ctype.h>
49	#include <linux/err.h>	49	#include <linux/err.h>
50	#include <linux/list.h>	50	#include <linux/list.h>
51	#include <linux/dmaengine.h>	51	#include <linux/dmaengine.h>
52	#include <linux/scatterlist.h>	52	#include <linux/scatterlist.h>
53		53
54	#ifdef CONFIG_SUPERH	54	#ifdef CONFIG_SUPERH
55	#include <asm/sh_bios.h>	55	#include <asm/sh_bios.h>
56	#endif	56	#endif
57		57
58	#ifdef CONFIG_H8300	58	#ifdef CONFIG_H8300
59	#include <asm/gpio.h>	59	#include <asm/gpio.h>
60	#endif	60	#endif
61		61
62	#include "sh-sci.h"	62	#include "sh-sci.h"
63		63
64	struct sci_port {	64	struct sci_port {
65	struct uart_port port;	65	struct uart_port port;
66		66
67	/* Port type */	67	/* Port type */
68	unsigned int type;	68	unsigned int type;
69		69
70	/* Port IRQs: ERI, RXI, TXI, BRI (optional) */	70	/* Port IRQs: ERI, RXI, TXI, BRI (optional) */
71	unsigned int irqs[SCIx_NR_IRQS];	71	unsigned int irqs[SCIx_NR_IRQS];
72		72
73	/* Port enable callback */	73	/* Port enable callback */
74	void (enable)(struct uart_port port);	74	void (enable)(struct uart_port port);
75		75
76	/* Port disable callback */	76	/* Port disable callback */
77	void (disable)(struct uart_port port);	77	void (disable)(struct uart_port port);
78		78
79	/* Break timer */	79	/* Break timer */
80	struct timer_list break_timer;	80	struct timer_list break_timer;
81	int break_flag;	81	int break_flag;
82		82
83	/* Interface clock */	83	/* Interface clock */
84	struct clk *iclk;	84	struct clk *iclk;
85	/* Data clock */	85	/* Data clock */
86	struct clk *dclk;	86	struct clk *dclk;
87		87
88	struct list_head node;	88	struct list_head node;
89	struct dma_chan *chan_tx;	89	struct dma_chan *chan_tx;
90	struct dma_chan *chan_rx;	90	struct dma_chan *chan_rx;
91	#ifdef CONFIG_SERIAL_SH_SCI_DMA	91	#ifdef CONFIG_SERIAL_SH_SCI_DMA
92	struct device *dma_dev;	92	struct device *dma_dev;
93	enum sh_dmae_slave_chan_id slave_tx;	93	enum sh_dmae_slave_chan_id slave_tx;
94	enum sh_dmae_slave_chan_id slave_rx;	94	enum sh_dmae_slave_chan_id slave_rx;
95	struct dma_async_tx_descriptor *desc_tx;	95	struct dma_async_tx_descriptor *desc_tx;
96	struct dma_async_tx_descriptor *desc_rx[2];	96	struct dma_async_tx_descriptor *desc_rx[2];
97	dma_cookie_t cookie_tx;	97	dma_cookie_t cookie_tx;
98	dma_cookie_t cookie_rx[2];	98	dma_cookie_t cookie_rx[2];
99	dma_cookie_t active_rx;	99	dma_cookie_t active_rx;
100	struct scatterlist sg_tx;	100	struct scatterlist sg_tx;
101	unsigned int sg_len_tx;	101	unsigned int sg_len_tx;
102	struct scatterlist sg_rx[2];	102	struct scatterlist sg_rx[2];
103	size_t buf_len_rx;	103	size_t buf_len_rx;
104	struct sh_dmae_slave param_tx;	104	struct sh_dmae_slave param_tx;
105	struct sh_dmae_slave param_rx;	105	struct sh_dmae_slave param_rx;
106	struct work_struct work_tx;	106	struct work_struct work_tx;
107	struct work_struct work_rx;	107	struct work_struct work_rx;
108	struct timer_list rx_timer;	108	struct timer_list rx_timer;
109	#endif	109	#endif
110	};	110	};
111		111
112	struct sh_sci_priv {	112	struct sh_sci_priv {
113	spinlock_t lock;	113	spinlock_t lock;
114	struct list_head ports;	114	struct list_head ports;
115	struct notifier_block clk_nb;	115	struct notifier_block clk_nb;
116	};	116	};
117		117
118	/* Function prototypes */	118	/* Function prototypes */
119	static void sci_stop_tx(struct uart_port *port);	119	static void sci_stop_tx(struct uart_port *port);
120		120
121	#define SCI_NPORTS CONFIG_SERIAL_SH_SCI_NR_UARTS	121	#define SCI_NPORTS CONFIG_SERIAL_SH_SCI_NR_UARTS
122		122
123	static struct sci_port sci_ports[SCI_NPORTS];	123	static struct sci_port sci_ports[SCI_NPORTS];
124	static struct uart_driver sci_uart_driver;	124	static struct uart_driver sci_uart_driver;
125		125
126	static inline struct sci_port *	126	static inline struct sci_port *
127	to_sci_port(struct uart_port *uart)	127	to_sci_port(struct uart_port *uart)
128	{	128	{
129	return container_of(uart, struct sci_port, port);	129	return container_of(uart, struct sci_port, port);
130	}	130	}
131		131
132	#if defined(CONFIG_CONSOLE_POLL) \|\| defined(CONFIG_SERIAL_SH_SCI_CONSOLE)	132	#if defined(CONFIG_CONSOLE_POLL) \|\| defined(CONFIG_SERIAL_SH_SCI_CONSOLE)
133		133
134	#ifdef CONFIG_CONSOLE_POLL	134	#ifdef CONFIG_CONSOLE_POLL
135	static inline void handle_error(struct uart_port *port)	135	static inline void handle_error(struct uart_port *port)
136	{	136	{
137	/* Clear error flags */	137	/* Clear error flags */
138	sci_out(port, SCxSR, SCxSR_ERROR_CLEAR(port));	138	sci_out(port, SCxSR, SCxSR_ERROR_CLEAR(port));
139	}	139	}
140		140
141	static int sci_poll_get_char(struct uart_port *port)	141	static int sci_poll_get_char(struct uart_port *port)
142	{	142	{
143	unsigned short status;	143	unsigned short status;
144	int c;	144	int c;
145		145
146	do {	146	do {
147	status = sci_in(port, SCxSR);	147	status = sci_in(port, SCxSR);
148	if (status & SCxSR_ERRORS(port)) {	148	if (status & SCxSR_ERRORS(port)) {
149	handle_error(port);	149	handle_error(port);
150	continue;	150	continue;
151	}	151	}
152	} while (!(status & SCxSR_RDxF(port)));	152	} while (!(status & SCxSR_RDxF(port)));
153		153
154	c = sci_in(port, SCxRDR);	154	c = sci_in(port, SCxRDR);
155		155
156	/* Dummy read */	156	/* Dummy read */
157	sci_in(port, SCxSR);	157	sci_in(port, SCxSR);
158	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));	158	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));
159		159
160	return c;	160	return c;
161	}	161	}
162	#endif	162	#endif
163		163
164	static void sci_poll_put_char(struct uart_port *port, unsigned char c)	164	static void sci_poll_put_char(struct uart_port *port, unsigned char c)
165	{	165	{
166	unsigned short status;	166	unsigned short status;
167		167
168	do {	168	do {
169	status = sci_in(port, SCxSR);	169	status = sci_in(port, SCxSR);
170	} while (!(status & SCxSR_TDxE(port)));	170	} while (!(status & SCxSR_TDxE(port)));
171		171
172	sci_out(port, SCxTDR, c);	172	sci_out(port, SCxTDR, c);
173	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port) & ~SCxSR_TEND(port));	173	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port) & ~SCxSR_TEND(port));
174	}	174	}
175	#endif /* CONFIG_CONSOLE_POLL \|\| CONFIG_SERIAL_SH_SCI_CONSOLE */	175	#endif /* CONFIG_CONSOLE_POLL \|\| CONFIG_SERIAL_SH_SCI_CONSOLE */
176		176
177	#if defined(__H8300H__) \|\| defined(__H8300S__)	177	#if defined(__H8300H__) \|\| defined(__H8300S__)
178	static void sci_init_pins(struct uart_port *port, unsigned int cflag)	178	static void sci_init_pins(struct uart_port *port, unsigned int cflag)
179	{	179	{
180	int ch = (port->mapbase - SMR0) >> 3;	180	int ch = (port->mapbase - SMR0) >> 3;
181		181
182	/* set DDR regs */	182	/* set DDR regs */
183	H8300_GPIO_DDR(h8300_sci_pins[ch].port,	183	H8300_GPIO_DDR(h8300_sci_pins[ch].port,
184	h8300_sci_pins[ch].rx,	184	h8300_sci_pins[ch].rx,
185	H8300_GPIO_INPUT);	185	H8300_GPIO_INPUT);
186	H8300_GPIO_DDR(h8300_sci_pins[ch].port,	186	H8300_GPIO_DDR(h8300_sci_pins[ch].port,
187	h8300_sci_pins[ch].tx,	187	h8300_sci_pins[ch].tx,
188	H8300_GPIO_OUTPUT);	188	H8300_GPIO_OUTPUT);
189		189
190	/* tx mark output*/	190	/* tx mark output*/
191	H8300_SCI_DR(ch) \|= h8300_sci_pins[ch].tx;	191	H8300_SCI_DR(ch) \|= h8300_sci_pins[ch].tx;
192	}	192	}
193	#elif defined(CONFIG_CPU_SUBTYPE_SH7710) \|\| defined(CONFIG_CPU_SUBTYPE_SH7712)	193	#elif defined(CONFIG_CPU_SUBTYPE_SH7710) \|\| defined(CONFIG_CPU_SUBTYPE_SH7712)
194	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	194	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
195	{	195	{
196	if (port->mapbase == 0xA4400000) {	196	if (port->mapbase == 0xA4400000) {
197	__raw_writew(__raw_readw(PACR) & 0xffc0, PACR);	197	__raw_writew(__raw_readw(PACR) & 0xffc0, PACR);
198	__raw_writew(__raw_readw(PBCR) & 0x0fff, PBCR);	198	__raw_writew(__raw_readw(PBCR) & 0x0fff, PBCR);
199	} else if (port->mapbase == 0xA4410000)	199	} else if (port->mapbase == 0xA4410000)
200	__raw_writew(__raw_readw(PBCR) & 0xf003, PBCR);	200	__raw_writew(__raw_readw(PBCR) & 0xf003, PBCR);
201	}	201	}
202	#elif defined(CONFIG_CPU_SUBTYPE_SH7720) \|\| defined(CONFIG_CPU_SUBTYPE_SH7721)	202	#elif defined(CONFIG_CPU_SUBTYPE_SH7720) \|\| defined(CONFIG_CPU_SUBTYPE_SH7721)
203	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	203	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
204	{	204	{
205	unsigned short data;	205	unsigned short data;
206		206
207	if (cflag & CRTSCTS) {	207	if (cflag & CRTSCTS) {
208	/* enable RTS/CTS */	208	/* enable RTS/CTS */
209	if (port->mapbase == 0xa4430000) { /* SCIF0 */	209	if (port->mapbase == 0xa4430000) { /* SCIF0 */
210	/* Clear PTCR bit 9-2; enable all scif pins but sck */	210	/* Clear PTCR bit 9-2; enable all scif pins but sck */
211	data = __raw_readw(PORT_PTCR);	211	data = __raw_readw(PORT_PTCR);
212	__raw_writew((data & 0xfc03), PORT_PTCR);	212	__raw_writew((data & 0xfc03), PORT_PTCR);
213	} else if (port->mapbase == 0xa4438000) { /* SCIF1 */	213	} else if (port->mapbase == 0xa4438000) { /* SCIF1 */
214	/* Clear PVCR bit 9-2 */	214	/* Clear PVCR bit 9-2 */
215	data = __raw_readw(PORT_PVCR);	215	data = __raw_readw(PORT_PVCR);
216	__raw_writew((data & 0xfc03), PORT_PVCR);	216	__raw_writew((data & 0xfc03), PORT_PVCR);
217	}	217	}
218	} else {	218	} else {
219	if (port->mapbase == 0xa4430000) { /* SCIF0 */	219	if (port->mapbase == 0xa4430000) { /* SCIF0 */
220	/* Clear PTCR bit 5-2; enable only tx and rx */	220	/* Clear PTCR bit 5-2; enable only tx and rx */
221	data = __raw_readw(PORT_PTCR);	221	data = __raw_readw(PORT_PTCR);
222	__raw_writew((data & 0xffc3), PORT_PTCR);	222	__raw_writew((data & 0xffc3), PORT_PTCR);
223	} else if (port->mapbase == 0xa4438000) { /* SCIF1 */	223	} else if (port->mapbase == 0xa4438000) { /* SCIF1 */
224	/* Clear PVCR bit 5-2 */	224	/* Clear PVCR bit 5-2 */
225	data = __raw_readw(PORT_PVCR);	225	data = __raw_readw(PORT_PVCR);
226	__raw_writew((data & 0xffc3), PORT_PVCR);	226	__raw_writew((data & 0xffc3), PORT_PVCR);
227	}	227	}
228	}	228	}
229	}	229	}
230	#elif defined(CONFIG_CPU_SH3)	230	#elif defined(CONFIG_CPU_SH3)
231	/* For SH7705, SH7706, SH7707, SH7709, SH7709A, SH7729 */	231	/* For SH7705, SH7706, SH7707, SH7709, SH7709A, SH7729 */
232	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	232	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
233	{	233	{
234	unsigned short data;	234	unsigned short data;
235		235
236	/* We need to set SCPCR to enable RTS/CTS */	236	/* We need to set SCPCR to enable RTS/CTS */
237	data = __raw_readw(SCPCR);	237	data = __raw_readw(SCPCR);
238	/* Clear out SCP7MD1,0, SCP6MD1,0, SCP4MD1,0*/	238	/* Clear out SCP7MD1,0, SCP6MD1,0, SCP4MD1,0*/
239	__raw_writew(data & 0x0fcf, SCPCR);	239	__raw_writew(data & 0x0fcf, SCPCR);
240		240
241	if (!(cflag & CRTSCTS)) {	241	if (!(cflag & CRTSCTS)) {
242	/* We need to set SCPCR to enable RTS/CTS */	242	/* We need to set SCPCR to enable RTS/CTS */
243	data = __raw_readw(SCPCR);	243	data = __raw_readw(SCPCR);
244	/* Clear out SCP7MD1,0, SCP4MD1,0,	244	/* Clear out SCP7MD1,0, SCP4MD1,0,
245	Set SCP6MD1,0 = {01} (output) */	245	Set SCP6MD1,0 = {01} (output) */
246	__raw_writew((data & 0x0fcf) \| 0x1000, SCPCR);	246	__raw_writew((data & 0x0fcf) \| 0x1000, SCPCR);
247		247
248	data = __raw_readb(SCPDR);	248	data = __raw_readb(SCPDR);
249	/* Set /RTS2 (bit6) = 0 */	249	/* Set /RTS2 (bit6) = 0 */
250	__raw_writeb(data & 0xbf, SCPDR);	250	__raw_writeb(data & 0xbf, SCPDR);
251	}	251	}
252	}	252	}
253	#elif defined(CONFIG_CPU_SUBTYPE_SH7722)	253	#elif defined(CONFIG_CPU_SUBTYPE_SH7722)
254	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	254	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
255	{	255	{
256	unsigned short data;	256	unsigned short data;
257		257
258	if (port->mapbase == 0xffe00000) {	258	if (port->mapbase == 0xffe00000) {
259	data = __raw_readw(PSCR);	259	data = __raw_readw(PSCR);
260	data &= ~0x03cf;	260	data &= ~0x03cf;
261	if (!(cflag & CRTSCTS))	261	if (!(cflag & CRTSCTS))
262	data \|= 0x0340;	262	data \|= 0x0340;
263		263
264	__raw_writew(data, PSCR);	264	__raw_writew(data, PSCR);
265	}	265	}
266	}	266	}
267	#elif defined(CONFIG_CPU_SUBTYPE_SH7757) \|\| \	267	#elif defined(CONFIG_CPU_SUBTYPE_SH7757) \|\| \
268	defined(CONFIG_CPU_SUBTYPE_SH7763) \|\| \	268	defined(CONFIG_CPU_SUBTYPE_SH7763) \|\| \
269	defined(CONFIG_CPU_SUBTYPE_SH7780) \|\| \	269	defined(CONFIG_CPU_SUBTYPE_SH7780) \|\| \
270	defined(CONFIG_CPU_SUBTYPE_SH7785) \|\| \	270	defined(CONFIG_CPU_SUBTYPE_SH7785) \|\| \
271	defined(CONFIG_CPU_SUBTYPE_SH7786) \|\| \	271	defined(CONFIG_CPU_SUBTYPE_SH7786) \|\| \
272	defined(CONFIG_CPU_SUBTYPE_SHX3)	272	defined(CONFIG_CPU_SUBTYPE_SHX3)
273	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	273	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
274	{	274	{
275	if (!(cflag & CRTSCTS))	275	if (!(cflag & CRTSCTS))
276	__raw_writew(0x0080, SCSPTR0); /* Set RTS = 1 */	276	__raw_writew(0x0080, SCSPTR0); /* Set RTS = 1 */
277	}	277	}
278	#elif defined(CONFIG_CPU_SH4) && !defined(CONFIG_CPU_SH4A)	278	#elif defined(CONFIG_CPU_SH4) && !defined(CONFIG_CPU_SH4A)
279	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	279	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
280	{	280	{
281	if (!(cflag & CRTSCTS))	281	if (!(cflag & CRTSCTS))
282	__raw_writew(0x0080, SCSPTR2); /* Set RTS = 1 */	282	__raw_writew(0x0080, SCSPTR2); /* Set RTS = 1 */
283	}	283	}
284	#else	284	#else
285	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)	285	static inline void sci_init_pins(struct uart_port *port, unsigned int cflag)
286	{	286	{
287	/* Nothing to do */	287	/* Nothing to do */
288	}	288	}
289	#endif	289	#endif
290		290
291	#if defined(CONFIG_CPU_SUBTYPE_SH7760) \|\| \	291	#if defined(CONFIG_CPU_SUBTYPE_SH7760) \|\| \
292	defined(CONFIG_CPU_SUBTYPE_SH7780) \|\| \	292	defined(CONFIG_CPU_SUBTYPE_SH7780) \|\| \
293	defined(CONFIG_CPU_SUBTYPE_SH7785) \|\| \	293	defined(CONFIG_CPU_SUBTYPE_SH7785) \|\| \
294	defined(CONFIG_CPU_SUBTYPE_SH7786)	294	defined(CONFIG_CPU_SUBTYPE_SH7786)
295	static int scif_txfill(struct uart_port *port)	295	static int scif_txfill(struct uart_port *port)
296	{	296	{
297	return sci_in(port, SCTFDR) & 0xff;	297	return sci_in(port, SCTFDR) & 0xff;
298	}	298	}
299		299
300	static int scif_txroom(struct uart_port *port)	300	static int scif_txroom(struct uart_port *port)
301	{	301	{
302	return SCIF_TXROOM_MAX - scif_txfill(port);	302	return SCIF_TXROOM_MAX - scif_txfill(port);
303	}	303	}
304		304
305	static int scif_rxfill(struct uart_port *port)	305	static int scif_rxfill(struct uart_port *port)
306	{	306	{
307	return sci_in(port, SCRFDR) & 0xff;	307	return sci_in(port, SCRFDR) & 0xff;
308	}	308	}
309	#elif defined(CONFIG_CPU_SUBTYPE_SH7763)	309	#elif defined(CONFIG_CPU_SUBTYPE_SH7763)
310	static int scif_txfill(struct uart_port *port)	310	static int scif_txfill(struct uart_port *port)
311	{	311	{
312	if (port->mapbase == 0xffe00000 \|\|	312	if (port->mapbase == 0xffe00000 \|\|
313	port->mapbase == 0xffe08000)	313	port->mapbase == 0xffe08000)
314	/* SCIF0/1*/	314	/* SCIF0/1*/
315	return sci_in(port, SCTFDR) & 0xff;	315	return sci_in(port, SCTFDR) & 0xff;
316	else	316	else
317	/* SCIF2 */	317	/* SCIF2 */
318	return sci_in(port, SCFDR) >> 8;	318	return sci_in(port, SCFDR) >> 8;
319	}	319	}
320		320
321	static int scif_txroom(struct uart_port *port)	321	static int scif_txroom(struct uart_port *port)
322	{	322	{
323	if (port->mapbase == 0xffe00000 \|\|	323	if (port->mapbase == 0xffe00000 \|\|
324	port->mapbase == 0xffe08000)	324	port->mapbase == 0xffe08000)
325	/* SCIF0/1*/	325	/* SCIF0/1*/
326	return SCIF_TXROOM_MAX - scif_txfill(port);	326	return SCIF_TXROOM_MAX - scif_txfill(port);
327	else	327	else
328	/* SCIF2 */	328	/* SCIF2 */
329	return SCIF2_TXROOM_MAX - scif_txfill(port);	329	return SCIF2_TXROOM_MAX - scif_txfill(port);
330	}	330	}
331		331
332	static int scif_rxfill(struct uart_port *port)	332	static int scif_rxfill(struct uart_port *port)
333	{	333	{
334	if ((port->mapbase == 0xffe00000) \|\|	334	if ((port->mapbase == 0xffe00000) \|\|
335	(port->mapbase == 0xffe08000)) {	335	(port->mapbase == 0xffe08000)) {
336	/* SCIF0/1*/	336	/* SCIF0/1*/
337	return sci_in(port, SCRFDR) & 0xff;	337	return sci_in(port, SCRFDR) & 0xff;
338	} else {	338	} else {
339	/* SCIF2 */	339	/* SCIF2 */
340	return sci_in(port, SCFDR) & SCIF2_RFDC_MASK;	340	return sci_in(port, SCFDR) & SCIF2_RFDC_MASK;
341	}	341	}
342	}	342	}
343	#else	343	#else
344	static int scif_txfill(struct uart_port *port)	344	static int scif_txfill(struct uart_port *port)
345	{	345	{
346	return sci_in(port, SCFDR) >> 8;	346	return sci_in(port, SCFDR) >> 8;
347	}	347	}
348		348
349	static int scif_txroom(struct uart_port *port)	349	static int scif_txroom(struct uart_port *port)
350	{	350	{
351	return SCIF_TXROOM_MAX - scif_txfill(port);	351	return SCIF_TXROOM_MAX - scif_txfill(port);
352	}	352	}
353		353
354	static int scif_rxfill(struct uart_port *port)	354	static int scif_rxfill(struct uart_port *port)
355	{	355	{
356	return sci_in(port, SCFDR) & SCIF_RFDC_MASK;	356	return sci_in(port, SCFDR) & SCIF_RFDC_MASK;
357	}	357	}
358	#endif	358	#endif
359		359
360	static int sci_txfill(struct uart_port *port)	360	static int sci_txfill(struct uart_port *port)
361	{	361	{
362	return !(sci_in(port, SCxSR) & SCI_TDRE);	362	return !(sci_in(port, SCxSR) & SCI_TDRE);
363	}	363	}
364		364
365	static int sci_txroom(struct uart_port *port)	365	static int sci_txroom(struct uart_port *port)
366	{	366	{
367	return !sci_txfill(port);	367	return !sci_txfill(port);
368	}	368	}
369		369
370	static int sci_rxfill(struct uart_port *port)	370	static int sci_rxfill(struct uart_port *port)
371	{	371	{
372	return (sci_in(port, SCxSR) & SCxSR_RDxF(port)) != 0;	372	return (sci_in(port, SCxSR) & SCxSR_RDxF(port)) != 0;
373	}	373	}
374		374
375	/* ********************************************************************** *	375	/* ********************************************************************** *
376	* the interrupt related routines *	376	* the interrupt related routines *
377	* ********************************************************************** */	377	* ********************************************************************** */
378		378
379	static void sci_transmit_chars(struct uart_port *port)	379	static void sci_transmit_chars(struct uart_port *port)
380	{	380	{
381	struct circ_buf *xmit = &port->state->xmit;	381	struct circ_buf *xmit = &port->state->xmit;
382	unsigned int stopped = uart_tx_stopped(port);	382	unsigned int stopped = uart_tx_stopped(port);
383	unsigned short status;	383	unsigned short status;
384	unsigned short ctrl;	384	unsigned short ctrl;
385	int count;	385	int count;
386		386
387	status = sci_in(port, SCxSR);	387	status = sci_in(port, SCxSR);
388	if (!(status & SCxSR_TDxE(port))) {	388	if (!(status & SCxSR_TDxE(port))) {
389	ctrl = sci_in(port, SCSCR);	389	ctrl = sci_in(port, SCSCR);
390	if (uart_circ_empty(xmit))	390	if (uart_circ_empty(xmit))
391	ctrl &= ~SCI_CTRL_FLAGS_TIE;	391	ctrl &= ~SCI_CTRL_FLAGS_TIE;
392	else	392	else
393	ctrl \|= SCI_CTRL_FLAGS_TIE;	393	ctrl \|= SCI_CTRL_FLAGS_TIE;
394	sci_out(port, SCSCR, ctrl);	394	sci_out(port, SCSCR, ctrl);
395	return;	395	return;
396	}	396	}
397		397
398	if (port->type == PORT_SCI)	398	if (port->type == PORT_SCI)
399	count = sci_txroom(port);	399	count = sci_txroom(port);
400	else	400	else
401	count = scif_txroom(port);	401	count = scif_txroom(port);
402		402
403	do {	403	do {
404	unsigned char c;	404	unsigned char c;
405		405
406	if (port->x_char) {	406	if (port->x_char) {
407	c = port->x_char;	407	c = port->x_char;
408	port->x_char = 0;	408	port->x_char = 0;
409	} else if (!uart_circ_empty(xmit) && !stopped) {	409	} else if (!uart_circ_empty(xmit) && !stopped) {
410	c = xmit->buf[xmit->tail];	410	c = xmit->buf[xmit->tail];
411	xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);	411	xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
412	} else {	412	} else {
413	break;	413	break;
414	}	414	}
415		415
416	sci_out(port, SCxTDR, c);	416	sci_out(port, SCxTDR, c);
417		417
418	port->icount.tx++;	418	port->icount.tx++;
419	} while (--count > 0);	419	} while (--count > 0);
420		420
421	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port));	421	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port));
422		422
423	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)	423	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
424	uart_write_wakeup(port);	424	uart_write_wakeup(port);
425	if (uart_circ_empty(xmit)) {	425	if (uart_circ_empty(xmit)) {
426	sci_stop_tx(port);	426	sci_stop_tx(port);
427	} else {	427	} else {
428	ctrl = sci_in(port, SCSCR);	428	ctrl = sci_in(port, SCSCR);
429		429
430	if (port->type != PORT_SCI) {	430	if (port->type != PORT_SCI) {
431	sci_in(port, SCxSR); /* Dummy read */	431	sci_in(port, SCxSR); /* Dummy read */
432	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port));	432	sci_out(port, SCxSR, SCxSR_TDxE_CLEAR(port));
433	}	433	}
434		434
435	ctrl \|= SCI_CTRL_FLAGS_TIE;	435	ctrl \|= SCI_CTRL_FLAGS_TIE;
436	sci_out(port, SCSCR, ctrl);	436	sci_out(port, SCSCR, ctrl);
437	}	437	}
438	}	438	}
439		439
440	/* On SH3, SCIF may read end-of-break as a space->mark char */	440	/* On SH3, SCIF may read end-of-break as a space->mark char */
441	#define STEPFN(c) ({int __c = (c); (((__c-1)\|(__c)) == -1); })	441	#define STEPFN(c) ({int __c = (c); (((__c-1)\|(__c)) == -1); })
442		442
443	static inline void sci_receive_chars(struct uart_port *port)	443	static inline void sci_receive_chars(struct uart_port *port)
444	{	444	{
445	struct sci_port *sci_port = to_sci_port(port);	445	struct sci_port *sci_port = to_sci_port(port);
446	struct tty_struct *tty = port->state->port.tty;	446	struct tty_struct *tty = port->state->port.tty;
447	int i, count, copied = 0;	447	int i, count, copied = 0;
448	unsigned short status;	448	unsigned short status;
449	unsigned char flag;	449	unsigned char flag;
450		450
451	status = sci_in(port, SCxSR);	451	status = sci_in(port, SCxSR);
452	if (!(status & SCxSR_RDxF(port)))	452	if (!(status & SCxSR_RDxF(port)))
453	return;	453	return;
454		454
455	while (1) {	455	while (1) {
456	if (port->type == PORT_SCI)	456	if (port->type == PORT_SCI)
457	count = sci_rxfill(port);	457	count = sci_rxfill(port);
458	else	458	else
459	count = scif_rxfill(port);	459	count = scif_rxfill(port);
460		460
461	/* Don't copy more bytes than there is room for in the buffer */	461	/* Don't copy more bytes than there is room for in the buffer */
462	count = tty_buffer_request_room(tty, count);	462	count = tty_buffer_request_room(tty, count);
463		463
464	/* If for any reason we can't copy more data, we're done! */	464	/* If for any reason we can't copy more data, we're done! */
465	if (count == 0)	465	if (count == 0)
466	break;	466	break;
467		467
468	if (port->type == PORT_SCI) {	468	if (port->type == PORT_SCI) {
469	char c = sci_in(port, SCxRDR);	469	char c = sci_in(port, SCxRDR);
470	if (uart_handle_sysrq_char(port, c) \|\|	470	if (uart_handle_sysrq_char(port, c) \|\|
471	sci_port->break_flag)	471	sci_port->break_flag)
472	count = 0;	472	count = 0;
473	else	473	else
474	tty_insert_flip_char(tty, c, TTY_NORMAL);	474	tty_insert_flip_char(tty, c, TTY_NORMAL);
475	} else {	475	} else {
476	for (i = 0; i < count; i++) {	476	for (i = 0; i < count; i++) {
477	char c = sci_in(port, SCxRDR);	477	char c = sci_in(port, SCxRDR);
478	status = sci_in(port, SCxSR);	478	status = sci_in(port, SCxSR);
479	#if defined(CONFIG_CPU_SH3)	479	#if defined(CONFIG_CPU_SH3)
480	/* Skip "chars" during break */	480	/* Skip "chars" during break */
481	if (sci_port->break_flag) {	481	if (sci_port->break_flag) {
482	if ((c == 0) &&	482	if ((c == 0) &&
483	(status & SCxSR_FER(port))) {	483	(status & SCxSR_FER(port))) {
484	count--; i--;	484	count--; i--;
485	continue;	485	continue;
486	}	486	}
487		487
488	/* Nonzero => end-of-break */	488	/* Nonzero => end-of-break */
489	dev_dbg(port->dev, "debounce<%02x>\n", c);	489	dev_dbg(port->dev, "debounce<%02x>\n", c);
490	sci_port->break_flag = 0;	490	sci_port->break_flag = 0;
491		491
492	if (STEPFN(c)) {	492	if (STEPFN(c)) {
493	count--; i--;	493	count--; i--;
494	continue;	494	continue;
495	}	495	}
496	}	496	}
497	#endif /* CONFIG_CPU_SH3 */	497	#endif /* CONFIG_CPU_SH3 */
498	if (uart_handle_sysrq_char(port, c)) {	498	if (uart_handle_sysrq_char(port, c)) {
499	count--; i--;	499	count--; i--;
500	continue;	500	continue;
501	}	501	}
502		502
503	/* Store data and status */	503	/* Store data and status */
504	if (status & SCxSR_FER(port)) {	504	if (status & SCxSR_FER(port)) {
505	flag = TTY_FRAME;	505	flag = TTY_FRAME;
506	dev_notice(port->dev, "frame error\n");	506	dev_notice(port->dev, "frame error\n");
507	} else if (status & SCxSR_PER(port)) {	507	} else if (status & SCxSR_PER(port)) {
508	flag = TTY_PARITY;	508	flag = TTY_PARITY;
509	dev_notice(port->dev, "parity error\n");	509	dev_notice(port->dev, "parity error\n");
510	} else	510	} else
511	flag = TTY_NORMAL;	511	flag = TTY_NORMAL;
512		512
513	tty_insert_flip_char(tty, c, flag);	513	tty_insert_flip_char(tty, c, flag);
514	}	514	}
515	}	515	}
516		516
517	sci_in(port, SCxSR); /* dummy read */	517	sci_in(port, SCxSR); /* dummy read */
518	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));	518	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));
519		519
520	copied += count;	520	copied += count;
521	port->icount.rx += count;	521	port->icount.rx += count;
522	}	522	}
523		523
524	if (copied) {	524	if (copied) {
525	/* Tell the rest of the system the news. New characters! */	525	/* Tell the rest of the system the news. New characters! */
526	tty_flip_buffer_push(tty);	526	tty_flip_buffer_push(tty);
527	} else {	527	} else {
528	sci_in(port, SCxSR); /* dummy read */	528	sci_in(port, SCxSR); /* dummy read */
529	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));	529	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));
530	}	530	}
531	}	531	}
532		532
533	#define SCI_BREAK_JIFFIES (HZ/20)	533	#define SCI_BREAK_JIFFIES (HZ/20)
534	/* The sci generates interrupts during the break,	534	/* The sci generates interrupts during the break,
535	* 1 per millisecond or so during the break period, for 9600 baud.	535	* 1 per millisecond or so during the break period, for 9600 baud.
536	* So dont bother disabling interrupts.	536	* So dont bother disabling interrupts.
537	* But dont want more than 1 break event.	537	* But dont want more than 1 break event.
538	* Use a kernel timer to periodically poll the rx line until	538	* Use a kernel timer to periodically poll the rx line until
539	* the break is finished.	539	* the break is finished.
540	*/	540	*/
541	static void sci_schedule_break_timer(struct sci_port *port)	541	static void sci_schedule_break_timer(struct sci_port *port)
542	{	542	{
543	port->break_timer.expires = jiffies + SCI_BREAK_JIFFIES;	543	port->break_timer.expires = jiffies + SCI_BREAK_JIFFIES;
544	add_timer(&port->break_timer);	544	add_timer(&port->break_timer);
545	}	545	}
546	/* Ensure that two consecutive samples find the break over. */	546	/* Ensure that two consecutive samples find the break over. */
547	static void sci_break_timer(unsigned long data)	547	static void sci_break_timer(unsigned long data)
548	{	548	{
549	struct sci_port port = (struct sci_port )data;	549	struct sci_port port = (struct sci_port )data;
550		550
551	if (sci_rxd_in(&port->port) == 0) {	551	if (sci_rxd_in(&port->port) == 0) {
552	port->break_flag = 1;	552	port->break_flag = 1;
553	sci_schedule_break_timer(port);	553	sci_schedule_break_timer(port);
554	} else if (port->break_flag == 1) {	554	} else if (port->break_flag == 1) {
555	/* break is over. */	555	/* break is over. */
556	port->break_flag = 2;	556	port->break_flag = 2;
557	sci_schedule_break_timer(port);	557	sci_schedule_break_timer(port);
558	} else	558	} else
559	port->break_flag = 0;	559	port->break_flag = 0;
560	}	560	}
561		561
562	static inline int sci_handle_errors(struct uart_port *port)	562	static inline int sci_handle_errors(struct uart_port *port)
563	{	563	{
564	int copied = 0;	564	int copied = 0;
565	unsigned short status = sci_in(port, SCxSR);	565	unsigned short status = sci_in(port, SCxSR);
566	struct tty_struct *tty = port->state->port.tty;	566	struct tty_struct *tty = port->state->port.tty;
567		567
568	if (status & SCxSR_ORER(port)) {	568	if (status & SCxSR_ORER(port)) {
569	/* overrun error */	569	/* overrun error */
570	if (tty_insert_flip_char(tty, 0, TTY_OVERRUN))	570	if (tty_insert_flip_char(tty, 0, TTY_OVERRUN))
571	copied++;	571	copied++;
572		572
573	dev_notice(port->dev, "overrun error");	573	dev_notice(port->dev, "overrun error");
574	}	574	}
575		575
576	if (status & SCxSR_FER(port)) {	576	if (status & SCxSR_FER(port)) {
577	if (sci_rxd_in(port) == 0) {	577	if (sci_rxd_in(port) == 0) {
578	/* Notify of BREAK */	578	/* Notify of BREAK */
579	struct sci_port *sci_port = to_sci_port(port);	579	struct sci_port *sci_port = to_sci_port(port);
580		580
581	if (!sci_port->break_flag) {	581	if (!sci_port->break_flag) {
582	sci_port->break_flag = 1;	582	sci_port->break_flag = 1;
583	sci_schedule_break_timer(sci_port);	583	sci_schedule_break_timer(sci_port);
584		584
585	/* Do sysrq handling. */	585	/* Do sysrq handling. */
586	if (uart_handle_break(port))	586	if (uart_handle_break(port))
587	return 0;	587	return 0;
588		588
589	dev_dbg(port->dev, "BREAK detected\n");	589	dev_dbg(port->dev, "BREAK detected\n");
590		590
591	if (tty_insert_flip_char(tty, 0, TTY_BREAK))	591	if (tty_insert_flip_char(tty, 0, TTY_BREAK))
592	copied++;	592	copied++;
593	}	593	}
594		594
595	} else {	595	} else {
596	/* frame error */	596	/* frame error */
597	if (tty_insert_flip_char(tty, 0, TTY_FRAME))	597	if (tty_insert_flip_char(tty, 0, TTY_FRAME))
598	copied++;	598	copied++;
599		599
600	dev_notice(port->dev, "frame error\n");	600	dev_notice(port->dev, "frame error\n");
601	}	601	}
602	}	602	}
603		603
604	if (status & SCxSR_PER(port)) {	604	if (status & SCxSR_PER(port)) {
605	/* parity error */	605	/* parity error */
606	if (tty_insert_flip_char(tty, 0, TTY_PARITY))	606	if (tty_insert_flip_char(tty, 0, TTY_PARITY))
607	copied++;	607	copied++;
608		608
609	dev_notice(port->dev, "parity error");	609	dev_notice(port->dev, "parity error");
610	}	610	}
611		611
612	if (copied)	612	if (copied)
613	tty_flip_buffer_push(tty);	613	tty_flip_buffer_push(tty);
614		614
615	return copied;	615	return copied;
616	}	616	}
617		617
618	static inline int sci_handle_fifo_overrun(struct uart_port *port)	618	static inline int sci_handle_fifo_overrun(struct uart_port *port)
619	{	619	{
620	struct tty_struct *tty = port->state->port.tty;	620	struct tty_struct *tty = port->state->port.tty;
621	int copied = 0;	621	int copied = 0;
622		622
623	if (port->type != PORT_SCIF)	623	if (port->type != PORT_SCIF)
624	return 0;	624	return 0;
625		625
626	if ((sci_in(port, SCLSR) & SCIF_ORER) != 0) {	626	if ((sci_in(port, SCLSR) & SCIF_ORER) != 0) {
627	sci_out(port, SCLSR, 0);	627	sci_out(port, SCLSR, 0);
628		628
629	tty_insert_flip_char(tty, 0, TTY_OVERRUN);	629	tty_insert_flip_char(tty, 0, TTY_OVERRUN);
630	tty_flip_buffer_push(tty);	630	tty_flip_buffer_push(tty);
631		631
632	dev_notice(port->dev, "overrun error\n");	632	dev_notice(port->dev, "overrun error\n");
633	copied++;	633	copied++;
634	}	634	}
635		635
636	return copied;	636	return copied;
637	}	637	}
638		638
639	static inline int sci_handle_breaks(struct uart_port *port)	639	static inline int sci_handle_breaks(struct uart_port *port)
640	{	640	{
641	int copied = 0;	641	int copied = 0;
642	unsigned short status = sci_in(port, SCxSR);	642	unsigned short status = sci_in(port, SCxSR);
643	struct tty_struct *tty = port->state->port.tty;	643	struct tty_struct *tty = port->state->port.tty;
644	struct sci_port *s = to_sci_port(port);	644	struct sci_port *s = to_sci_port(port);
645		645
646	if (uart_handle_break(port))	646	if (uart_handle_break(port))
647	return 0;	647	return 0;
648		648
649	if (!s->break_flag && status & SCxSR_BRK(port)) {	649	if (!s->break_flag && status & SCxSR_BRK(port)) {
650	#if defined(CONFIG_CPU_SH3)	650	#if defined(CONFIG_CPU_SH3)
651	/* Debounce break */	651	/* Debounce break */
652	s->break_flag = 1;	652	s->break_flag = 1;
653	#endif	653	#endif
654	/* Notify of BREAK */	654	/* Notify of BREAK */
655	if (tty_insert_flip_char(tty, 0, TTY_BREAK))	655	if (tty_insert_flip_char(tty, 0, TTY_BREAK))
656	copied++;	656	copied++;
657		657
658	dev_dbg(port->dev, "BREAK detected\n");	658	dev_dbg(port->dev, "BREAK detected\n");
659	}	659	}
660		660
661	if (copied)	661	if (copied)
662	tty_flip_buffer_push(tty);	662	tty_flip_buffer_push(tty);
663		663
664	copied += sci_handle_fifo_overrun(port);	664	copied += sci_handle_fifo_overrun(port);
665		665
666	return copied;	666	return copied;
667	}	667	}
668		668
669	static irqreturn_t sci_rx_interrupt(int irq, void *ptr)	669	static irqreturn_t sci_rx_interrupt(int irq, void *ptr)
670	{	670	{
671	#ifdef CONFIG_SERIAL_SH_SCI_DMA	671	#ifdef CONFIG_SERIAL_SH_SCI_DMA
672	struct uart_port *port = ptr;	672	struct uart_port *port = ptr;
673	struct sci_port *s = to_sci_port(port);	673	struct sci_port *s = to_sci_port(port);
674		674
675	if (s->chan_rx) {	675	if (s->chan_rx) {
676	unsigned long tout;	676	unsigned long tout;
677	u16 scr = sci_in(port, SCSCR);	677	u16 scr = sci_in(port, SCSCR);
678	u16 ssr = sci_in(port, SCxSR);	678	u16 ssr = sci_in(port, SCxSR);
679		679
680	/* Disable future Rx interrupts */	680	/* Disable future Rx interrupts */
681	sci_out(port, SCSCR, scr & ~SCI_CTRL_FLAGS_RIE);	681	sci_out(port, SCSCR, scr & ~SCI_CTRL_FLAGS_RIE);
682	/* Clear current interrupt */	682	/* Clear current interrupt */
683	sci_out(port, SCxSR, ssr & ~(1 \| SCxSR_RDxF(port)));	683	sci_out(port, SCxSR, ssr & ~(1 \| SCxSR_RDxF(port)));
684	/* Calculate delay for 1.5 DMA buffers */	684	/* Calculate delay for 1.5 DMA buffers */
685	tout = (port->timeout - HZ / 50) * s->buf_len_rx * 3 /	685	tout = (port->timeout - HZ / 50) * s->buf_len_rx * 3 /
686	port->fifosize / 2;	686	port->fifosize / 2;
687	dev_dbg(port->dev, "Rx IRQ: setup timeout in %lu ms\n",	687	dev_dbg(port->dev, "Rx IRQ: setup timeout in %lu ms\n",
688	tout * 1000 / HZ);	688	tout * 1000 / HZ);
689	if (tout < 2)	689	if (tout < 2)
690	tout = 2;	690	tout = 2;
691	mod_timer(&s->rx_timer, jiffies + tout);	691	mod_timer(&s->rx_timer, jiffies + tout);
692		692
693	return IRQ_HANDLED;	693	return IRQ_HANDLED;
694	}	694	}
695	#endif	695	#endif
696		696
697	/* I think sci_receive_chars has to be called irrespective	697	/* I think sci_receive_chars has to be called irrespective
698	* of whether the I_IXOFF is set, otherwise, how is the interrupt	698	* of whether the I_IXOFF is set, otherwise, how is the interrupt
699	* to be disabled?	699	* to be disabled?
700	*/	700	*/
701	sci_receive_chars(ptr);	701	sci_receive_chars(ptr);
702		702
703	return IRQ_HANDLED;	703	return IRQ_HANDLED;
704	}	704	}
705		705
706	static irqreturn_t sci_tx_interrupt(int irq, void *ptr)	706	static irqreturn_t sci_tx_interrupt(int irq, void *ptr)
707	{	707	{
708	struct uart_port *port = ptr;	708	struct uart_port *port = ptr;
709	unsigned long flags;	709	unsigned long flags;
710		710
711	spin_lock_irqsave(&port->lock, flags);	711	spin_lock_irqsave(&port->lock, flags);
712	sci_transmit_chars(port);	712	sci_transmit_chars(port);
713	spin_unlock_irqrestore(&port->lock, flags);	713	spin_unlock_irqrestore(&port->lock, flags);
714		714
715	return IRQ_HANDLED;	715	return IRQ_HANDLED;
716	}	716	}
717		717
718	static irqreturn_t sci_er_interrupt(int irq, void *ptr)	718	static irqreturn_t sci_er_interrupt(int irq, void *ptr)
719	{	719	{
720	struct uart_port *port = ptr;	720	struct uart_port *port = ptr;
721		721
722	/* Handle errors */	722	/* Handle errors */
723	if (port->type == PORT_SCI) {	723	if (port->type == PORT_SCI) {
724	if (sci_handle_errors(port)) {	724	if (sci_handle_errors(port)) {
725	/* discard character in rx buffer */	725	/* discard character in rx buffer */
726	sci_in(port, SCxSR);	726	sci_in(port, SCxSR);
727	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));	727	sci_out(port, SCxSR, SCxSR_RDxF_CLEAR(port));
728	}	728	}
729	} else {	729	} else {
730	sci_handle_fifo_overrun(port);	730	sci_handle_fifo_overrun(port);
731	sci_rx_interrupt(irq, ptr);	731	sci_rx_interrupt(irq, ptr);
732	}	732	}
733		733
734	sci_out(port, SCxSR, SCxSR_ERROR_CLEAR(port));	734	sci_out(port, SCxSR, SCxSR_ERROR_CLEAR(port));
735		735
736	/* Kick the transmission */	736	/* Kick the transmission */
737	sci_tx_interrupt(irq, ptr);	737	sci_tx_interrupt(irq, ptr);
738		738
739	return IRQ_HANDLED;	739	return IRQ_HANDLED;
740	}	740	}
741		741
742	static irqreturn_t sci_br_interrupt(int irq, void *ptr)	742	static irqreturn_t sci_br_interrupt(int irq, void *ptr)
743	{	743	{
744	struct uart_port *port = ptr;	744	struct uart_port *port = ptr;
745		745
746	/* Handle BREAKs */	746	/* Handle BREAKs */
747	sci_handle_breaks(port);	747	sci_handle_breaks(port);
748	sci_out(port, SCxSR, SCxSR_BREAK_CLEAR(port));	748	sci_out(port, SCxSR, SCxSR_BREAK_CLEAR(port));
749		749
750	return IRQ_HANDLED;	750	return IRQ_HANDLED;
751	}	751	}
752		752
753	static irqreturn_t sci_mpxed_interrupt(int irq, void *ptr)	753	static irqreturn_t sci_mpxed_interrupt(int irq, void *ptr)
754	{	754	{
755	unsigned short ssr_status, scr_status, err_enabled;	755	unsigned short ssr_status, scr_status, err_enabled;
756	struct uart_port *port = ptr;	756	struct uart_port *port = ptr;
757	struct sci_port *s = to_sci_port(port);	757	struct sci_port *s = to_sci_port(port);
758	irqreturn_t ret = IRQ_NONE;	758	irqreturn_t ret = IRQ_NONE;
759		759
760	ssr_status = sci_in(port, SCxSR);	760	ssr_status = sci_in(port, SCxSR);
761	scr_status = sci_in(port, SCSCR);	761	scr_status = sci_in(port, SCSCR);
762	err_enabled = scr_status & (SCI_CTRL_FLAGS_REIE \| SCI_CTRL_FLAGS_RIE);	762	err_enabled = scr_status & (SCI_CTRL_FLAGS_REIE \| SCI_CTRL_FLAGS_RIE);
763		763
764	/* Tx Interrupt */	764	/* Tx Interrupt */
765	if ((ssr_status & SCxSR_TDxE(port)) && (scr_status & SCI_CTRL_FLAGS_TIE) &&	765	if ((ssr_status & SCxSR_TDxE(port)) && (scr_status & SCI_CTRL_FLAGS_TIE) &&
766	!s->chan_tx)	766	!s->chan_tx)
767	ret = sci_tx_interrupt(irq, ptr);	767	ret = sci_tx_interrupt(irq, ptr);
768	/*	768	/*
769	* Rx Interrupt: if we're using DMA, the DMA controller clears RDF /	769	* Rx Interrupt: if we're using DMA, the DMA controller clears RDF /
770	* DR flags	770	* DR flags
771	*/	771	*/
772	if (((ssr_status & SCxSR_RDxF(port)) \|\| s->chan_rx) &&	772	if (((ssr_status & SCxSR_RDxF(port)) \|\| s->chan_rx) &&
773	(scr_status & SCI_CTRL_FLAGS_RIE))	773	(scr_status & SCI_CTRL_FLAGS_RIE))
774	ret = sci_rx_interrupt(irq, ptr);	774	ret = sci_rx_interrupt(irq, ptr);
775	/* Error Interrupt */	775	/* Error Interrupt */
776	if ((ssr_status & SCxSR_ERRORS(port)) && err_enabled)	776	if ((ssr_status & SCxSR_ERRORS(port)) && err_enabled)
777	ret = sci_er_interrupt(irq, ptr);	777	ret = sci_er_interrupt(irq, ptr);
778	/* Break Interrupt */	778	/* Break Interrupt */
779	if ((ssr_status & SCxSR_BRK(port)) && err_enabled)	779	if ((ssr_status & SCxSR_BRK(port)) && err_enabled)
780	ret = sci_br_interrupt(irq, ptr);	780	ret = sci_br_interrupt(irq, ptr);
781		781
782	WARN_ONCE(ret == IRQ_NONE,	782	WARN_ONCE(ret == IRQ_NONE,
783	"%s: %d IRQ %d, status %x, control %x\n", __func__,	783	"%s: %d IRQ %d, status %x, control %x\n", __func__,
784	irq, port->line, ssr_status, scr_status);	784	irq, port->line, ssr_status, scr_status);
785		785
786	return ret;	786	return ret;
787	}	787	}
788		788
789	/*	789	/*
790	* Here we define a transistion notifier so that we can update all of our	790	* Here we define a transistion notifier so that we can update all of our
791	* ports' baud rate when the peripheral clock changes.	791	* ports' baud rate when the peripheral clock changes.
792	*/	792	*/
793	static int sci_notifier(struct notifier_block *self,	793	static int sci_notifier(struct notifier_block *self,
794	unsigned long phase, void *p)	794	unsigned long phase, void *p)
795	{	795	{
796	struct sh_sci_priv *priv = container_of(self,	796	struct sh_sci_priv *priv = container_of(self,
797	struct sh_sci_priv, clk_nb);	797	struct sh_sci_priv, clk_nb);
798	struct sci_port *sci_port;	798	struct sci_port *sci_port;
799	unsigned long flags;	799	unsigned long flags;
800		800
801	if ((phase == CPUFREQ_POSTCHANGE) \|\|	801	if ((phase == CPUFREQ_POSTCHANGE) \|\|
802	(phase == CPUFREQ_RESUMECHANGE)) {	802	(phase == CPUFREQ_RESUMECHANGE)) {
803	spin_lock_irqsave(&priv->lock, flags);	803	spin_lock_irqsave(&priv->lock, flags);
804	list_for_each_entry(sci_port, &priv->ports, node)	804	list_for_each_entry(sci_port, &priv->ports, node)
805	sci_port->port.uartclk = clk_get_rate(sci_port->dclk);	805	sci_port->port.uartclk = clk_get_rate(sci_port->dclk);
806	spin_unlock_irqrestore(&priv->lock, flags);	806	spin_unlock_irqrestore(&priv->lock, flags);
807	}	807	}
808		808
809	return NOTIFY_OK;	809	return NOTIFY_OK;
810	}	810	}
811		811
812	static void sci_clk_enable(struct uart_port *port)	812	static void sci_clk_enable(struct uart_port *port)
813	{	813	{
814	struct sci_port *sci_port = to_sci_port(port);	814	struct sci_port *sci_port = to_sci_port(port);
815		815
816	clk_enable(sci_port->dclk);	816	clk_enable(sci_port->dclk);
817	sci_port->port.uartclk = clk_get_rate(sci_port->dclk);	817	sci_port->port.uartclk = clk_get_rate(sci_port->dclk);
818		818
819	if (sci_port->iclk)	819	if (sci_port->iclk)
820	clk_enable(sci_port->iclk);	820	clk_enable(sci_port->iclk);
821	}	821	}
822		822
823	static void sci_clk_disable(struct uart_port *port)	823	static void sci_clk_disable(struct uart_port *port)
824	{	824	{
825	struct sci_port *sci_port = to_sci_port(port);	825	struct sci_port *sci_port = to_sci_port(port);
826		826
827	if (sci_port->iclk)	827	if (sci_port->iclk)
828	clk_disable(sci_port->iclk);	828	clk_disable(sci_port->iclk);
829		829
830	clk_disable(sci_port->dclk);	830	clk_disable(sci_port->dclk);
831	}	831	}
832		832
833	static int sci_request_irq(struct sci_port *port)	833	static int sci_request_irq(struct sci_port *port)
834	{	834	{
835	int i;	835	int i;
836	irqreturn_t (handlers[4])(int irq, void ptr) = {	836	irqreturn_t (handlers[4])(int irq, void ptr) = {
837	sci_er_interrupt, sci_rx_interrupt, sci_tx_interrupt,	837	sci_er_interrupt, sci_rx_interrupt, sci_tx_interrupt,
838	sci_br_interrupt,	838	sci_br_interrupt,
839	};	839	};
840	const char *desc[] = { "SCI Receive Error", "SCI Receive Data Full",	840	const char *desc[] = { "SCI Receive Error", "SCI Receive Data Full",
841	"SCI Transmit Data Empty", "SCI Break" };	841	"SCI Transmit Data Empty", "SCI Break" };
842		842
843	if (port->irqs[0] == port->irqs[1]) {	843	if (port->irqs[0] == port->irqs[1]) {
844	if (unlikely(!port->irqs[0]))	844	if (unlikely(!port->irqs[0]))
845	return -ENODEV;	845	return -ENODEV;
846		846
847	if (request_irq(port->irqs[0], sci_mpxed_interrupt,	847	if (request_irq(port->irqs[0], sci_mpxed_interrupt,
848	IRQF_DISABLED, "sci", port)) {	848	IRQF_DISABLED, "sci", port)) {
849	dev_err(port->port.dev, "Can't allocate IRQ\n");	849	dev_err(port->port.dev, "Can't allocate IRQ\n");
850	return -ENODEV;	850	return -ENODEV;
851	}	851	}
852	} else {	852	} else {
853	for (i = 0; i < ARRAY_SIZE(handlers); i++) {	853	for (i = 0; i < ARRAY_SIZE(handlers); i++) {
854	if (unlikely(!port->irqs[i]))	854	if (unlikely(!port->irqs[i]))
855	continue;	855	continue;
856		856
857	if (request_irq(port->irqs[i], handlers[i],	857	if (request_irq(port->irqs[i], handlers[i],
858	IRQF_DISABLED, desc[i], port)) {	858	IRQF_DISABLED, desc[i], port)) {
859	dev_err(port->port.dev, "Can't allocate IRQ\n");	859	dev_err(port->port.dev, "Can't allocate IRQ\n");
860	return -ENODEV;	860	return -ENODEV;
861	}	861	}
862	}	862	}
863	}	863	}
864		864
865	return 0;	865	return 0;
866	}	866	}
867		867
868	static void sci_free_irq(struct sci_port *port)	868	static void sci_free_irq(struct sci_port *port)
869	{	869	{
870	int i;	870	int i;
871		871
872	if (port->irqs[0] == port->irqs[1])	872	if (port->irqs[0] == port->irqs[1])
873	free_irq(port->irqs[0], port);	873	free_irq(port->irqs[0], port);
874	else {	874	else {
875	for (i = 0; i < ARRAY_SIZE(port->irqs); i++) {	875	for (i = 0; i < ARRAY_SIZE(port->irqs); i++) {
876	if (!port->irqs[i])	876	if (!port->irqs[i])
877	continue;	877	continue;
878		878
879	free_irq(port->irqs[i], port);	879	free_irq(port->irqs[i], port);
880	}	880	}
881	}	881	}
882	}	882	}
883		883
884	static unsigned int sci_tx_empty(struct uart_port *port)	884	static unsigned int sci_tx_empty(struct uart_port *port)
885	{	885	{
886	unsigned short status = sci_in(port, SCxSR);	886	unsigned short status = sci_in(port, SCxSR);
887	unsigned short in_tx_fifo = scif_txfill(port);	887	unsigned short in_tx_fifo = scif_txfill(port);
888		888
889	return (status & SCxSR_TEND(port)) && !in_tx_fifo ? TIOCSER_TEMT : 0;	889	return (status & SCxSR_TEND(port)) && !in_tx_fifo ? TIOCSER_TEMT : 0;
890	}	890	}
891		891
892	static void sci_set_mctrl(struct uart_port *port, unsigned int mctrl)	892	static void sci_set_mctrl(struct uart_port *port, unsigned int mctrl)
893	{	893	{
894	/* This routine is used for seting signals of: DTR, DCD, CTS/RTS */	894	/* This routine is used for seting signals of: DTR, DCD, CTS/RTS */
895	/* We use SCIF's hardware for CTS/RTS, so don't need any for that. */	895	/* We use SCIF's hardware for CTS/RTS, so don't need any for that. */
896	/* If you have signals for DTR and DCD, please implement here. */	896	/* If you have signals for DTR and DCD, please implement here. */
897	}	897	}
898		898
899	static unsigned int sci_get_mctrl(struct uart_port *port)	899	static unsigned int sci_get_mctrl(struct uart_port *port)
900	{	900	{
901	/* This routine is used for getting signals of: DTR, DCD, DSR, RI,	901	/* This routine is used for getting signals of: DTR, DCD, DSR, RI,
902	and CTS/RTS */	902	and CTS/RTS */
903		903
904	return TIOCM_DTR \| TIOCM_RTS \| TIOCM_DSR;	904	return TIOCM_DTR \| TIOCM_RTS \| TIOCM_DSR;
905	}	905	}
906		906
907	#ifdef CONFIG_SERIAL_SH_SCI_DMA	907	#ifdef CONFIG_SERIAL_SH_SCI_DMA
908	static void sci_dma_tx_complete(void *arg)	908	static void sci_dma_tx_complete(void *arg)
909	{	909	{
910	struct sci_port *s = arg;	910	struct sci_port *s = arg;
911	struct uart_port *port = &s->port;	911	struct uart_port *port = &s->port;
912	struct circ_buf *xmit = &port->state->xmit;	912	struct circ_buf *xmit = &port->state->xmit;
913	unsigned long flags;	913	unsigned long flags;
914		914
915	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);	915	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);
916		916
917	spin_lock_irqsave(&port->lock, flags);	917	spin_lock_irqsave(&port->lock, flags);
918		918
919	xmit->tail += s->sg_tx.length;	919	xmit->tail += s->sg_tx.length;
920	xmit->tail &= UART_XMIT_SIZE - 1;	920	xmit->tail &= UART_XMIT_SIZE - 1;
921		921
922	port->icount.tx += s->sg_tx.length;	922	port->icount.tx += s->sg_tx.length;
923		923
924	async_tx_ack(s->desc_tx);	924	async_tx_ack(s->desc_tx);
925	s->cookie_tx = -EINVAL;	925	s->cookie_tx = -EINVAL;
926	s->desc_tx = NULL;	926	s->desc_tx = NULL;
927		927
928	spin_unlock_irqrestore(&port->lock, flags);	928	spin_unlock_irqrestore(&port->lock, flags);
929		929
930	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)	930	if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
931	uart_write_wakeup(port);	931	uart_write_wakeup(port);
932		932
933	if (uart_circ_chars_pending(xmit))	933	if (uart_circ_chars_pending(xmit))
934	schedule_work(&s->work_tx);	934	schedule_work(&s->work_tx);
935	}	935	}
936		936
937	/* Locking: called with port lock held */	937	/* Locking: called with port lock held */
938	static int sci_dma_rx_push(struct sci_port s, struct tty_struct tty,	938	static int sci_dma_rx_push(struct sci_port s, struct tty_struct tty,
939	size_t count)	939	size_t count)
940	{	940	{
941	struct uart_port *port = &s->port;	941	struct uart_port *port = &s->port;
942	int i, active, room;	942	int i, active, room;
943		943
944	room = tty_buffer_request_room(tty, count);	944	room = tty_buffer_request_room(tty, count);
945		945
946	if (s->active_rx == s->cookie_rx[0]) {	946	if (s->active_rx == s->cookie_rx[0]) {
947	active = 0;	947	active = 0;
948	} else if (s->active_rx == s->cookie_rx[1]) {	948	} else if (s->active_rx == s->cookie_rx[1]) {
949	active = 1;	949	active = 1;
950	} else {	950	} else {
951	dev_err(port->dev, "cookie %d not found!\n", s->active_rx);	951	dev_err(port->dev, "cookie %d not found!\n", s->active_rx);
952	return 0;	952	return 0;
953	}	953	}
954		954
955	if (room < count)	955	if (room < count)
956	dev_warn(port->dev, "Rx overrun: dropping %u bytes\n",	956	dev_warn(port->dev, "Rx overrun: dropping %u bytes\n",
957	count - room);	957	count - room);
958	if (!room)	958	if (!room)
959	return room;	959	return room;
960		960
961	for (i = 0; i < room; i++)	961	for (i = 0; i < room; i++)
962	tty_insert_flip_char(tty, ((u8 *)sg_virt(&s->sg_rx[active]))[i],	962	tty_insert_flip_char(tty, ((u8 *)sg_virt(&s->sg_rx[active]))[i],
963	TTY_NORMAL);	963	TTY_NORMAL);
964		964
965	port->icount.rx += room;	965	port->icount.rx += room;
966		966
967	return room;	967	return room;
968	}	968	}
969		969
970	static void sci_dma_rx_complete(void *arg)	970	static void sci_dma_rx_complete(void *arg)
971	{	971	{
972	struct sci_port *s = arg;	972	struct sci_port *s = arg;
973	struct uart_port *port = &s->port;	973	struct uart_port *port = &s->port;
974	struct tty_struct *tty = port->state->port.tty;	974	struct tty_struct *tty = port->state->port.tty;
975	unsigned long flags;	975	unsigned long flags;
976	int count;	976	int count;
977		977
978	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);	978	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);
979		979
980	spin_lock_irqsave(&port->lock, flags);	980	spin_lock_irqsave(&port->lock, flags);
981		981
982	count = sci_dma_rx_push(s, tty, s->buf_len_rx);	982	count = sci_dma_rx_push(s, tty, s->buf_len_rx);
983		983
984	mod_timer(&s->rx_timer, jiffies + msecs_to_jiffies(5));	984	mod_timer(&s->rx_timer, jiffies + msecs_to_jiffies(5));
985		985
986	spin_unlock_irqrestore(&port->lock, flags);	986	spin_unlock_irqrestore(&port->lock, flags);
987		987
988	if (count)	988	if (count)
989	tty_flip_buffer_push(tty);	989	tty_flip_buffer_push(tty);
990		990
991	schedule_work(&s->work_rx);	991	schedule_work(&s->work_rx);
992	}	992	}
993		993
994	static void sci_start_rx(struct uart_port *port);	994	static void sci_start_rx(struct uart_port *port);
995	static void sci_start_tx(struct uart_port *port);	995	static void sci_start_tx(struct uart_port *port);
996		996
997	static void sci_rx_dma_release(struct sci_port *s, bool enable_pio)	997	static void sci_rx_dma_release(struct sci_port *s, bool enable_pio)
998	{	998	{
999	struct dma_chan *chan = s->chan_rx;	999	struct dma_chan *chan = s->chan_rx;
1000	struct uart_port *port = &s->port;	1000	struct uart_port *port = &s->port;
1001		1001
1002	s->chan_rx = NULL;	1002	s->chan_rx = NULL;
1003	s->cookie_rx[0] = s->cookie_rx[1] = -EINVAL;	1003	s->cookie_rx[0] = s->cookie_rx[1] = -EINVAL;
1004	dma_release_channel(chan);	1004	dma_release_channel(chan);
1005	dma_free_coherent(port->dev, s->buf_len_rx * 2,	1005	dma_free_coherent(port->dev, s->buf_len_rx * 2,
1006	sg_virt(&s->sg_rx[0]), sg_dma_address(&s->sg_rx[0]));	1006	sg_virt(&s->sg_rx[0]), sg_dma_address(&s->sg_rx[0]));
1007	if (enable_pio)	1007	if (enable_pio)
1008	sci_start_rx(port);	1008	sci_start_rx(port);
1009	}	1009	}
1010		1010
1011	static void sci_tx_dma_release(struct sci_port *s, bool enable_pio)	1011	static void sci_tx_dma_release(struct sci_port *s, bool enable_pio)
1012	{	1012	{
1013	struct dma_chan *chan = s->chan_tx;	1013	struct dma_chan *chan = s->chan_tx;
1014	struct uart_port *port = &s->port;	1014	struct uart_port *port = &s->port;
1015		1015
1016	s->chan_tx = NULL;	1016	s->chan_tx = NULL;
1017	s->cookie_tx = -EINVAL;	1017	s->cookie_tx = -EINVAL;
1018	dma_release_channel(chan);	1018	dma_release_channel(chan);
1019	if (enable_pio)	1019	if (enable_pio)
1020	sci_start_tx(port);	1020	sci_start_tx(port);
1021	}	1021	}
1022		1022
1023	static void sci_submit_rx(struct sci_port *s)	1023	static void sci_submit_rx(struct sci_port *s)
1024	{	1024	{
1025	struct dma_chan *chan = s->chan_rx;	1025	struct dma_chan *chan = s->chan_rx;
1026	int i;	1026	int i;
1027		1027
1028	for (i = 0; i < 2; i++) {	1028	for (i = 0; i < 2; i++) {
1029	struct scatterlist *sg = &s->sg_rx[i];	1029	struct scatterlist *sg = &s->sg_rx[i];
1030	struct dma_async_tx_descriptor *desc;	1030	struct dma_async_tx_descriptor *desc;
1031		1031
1032	desc = chan->device->device_prep_slave_sg(chan,	1032	desc = chan->device->device_prep_slave_sg(chan,
1033	sg, 1, DMA_FROM_DEVICE, DMA_PREP_INTERRUPT);	1033	sg, 1, DMA_FROM_DEVICE, DMA_PREP_INTERRUPT);
1034		1034
1035	if (desc) {	1035	if (desc) {
1036	s->desc_rx[i] = desc;	1036	s->desc_rx[i] = desc;
1037	desc->callback = sci_dma_rx_complete;	1037	desc->callback = sci_dma_rx_complete;
1038	desc->callback_param = s;	1038	desc->callback_param = s;
1039	s->cookie_rx[i] = desc->tx_submit(desc);	1039	s->cookie_rx[i] = desc->tx_submit(desc);
1040	}	1040	}
1041		1041
1042	if (!desc \|\| s->cookie_rx[i] < 0) {	1042	if (!desc \|\| s->cookie_rx[i] < 0) {
1043	if (i) {	1043	if (i) {
1044	async_tx_ack(s->desc_rx[0]);	1044	async_tx_ack(s->desc_rx[0]);
1045	s->cookie_rx[0] = -EINVAL;	1045	s->cookie_rx[0] = -EINVAL;
1046	}	1046	}
1047	if (desc) {	1047	if (desc) {
1048	async_tx_ack(desc);	1048	async_tx_ack(desc);
1049	s->cookie_rx[i] = -EINVAL;	1049	s->cookie_rx[i] = -EINVAL;
1050	}	1050	}
1051	dev_warn(s->port.dev,	1051	dev_warn(s->port.dev,
1052	"failed to re-start DMA, using PIO\n");	1052	"failed to re-start DMA, using PIO\n");
1053	sci_rx_dma_release(s, true);	1053	sci_rx_dma_release(s, true);
1054	return;	1054	return;
1055	}	1055	}
1056	}	1056	}
1057		1057
1058	s->active_rx = s->cookie_rx[0];	1058	s->active_rx = s->cookie_rx[0];
1059		1059
1060	dma_async_issue_pending(chan);	1060	dma_async_issue_pending(chan);
1061	}	1061	}
1062		1062
1063	static void work_fn_rx(struct work_struct *work)	1063	static void work_fn_rx(struct work_struct *work)
1064	{	1064	{
1065	struct sci_port *s = container_of(work, struct sci_port, work_rx);	1065	struct sci_port *s = container_of(work, struct sci_port, work_rx);
1066	struct uart_port *port = &s->port;	1066	struct uart_port *port = &s->port;
1067	struct dma_async_tx_descriptor *desc;	1067	struct dma_async_tx_descriptor *desc;
1068	int new;	1068	int new;
1069		1069
1070	if (s->active_rx == s->cookie_rx[0]) {	1070	if (s->active_rx == s->cookie_rx[0]) {
1071	new = 0;	1071	new = 0;
1072	} else if (s->active_rx == s->cookie_rx[1]) {	1072	} else if (s->active_rx == s->cookie_rx[1]) {
1073	new = 1;	1073	new = 1;
1074	} else {	1074	} else {
1075	dev_err(port->dev, "cookie %d not found!\n", s->active_rx);	1075	dev_err(port->dev, "cookie %d not found!\n", s->active_rx);
1076	return;	1076	return;
1077	}	1077	}
1078	desc = s->desc_rx[new];	1078	desc = s->desc_rx[new];
1079		1079
1080	if (dma_async_is_tx_complete(s->chan_rx, s->active_rx, NULL, NULL) !=	1080	if (dma_async_is_tx_complete(s->chan_rx, s->active_rx, NULL, NULL) !=
1081	DMA_SUCCESS) {	1081	DMA_SUCCESS) {
1082	/* Handle incomplete DMA receive */	1082	/* Handle incomplete DMA receive */
1083	struct tty_struct *tty = port->state->port.tty;	1083	struct tty_struct *tty = port->state->port.tty;
1084	struct dma_chan *chan = s->chan_rx;	1084	struct dma_chan *chan = s->chan_rx;
1085	struct sh_desc *sh_desc = container_of(desc, struct sh_desc,	1085	struct sh_desc *sh_desc = container_of(desc, struct sh_desc,
1086	async_tx);	1086	async_tx);
1087	unsigned long flags;	1087	unsigned long flags;
1088	int count;	1088	int count;
1089		1089
1090	chan->device->device_control(chan, DMA_TERMINATE_ALL);	1090	chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
1091	dev_dbg(port->dev, "Read %u bytes with cookie %d\n",	1091	dev_dbg(port->dev, "Read %u bytes with cookie %d\n",
1092	sh_desc->partial, sh_desc->cookie);	1092	sh_desc->partial, sh_desc->cookie);
1093		1093
1094	spin_lock_irqsave(&port->lock, flags);	1094	spin_lock_irqsave(&port->lock, flags);
1095	count = sci_dma_rx_push(s, tty, sh_desc->partial);	1095	count = sci_dma_rx_push(s, tty, sh_desc->partial);
1096	spin_unlock_irqrestore(&port->lock, flags);	1096	spin_unlock_irqrestore(&port->lock, flags);
1097		1097
1098	if (count)	1098	if (count)
1099	tty_flip_buffer_push(tty);	1099	tty_flip_buffer_push(tty);
1100		1100
1101	sci_submit_rx(s);	1101	sci_submit_rx(s);
1102		1102
1103	return;	1103	return;
1104	}	1104	}
1105		1105
1106	s->cookie_rx[new] = desc->tx_submit(desc);	1106	s->cookie_rx[new] = desc->tx_submit(desc);
1107	if (s->cookie_rx[new] < 0) {	1107	if (s->cookie_rx[new] < 0) {
1108	dev_warn(port->dev, "Failed submitting Rx DMA descriptor\n");	1108	dev_warn(port->dev, "Failed submitting Rx DMA descriptor\n");
1109	sci_rx_dma_release(s, true);	1109	sci_rx_dma_release(s, true);
1110	return;	1110	return;
1111	}	1111	}
1112		1112
1113	dev_dbg(port->dev, "%s: cookie %d #%d\n", __func__,	1113	dev_dbg(port->dev, "%s: cookie %d #%d\n", __func__,
1114	s->cookie_rx[new], new);	1114	s->cookie_rx[new], new);
1115		1115
1116	s->active_rx = s->cookie_rx[!new];	1116	s->active_rx = s->cookie_rx[!new];
1117	}	1117	}
1118		1118
1119	static void work_fn_tx(struct work_struct *work)	1119	static void work_fn_tx(struct work_struct *work)
1120	{	1120	{
1121	struct sci_port *s = container_of(work, struct sci_port, work_tx);	1121	struct sci_port *s = container_of(work, struct sci_port, work_tx);
1122	struct dma_async_tx_descriptor *desc;	1122	struct dma_async_tx_descriptor *desc;
1123	struct dma_chan *chan = s->chan_tx;	1123	struct dma_chan *chan = s->chan_tx;
1124	struct uart_port *port = &s->port;	1124	struct uart_port *port = &s->port;
1125	struct circ_buf *xmit = &port->state->xmit;	1125	struct circ_buf *xmit = &port->state->xmit;
1126	struct scatterlist *sg = &s->sg_tx;	1126	struct scatterlist *sg = &s->sg_tx;
1127		1127
1128	/*	1128	/*
1129	* DMA is idle now.	1129	* DMA is idle now.
1130	* Port xmit buffer is already mapped, and it is one page... Just adjust	1130	* Port xmit buffer is already mapped, and it is one page... Just adjust
1131	* offsets and lengths. Since it is a circular buffer, we have to	1131	* offsets and lengths. Since it is a circular buffer, we have to
1132	* transmit till the end, and then the rest. Take the port lock to get a	1132	* transmit till the end, and then the rest. Take the port lock to get a
1133	* consistent xmit buffer state.	1133	* consistent xmit buffer state.
1134	*/	1134	*/
1135	spin_lock_irq(&port->lock);	1135	spin_lock_irq(&port->lock);
1136	sg->offset = xmit->tail & (UART_XMIT_SIZE - 1);	1136	sg->offset = xmit->tail & (UART_XMIT_SIZE - 1);
1137	sg->dma_address = (sg_dma_address(sg) & ~(UART_XMIT_SIZE - 1)) +	1137	sg->dma_address = (sg_dma_address(sg) & ~(UART_XMIT_SIZE - 1)) +
1138	sg->offset;	1138	sg->offset;
1139	sg->length = min((int)CIRC_CNT(xmit->head, xmit->tail, UART_XMIT_SIZE),	1139	sg->length = min((int)CIRC_CNT(xmit->head, xmit->tail, UART_XMIT_SIZE),
1140	CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE));	1140	CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE));
1141	sg->dma_length = sg->length;	1141	sg->dma_length = sg->length;
1142	spin_unlock_irq(&port->lock);	1142	spin_unlock_irq(&port->lock);
1143		1143
1144	BUG_ON(!sg->length);	1144	BUG_ON(!sg->length);
1145		1145
1146	desc = chan->device->device_prep_slave_sg(chan,	1146	desc = chan->device->device_prep_slave_sg(chan,
1147	sg, s->sg_len_tx, DMA_TO_DEVICE,	1147	sg, s->sg_len_tx, DMA_TO_DEVICE,
1148	DMA_PREP_INTERRUPT \| DMA_CTRL_ACK);	1148	DMA_PREP_INTERRUPT \| DMA_CTRL_ACK);
1149	if (!desc) {	1149	if (!desc) {
1150	/* switch to PIO */	1150	/* switch to PIO */
1151	sci_tx_dma_release(s, true);	1151	sci_tx_dma_release(s, true);
1152	return;	1152	return;
1153	}	1153	}
1154		1154
1155	dma_sync_sg_for_device(port->dev, sg, 1, DMA_TO_DEVICE);	1155	dma_sync_sg_for_device(port->dev, sg, 1, DMA_TO_DEVICE);
1156		1156
1157	spin_lock_irq(&port->lock);	1157	spin_lock_irq(&port->lock);
1158	s->desc_tx = desc;	1158	s->desc_tx = desc;
1159	desc->callback = sci_dma_tx_complete;	1159	desc->callback = sci_dma_tx_complete;
1160	desc->callback_param = s;	1160	desc->callback_param = s;
1161	spin_unlock_irq(&port->lock);	1161	spin_unlock_irq(&port->lock);
1162	s->cookie_tx = desc->tx_submit(desc);	1162	s->cookie_tx = desc->tx_submit(desc);
1163	if (s->cookie_tx < 0) {	1163	if (s->cookie_tx < 0) {
1164	dev_warn(port->dev, "Failed submitting Tx DMA descriptor\n");	1164	dev_warn(port->dev, "Failed submitting Tx DMA descriptor\n");
1165	/* switch to PIO */	1165	/* switch to PIO */
1166	sci_tx_dma_release(s, true);	1166	sci_tx_dma_release(s, true);
1167	return;	1167	return;
1168	}	1168	}
1169		1169
1170	dev_dbg(port->dev, "%s: %p: %d...%d, cookie %d\n", __func__,	1170	dev_dbg(port->dev, "%s: %p: %d...%d, cookie %d\n", __func__,
1171	xmit->buf, xmit->tail, xmit->head, s->cookie_tx);	1171	xmit->buf, xmit->tail, xmit->head, s->cookie_tx);
1172		1172
1173	dma_async_issue_pending(chan);	1173	dma_async_issue_pending(chan);
1174	}	1174	}
1175	#endif	1175	#endif
1176		1176
1177	static void sci_start_tx(struct uart_port *port)	1177	static void sci_start_tx(struct uart_port *port)
1178	{	1178	{
1179	unsigned short ctrl;	1179	unsigned short ctrl;
1180		1180
1181	#ifdef CONFIG_SERIAL_SH_SCI_DMA	1181	#ifdef CONFIG_SERIAL_SH_SCI_DMA
1182	struct sci_port *s = to_sci_port(port);	1182	struct sci_port *s = to_sci_port(port);
1183		1183
1184	if (s->chan_tx) {	1184	if (s->chan_tx) {
1185	if (!uart_circ_empty(&s->port.state->xmit) && s->cookie_tx < 0)	1185	if (!uart_circ_empty(&s->port.state->xmit) && s->cookie_tx < 0)
1186	schedule_work(&s->work_tx);	1186	schedule_work(&s->work_tx);
1187		1187
1188	return;	1188	return;
1189	}	1189	}
1190	#endif	1190	#endif
1191		1191
1192	/* Set TIE (Transmit Interrupt Enable) bit in SCSCR */	1192	/* Set TIE (Transmit Interrupt Enable) bit in SCSCR */
1193	ctrl = sci_in(port, SCSCR);	1193	ctrl = sci_in(port, SCSCR);
1194	ctrl \|= SCI_CTRL_FLAGS_TIE;	1194	ctrl \|= SCI_CTRL_FLAGS_TIE;
1195	sci_out(port, SCSCR, ctrl);	1195	sci_out(port, SCSCR, ctrl);
1196	}	1196	}
1197		1197
1198	static void sci_stop_tx(struct uart_port *port)	1198	static void sci_stop_tx(struct uart_port *port)
1199	{	1199	{
1200	unsigned short ctrl;	1200	unsigned short ctrl;
1201		1201
1202	/* Clear TIE (Transmit Interrupt Enable) bit in SCSCR */	1202	/* Clear TIE (Transmit Interrupt Enable) bit in SCSCR */
1203	ctrl = sci_in(port, SCSCR);	1203	ctrl = sci_in(port, SCSCR);
1204	ctrl &= ~SCI_CTRL_FLAGS_TIE;	1204	ctrl &= ~SCI_CTRL_FLAGS_TIE;
1205	sci_out(port, SCSCR, ctrl);	1205	sci_out(port, SCSCR, ctrl);
1206	}	1206	}
1207		1207
1208	static void sci_start_rx(struct uart_port *port)	1208	static void sci_start_rx(struct uart_port *port)
1209	{	1209	{
1210	unsigned short ctrl = SCI_CTRL_FLAGS_RIE \| SCI_CTRL_FLAGS_REIE;	1210	unsigned short ctrl = SCI_CTRL_FLAGS_RIE \| SCI_CTRL_FLAGS_REIE;
1211		1211
1212	/* Set RIE (Receive Interrupt Enable) bit in SCSCR */	1212	/* Set RIE (Receive Interrupt Enable) bit in SCSCR */
1213	ctrl \|= sci_in(port, SCSCR);	1213	ctrl \|= sci_in(port, SCSCR);
1214	sci_out(port, SCSCR, ctrl);	1214	sci_out(port, SCSCR, ctrl);
1215	}	1215	}
1216		1216
1217	static void sci_stop_rx(struct uart_port *port)	1217	static void sci_stop_rx(struct uart_port *port)
1218	{	1218	{
1219	unsigned short ctrl;	1219	unsigned short ctrl;
1220		1220
1221	/* Clear RIE (Receive Interrupt Enable) bit in SCSCR */	1221	/* Clear RIE (Receive Interrupt Enable) bit in SCSCR */
1222	ctrl = sci_in(port, SCSCR);	1222	ctrl = sci_in(port, SCSCR);
1223	ctrl &= ~(SCI_CTRL_FLAGS_RIE \| SCI_CTRL_FLAGS_REIE);	1223	ctrl &= ~(SCI_CTRL_FLAGS_RIE \| SCI_CTRL_FLAGS_REIE);
1224	sci_out(port, SCSCR, ctrl);	1224	sci_out(port, SCSCR, ctrl);
1225	}	1225	}
1226		1226
1227	static void sci_enable_ms(struct uart_port *port)	1227	static void sci_enable_ms(struct uart_port *port)
1228	{	1228	{
1229	/* Nothing here yet .. */	1229	/* Nothing here yet .. */
1230	}	1230	}
1231		1231
1232	static void sci_break_ctl(struct uart_port *port, int break_state)	1232	static void sci_break_ctl(struct uart_port *port, int break_state)
1233	{	1233	{
1234	/* Nothing here yet .. */	1234	/* Nothing here yet .. */
1235	}	1235	}
1236		1236
1237	#ifdef CONFIG_SERIAL_SH_SCI_DMA	1237	#ifdef CONFIG_SERIAL_SH_SCI_DMA
1238	static bool filter(struct dma_chan chan, void slave)	1238	static bool filter(struct dma_chan chan, void slave)
1239	{	1239	{
1240	struct sh_dmae_slave *param = slave;	1240	struct sh_dmae_slave *param = slave;
1241		1241
1242	dev_dbg(chan->device->dev, "%s: slave ID %d\n", __func__,	1242	dev_dbg(chan->device->dev, "%s: slave ID %d\n", __func__,
1243	param->slave_id);	1243	param->slave_id);
1244		1244
1245	if (param->dma_dev == chan->device->dev) {	1245	if (param->dma_dev == chan->device->dev) {
1246	chan->private = param;	1246	chan->private = param;
1247	return true;	1247	return true;
1248	} else {	1248	} else {
1249	return false;	1249	return false;
1250	}	1250	}
1251	}	1251	}
1252		1252
1253	static void rx_timer_fn(unsigned long arg)	1253	static void rx_timer_fn(unsigned long arg)
1254	{	1254	{
1255	struct sci_port s = (struct sci_port )arg;	1255	struct sci_port s = (struct sci_port )arg;
1256	struct uart_port *port = &s->port;	1256	struct uart_port *port = &s->port;
1257		1257
1258	u16 scr = sci_in(port, SCSCR);	1258	u16 scr = sci_in(port, SCSCR);
1259	sci_out(port, SCSCR, scr \| SCI_CTRL_FLAGS_RIE);	1259	sci_out(port, SCSCR, scr \| SCI_CTRL_FLAGS_RIE);
1260	dev_dbg(port->dev, "DMA Rx timed out\n");	1260	dev_dbg(port->dev, "DMA Rx timed out\n");
1261	schedule_work(&s->work_rx);	1261	schedule_work(&s->work_rx);
1262	}	1262	}
1263		1263
1264	static void sci_request_dma(struct uart_port *port)	1264	static void sci_request_dma(struct uart_port *port)
1265	{	1265	{
1266	struct sci_port *s = to_sci_port(port);	1266	struct sci_port *s = to_sci_port(port);
1267	struct sh_dmae_slave *param;	1267	struct sh_dmae_slave *param;
1268	struct dma_chan *chan;	1268	struct dma_chan *chan;
1269	dma_cap_mask_t mask;	1269	dma_cap_mask_t mask;
1270	int nent;	1270	int nent;
1271		1271
1272	dev_dbg(port->dev, "%s: port %d DMA %p\n", __func__,	1272	dev_dbg(port->dev, "%s: port %d DMA %p\n", __func__,
1273	port->line, s->dma_dev);	1273	port->line, s->dma_dev);
1274		1274
1275	if (!s->dma_dev)	1275	if (!s->dma_dev)
1276	return;	1276	return;
1277		1277
1278	dma_cap_zero(mask);	1278	dma_cap_zero(mask);
1279	dma_cap_set(DMA_SLAVE, mask);	1279	dma_cap_set(DMA_SLAVE, mask);
1280		1280
1281	param = &s->param_tx;	1281	param = &s->param_tx;
1282		1282
1283	/* Slave ID, e.g., SHDMA_SLAVE_SCIF0_TX */	1283	/* Slave ID, e.g., SHDMA_SLAVE_SCIF0_TX */
1284	param->slave_id = s->slave_tx;	1284	param->slave_id = s->slave_tx;
1285	param->dma_dev = s->dma_dev;	1285	param->dma_dev = s->dma_dev;
1286		1286
1287	s->cookie_tx = -EINVAL;	1287	s->cookie_tx = -EINVAL;
1288	chan = dma_request_channel(mask, filter, param);	1288	chan = dma_request_channel(mask, filter, param);
1289	dev_dbg(port->dev, "%s: TX: got channel %p\n", __func__, chan);	1289	dev_dbg(port->dev, "%s: TX: got channel %p\n", __func__, chan);
1290	if (chan) {	1290	if (chan) {
1291	s->chan_tx = chan;	1291	s->chan_tx = chan;
1292	sg_init_table(&s->sg_tx, 1);	1292	sg_init_table(&s->sg_tx, 1);
1293	/* UART circular tx buffer is an aligned page. */	1293	/* UART circular tx buffer is an aligned page. */
1294	BUG_ON((int)port->state->xmit.buf & ~PAGE_MASK);	1294	BUG_ON((int)port->state->xmit.buf & ~PAGE_MASK);
1295	sg_set_page(&s->sg_tx, virt_to_page(port->state->xmit.buf),	1295	sg_set_page(&s->sg_tx, virt_to_page(port->state->xmit.buf),
1296	UART_XMIT_SIZE, (int)port->state->xmit.buf & ~PAGE_MASK);	1296	UART_XMIT_SIZE, (int)port->state->xmit.buf & ~PAGE_MASK);
1297	nent = dma_map_sg(port->dev, &s->sg_tx, 1, DMA_TO_DEVICE);	1297	nent = dma_map_sg(port->dev, &s->sg_tx, 1, DMA_TO_DEVICE);
1298	if (!nent)	1298	if (!nent)
1299	sci_tx_dma_release(s, false);	1299	sci_tx_dma_release(s, false);
1300	else	1300	else
1301	dev_dbg(port->dev, "%s: mapped %d@%p to %x\n", __func__,	1301	dev_dbg(port->dev, "%s: mapped %d@%p to %x\n", __func__,
1302	sg_dma_len(&s->sg_tx),	1302	sg_dma_len(&s->sg_tx),
1303	port->state->xmit.buf, sg_dma_address(&s->sg_tx));	1303	port->state->xmit.buf, sg_dma_address(&s->sg_tx));
1304		1304
1305	s->sg_len_tx = nent;	1305	s->sg_len_tx = nent;
1306		1306
1307	INIT_WORK(&s->work_tx, work_fn_tx);	1307	INIT_WORK(&s->work_tx, work_fn_tx);
1308	}	1308	}
1309		1309
1310	param = &s->param_rx;	1310	param = &s->param_rx;
1311		1311
1312	/* Slave ID, e.g., SHDMA_SLAVE_SCIF0_RX */	1312	/* Slave ID, e.g., SHDMA_SLAVE_SCIF0_RX */
1313	param->slave_id = s->slave_rx;	1313	param->slave_id = s->slave_rx;
1314	param->dma_dev = s->dma_dev;	1314	param->dma_dev = s->dma_dev;
1315		1315
1316	chan = dma_request_channel(mask, filter, param);	1316	chan = dma_request_channel(mask, filter, param);
1317	dev_dbg(port->dev, "%s: RX: got channel %p\n", __func__, chan);	1317	dev_dbg(port->dev, "%s: RX: got channel %p\n", __func__, chan);
1318	if (chan) {	1318	if (chan) {
1319	dma_addr_t dma[2];	1319	dma_addr_t dma[2];
1320	void *buf[2];	1320	void *buf[2];
1321	int i;	1321	int i;
1322		1322
1323	s->chan_rx = chan;	1323	s->chan_rx = chan;
1324		1324
1325	s->buf_len_rx = 2 * max(16, (int)port->fifosize);	1325	s->buf_len_rx = 2 * max(16, (int)port->fifosize);
1326	buf[0] = dma_alloc_coherent(port->dev, s->buf_len_rx * 2,	1326	buf[0] = dma_alloc_coherent(port->dev, s->buf_len_rx * 2,
1327	&dma[0], GFP_KERNEL);	1327	&dma[0], GFP_KERNEL);
1328		1328
1329	if (!buf[0]) {	1329	if (!buf[0]) {
1330	dev_warn(port->dev,	1330	dev_warn(port->dev,
1331	"failed to allocate dma buffer, using PIO\n");	1331	"failed to allocate dma buffer, using PIO\n");
1332	sci_rx_dma_release(s, true);	1332	sci_rx_dma_release(s, true);
1333	return;	1333	return;
1334	}	1334	}
1335		1335
1336	buf[1] = buf[0] + s->buf_len_rx;	1336	buf[1] = buf[0] + s->buf_len_rx;
1337	dma[1] = dma[0] + s->buf_len_rx;	1337	dma[1] = dma[0] + s->buf_len_rx;
1338		1338
1339	for (i = 0; i < 2; i++) {	1339	for (i = 0; i < 2; i++) {
1340	struct scatterlist *sg = &s->sg_rx[i];	1340	struct scatterlist *sg = &s->sg_rx[i];
1341		1341
1342	sg_init_table(sg, 1);	1342	sg_init_table(sg, 1);
1343	sg_set_page(sg, virt_to_page(buf[i]), s->buf_len_rx,	1343	sg_set_page(sg, virt_to_page(buf[i]), s->buf_len_rx,
1344	(int)buf[i] & ~PAGE_MASK);	1344	(int)buf[i] & ~PAGE_MASK);
1345	sg->dma_address = dma[i];	1345	sg->dma_address = dma[i];
1346	sg->dma_length = sg->length;	1346	sg->dma_length = sg->length;
1347	}	1347	}
1348		1348
1349	INIT_WORK(&s->work_rx, work_fn_rx);	1349	INIT_WORK(&s->work_rx, work_fn_rx);
1350	setup_timer(&s->rx_timer, rx_timer_fn, (unsigned long)s);	1350	setup_timer(&s->rx_timer, rx_timer_fn, (unsigned long)s);
1351		1351
1352	sci_submit_rx(s);	1352	sci_submit_rx(s);
1353	}	1353	}
1354	}	1354	}
1355		1355
1356	static void sci_free_dma(struct uart_port *port)	1356	static void sci_free_dma(struct uart_port *port)
1357	{	1357	{
1358	struct sci_port *s = to_sci_port(port);	1358	struct sci_port *s = to_sci_port(port);
1359		1359
1360	if (!s->dma_dev)	1360	if (!s->dma_dev)
1361	return;	1361	return;
1362		1362
1363	if (s->chan_tx)	1363	if (s->chan_tx)
1364	sci_tx_dma_release(s, false);	1364	sci_tx_dma_release(s, false);
1365	if (s->chan_rx)	1365	if (s->chan_rx)
1366	sci_rx_dma_release(s, false);	1366	sci_rx_dma_release(s, false);
1367	}	1367	}
1368	#endif	1368	#endif
1369		1369
1370	static int sci_startup(struct uart_port *port)	1370	static int sci_startup(struct uart_port *port)
1371	{	1371	{
1372	struct sci_port *s = to_sci_port(port);	1372	struct sci_port *s = to_sci_port(port);
1373		1373
1374	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);	1374	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);
1375		1375
1376	if (s->enable)	1376	if (s->enable)
1377	s->enable(port);	1377	s->enable(port);
1378		1378
1379	sci_request_irq(s);	1379	sci_request_irq(s);
1380	#ifdef CONFIG_SERIAL_SH_SCI_DMA	1380	#ifdef CONFIG_SERIAL_SH_SCI_DMA
1381	sci_request_dma(port);	1381	sci_request_dma(port);
1382	#endif	1382	#endif
1383	sci_start_tx(port);	1383	sci_start_tx(port);
1384	sci_start_rx(port);	1384	sci_start_rx(port);
1385		1385
1386	return 0;	1386	return 0;
1387	}	1387	}
1388		1388
1389	static void sci_shutdown(struct uart_port *port)	1389	static void sci_shutdown(struct uart_port *port)
1390	{	1390	{
1391	struct sci_port *s = to_sci_port(port);	1391	struct sci_port *s = to_sci_port(port);
1392		1392
1393	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);	1393	dev_dbg(port->dev, "%s(%d)\n", __func__, port->line);
1394		1394
1395	sci_stop_rx(port);	1395	sci_stop_rx(port);
1396	sci_stop_tx(port);	1396	sci_stop_tx(port);
1397	#ifdef CONFIG_SERIAL_SH_SCI_DMA	1397	#ifdef CONFIG_SERIAL_SH_SCI_DMA
1398	sci_free_dma(port);	1398	sci_free_dma(port);
1399	#endif	1399	#endif
1400	sci_free_irq(s);	1400	sci_free_irq(s);
1401		1401
1402	if (s->disable)	1402	if (s->disable)
1403	s->disable(port);	1403	s->disable(port);
1404	}	1404	}
1405		1405
1406	static void sci_set_termios(struct uart_port port, struct ktermios termios,	1406	static void sci_set_termios(struct uart_port port, struct ktermios termios,
1407	struct ktermios *old)	1407	struct ktermios *old)
1408	{	1408	{
1409	unsigned int status, baud, smr_val, max_baud;	1409	unsigned int status, baud, smr_val, max_baud;
1410	int t = -1;	1410	int t = -1;
1411		1411
1412	/*	1412	/*
1413	* earlyprintk comes here early on with port->uartclk set to zero.	1413	* earlyprintk comes here early on with port->uartclk set to zero.
1414	* the clock framework is not up and running at this point so here	1414	* the clock framework is not up and running at this point so here
1415	* we assume that 115200 is the maximum baud rate. please note that	1415	* we assume that 115200 is the maximum baud rate. please note that
1416	* the baud rate is not programmed during earlyprintk - it is assumed	1416	* the baud rate is not programmed during earlyprintk - it is assumed
1417	* that the previous boot loader has enabled required clocks and	1417	* that the previous boot loader has enabled required clocks and
1418	* setup the baud rate generator hardware for us already.	1418	* setup the baud rate generator hardware for us already.
1419	*/	1419	*/
1420	max_baud = port->uartclk ? port->uartclk / 16 : 115200;	1420	max_baud = port->uartclk ? port->uartclk / 16 : 115200;
1421		1421
1422	baud = uart_get_baud_rate(port, termios, old, 0, max_baud);	1422	baud = uart_get_baud_rate(port, termios, old, 0, max_baud);
1423	if (likely(baud && port->uartclk))	1423	if (likely(baud && port->uartclk))
1424	t = SCBRR_VALUE(baud, port->uartclk);	1424	t = SCBRR_VALUE(baud, port->uartclk);
1425		1425
1426	do {	1426	do {
1427	status = sci_in(port, SCxSR);	1427	status = sci_in(port, SCxSR);
1428	} while (!(status & SCxSR_TEND(port)));	1428	} while (!(status & SCxSR_TEND(port)));
1429		1429
1430	sci_out(port, SCSCR, 0x00); /* TE=0, RE=0, CKE1=0 */	1430	sci_out(port, SCSCR, 0x00); /* TE=0, RE=0, CKE1=0 */
1431		1431
1432	if (port->type != PORT_SCI)	1432	if (port->type != PORT_SCI)
1433	sci_out(port, SCFCR, SCFCR_RFRST \| SCFCR_TFRST);	1433	sci_out(port, SCFCR, SCFCR_RFRST \| SCFCR_TFRST);
1434		1434
1435	smr_val = sci_in(port, SCSMR) & 3;	1435	smr_val = sci_in(port, SCSMR) & 3;
1436	if ((termios->c_cflag & CSIZE) == CS7)	1436	if ((termios->c_cflag & CSIZE) == CS7)
1437	smr_val \|= 0x40;	1437	smr_val \|= 0x40;
1438	if (termios->c_cflag & PARENB)	1438	if (termios->c_cflag & PARENB)
1439	smr_val \|= 0x20;	1439	smr_val \|= 0x20;
1440	if (termios->c_cflag & PARODD)	1440	if (termios->c_cflag & PARODD)
1441	smr_val \|= 0x30;	1441	smr_val \|= 0x30;
1442	if (termios->c_cflag & CSTOPB)	1442	if (termios->c_cflag & CSTOPB)
1443	smr_val \|= 0x08;	1443	smr_val \|= 0x08;
1444		1444
1445	uart_update_timeout(port, termios->c_cflag, baud);	1445	uart_update_timeout(port, termios->c_cflag, baud);
1446		1446
1447	sci_out(port, SCSMR, smr_val);	1447	sci_out(port, SCSMR, smr_val);
1448		1448
1449	dev_dbg(port->dev, "%s: SMR %x, t %x, SCSCR %x\n", __func__, smr_val, t,	1449	dev_dbg(port->dev, "%s: SMR %x, t %x, SCSCR %x\n", __func__, smr_val, t,
1450	SCSCR_INIT(port));	1450	SCSCR_INIT(port));
1451		1451
1452	if (t > 0) {	1452	if (t > 0) {
1453	if (t >= 256) {	1453	if (t >= 256) {
1454	sci_out(port, SCSMR, (sci_in(port, SCSMR) & ~3) \| 1);	1454	sci_out(port, SCSMR, (sci_in(port, SCSMR) & ~3) \| 1);
1455	t >>= 2;	1455	t >>= 2;
1456	} else	1456	} else
1457	sci_out(port, SCSMR, sci_in(port, SCSMR) & ~3);	1457	sci_out(port, SCSMR, sci_in(port, SCSMR) & ~3);
1458		1458
1459	sci_out(port, SCBRR, t);	1459	sci_out(port, SCBRR, t);
1460	udelay((1000000+(baud-1)) / baud); /* Wait one bit interval */	1460	udelay((1000000+(baud-1)) / baud); /* Wait one bit interval */
1461	}	1461	}
1462		1462
1463	sci_init_pins(port, termios->c_cflag);	1463	sci_init_pins(port, termios->c_cflag);
1464	sci_out(port, SCFCR, (termios->c_cflag & CRTSCTS) ? SCFCR_MCE : 0);	1464	sci_out(port, SCFCR, (termios->c_cflag & CRTSCTS) ? SCFCR_MCE : 0);
1465		1465
1466	sci_out(port, SCSCR, SCSCR_INIT(port));	1466	sci_out(port, SCSCR, SCSCR_INIT(port));
1467		1467
1468	if ((termios->c_cflag & CREAD) != 0)	1468	if ((termios->c_cflag & CREAD) != 0)
1469	sci_start_rx(port);	1469	sci_start_rx(port);
1470	}	1470	}
1471		1471
1472	static const char sci_type(struct uart_port port)	1472	static const char sci_type(struct uart_port port)
1473	{	1473	{
1474	switch (port->type) {	1474	switch (port->type) {
1475	case PORT_IRDA:	1475	case PORT_IRDA:
1476	return "irda";	1476	return "irda";
1477	case PORT_SCI:	1477	case PORT_SCI:
1478	return "sci";	1478	return "sci";
1479	case PORT_SCIF:	1479	case PORT_SCIF:
1480	return "scif";	1480	return "scif";
1481	case PORT_SCIFA:	1481	case PORT_SCIFA:
1482	return "scifa";	1482	return "scifa";
1483	}	1483	}
1484		1484
1485	return NULL;	1485	return NULL;
1486	}	1486	}
1487		1487
1488	static void sci_release_port(struct uart_port *port)	1488	static void sci_release_port(struct uart_port *port)
1489	{	1489	{
1490	/* Nothing here yet .. */	1490	/* Nothing here yet .. */
1491	}	1491	}
1492		1492
1493	static int sci_request_port(struct uart_port *port)	1493	static int sci_request_port(struct uart_port *port)
1494	{	1494	{
1495	/* Nothing here yet .. */	1495	/* Nothing here yet .. */
1496	return 0;	1496	return 0;
1497	}	1497	}
1498		1498
1499	static void sci_config_port(struct uart_port *port, int flags)	1499	static void sci_config_port(struct uart_port *port, int flags)
1500	{	1500	{
1501	struct sci_port *s = to_sci_port(port);	1501	struct sci_port *s = to_sci_port(port);
1502		1502
1503	port->type = s->type;	1503	port->type = s->type;
1504		1504
1505	if (port->membase)	1505	if (port->membase)
1506	return;	1506	return;
1507		1507
1508	if (port->flags & UPF_IOREMAP) {	1508	if (port->flags & UPF_IOREMAP) {
1509	port->membase = ioremap_nocache(port->mapbase, 0x40);	1509	port->membase = ioremap_nocache(port->mapbase, 0x40);
1510		1510
1511	if (IS_ERR(port->membase))	1511	if (IS_ERR(port->membase))
1512	dev_err(port->dev, "can't remap port#%d\n", port->line);	1512	dev_err(port->dev, "can't remap port#%d\n", port->line);
1513	} else {	1513	} else {
1514	/*	1514	/*
1515	* For the simple (and majority of) cases where we don't	1515	* For the simple (and majority of) cases where we don't
1516	* need to do any remapping, just cast the cookie	1516	* need to do any remapping, just cast the cookie
1517	* directly.	1517	* directly.
1518	*/	1518	*/
1519	port->membase = (void __iomem *)port->mapbase;	1519	port->membase = (void __iomem *)port->mapbase;
1520	}	1520	}
1521	}	1521	}
1522		1522
1523	static int sci_verify_port(struct uart_port port, struct serial_struct ser)	1523	static int sci_verify_port(struct uart_port port, struct serial_struct ser)
1524	{	1524	{
1525	struct sci_port *s = to_sci_port(port);	1525	struct sci_port *s = to_sci_port(port);
1526		1526
1527	if (ser->irq != s->irqs[SCIx_TXI_IRQ] \|\| ser->irq > nr_irqs)	1527	if (ser->irq != s->irqs[SCIx_TXI_IRQ] \|\| ser->irq > nr_irqs)
1528	return -EINVAL;	1528	return -EINVAL;
1529	if (ser->baud_base < 2400)	1529	if (ser->baud_base < 2400)
1530	/* No paper tape reader for Mitch.. */	1530	/* No paper tape reader for Mitch.. */
1531	return -EINVAL;	1531	return -EINVAL;
1532		1532
1533	return 0;	1533	return 0;
1534	}	1534	}
1535		1535
1536	static struct uart_ops sci_uart_ops = {	1536	static struct uart_ops sci_uart_ops = {
1537	.tx_empty = sci_tx_empty,	1537	.tx_empty = sci_tx_empty,
1538	.set_mctrl = sci_set_mctrl,	1538	.set_mctrl = sci_set_mctrl,
1539	.get_mctrl = sci_get_mctrl,	1539	.get_mctrl = sci_get_mctrl,
1540	.start_tx = sci_start_tx,	1540	.start_tx = sci_start_tx,
1541	.stop_tx = sci_stop_tx,	1541	.stop_tx = sci_stop_tx,
1542	.stop_rx = sci_stop_rx,	1542	.stop_rx = sci_stop_rx,
1543	.enable_ms = sci_enable_ms,	1543	.enable_ms = sci_enable_ms,
1544	.break_ctl = sci_break_ctl,	1544	.break_ctl = sci_break_ctl,
1545	.startup = sci_startup,	1545	.startup = sci_startup,
1546	.shutdown = sci_shutdown,	1546	.shutdown = sci_shutdown,
1547	.set_termios = sci_set_termios,	1547	.set_termios = sci_set_termios,
1548	.type = sci_type,	1548	.type = sci_type,
1549	.release_port = sci_release_port,	1549	.release_port = sci_release_port,
1550	.request_port = sci_request_port,	1550	.request_port = sci_request_port,
1551	.config_port = sci_config_port,	1551	.config_port = sci_config_port,
1552	.verify_port = sci_verify_port,	1552	.verify_port = sci_verify_port,
1553	#ifdef CONFIG_CONSOLE_POLL	1553	#ifdef CONFIG_CONSOLE_POLL
1554	.poll_get_char = sci_poll_get_char,	1554	.poll_get_char = sci_poll_get_char,
1555	.poll_put_char = sci_poll_put_char,	1555	.poll_put_char = sci_poll_put_char,
1556	#endif	1556	#endif
1557	};	1557	};
1558		1558
1559	static void __devinit sci_init_single(struct platform_device *dev,	1559	static void __devinit sci_init_single(struct platform_device *dev,
1560	struct sci_port *sci_port,	1560	struct sci_port *sci_port,
1561	unsigned int index,	1561	unsigned int index,
1562	struct plat_sci_port *p)	1562	struct plat_sci_port *p)
1563	{	1563	{
1564	struct uart_port *port = &sci_port->port;	1564	struct uart_port *port = &sci_port->port;
1565		1565
1566	port->ops = &sci_uart_ops;	1566	port->ops = &sci_uart_ops;
1567	port->iotype = UPIO_MEM;	1567	port->iotype = UPIO_MEM;
1568	port->line = index;	1568	port->line = index;
1569		1569
1570	switch (p->type) {	1570	switch (p->type) {
1571	case PORT_SCIFA:	1571	case PORT_SCIFA:
1572	port->fifosize = 64;	1572	port->fifosize = 64;
1573	break;	1573	break;
1574	case PORT_SCIF:	1574	case PORT_SCIF:
1575	port->fifosize = 16;	1575	port->fifosize = 16;
1576	break;	1576	break;
1577	default:	1577	default:
1578	port->fifosize = 1;	1578	port->fifosize = 1;
1579	break;	1579	break;
1580	}	1580	}
1581		1581
1582	if (dev) {	1582	if (dev) {
1583	sci_port->iclk = p->clk ? clk_get(&dev->dev, p->clk) : NULL;	1583	sci_port->iclk = p->clk ? clk_get(&dev->dev, p->clk) : NULL;
1584	sci_port->dclk = clk_get(&dev->dev, "peripheral_clk");	1584	sci_port->dclk = clk_get(&dev->dev, "peripheral_clk");
1585	sci_port->enable = sci_clk_enable;	1585	sci_port->enable = sci_clk_enable;
1586	sci_port->disable = sci_clk_disable;	1586	sci_port->disable = sci_clk_disable;
1587	port->dev = &dev->dev;	1587	port->dev = &dev->dev;
1588	}	1588	}
1589		1589
1590	sci_port->break_timer.data = (unsigned long)sci_port;	1590	sci_port->break_timer.data = (unsigned long)sci_port;
1591	sci_port->break_timer.function = sci_break_timer;	1591	sci_port->break_timer.function = sci_break_timer;
1592	init_timer(&sci_port->break_timer);	1592	init_timer(&sci_port->break_timer);
1593		1593
1594	port->mapbase = p->mapbase;	1594	port->mapbase = p->mapbase;
1595	port->membase = p->membase;	1595	port->membase = p->membase;
1596		1596
1597	port->irq = p->irqs[SCIx_TXI_IRQ];	1597	port->irq = p->irqs[SCIx_TXI_IRQ];
1598	port->flags = p->flags;	1598	port->flags = p->flags;
1599	sci_port->type = port->type = p->type;	1599	sci_port->type = port->type = p->type;
1600		1600
1601	#ifdef CONFIG_SERIAL_SH_SCI_DMA	1601	#ifdef CONFIG_SERIAL_SH_SCI_DMA
1602	sci_port->dma_dev = p->dma_dev;	1602	sci_port->dma_dev = p->dma_dev;
1603	sci_port->slave_tx = p->dma_slave_tx;	1603	sci_port->slave_tx = p->dma_slave_tx;
1604	sci_port->slave_rx = p->dma_slave_rx;	1604	sci_port->slave_rx = p->dma_slave_rx;
1605		1605
1606	dev_dbg(port->dev, "%s: DMA device %p, tx %d, rx %d\n", __func__,	1606	dev_dbg(port->dev, "%s: DMA device %p, tx %d, rx %d\n", __func__,
1607	p->dma_dev, p->dma_slave_tx, p->dma_slave_rx);	1607	p->dma_dev, p->dma_slave_tx, p->dma_slave_rx);
1608	#endif	1608	#endif
1609		1609
1610	memcpy(&sci_port->irqs, &p->irqs, sizeof(p->irqs));	1610	memcpy(&sci_port->irqs, &p->irqs, sizeof(p->irqs));
1611	}	1611	}
1612		1612
1613	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE	1613	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE
1614	static struct tty_driver serial_console_device(struct console co, int *index)	1614	static struct tty_driver serial_console_device(struct console co, int *index)
1615	{	1615	{
1616	struct uart_driver *p = &sci_uart_driver;	1616	struct uart_driver *p = &sci_uart_driver;
1617	*index = co->index;	1617	*index = co->index;
1618	return p->tty_driver;	1618	return p->tty_driver;
1619	}	1619	}
1620		1620
1621	static void serial_console_putchar(struct uart_port *port, int ch)	1621	static void serial_console_putchar(struct uart_port *port, int ch)
1622	{	1622	{
1623	sci_poll_put_char(port, ch);	1623	sci_poll_put_char(port, ch);
1624	}	1624	}
1625		1625
1626	/*	1626	/*
1627	* Print a string to the serial port trying not to disturb	1627	* Print a string to the serial port trying not to disturb
1628	* any possible real use of the port...	1628	* any possible real use of the port...
1629	*/	1629	*/
1630	static void serial_console_write(struct console co, const char s,	1630	static void serial_console_write(struct console co, const char s,
1631	unsigned count)	1631	unsigned count)
1632	{	1632	{
1633	struct uart_port *port = co->data;	1633	struct uart_port *port = co->data;
1634	struct sci_port *sci_port = to_sci_port(port);	1634	struct sci_port *sci_port = to_sci_port(port);
1635	unsigned short bits;	1635	unsigned short bits;
1636		1636
1637	if (sci_port->enable)	1637	if (sci_port->enable)
1638	sci_port->enable(port);	1638	sci_port->enable(port);
1639		1639
1640	uart_console_write(port, s, count, serial_console_putchar);	1640	uart_console_write(port, s, count, serial_console_putchar);
1641		1641
1642	/* wait until fifo is empty and last bit has been transmitted */	1642	/* wait until fifo is empty and last bit has been transmitted */
1643	bits = SCxSR_TDxE(port) \| SCxSR_TEND(port);	1643	bits = SCxSR_TDxE(port) \| SCxSR_TEND(port);
1644	while ((sci_in(port, SCxSR) & bits) != bits)	1644	while ((sci_in(port, SCxSR) & bits) != bits)
1645	cpu_relax();	1645	cpu_relax();
1646		1646
1647	if (sci_port->disable)	1647	if (sci_port->disable)
1648	sci_port->disable(port);	1648	sci_port->disable(port);
1649	}	1649	}
1650		1650
1651	static int __devinit serial_console_setup(struct console co, char options)	1651	static int __devinit serial_console_setup(struct console co, char options)
1652	{	1652	{
1653	struct sci_port *sci_port;	1653	struct sci_port *sci_port;
1654	struct uart_port *port;	1654	struct uart_port *port;
1655	int baud = 115200;	1655	int baud = 115200;
1656	int bits = 8;	1656	int bits = 8;
1657	int parity = 'n';	1657	int parity = 'n';
1658	int flow = 'n';	1658	int flow = 'n';
1659	int ret;	1659	int ret;
1660		1660
1661	/*	1661	/*
1662	* Check whether an invalid uart number has been specified, and	1662	* Check whether an invalid uart number has been specified, and
1663	* if so, search for the first available port that does have	1663	* if so, search for the first available port that does have
1664	* console support.	1664	* console support.
1665	*/	1665	*/
1666	if (co->index >= SCI_NPORTS)	1666	if (co->index >= SCI_NPORTS)
1667	co->index = 0;	1667	co->index = 0;
1668		1668
1669	if (co->data) {	1669	if (co->data) {
1670	port = co->data;	1670	port = co->data;
1671	sci_port = to_sci_port(port);	1671	sci_port = to_sci_port(port);
1672	} else {	1672	} else {
1673	sci_port = &sci_ports[co->index];	1673	sci_port = &sci_ports[co->index];
1674	port = &sci_port->port;	1674	port = &sci_port->port;
1675	co->data = port;	1675	co->data = port;
1676	}	1676	}
1677		1677
1678	/*	1678	/*
1679	* Also need to check port->type, we don't actually have any	1679	* Also need to check port->type, we don't actually have any
1680	* UPIO_PORT ports, but uart_report_port() handily misreports	1680	* UPIO_PORT ports, but uart_report_port() handily misreports
1681	* it anyways if we don't have a port available by the time this is	1681	* it anyways if we don't have a port available by the time this is
1682	* called.	1682	* called.
1683	*/	1683	*/
1684	if (!port->type)	1684	if (!port->type)
1685	return -ENODEV;	1685	return -ENODEV;
1686		1686
1687	sci_config_port(port, 0);	1687	sci_config_port(port, 0);
1688		1688
1689	if (sci_port->enable)	1689	if (sci_port->enable)
1690	sci_port->enable(port);	1690	sci_port->enable(port);
1691		1691
1692	if (options)	1692	if (options)
1693	uart_parse_options(options, &baud, &parity, &bits, &flow);	1693	uart_parse_options(options, &baud, &parity, &bits, &flow);
1694		1694
1695	ret = uart_set_options(port, co, baud, parity, bits, flow);	1695	ret = uart_set_options(port, co, baud, parity, bits, flow);
1696	#if defined(__H8300H__) \|\| defined(__H8300S__)	1696	#if defined(__H8300H__) \|\| defined(__H8300S__)
1697	/* disable rx interrupt */	1697	/* disable rx interrupt */
1698	if (ret == 0)	1698	if (ret == 0)
1699	sci_stop_rx(port);	1699	sci_stop_rx(port);
1700	#endif	1700	#endif
1701	/* TODO: disable clock */	1701	/* TODO: disable clock */
1702	return ret;	1702	return ret;
1703	}	1703	}
1704		1704
1705	static struct console serial_console = {	1705	static struct console serial_console = {
1706	.name = "ttySC",	1706	.name = "ttySC",
1707	.device = serial_console_device,	1707	.device = serial_console_device,
1708	.write = serial_console_write,	1708	.write = serial_console_write,
1709	.setup = serial_console_setup,	1709	.setup = serial_console_setup,
1710	.flags = CON_PRINTBUFFER,	1710	.flags = CON_PRINTBUFFER,
1711	.index = -1,	1711	.index = -1,
1712	};	1712	};
1713		1713
1714	static int __init sci_console_init(void)	1714	static int __init sci_console_init(void)
1715	{	1715	{
1716	register_console(&serial_console);	1716	register_console(&serial_console);
1717	return 0;	1717	return 0;
1718	}	1718	}
1719	console_initcall(sci_console_init);	1719	console_initcall(sci_console_init);
1720		1720
1721	static struct sci_port early_serial_port;	1721	static struct sci_port early_serial_port;
1722	static struct console early_serial_console = {	1722	static struct console early_serial_console = {
1723	.name = "early_ttySC",	1723	.name = "early_ttySC",
1724	.write = serial_console_write,	1724	.write = serial_console_write,
1725	.flags = CON_PRINTBUFFER,	1725	.flags = CON_PRINTBUFFER,
1726	};	1726	};
1727	static char early_serial_buf[32];	1727	static char early_serial_buf[32];
1728		1728
1729	#endif /* CONFIG_SERIAL_SH_SCI_CONSOLE */	1729	#endif /* CONFIG_SERIAL_SH_SCI_CONSOLE */
1730		1730
1731	#if defined(CONFIG_SERIAL_SH_SCI_CONSOLE)	1731	#if defined(CONFIG_SERIAL_SH_SCI_CONSOLE)
1732	#define SCI_CONSOLE (&serial_console)	1732	#define SCI_CONSOLE (&serial_console)
1733	#else	1733	#else
1734	#define SCI_CONSOLE 0	1734	#define SCI_CONSOLE 0
1735	#endif	1735	#endif
1736		1736
1737	static char banner[] __initdata =	1737	static char banner[] __initdata =
1738	KERN_INFO "SuperH SCI(F) driver initialized\n";	1738	KERN_INFO "SuperH SCI(F) driver initialized\n";
1739		1739
1740	static struct uart_driver sci_uart_driver = {	1740	static struct uart_driver sci_uart_driver = {
1741	.owner = THIS_MODULE,	1741	.owner = THIS_MODULE,
1742	.driver_name = "sci",	1742	.driver_name = "sci",
1743	.dev_name = "ttySC",	1743	.dev_name = "ttySC",
1744	.major = SCI_MAJOR,	1744	.major = SCI_MAJOR,
1745	.minor = SCI_MINOR_START,	1745	.minor = SCI_MINOR_START,
1746	.nr = SCI_NPORTS,	1746	.nr = SCI_NPORTS,
1747	.cons = SCI_CONSOLE,	1747	.cons = SCI_CONSOLE,
1748	};	1748	};
1749		1749
1750		1750
1751	static int sci_remove(struct platform_device *dev)	1751	static int sci_remove(struct platform_device *dev)
1752	{	1752	{
1753	struct sh_sci_priv *priv = platform_get_drvdata(dev);	1753	struct sh_sci_priv *priv = platform_get_drvdata(dev);
1754	struct sci_port *p;	1754	struct sci_port *p;
1755	unsigned long flags;	1755	unsigned long flags;
1756		1756
1757	cpufreq_unregister_notifier(&priv->clk_nb, CPUFREQ_TRANSITION_NOTIFIER);	1757	cpufreq_unregister_notifier(&priv->clk_nb, CPUFREQ_TRANSITION_NOTIFIER);
1758		1758
1759	spin_lock_irqsave(&priv->lock, flags);	1759	spin_lock_irqsave(&priv->lock, flags);
1760	list_for_each_entry(p, &priv->ports, node)	1760	list_for_each_entry(p, &priv->ports, node)
1761	uart_remove_one_port(&sci_uart_driver, &p->port);	1761	uart_remove_one_port(&sci_uart_driver, &p->port);
1762	spin_unlock_irqrestore(&priv->lock, flags);	1762	spin_unlock_irqrestore(&priv->lock, flags);
1763		1763
1764	kfree(priv);	1764	kfree(priv);
1765	return 0;	1765	return 0;
1766	}	1766	}
1767		1767
1768	static int __devinit sci_probe_single(struct platform_device *dev,	1768	static int __devinit sci_probe_single(struct platform_device *dev,
1769	unsigned int index,	1769	unsigned int index,
1770	struct plat_sci_port *p,	1770	struct plat_sci_port *p,
1771	struct sci_port *sciport)	1771	struct sci_port *sciport)
1772	{	1772	{
1773	struct sh_sci_priv *priv = platform_get_drvdata(dev);	1773	struct sh_sci_priv *priv = platform_get_drvdata(dev);
1774	unsigned long flags;	1774	unsigned long flags;
1775	int ret;	1775	int ret;
1776		1776
1777	/* Sanity check */	1777	/* Sanity check */
1778	if (unlikely(index >= SCI_NPORTS)) {	1778	if (unlikely(index >= SCI_NPORTS)) {
1779	dev_notice(&dev->dev, "Attempting to register port "	1779	dev_notice(&dev->dev, "Attempting to register port "
1780	"%d when only %d are available.\n",	1780	"%d when only %d are available.\n",
1781	index+1, SCI_NPORTS);	1781	index+1, SCI_NPORTS);
1782	dev_notice(&dev->dev, "Consider bumping "	1782	dev_notice(&dev->dev, "Consider bumping "
1783	"CONFIG_SERIAL_SH_SCI_NR_UARTS!\n");	1783	"CONFIG_SERIAL_SH_SCI_NR_UARTS!\n");
1784	return 0;	1784	return 0;
1785	}	1785	}
1786		1786
1787	sci_init_single(dev, sciport, index, p);	1787	sci_init_single(dev, sciport, index, p);
1788		1788
1789	ret = uart_add_one_port(&sci_uart_driver, &sciport->port);	1789	ret = uart_add_one_port(&sci_uart_driver, &sciport->port);
1790	if (ret)	1790	if (ret)
1791	return ret;	1791	return ret;
1792		1792
1793	INIT_LIST_HEAD(&sciport->node);	1793	INIT_LIST_HEAD(&sciport->node);
1794		1794
1795	spin_lock_irqsave(&priv->lock, flags);	1795	spin_lock_irqsave(&priv->lock, flags);
1796	list_add(&sciport->node, &priv->ports);	1796	list_add(&sciport->node, &priv->ports);
1797	spin_unlock_irqrestore(&priv->lock, flags);	1797	spin_unlock_irqrestore(&priv->lock, flags);
1798		1798
1799	return 0;	1799	return 0;
1800	}	1800	}
1801		1801
1802	/*	1802	/*
1803	* Register a set of serial devices attached to a platform device. The	1803	* Register a set of serial devices attached to a platform device. The
1804	* list is terminated with a zero flags entry, which means we expect	1804	* list is terminated with a zero flags entry, which means we expect
1805	* all entries to have at least UPF_BOOT_AUTOCONF set. Platforms that need	1805	* all entries to have at least UPF_BOOT_AUTOCONF set. Platforms that need
1806	* remapping (such as sh64) should also set UPF_IOREMAP.	1806	* remapping (such as sh64) should also set UPF_IOREMAP.
1807	*/	1807	*/
1808	static int __devinit sci_probe(struct platform_device *dev)	1808	static int __devinit sci_probe(struct platform_device *dev)
1809	{	1809	{
1810	struct plat_sci_port *p = dev->dev.platform_data;	1810	struct plat_sci_port *p = dev->dev.platform_data;
1811	struct sh_sci_priv *priv;	1811	struct sh_sci_priv *priv;
1812	int i, ret = -EINVAL;	1812	int i, ret = -EINVAL;
1813		1813
1814	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE	1814	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE
1815	if (is_early_platform_device(dev)) {	1815	if (is_early_platform_device(dev)) {
1816	if (dev->id == -1)	1816	if (dev->id == -1)
1817	return -ENOTSUPP;	1817	return -ENOTSUPP;
1818	early_serial_console.index = dev->id;	1818	early_serial_console.index = dev->id;
1819	early_serial_console.data = &early_serial_port.port;	1819	early_serial_console.data = &early_serial_port.port;
1820	sci_init_single(NULL, &early_serial_port, dev->id, p);	1820	sci_init_single(NULL, &early_serial_port, dev->id, p);
1821	serial_console_setup(&early_serial_console, early_serial_buf);	1821	serial_console_setup(&early_serial_console, early_serial_buf);
1822	if (!strstr(early_serial_buf, "keep"))	1822	if (!strstr(early_serial_buf, "keep"))
1823	early_serial_console.flags \|= CON_BOOT;	1823	early_serial_console.flags \|= CON_BOOT;
1824	register_console(&early_serial_console);	1824	register_console(&early_serial_console);
1825	return 0;	1825	return 0;
1826	}	1826	}
1827	#endif	1827	#endif
1828		1828
1829	priv = kzalloc(sizeof(*priv), GFP_KERNEL);	1829	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1830	if (!priv)	1830	if (!priv)
1831	return -ENOMEM;	1831	return -ENOMEM;
1832		1832
1833	INIT_LIST_HEAD(&priv->ports);	1833	INIT_LIST_HEAD(&priv->ports);
1834	spin_lock_init(&priv->lock);	1834	spin_lock_init(&priv->lock);
1835	platform_set_drvdata(dev, priv);	1835	platform_set_drvdata(dev, priv);
1836		1836
1837	priv->clk_nb.notifier_call = sci_notifier;	1837	priv->clk_nb.notifier_call = sci_notifier;
1838	cpufreq_register_notifier(&priv->clk_nb, CPUFREQ_TRANSITION_NOTIFIER);	1838	cpufreq_register_notifier(&priv->clk_nb, CPUFREQ_TRANSITION_NOTIFIER);
1839		1839
1840	if (dev->id != -1) {	1840	if (dev->id != -1) {
1841	ret = sci_probe_single(dev, dev->id, p, &sci_ports[dev->id]);	1841	ret = sci_probe_single(dev, dev->id, p, &sci_ports[dev->id]);
1842	if (ret)	1842	if (ret)
1843	goto err_unreg;	1843	goto err_unreg;
1844	} else {	1844	} else {
1845	for (i = 0; p && p->flags != 0; p++, i++) {	1845	for (i = 0; p && p->flags != 0; p++, i++) {
1846	ret = sci_probe_single(dev, i, p, &sci_ports[i]);	1846	ret = sci_probe_single(dev, i, p, &sci_ports[i]);
1847	if (ret)	1847	if (ret)
1848	goto err_unreg;	1848	goto err_unreg;
1849	}	1849	}
1850	}	1850	}
1851		1851
1852	#ifdef CONFIG_SH_STANDARD_BIOS	1852	#ifdef CONFIG_SH_STANDARD_BIOS
1853	sh_bios_gdb_detach();	1853	sh_bios_gdb_detach();
1854	#endif	1854	#endif
1855		1855
1856	return 0;	1856	return 0;
1857		1857
1858	err_unreg:	1858	err_unreg:
1859	sci_remove(dev);	1859	sci_remove(dev);
1860	return ret;	1860	return ret;
1861	}	1861	}
1862		1862
1863	static int sci_suspend(struct device *dev)	1863	static int sci_suspend(struct device *dev)
1864	{	1864	{
1865	struct sh_sci_priv *priv = dev_get_drvdata(dev);	1865	struct sh_sci_priv *priv = dev_get_drvdata(dev);
1866	struct sci_port *p;	1866	struct sci_port *p;
1867	unsigned long flags;	1867	unsigned long flags;
1868		1868
1869	spin_lock_irqsave(&priv->lock, flags);	1869	spin_lock_irqsave(&priv->lock, flags);
1870	list_for_each_entry(p, &priv->ports, node)	1870	list_for_each_entry(p, &priv->ports, node)
1871	uart_suspend_port(&sci_uart_driver, &p->port);	1871	uart_suspend_port(&sci_uart_driver, &p->port);
1872	spin_unlock_irqrestore(&priv->lock, flags);	1872	spin_unlock_irqrestore(&priv->lock, flags);
1873		1873
1874	return 0;	1874	return 0;
1875	}	1875	}
1876		1876
1877	static int sci_resume(struct device *dev)	1877	static int sci_resume(struct device *dev)
1878	{	1878	{
1879	struct sh_sci_priv *priv = dev_get_drvdata(dev);	1879	struct sh_sci_priv *priv = dev_get_drvdata(dev);
1880	struct sci_port *p;	1880	struct sci_port *p;
1881	unsigned long flags;	1881	unsigned long flags;
1882		1882
1883	spin_lock_irqsave(&priv->lock, flags);	1883	spin_lock_irqsave(&priv->lock, flags);
1884	list_for_each_entry(p, &priv->ports, node)	1884	list_for_each_entry(p, &priv->ports, node)
1885	uart_resume_port(&sci_uart_driver, &p->port);	1885	uart_resume_port(&sci_uart_driver, &p->port);
1886	spin_unlock_irqrestore(&priv->lock, flags);	1886	spin_unlock_irqrestore(&priv->lock, flags);
1887		1887
1888	return 0;	1888	return 0;
1889	}	1889	}
1890		1890
1891	static const struct dev_pm_ops sci_dev_pm_ops = {	1891	static const struct dev_pm_ops sci_dev_pm_ops = {
1892	.suspend = sci_suspend,	1892	.suspend = sci_suspend,
1893	.resume = sci_resume,	1893	.resume = sci_resume,
1894	};	1894	};
1895		1895
1896	static struct platform_driver sci_driver = {	1896	static struct platform_driver sci_driver = {
1897	.probe = sci_probe,	1897	.probe = sci_probe,
1898	.remove = sci_remove,	1898	.remove = sci_remove,
1899	.driver = {	1899	.driver = {
1900	.name = "sh-sci",	1900	.name = "sh-sci",
1901	.owner = THIS_MODULE,	1901	.owner = THIS_MODULE,
1902	.pm = &sci_dev_pm_ops,	1902	.pm = &sci_dev_pm_ops,
1903	},	1903	},
1904	};	1904	};
1905		1905
1906	static int __init sci_init(void)	1906	static int __init sci_init(void)
1907	{	1907	{
1908	int ret;	1908	int ret;
1909		1909
1910	printk(banner);	1910	printk(banner);
1911		1911
1912	ret = uart_register_driver(&sci_uart_driver);	1912	ret = uart_register_driver(&sci_uart_driver);
1913	if (likely(ret == 0)) {	1913	if (likely(ret == 0)) {
1914	ret = platform_driver_register(&sci_driver);	1914	ret = platform_driver_register(&sci_driver);
1915	if (unlikely(ret))	1915	if (unlikely(ret))
1916	uart_unregister_driver(&sci_uart_driver);	1916	uart_unregister_driver(&sci_uart_driver);
1917	}	1917	}
1918		1918
1919	return ret;	1919	return ret;
1920	}	1920	}
1921		1921
1922	static void __exit sci_exit(void)	1922	static void __exit sci_exit(void)
1923	{	1923	{
1924	platform_driver_unregister(&sci_driver);	1924	platform_driver_unregister(&sci_driver);
1925	uart_unregister_driver(&sci_uart_driver);	1925	uart_unregister_driver(&sci_uart_driver);
1926	}	1926	}
1927		1927
1928	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE	1928	#ifdef CONFIG_SERIAL_SH_SCI_CONSOLE
1929	early_platform_init_buffer("earlyprintk", &sci_driver,	1929	early_platform_init_buffer("earlyprintk", &sci_driver,
1930	early_serial_buf, ARRAY_SIZE(early_serial_buf));	1930	early_serial_buf, ARRAY_SIZE(early_serial_buf));
1931	#endif	1931	#endif
1932	module_init(sci_init);	1932	module_init(sci_init);
1933	module_exit(sci_exit);	1933	module_exit(sci_exit);
1934		1934
1935	MODULE_LICENSE("GPL");	1935	MODULE_LICENSE("GPL");
1936	MODULE_ALIAS("platform:sh-sci");	1936	MODULE_ALIAS("platform:sh-sci");
1937		1937

drivers/video/mx3fb.c

Diff comments View file @ 0582763

1	/*	1	/*
2	* Copyright (C) 2008	2	* Copyright (C) 2008
3	* Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>	3	* Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
4	*	4	*
5	* Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.	5	* Copyright 2004-2007 Freescale Semiconductor, Inc. All Rights Reserved.
6	*	6	*
7	* This program is free software; you can redistribute it and/or modify	7	* This program is free software; you can redistribute it and/or modify
8	* it under the terms of the GNU General Public License version 2 as	8	* it under the terms of the GNU General Public License version 2 as
9	* published by the Free Software Foundation.	9	* published by the Free Software Foundation.
10	*/	10	*/
11		11
12	#include <linux/module.h>	12	#include <linux/module.h>
13	#include <linux/kernel.h>	13	#include <linux/kernel.h>
14	#include <linux/platform_device.h>	14	#include <linux/platform_device.h>
15	#include <linux/sched.h>	15	#include <linux/sched.h>
16	#include <linux/errno.h>	16	#include <linux/errno.h>
17	#include <linux/string.h>	17	#include <linux/string.h>
18	#include <linux/interrupt.h>	18	#include <linux/interrupt.h>
19	#include <linux/slab.h>	19	#include <linux/slab.h>
20	#include <linux/fb.h>	20	#include <linux/fb.h>
21	#include <linux/delay.h>	21	#include <linux/delay.h>
22	#include <linux/init.h>	22	#include <linux/init.h>
23	#include <linux/ioport.h>	23	#include <linux/ioport.h>
24	#include <linux/dma-mapping.h>	24	#include <linux/dma-mapping.h>
25	#include <linux/dmaengine.h>	25	#include <linux/dmaengine.h>
26	#include <linux/console.h>	26	#include <linux/console.h>
27	#include <linux/clk.h>	27	#include <linux/clk.h>
28	#include <linux/mutex.h>	28	#include <linux/mutex.h>
29		29
30	#include <mach/hardware.h>	30	#include <mach/hardware.h>
31	#include <mach/ipu.h>	31	#include <mach/ipu.h>
32	#include <mach/mx3fb.h>	32	#include <mach/mx3fb.h>
33		33
34	#include <asm/io.h>	34	#include <asm/io.h>
35	#include <asm/uaccess.h>	35	#include <asm/uaccess.h>
36		36
37	#define MX3FB_NAME "mx3_sdc_fb"	37	#define MX3FB_NAME "mx3_sdc_fb"
38		38
39	#define MX3FB_REG_OFFSET 0xB4	39	#define MX3FB_REG_OFFSET 0xB4
40		40
41	/* SDC Registers */	41	/* SDC Registers */
42	#define SDC_COM_CONF (0xB4 - MX3FB_REG_OFFSET)	42	#define SDC_COM_CONF (0xB4 - MX3FB_REG_OFFSET)
43	#define SDC_GW_CTRL (0xB8 - MX3FB_REG_OFFSET)	43	#define SDC_GW_CTRL (0xB8 - MX3FB_REG_OFFSET)
44	#define SDC_FG_POS (0xBC - MX3FB_REG_OFFSET)	44	#define SDC_FG_POS (0xBC - MX3FB_REG_OFFSET)
45	#define SDC_BG_POS (0xC0 - MX3FB_REG_OFFSET)	45	#define SDC_BG_POS (0xC0 - MX3FB_REG_OFFSET)
46	#define SDC_CUR_POS (0xC4 - MX3FB_REG_OFFSET)	46	#define SDC_CUR_POS (0xC4 - MX3FB_REG_OFFSET)
47	#define SDC_PWM_CTRL (0xC8 - MX3FB_REG_OFFSET)	47	#define SDC_PWM_CTRL (0xC8 - MX3FB_REG_OFFSET)
48	#define SDC_CUR_MAP (0xCC - MX3FB_REG_OFFSET)	48	#define SDC_CUR_MAP (0xCC - MX3FB_REG_OFFSET)
49	#define SDC_HOR_CONF (0xD0 - MX3FB_REG_OFFSET)	49	#define SDC_HOR_CONF (0xD0 - MX3FB_REG_OFFSET)
50	#define SDC_VER_CONF (0xD4 - MX3FB_REG_OFFSET)	50	#define SDC_VER_CONF (0xD4 - MX3FB_REG_OFFSET)
51	#define SDC_SHARP_CONF_1 (0xD8 - MX3FB_REG_OFFSET)	51	#define SDC_SHARP_CONF_1 (0xD8 - MX3FB_REG_OFFSET)
52	#define SDC_SHARP_CONF_2 (0xDC - MX3FB_REG_OFFSET)	52	#define SDC_SHARP_CONF_2 (0xDC - MX3FB_REG_OFFSET)
53		53
54	/* Register bits */	54	/* Register bits */
55	#define SDC_COM_TFT_COLOR 0x00000001UL	55	#define SDC_COM_TFT_COLOR 0x00000001UL
56	#define SDC_COM_FG_EN 0x00000010UL	56	#define SDC_COM_FG_EN 0x00000010UL
57	#define SDC_COM_GWSEL 0x00000020UL	57	#define SDC_COM_GWSEL 0x00000020UL
58	#define SDC_COM_GLB_A 0x00000040UL	58	#define SDC_COM_GLB_A 0x00000040UL
59	#define SDC_COM_KEY_COLOR_G 0x00000080UL	59	#define SDC_COM_KEY_COLOR_G 0x00000080UL
60	#define SDC_COM_BG_EN 0x00000200UL	60	#define SDC_COM_BG_EN 0x00000200UL
61	#define SDC_COM_SHARP 0x00001000UL	61	#define SDC_COM_SHARP 0x00001000UL
62		62
63	#define SDC_V_SYNC_WIDTH_L 0x00000001UL	63	#define SDC_V_SYNC_WIDTH_L 0x00000001UL
64		64
65	/* Display Interface registers */	65	/* Display Interface registers */
66	#define DI_DISP_IF_CONF (0x0124 - MX3FB_REG_OFFSET)	66	#define DI_DISP_IF_CONF (0x0124 - MX3FB_REG_OFFSET)
67	#define DI_DISP_SIG_POL (0x0128 - MX3FB_REG_OFFSET)	67	#define DI_DISP_SIG_POL (0x0128 - MX3FB_REG_OFFSET)
68	#define DI_SER_DISP1_CONF (0x012C - MX3FB_REG_OFFSET)	68	#define DI_SER_DISP1_CONF (0x012C - MX3FB_REG_OFFSET)
69	#define DI_SER_DISP2_CONF (0x0130 - MX3FB_REG_OFFSET)	69	#define DI_SER_DISP2_CONF (0x0130 - MX3FB_REG_OFFSET)
70	#define DI_HSP_CLK_PER (0x0134 - MX3FB_REG_OFFSET)	70	#define DI_HSP_CLK_PER (0x0134 - MX3FB_REG_OFFSET)
71	#define DI_DISP0_TIME_CONF_1 (0x0138 - MX3FB_REG_OFFSET)	71	#define DI_DISP0_TIME_CONF_1 (0x0138 - MX3FB_REG_OFFSET)
72	#define DI_DISP0_TIME_CONF_2 (0x013C - MX3FB_REG_OFFSET)	72	#define DI_DISP0_TIME_CONF_2 (0x013C - MX3FB_REG_OFFSET)
73	#define DI_DISP0_TIME_CONF_3 (0x0140 - MX3FB_REG_OFFSET)	73	#define DI_DISP0_TIME_CONF_3 (0x0140 - MX3FB_REG_OFFSET)
74	#define DI_DISP1_TIME_CONF_1 (0x0144 - MX3FB_REG_OFFSET)	74	#define DI_DISP1_TIME_CONF_1 (0x0144 - MX3FB_REG_OFFSET)
75	#define DI_DISP1_TIME_CONF_2 (0x0148 - MX3FB_REG_OFFSET)	75	#define DI_DISP1_TIME_CONF_2 (0x0148 - MX3FB_REG_OFFSET)
76	#define DI_DISP1_TIME_CONF_3 (0x014C - MX3FB_REG_OFFSET)	76	#define DI_DISP1_TIME_CONF_3 (0x014C - MX3FB_REG_OFFSET)
77	#define DI_DISP2_TIME_CONF_1 (0x0150 - MX3FB_REG_OFFSET)	77	#define DI_DISP2_TIME_CONF_1 (0x0150 - MX3FB_REG_OFFSET)
78	#define DI_DISP2_TIME_CONF_2 (0x0154 - MX3FB_REG_OFFSET)	78	#define DI_DISP2_TIME_CONF_2 (0x0154 - MX3FB_REG_OFFSET)
79	#define DI_DISP2_TIME_CONF_3 (0x0158 - MX3FB_REG_OFFSET)	79	#define DI_DISP2_TIME_CONF_3 (0x0158 - MX3FB_REG_OFFSET)
80	#define DI_DISP3_TIME_CONF (0x015C - MX3FB_REG_OFFSET)	80	#define DI_DISP3_TIME_CONF (0x015C - MX3FB_REG_OFFSET)
81	#define DI_DISP0_DB0_MAP (0x0160 - MX3FB_REG_OFFSET)	81	#define DI_DISP0_DB0_MAP (0x0160 - MX3FB_REG_OFFSET)
82	#define DI_DISP0_DB1_MAP (0x0164 - MX3FB_REG_OFFSET)	82	#define DI_DISP0_DB1_MAP (0x0164 - MX3FB_REG_OFFSET)
83	#define DI_DISP0_DB2_MAP (0x0168 - MX3FB_REG_OFFSET)	83	#define DI_DISP0_DB2_MAP (0x0168 - MX3FB_REG_OFFSET)
84	#define DI_DISP0_CB0_MAP (0x016C - MX3FB_REG_OFFSET)	84	#define DI_DISP0_CB0_MAP (0x016C - MX3FB_REG_OFFSET)
85	#define DI_DISP0_CB1_MAP (0x0170 - MX3FB_REG_OFFSET)	85	#define DI_DISP0_CB1_MAP (0x0170 - MX3FB_REG_OFFSET)
86	#define DI_DISP0_CB2_MAP (0x0174 - MX3FB_REG_OFFSET)	86	#define DI_DISP0_CB2_MAP (0x0174 - MX3FB_REG_OFFSET)
87	#define DI_DISP1_DB0_MAP (0x0178 - MX3FB_REG_OFFSET)	87	#define DI_DISP1_DB0_MAP (0x0178 - MX3FB_REG_OFFSET)
88	#define DI_DISP1_DB1_MAP (0x017C - MX3FB_REG_OFFSET)	88	#define DI_DISP1_DB1_MAP (0x017C - MX3FB_REG_OFFSET)
89	#define DI_DISP1_DB2_MAP (0x0180 - MX3FB_REG_OFFSET)	89	#define DI_DISP1_DB2_MAP (0x0180 - MX3FB_REG_OFFSET)
90	#define DI_DISP1_CB0_MAP (0x0184 - MX3FB_REG_OFFSET)	90	#define DI_DISP1_CB0_MAP (0x0184 - MX3FB_REG_OFFSET)
91	#define DI_DISP1_CB1_MAP (0x0188 - MX3FB_REG_OFFSET)	91	#define DI_DISP1_CB1_MAP (0x0188 - MX3FB_REG_OFFSET)
92	#define DI_DISP1_CB2_MAP (0x018C - MX3FB_REG_OFFSET)	92	#define DI_DISP1_CB2_MAP (0x018C - MX3FB_REG_OFFSET)
93	#define DI_DISP2_DB0_MAP (0x0190 - MX3FB_REG_OFFSET)	93	#define DI_DISP2_DB0_MAP (0x0190 - MX3FB_REG_OFFSET)
94	#define DI_DISP2_DB1_MAP (0x0194 - MX3FB_REG_OFFSET)	94	#define DI_DISP2_DB1_MAP (0x0194 - MX3FB_REG_OFFSET)
95	#define DI_DISP2_DB2_MAP (0x0198 - MX3FB_REG_OFFSET)	95	#define DI_DISP2_DB2_MAP (0x0198 - MX3FB_REG_OFFSET)
96	#define DI_DISP2_CB0_MAP (0x019C - MX3FB_REG_OFFSET)	96	#define DI_DISP2_CB0_MAP (0x019C - MX3FB_REG_OFFSET)
97	#define DI_DISP2_CB1_MAP (0x01A0 - MX3FB_REG_OFFSET)	97	#define DI_DISP2_CB1_MAP (0x01A0 - MX3FB_REG_OFFSET)
98	#define DI_DISP2_CB2_MAP (0x01A4 - MX3FB_REG_OFFSET)	98	#define DI_DISP2_CB2_MAP (0x01A4 - MX3FB_REG_OFFSET)
99	#define DI_DISP3_B0_MAP (0x01A8 - MX3FB_REG_OFFSET)	99	#define DI_DISP3_B0_MAP (0x01A8 - MX3FB_REG_OFFSET)
100	#define DI_DISP3_B1_MAP (0x01AC - MX3FB_REG_OFFSET)	100	#define DI_DISP3_B1_MAP (0x01AC - MX3FB_REG_OFFSET)
101	#define DI_DISP3_B2_MAP (0x01B0 - MX3FB_REG_OFFSET)	101	#define DI_DISP3_B2_MAP (0x01B0 - MX3FB_REG_OFFSET)
102	#define DI_DISP_ACC_CC (0x01B4 - MX3FB_REG_OFFSET)	102	#define DI_DISP_ACC_CC (0x01B4 - MX3FB_REG_OFFSET)
103	#define DI_DISP_LLA_CONF (0x01B8 - MX3FB_REG_OFFSET)	103	#define DI_DISP_LLA_CONF (0x01B8 - MX3FB_REG_OFFSET)
104	#define DI_DISP_LLA_DATA (0x01BC - MX3FB_REG_OFFSET)	104	#define DI_DISP_LLA_DATA (0x01BC - MX3FB_REG_OFFSET)
105		105
106	/* DI_DISP_SIG_POL bits */	106	/* DI_DISP_SIG_POL bits */
107	#define DI_D3_VSYNC_POL_SHIFT 28	107	#define DI_D3_VSYNC_POL_SHIFT 28
108	#define DI_D3_HSYNC_POL_SHIFT 27	108	#define DI_D3_HSYNC_POL_SHIFT 27
109	#define DI_D3_DRDY_SHARP_POL_SHIFT 26	109	#define DI_D3_DRDY_SHARP_POL_SHIFT 26
110	#define DI_D3_CLK_POL_SHIFT 25	110	#define DI_D3_CLK_POL_SHIFT 25
111	#define DI_D3_DATA_POL_SHIFT 24	111	#define DI_D3_DATA_POL_SHIFT 24
112		112
113	/* DI_DISP_IF_CONF bits */	113	/* DI_DISP_IF_CONF bits */
114	#define DI_D3_CLK_IDLE_SHIFT 26	114	#define DI_D3_CLK_IDLE_SHIFT 26
115	#define DI_D3_CLK_SEL_SHIFT 25	115	#define DI_D3_CLK_SEL_SHIFT 25
116	#define DI_D3_DATAMSK_SHIFT 24	116	#define DI_D3_DATAMSK_SHIFT 24
117		117
118	enum ipu_panel {	118	enum ipu_panel {
119	IPU_PANEL_SHARP_TFT,	119	IPU_PANEL_SHARP_TFT,
120	IPU_PANEL_TFT,	120	IPU_PANEL_TFT,
121	};	121	};
122		122
123	struct ipu_di_signal_cfg {	123	struct ipu_di_signal_cfg {
124	unsigned datamask_en:1;	124	unsigned datamask_en:1;
125	unsigned clksel_en:1;	125	unsigned clksel_en:1;
126	unsigned clkidle_en:1;	126	unsigned clkidle_en:1;
127	unsigned data_pol:1; /* true = inverted */	127	unsigned data_pol:1; /* true = inverted */
128	unsigned clk_pol:1; /* true = rising edge */	128	unsigned clk_pol:1; /* true = rising edge */
129	unsigned enable_pol:1;	129	unsigned enable_pol:1;
130	unsigned Hsync_pol:1; /* true = active high */	130	unsigned Hsync_pol:1; /* true = active high */
131	unsigned Vsync_pol:1;	131	unsigned Vsync_pol:1;
132	};	132	};
133		133
134	static const struct fb_videomode mx3fb_modedb[] = {	134	static const struct fb_videomode mx3fb_modedb[] = {
135	{	135	{
136	/* 240x320 @ 60 Hz */	136	/* 240x320 @ 60 Hz */
137	.name = "Sharp-QVGA",	137	.name = "Sharp-QVGA",
138	.refresh = 60,	138	.refresh = 60,
139	.xres = 240,	139	.xres = 240,
140	.yres = 320,	140	.yres = 320,
141	.pixclock = 185925,	141	.pixclock = 185925,
142	.left_margin = 9,	142	.left_margin = 9,
143	.right_margin = 16,	143	.right_margin = 16,
144	.upper_margin = 7,	144	.upper_margin = 7,
145	.lower_margin = 9,	145	.lower_margin = 9,
146	.hsync_len = 1,	146	.hsync_len = 1,
147	.vsync_len = 1,	147	.vsync_len = 1,
148	.sync = FB_SYNC_HOR_HIGH_ACT \| FB_SYNC_SHARP_MODE \|	148	.sync = FB_SYNC_HOR_HIGH_ACT \| FB_SYNC_SHARP_MODE \|
149	FB_SYNC_CLK_INVERT \| FB_SYNC_DATA_INVERT \|	149	FB_SYNC_CLK_INVERT \| FB_SYNC_DATA_INVERT \|
150	FB_SYNC_CLK_IDLE_EN,	150	FB_SYNC_CLK_IDLE_EN,
151	.vmode = FB_VMODE_NONINTERLACED,	151	.vmode = FB_VMODE_NONINTERLACED,
152	.flag = 0,	152	.flag = 0,
153	}, {	153	}, {
154	/* 240x33 @ 60 Hz */	154	/* 240x33 @ 60 Hz */
155	.name = "Sharp-CLI",	155	.name = "Sharp-CLI",
156	.refresh = 60,	156	.refresh = 60,
157	.xres = 240,	157	.xres = 240,
158	.yres = 33,	158	.yres = 33,
159	.pixclock = 185925,	159	.pixclock = 185925,
160	.left_margin = 9,	160	.left_margin = 9,
161	.right_margin = 16,	161	.right_margin = 16,
162	.upper_margin = 7,	162	.upper_margin = 7,
163	.lower_margin = 9 + 287,	163	.lower_margin = 9 + 287,
164	.hsync_len = 1,	164	.hsync_len = 1,
165	.vsync_len = 1,	165	.vsync_len = 1,
166	.sync = FB_SYNC_HOR_HIGH_ACT \| FB_SYNC_SHARP_MODE \|	166	.sync = FB_SYNC_HOR_HIGH_ACT \| FB_SYNC_SHARP_MODE \|
167	FB_SYNC_CLK_INVERT \| FB_SYNC_DATA_INVERT \|	167	FB_SYNC_CLK_INVERT \| FB_SYNC_DATA_INVERT \|
168	FB_SYNC_CLK_IDLE_EN,	168	FB_SYNC_CLK_IDLE_EN,
169	.vmode = FB_VMODE_NONINTERLACED,	169	.vmode = FB_VMODE_NONINTERLACED,
170	.flag = 0,	170	.flag = 0,
171	}, {	171	}, {
172	/* 640x480 @ 60 Hz */	172	/* 640x480 @ 60 Hz */
173	.name = "NEC-VGA",	173	.name = "NEC-VGA",
174	.refresh = 60,	174	.refresh = 60,
175	.xres = 640,	175	.xres = 640,
176	.yres = 480,	176	.yres = 480,
177	.pixclock = 38255,	177	.pixclock = 38255,
178	.left_margin = 144,	178	.left_margin = 144,
179	.right_margin = 0,	179	.right_margin = 0,
180	.upper_margin = 34,	180	.upper_margin = 34,
181	.lower_margin = 40,	181	.lower_margin = 40,
182	.hsync_len = 1,	182	.hsync_len = 1,
183	.vsync_len = 1,	183	.vsync_len = 1,
184	.sync = FB_SYNC_VERT_HIGH_ACT \| FB_SYNC_OE_ACT_HIGH,	184	.sync = FB_SYNC_VERT_HIGH_ACT \| FB_SYNC_OE_ACT_HIGH,
185	.vmode = FB_VMODE_NONINTERLACED,	185	.vmode = FB_VMODE_NONINTERLACED,
186	.flag = 0,	186	.flag = 0,
187	}, {	187	}, {
188	/* NTSC TV output */	188	/* NTSC TV output */
189	.name = "TV-NTSC",	189	.name = "TV-NTSC",
190	.refresh = 60,	190	.refresh = 60,
191	.xres = 640,	191	.xres = 640,
192	.yres = 480,	192	.yres = 480,
193	.pixclock = 37538,	193	.pixclock = 37538,
194	.left_margin = 38,	194	.left_margin = 38,
195	.right_margin = 858 - 640 - 38 - 3,	195	.right_margin = 858 - 640 - 38 - 3,
196	.upper_margin = 36,	196	.upper_margin = 36,
197	.lower_margin = 518 - 480 - 36 - 1,	197	.lower_margin = 518 - 480 - 36 - 1,
198	.hsync_len = 3,	198	.hsync_len = 3,
199	.vsync_len = 1,	199	.vsync_len = 1,
200	.sync = 0,	200	.sync = 0,
201	.vmode = FB_VMODE_NONINTERLACED,	201	.vmode = FB_VMODE_NONINTERLACED,
202	.flag = 0,	202	.flag = 0,
203	}, {	203	}, {
204	/* PAL TV output */	204	/* PAL TV output */
205	.name = "TV-PAL",	205	.name = "TV-PAL",
206	.refresh = 50,	206	.refresh = 50,
207	.xres = 640,	207	.xres = 640,
208	.yres = 480,	208	.yres = 480,
209	.pixclock = 37538,	209	.pixclock = 37538,
210	.left_margin = 38,	210	.left_margin = 38,
211	.right_margin = 960 - 640 - 38 - 32,	211	.right_margin = 960 - 640 - 38 - 32,
212	.upper_margin = 32,	212	.upper_margin = 32,
213	.lower_margin = 555 - 480 - 32 - 3,	213	.lower_margin = 555 - 480 - 32 - 3,
214	.hsync_len = 32,	214	.hsync_len = 32,
215	.vsync_len = 3,	215	.vsync_len = 3,
216	.sync = 0,	216	.sync = 0,
217	.vmode = FB_VMODE_NONINTERLACED,	217	.vmode = FB_VMODE_NONINTERLACED,
218	.flag = 0,	218	.flag = 0,
219	}, {	219	}, {
220	/* TV output VGA mode, 640x480 @ 65 Hz */	220	/* TV output VGA mode, 640x480 @ 65 Hz */
221	.name = "TV-VGA",	221	.name = "TV-VGA",
222	.refresh = 60,	222	.refresh = 60,
223	.xres = 640,	223	.xres = 640,
224	.yres = 480,	224	.yres = 480,
225	.pixclock = 40574,	225	.pixclock = 40574,
226	.left_margin = 35,	226	.left_margin = 35,
227	.right_margin = 45,	227	.right_margin = 45,
228	.upper_margin = 9,	228	.upper_margin = 9,
229	.lower_margin = 1,	229	.lower_margin = 1,
230	.hsync_len = 46,	230	.hsync_len = 46,
231	.vsync_len = 5,	231	.vsync_len = 5,
232	.sync = 0,	232	.sync = 0,
233	.vmode = FB_VMODE_NONINTERLACED,	233	.vmode = FB_VMODE_NONINTERLACED,
234	.flag = 0,	234	.flag = 0,
235	},	235	},
236	};	236	};
237		237
238	struct mx3fb_data {	238	struct mx3fb_data {
239	struct fb_info *fbi;	239	struct fb_info *fbi;
240	int backlight_level;	240	int backlight_level;
241	void __iomem *reg_base;	241	void __iomem *reg_base;
242	spinlock_t lock;	242	spinlock_t lock;
243	struct device *dev;	243	struct device *dev;
244		244
245	uint32_t h_start_width;	245	uint32_t h_start_width;
246	uint32_t v_start_width;	246	uint32_t v_start_width;
247	};	247	};
248		248
249	struct dma_chan_request {	249	struct dma_chan_request {
250	struct mx3fb_data *mx3fb;	250	struct mx3fb_data *mx3fb;
251	enum ipu_channel id;	251	enum ipu_channel id;
252	};	252	};
253		253
254	/* MX3 specific framebuffer information. */	254	/* MX3 specific framebuffer information. */
255	struct mx3fb_info {	255	struct mx3fb_info {
256	int blank;	256	int blank;
257	enum ipu_channel ipu_ch;	257	enum ipu_channel ipu_ch;
258	uint32_t cur_ipu_buf;	258	uint32_t cur_ipu_buf;
259		259
260	u32 pseudo_palette[16];	260	u32 pseudo_palette[16];
261		261
262	struct completion flip_cmpl;	262	struct completion flip_cmpl;
263	struct mutex mutex; /* Protects fb-ops */	263	struct mutex mutex; /* Protects fb-ops */
264	struct mx3fb_data *mx3fb;	264	struct mx3fb_data *mx3fb;
265	struct idmac_channel *idmac_channel;	265	struct idmac_channel *idmac_channel;
266	struct dma_async_tx_descriptor *txd;	266	struct dma_async_tx_descriptor *txd;
267	dma_cookie_t cookie;	267	dma_cookie_t cookie;
268	struct scatterlist sg[2];	268	struct scatterlist sg[2];
269		269
270	u32 sync; /* preserve var->sync flags */	270	u32 sync; /* preserve var->sync flags */
271	};	271	};
272		272
273	static void mx3fb_dma_done(void *);	273	static void mx3fb_dma_done(void *);
274		274
275	/* Used fb-mode and bpp. Can be set on kernel command line, therefore file-static. */	275	/* Used fb-mode and bpp. Can be set on kernel command line, therefore file-static. */
276	static const char *fb_mode;	276	static const char *fb_mode;
277	static unsigned long default_bpp = 16;	277	static unsigned long default_bpp = 16;
278		278
279	static u32 mx3fb_read_reg(struct mx3fb_data *mx3fb, unsigned long reg)	279	static u32 mx3fb_read_reg(struct mx3fb_data *mx3fb, unsigned long reg)
280	{	280	{
281	return __raw_readl(mx3fb->reg_base + reg);	281	return __raw_readl(mx3fb->reg_base + reg);
282	}	282	}
283		283
284	static void mx3fb_write_reg(struct mx3fb_data *mx3fb, u32 value, unsigned long reg)	284	static void mx3fb_write_reg(struct mx3fb_data *mx3fb, u32 value, unsigned long reg)
285	{	285	{
286	__raw_writel(value, mx3fb->reg_base + reg);	286	__raw_writel(value, mx3fb->reg_base + reg);
287	}	287	}
288		288
289	static const uint32_t di_mappings[] = {	289	static const uint32_t di_mappings[] = {
290	0x1600AAAA, 0x00E05555, 0x00070000, 3, /* RGB888 */	290	0x1600AAAA, 0x00E05555, 0x00070000, 3, /* RGB888 */
291	0x0005000F, 0x000B000F, 0x0011000F, 1, /* RGB666 */	291	0x0005000F, 0x000B000F, 0x0011000F, 1, /* RGB666 */
292	0x0011000F, 0x000B000F, 0x0005000F, 1, /* BGR666 */	292	0x0011000F, 0x000B000F, 0x0005000F, 1, /* BGR666 */
293	0x0004003F, 0x000A000F, 0x000F003F, 1 /* RGB565 */	293	0x0004003F, 0x000A000F, 0x000F003F, 1 /* RGB565 */
294	};	294	};
295		295
296	static void sdc_fb_init(struct mx3fb_info *fbi)	296	static void sdc_fb_init(struct mx3fb_info *fbi)
297	{	297	{
298	struct mx3fb_data *mx3fb = fbi->mx3fb;	298	struct mx3fb_data *mx3fb = fbi->mx3fb;
299	uint32_t reg;	299	uint32_t reg;
300		300
301	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);	301	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);
302		302
303	mx3fb_write_reg(mx3fb, reg \| SDC_COM_BG_EN, SDC_COM_CONF);	303	mx3fb_write_reg(mx3fb, reg \| SDC_COM_BG_EN, SDC_COM_CONF);
304	}	304	}
305		305
306	/* Returns enabled flag before uninit */	306	/* Returns enabled flag before uninit */
307	static uint32_t sdc_fb_uninit(struct mx3fb_info *fbi)	307	static uint32_t sdc_fb_uninit(struct mx3fb_info *fbi)
308	{	308	{
309	struct mx3fb_data *mx3fb = fbi->mx3fb;	309	struct mx3fb_data *mx3fb = fbi->mx3fb;
310	uint32_t reg;	310	uint32_t reg;
311		311
312	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);	312	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);
313		313
314	mx3fb_write_reg(mx3fb, reg & ~SDC_COM_BG_EN, SDC_COM_CONF);	314	mx3fb_write_reg(mx3fb, reg & ~SDC_COM_BG_EN, SDC_COM_CONF);
315		315
316	return reg & SDC_COM_BG_EN;	316	return reg & SDC_COM_BG_EN;
317	}	317	}
318		318
319	static void sdc_enable_channel(struct mx3fb_info *mx3_fbi)	319	static void sdc_enable_channel(struct mx3fb_info *mx3_fbi)
320	{	320	{
321	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;	321	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;
322	struct idmac_channel *ichan = mx3_fbi->idmac_channel;	322	struct idmac_channel *ichan = mx3_fbi->idmac_channel;
323	struct dma_chan *dma_chan = &ichan->dma_chan;	323	struct dma_chan *dma_chan = &ichan->dma_chan;
324	unsigned long flags;	324	unsigned long flags;
325	dma_cookie_t cookie;	325	dma_cookie_t cookie;
326		326
327	if (mx3_fbi->txd)	327	if (mx3_fbi->txd)
328	dev_dbg(mx3fb->dev, "mx3fbi %p, desc %p, sg %p\n", mx3_fbi,	328	dev_dbg(mx3fb->dev, "mx3fbi %p, desc %p, sg %p\n", mx3_fbi,
329	to_tx_desc(mx3_fbi->txd), to_tx_desc(mx3_fbi->txd)->sg);	329	to_tx_desc(mx3_fbi->txd), to_tx_desc(mx3_fbi->txd)->sg);
330	else	330	else
331	dev_dbg(mx3fb->dev, "mx3fbi %p, txd = NULL\n", mx3_fbi);	331	dev_dbg(mx3fb->dev, "mx3fbi %p, txd = NULL\n", mx3_fbi);
332		332
333	/* This enables the channel */	333	/* This enables the channel */
334	if (mx3_fbi->cookie < 0) {	334	if (mx3_fbi->cookie < 0) {
335	mx3_fbi->txd = dma_chan->device->device_prep_slave_sg(dma_chan,	335	mx3_fbi->txd = dma_chan->device->device_prep_slave_sg(dma_chan,
336	&mx3_fbi->sg[0], 1, DMA_TO_DEVICE, DMA_PREP_INTERRUPT);	336	&mx3_fbi->sg[0], 1, DMA_TO_DEVICE, DMA_PREP_INTERRUPT);
337	if (!mx3_fbi->txd) {	337	if (!mx3_fbi->txd) {
338	dev_err(mx3fb->dev, "Cannot allocate descriptor on %d\n",	338	dev_err(mx3fb->dev, "Cannot allocate descriptor on %d\n",
339	dma_chan->chan_id);	339	dma_chan->chan_id);
340	return;	340	return;
341	}	341	}
342		342
343	mx3_fbi->txd->callback_param = mx3_fbi->txd;	343	mx3_fbi->txd->callback_param = mx3_fbi->txd;
344	mx3_fbi->txd->callback = mx3fb_dma_done;	344	mx3_fbi->txd->callback = mx3fb_dma_done;
345		345
346	cookie = mx3_fbi->txd->tx_submit(mx3_fbi->txd);	346	cookie = mx3_fbi->txd->tx_submit(mx3_fbi->txd);
347	dev_dbg(mx3fb->dev, "%d: Submit %p #%d [%c]\n", __LINE__,	347	dev_dbg(mx3fb->dev, "%d: Submit %p #%d [%c]\n", __LINE__,
348	mx3_fbi->txd, cookie, list_empty(&ichan->queue) ? '-' : '+');	348	mx3_fbi->txd, cookie, list_empty(&ichan->queue) ? '-' : '+');
349	} else {	349	} else {
350	if (!mx3_fbi->txd \|\| !mx3_fbi->txd->tx_submit) {	350	if (!mx3_fbi->txd \|\| !mx3_fbi->txd->tx_submit) {
351	dev_err(mx3fb->dev, "Cannot enable channel %d\n",	351	dev_err(mx3fb->dev, "Cannot enable channel %d\n",
352	dma_chan->chan_id);	352	dma_chan->chan_id);
353	return;	353	return;
354	}	354	}
355		355
356	/* Just re-activate the same buffer */	356	/* Just re-activate the same buffer */
357	dma_async_issue_pending(dma_chan);	357	dma_async_issue_pending(dma_chan);
358	cookie = mx3_fbi->cookie;	358	cookie = mx3_fbi->cookie;
359	dev_dbg(mx3fb->dev, "%d: Re-submit %p #%d [%c]\n", __LINE__,	359	dev_dbg(mx3fb->dev, "%d: Re-submit %p #%d [%c]\n", __LINE__,
360	mx3_fbi->txd, cookie, list_empty(&ichan->queue) ? '-' : '+');	360	mx3_fbi->txd, cookie, list_empty(&ichan->queue) ? '-' : '+');
361	}	361	}
362		362
363	if (cookie >= 0) {	363	if (cookie >= 0) {
364	spin_lock_irqsave(&mx3fb->lock, flags);	364	spin_lock_irqsave(&mx3fb->lock, flags);
365	sdc_fb_init(mx3_fbi);	365	sdc_fb_init(mx3_fbi);
366	mx3_fbi->cookie = cookie;	366	mx3_fbi->cookie = cookie;
367	spin_unlock_irqrestore(&mx3fb->lock, flags);	367	spin_unlock_irqrestore(&mx3fb->lock, flags);
368	}	368	}
369		369
370	/*	370	/*
371	* Attention! Without this msleep the channel keeps generating	371	* Attention! Without this msleep the channel keeps generating
372	* interrupts. Next sdc_set_brightness() is going to be called	372	* interrupts. Next sdc_set_brightness() is going to be called
373	* from mx3fb_blank().	373	* from mx3fb_blank().
374	*/	374	*/
375	msleep(2);	375	msleep(2);
376	}	376	}
377		377
378	static void sdc_disable_channel(struct mx3fb_info *mx3_fbi)	378	static void sdc_disable_channel(struct mx3fb_info *mx3_fbi)
379	{	379	{
380	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;	380	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;
381	uint32_t enabled;	381	uint32_t enabled;
382	unsigned long flags;	382	unsigned long flags;
383		383
384	spin_lock_irqsave(&mx3fb->lock, flags);	384	spin_lock_irqsave(&mx3fb->lock, flags);
385		385
386	enabled = sdc_fb_uninit(mx3_fbi);	386	enabled = sdc_fb_uninit(mx3_fbi);
387		387
388	spin_unlock_irqrestore(&mx3fb->lock, flags);	388	spin_unlock_irqrestore(&mx3fb->lock, flags);
389		389
390	mx3_fbi->txd->chan->device->device_control(mx3_fbi->txd->chan,	390	mx3_fbi->txd->chan->device->device_control(mx3_fbi->txd->chan,
391	DMA_TERMINATE_ALL);	391	DMA_TERMINATE_ALL, 0);
392	mx3_fbi->txd = NULL;	392	mx3_fbi->txd = NULL;
393	mx3_fbi->cookie = -EINVAL;	393	mx3_fbi->cookie = -EINVAL;
394	}	394	}
395		395
396	/**	396	/**
397	* sdc_set_window_pos() - set window position of the respective plane.	397	* sdc_set_window_pos() - set window position of the respective plane.
398	* @mx3fb: mx3fb context.	398	* @mx3fb: mx3fb context.
399	* @channel: IPU DMAC channel ID.	399	* @channel: IPU DMAC channel ID.
400	* @x_pos: X coordinate relative to the top left corner to place window at.	400	* @x_pos: X coordinate relative to the top left corner to place window at.
401	* @y_pos: Y coordinate relative to the top left corner to place window at.	401	* @y_pos: Y coordinate relative to the top left corner to place window at.
402	* @return: 0 on success or negative error code on failure.	402	* @return: 0 on success or negative error code on failure.
403	*/	403	*/
404	static int sdc_set_window_pos(struct mx3fb_data *mx3fb, enum ipu_channel channel,	404	static int sdc_set_window_pos(struct mx3fb_data *mx3fb, enum ipu_channel channel,
405	int16_t x_pos, int16_t y_pos)	405	int16_t x_pos, int16_t y_pos)
406	{	406	{
407	if (channel != IDMAC_SDC_0)	407	if (channel != IDMAC_SDC_0)
408	return -EINVAL;	408	return -EINVAL;
409		409
410	x_pos += mx3fb->h_start_width;	410	x_pos += mx3fb->h_start_width;
411	y_pos += mx3fb->v_start_width;	411	y_pos += mx3fb->v_start_width;
412		412
413	mx3fb_write_reg(mx3fb, (x_pos << 16) \| y_pos, SDC_BG_POS);	413	mx3fb_write_reg(mx3fb, (x_pos << 16) \| y_pos, SDC_BG_POS);
414	return 0;	414	return 0;
415	}	415	}
416		416
417	/**	417	/**
418	* sdc_init_panel() - initialize a synchronous LCD panel.	418	* sdc_init_panel() - initialize a synchronous LCD panel.
419	* @mx3fb: mx3fb context.	419	* @mx3fb: mx3fb context.
420	* @panel: panel type.	420	* @panel: panel type.
421	* @pixel_clk: desired pixel clock frequency in Hz.	421	* @pixel_clk: desired pixel clock frequency in Hz.
422	* @width: width of panel in pixels.	422	* @width: width of panel in pixels.
423	* @height: height of panel in pixels.	423	* @height: height of panel in pixels.
424	* @pixel_fmt: pixel format of buffer as FOURCC ASCII code.	424	* @pixel_fmt: pixel format of buffer as FOURCC ASCII code.
425	* @h_start_width: number of pixel clocks between the HSYNC signal pulse	425	* @h_start_width: number of pixel clocks between the HSYNC signal pulse
426	* and the start of valid data.	426	* and the start of valid data.
427	* @h_sync_width: width of the HSYNC signal in units of pixel clocks.	427	* @h_sync_width: width of the HSYNC signal in units of pixel clocks.
428	* @h_end_width: number of pixel clocks between the end of valid data	428	* @h_end_width: number of pixel clocks between the end of valid data
429	* and the HSYNC signal for next line.	429	* and the HSYNC signal for next line.
430	* @v_start_width: number of lines between the VSYNC signal pulse and the	430	* @v_start_width: number of lines between the VSYNC signal pulse and the
431	* start of valid data.	431	* start of valid data.
432	* @v_sync_width: width of the VSYNC signal in units of lines	432	* @v_sync_width: width of the VSYNC signal in units of lines
433	* @v_end_width: number of lines between the end of valid data and the	433	* @v_end_width: number of lines between the end of valid data and the
434	* VSYNC signal for next frame.	434	* VSYNC signal for next frame.
435	* @sig: bitfield of signal polarities for LCD interface.	435	* @sig: bitfield of signal polarities for LCD interface.
436	* @return: 0 on success or negative error code on failure.	436	* @return: 0 on success or negative error code on failure.
437	*/	437	*/
438	static int sdc_init_panel(struct mx3fb_data *mx3fb, enum ipu_panel panel,	438	static int sdc_init_panel(struct mx3fb_data *mx3fb, enum ipu_panel panel,
439	uint32_t pixel_clk,	439	uint32_t pixel_clk,
440	uint16_t width, uint16_t height,	440	uint16_t width, uint16_t height,
441	enum pixel_fmt pixel_fmt,	441	enum pixel_fmt pixel_fmt,
442	uint16_t h_start_width, uint16_t h_sync_width,	442	uint16_t h_start_width, uint16_t h_sync_width,
443	uint16_t h_end_width, uint16_t v_start_width,	443	uint16_t h_end_width, uint16_t v_start_width,
444	uint16_t v_sync_width, uint16_t v_end_width,	444	uint16_t v_sync_width, uint16_t v_end_width,
445	struct ipu_di_signal_cfg sig)	445	struct ipu_di_signal_cfg sig)
446	{	446	{
447	unsigned long lock_flags;	447	unsigned long lock_flags;
448	uint32_t reg;	448	uint32_t reg;
449	uint32_t old_conf;	449	uint32_t old_conf;
450	uint32_t div;	450	uint32_t div;
451	struct clk *ipu_clk;	451	struct clk *ipu_clk;
452		452
453	dev_dbg(mx3fb->dev, "panel size = %d x %d", width, height);	453	dev_dbg(mx3fb->dev, "panel size = %d x %d", width, height);
454		454
455	if (v_sync_width == 0 \|\| h_sync_width == 0)	455	if (v_sync_width == 0 \|\| h_sync_width == 0)
456	return -EINVAL;	456	return -EINVAL;
457		457
458	/* Init panel size and blanking periods */	458	/* Init panel size and blanking periods */
459	reg = ((uint32_t) (h_sync_width - 1) << 26) \|	459	reg = ((uint32_t) (h_sync_width - 1) << 26) \|
460	((uint32_t) (width + h_start_width + h_end_width - 1) << 16);	460	((uint32_t) (width + h_start_width + h_end_width - 1) << 16);
461	mx3fb_write_reg(mx3fb, reg, SDC_HOR_CONF);	461	mx3fb_write_reg(mx3fb, reg, SDC_HOR_CONF);
462		462
463	#ifdef DEBUG	463	#ifdef DEBUG
464	printk(KERN_CONT " hor_conf %x,", reg);	464	printk(KERN_CONT " hor_conf %x,", reg);
465	#endif	465	#endif
466		466
467	reg = ((uint32_t) (v_sync_width - 1) << 26) \| SDC_V_SYNC_WIDTH_L \|	467	reg = ((uint32_t) (v_sync_width - 1) << 26) \| SDC_V_SYNC_WIDTH_L \|
468	((uint32_t) (height + v_start_width + v_end_width - 1) << 16);	468	((uint32_t) (height + v_start_width + v_end_width - 1) << 16);
469	mx3fb_write_reg(mx3fb, reg, SDC_VER_CONF);	469	mx3fb_write_reg(mx3fb, reg, SDC_VER_CONF);
470		470
471	#ifdef DEBUG	471	#ifdef DEBUG
472	printk(KERN_CONT " ver_conf %x\n", reg);	472	printk(KERN_CONT " ver_conf %x\n", reg);
473	#endif	473	#endif
474		474
475	mx3fb->h_start_width = h_start_width;	475	mx3fb->h_start_width = h_start_width;
476	mx3fb->v_start_width = v_start_width;	476	mx3fb->v_start_width = v_start_width;
477		477
478	switch (panel) {	478	switch (panel) {
479	case IPU_PANEL_SHARP_TFT:	479	case IPU_PANEL_SHARP_TFT:
480	mx3fb_write_reg(mx3fb, 0x00FD0102L, SDC_SHARP_CONF_1);	480	mx3fb_write_reg(mx3fb, 0x00FD0102L, SDC_SHARP_CONF_1);
481	mx3fb_write_reg(mx3fb, 0x00F500F4L, SDC_SHARP_CONF_2);	481	mx3fb_write_reg(mx3fb, 0x00F500F4L, SDC_SHARP_CONF_2);
482	mx3fb_write_reg(mx3fb, SDC_COM_SHARP \| SDC_COM_TFT_COLOR, SDC_COM_CONF);	482	mx3fb_write_reg(mx3fb, SDC_COM_SHARP \| SDC_COM_TFT_COLOR, SDC_COM_CONF);
483	break;	483	break;
484	case IPU_PANEL_TFT:	484	case IPU_PANEL_TFT:
485	mx3fb_write_reg(mx3fb, SDC_COM_TFT_COLOR, SDC_COM_CONF);	485	mx3fb_write_reg(mx3fb, SDC_COM_TFT_COLOR, SDC_COM_CONF);
486	break;	486	break;
487	default:	487	default:
488	return -EINVAL;	488	return -EINVAL;
489	}	489	}
490		490
491	/* Init clocking */	491	/* Init clocking */
492		492
493	/*	493	/*
494	* Calculate divider: fractional part is 4 bits so simply multiple by	494	* Calculate divider: fractional part is 4 bits so simply multiple by
495	* 2^4 to get fractional part, as long as we stay under ~250MHz and on	495	* 2^4 to get fractional part, as long as we stay under ~250MHz and on
496	* i.MX31 it (HSP_CLK) is <= 178MHz. Currently 128.267MHz	496	* i.MX31 it (HSP_CLK) is <= 178MHz. Currently 128.267MHz
497	*/	497	*/
498	ipu_clk = clk_get(mx3fb->dev, NULL);	498	ipu_clk = clk_get(mx3fb->dev, NULL);
499	if (!IS_ERR(ipu_clk)) {	499	if (!IS_ERR(ipu_clk)) {
500	div = clk_get_rate(ipu_clk) * 16 / pixel_clk;	500	div = clk_get_rate(ipu_clk) * 16 / pixel_clk;
501	clk_put(ipu_clk);	501	clk_put(ipu_clk);
502	} else {	502	} else {
503	div = 0;	503	div = 0;
504	}	504	}
505		505
506	if (div < 0x40) { /* Divider less than 4 */	506	if (div < 0x40) { /* Divider less than 4 */
507	dev_dbg(mx3fb->dev,	507	dev_dbg(mx3fb->dev,
508	"InitPanel() - Pixel clock divider less than 4\n");	508	"InitPanel() - Pixel clock divider less than 4\n");
509	div = 0x40;	509	div = 0x40;
510	}	510	}
511		511
512	dev_dbg(mx3fb->dev, "pixel clk = %u, divider %u.%u\n",	512	dev_dbg(mx3fb->dev, "pixel clk = %u, divider %u.%u\n",
513	pixel_clk, div >> 4, (div & 7) * 125);	513	pixel_clk, div >> 4, (div & 7) * 125);
514		514
515	spin_lock_irqsave(&mx3fb->lock, lock_flags);	515	spin_lock_irqsave(&mx3fb->lock, lock_flags);
516		516
517	/*	517	/*
518	* DISP3_IF_CLK_DOWN_WR is half the divider value and 2 fraction bits	518	* DISP3_IF_CLK_DOWN_WR is half the divider value and 2 fraction bits
519	* fewer. Subtract 1 extra from DISP3_IF_CLK_DOWN_WR based on timing	519	* fewer. Subtract 1 extra from DISP3_IF_CLK_DOWN_WR based on timing
520	* debug. DISP3_IF_CLK_UP_WR is 0	520	* debug. DISP3_IF_CLK_UP_WR is 0
521	*/	521	*/
522	mx3fb_write_reg(mx3fb, (((div / 8) - 1) << 22) \| div, DI_DISP3_TIME_CONF);	522	mx3fb_write_reg(mx3fb, (((div / 8) - 1) << 22) \| div, DI_DISP3_TIME_CONF);
523		523
524	/* DI settings */	524	/* DI settings */
525	old_conf = mx3fb_read_reg(mx3fb, DI_DISP_IF_CONF) & 0x78FFFFFF;	525	old_conf = mx3fb_read_reg(mx3fb, DI_DISP_IF_CONF) & 0x78FFFFFF;
526	old_conf \|= sig.datamask_en << DI_D3_DATAMSK_SHIFT \|	526	old_conf \|= sig.datamask_en << DI_D3_DATAMSK_SHIFT \|
527	sig.clksel_en << DI_D3_CLK_SEL_SHIFT \|	527	sig.clksel_en << DI_D3_CLK_SEL_SHIFT \|
528	sig.clkidle_en << DI_D3_CLK_IDLE_SHIFT;	528	sig.clkidle_en << DI_D3_CLK_IDLE_SHIFT;
529	mx3fb_write_reg(mx3fb, old_conf, DI_DISP_IF_CONF);	529	mx3fb_write_reg(mx3fb, old_conf, DI_DISP_IF_CONF);
530		530
531	old_conf = mx3fb_read_reg(mx3fb, DI_DISP_SIG_POL) & 0xE0FFFFFF;	531	old_conf = mx3fb_read_reg(mx3fb, DI_DISP_SIG_POL) & 0xE0FFFFFF;
532	old_conf \|= sig.data_pol << DI_D3_DATA_POL_SHIFT \|	532	old_conf \|= sig.data_pol << DI_D3_DATA_POL_SHIFT \|
533	sig.clk_pol << DI_D3_CLK_POL_SHIFT \|	533	sig.clk_pol << DI_D3_CLK_POL_SHIFT \|
534	sig.enable_pol << DI_D3_DRDY_SHARP_POL_SHIFT \|	534	sig.enable_pol << DI_D3_DRDY_SHARP_POL_SHIFT \|
535	sig.Hsync_pol << DI_D3_HSYNC_POL_SHIFT \|	535	sig.Hsync_pol << DI_D3_HSYNC_POL_SHIFT \|
536	sig.Vsync_pol << DI_D3_VSYNC_POL_SHIFT;	536	sig.Vsync_pol << DI_D3_VSYNC_POL_SHIFT;
537	mx3fb_write_reg(mx3fb, old_conf, DI_DISP_SIG_POL);	537	mx3fb_write_reg(mx3fb, old_conf, DI_DISP_SIG_POL);
538		538
539	switch (pixel_fmt) {	539	switch (pixel_fmt) {
540	case IPU_PIX_FMT_RGB24:	540	case IPU_PIX_FMT_RGB24:
541	mx3fb_write_reg(mx3fb, di_mappings[0], DI_DISP3_B0_MAP);	541	mx3fb_write_reg(mx3fb, di_mappings[0], DI_DISP3_B0_MAP);
542	mx3fb_write_reg(mx3fb, di_mappings[1], DI_DISP3_B1_MAP);	542	mx3fb_write_reg(mx3fb, di_mappings[1], DI_DISP3_B1_MAP);
543	mx3fb_write_reg(mx3fb, di_mappings[2], DI_DISP3_B2_MAP);	543	mx3fb_write_reg(mx3fb, di_mappings[2], DI_DISP3_B2_MAP);
544	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|	544	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|
545	((di_mappings[3] - 1) << 12), DI_DISP_ACC_CC);	545	((di_mappings[3] - 1) << 12), DI_DISP_ACC_CC);
546	break;	546	break;
547	case IPU_PIX_FMT_RGB666:	547	case IPU_PIX_FMT_RGB666:
548	mx3fb_write_reg(mx3fb, di_mappings[4], DI_DISP3_B0_MAP);	548	mx3fb_write_reg(mx3fb, di_mappings[4], DI_DISP3_B0_MAP);
549	mx3fb_write_reg(mx3fb, di_mappings[5], DI_DISP3_B1_MAP);	549	mx3fb_write_reg(mx3fb, di_mappings[5], DI_DISP3_B1_MAP);
550	mx3fb_write_reg(mx3fb, di_mappings[6], DI_DISP3_B2_MAP);	550	mx3fb_write_reg(mx3fb, di_mappings[6], DI_DISP3_B2_MAP);
551	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|	551	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|
552	((di_mappings[7] - 1) << 12), DI_DISP_ACC_CC);	552	((di_mappings[7] - 1) << 12), DI_DISP_ACC_CC);
553	break;	553	break;
554	case IPU_PIX_FMT_BGR666:	554	case IPU_PIX_FMT_BGR666:
555	mx3fb_write_reg(mx3fb, di_mappings[8], DI_DISP3_B0_MAP);	555	mx3fb_write_reg(mx3fb, di_mappings[8], DI_DISP3_B0_MAP);
556	mx3fb_write_reg(mx3fb, di_mappings[9], DI_DISP3_B1_MAP);	556	mx3fb_write_reg(mx3fb, di_mappings[9], DI_DISP3_B1_MAP);
557	mx3fb_write_reg(mx3fb, di_mappings[10], DI_DISP3_B2_MAP);	557	mx3fb_write_reg(mx3fb, di_mappings[10], DI_DISP3_B2_MAP);
558	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|	558	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|
559	((di_mappings[11] - 1) << 12), DI_DISP_ACC_CC);	559	((di_mappings[11] - 1) << 12), DI_DISP_ACC_CC);
560	break;	560	break;
561	default:	561	default:
562	mx3fb_write_reg(mx3fb, di_mappings[12], DI_DISP3_B0_MAP);	562	mx3fb_write_reg(mx3fb, di_mappings[12], DI_DISP3_B0_MAP);
563	mx3fb_write_reg(mx3fb, di_mappings[13], DI_DISP3_B1_MAP);	563	mx3fb_write_reg(mx3fb, di_mappings[13], DI_DISP3_B1_MAP);
564	mx3fb_write_reg(mx3fb, di_mappings[14], DI_DISP3_B2_MAP);	564	mx3fb_write_reg(mx3fb, di_mappings[14], DI_DISP3_B2_MAP);
565	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|	565	mx3fb_write_reg(mx3fb, mx3fb_read_reg(mx3fb, DI_DISP_ACC_CC) \|
566	((di_mappings[15] - 1) << 12), DI_DISP_ACC_CC);	566	((di_mappings[15] - 1) << 12), DI_DISP_ACC_CC);
567	break;	567	break;
568	}	568	}
569		569
570	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);	570	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);
571		571
572	dev_dbg(mx3fb->dev, "DI_DISP_IF_CONF = 0x%08X\n",	572	dev_dbg(mx3fb->dev, "DI_DISP_IF_CONF = 0x%08X\n",
573	mx3fb_read_reg(mx3fb, DI_DISP_IF_CONF));	573	mx3fb_read_reg(mx3fb, DI_DISP_IF_CONF));
574	dev_dbg(mx3fb->dev, "DI_DISP_SIG_POL = 0x%08X\n",	574	dev_dbg(mx3fb->dev, "DI_DISP_SIG_POL = 0x%08X\n",
575	mx3fb_read_reg(mx3fb, DI_DISP_SIG_POL));	575	mx3fb_read_reg(mx3fb, DI_DISP_SIG_POL));
576	dev_dbg(mx3fb->dev, "DI_DISP3_TIME_CONF = 0x%08X\n",	576	dev_dbg(mx3fb->dev, "DI_DISP3_TIME_CONF = 0x%08X\n",
577	mx3fb_read_reg(mx3fb, DI_DISP3_TIME_CONF));	577	mx3fb_read_reg(mx3fb, DI_DISP3_TIME_CONF));
578		578
579	return 0;	579	return 0;
580	}	580	}
581		581
582	/**	582	/**
583	* sdc_set_color_key() - set the transparent color key for SDC graphic plane.	583	* sdc_set_color_key() - set the transparent color key for SDC graphic plane.
584	* @mx3fb: mx3fb context.	584	* @mx3fb: mx3fb context.
585	* @channel: IPU DMAC channel ID.	585	* @channel: IPU DMAC channel ID.
586	* @enable: boolean to enable or disable color keyl.	586	* @enable: boolean to enable or disable color keyl.
587	* @color_key: 24-bit RGB color to use as transparent color key.	587	* @color_key: 24-bit RGB color to use as transparent color key.
588	* @return: 0 on success or negative error code on failure.	588	* @return: 0 on success or negative error code on failure.
589	*/	589	*/
590	static int sdc_set_color_key(struct mx3fb_data *mx3fb, enum ipu_channel channel,	590	static int sdc_set_color_key(struct mx3fb_data *mx3fb, enum ipu_channel channel,
591	bool enable, uint32_t color_key)	591	bool enable, uint32_t color_key)
592	{	592	{
593	uint32_t reg, sdc_conf;	593	uint32_t reg, sdc_conf;
594	unsigned long lock_flags;	594	unsigned long lock_flags;
595		595
596	spin_lock_irqsave(&mx3fb->lock, lock_flags);	596	spin_lock_irqsave(&mx3fb->lock, lock_flags);
597		597
598	sdc_conf = mx3fb_read_reg(mx3fb, SDC_COM_CONF);	598	sdc_conf = mx3fb_read_reg(mx3fb, SDC_COM_CONF);
599	if (channel == IDMAC_SDC_0)	599	if (channel == IDMAC_SDC_0)
600	sdc_conf &= ~SDC_COM_GWSEL;	600	sdc_conf &= ~SDC_COM_GWSEL;
601	else	601	else
602	sdc_conf \|= SDC_COM_GWSEL;	602	sdc_conf \|= SDC_COM_GWSEL;
603		603
604	if (enable) {	604	if (enable) {
605	reg = mx3fb_read_reg(mx3fb, SDC_GW_CTRL) & 0xFF000000L;	605	reg = mx3fb_read_reg(mx3fb, SDC_GW_CTRL) & 0xFF000000L;
606	mx3fb_write_reg(mx3fb, reg \| (color_key & 0x00FFFFFFL),	606	mx3fb_write_reg(mx3fb, reg \| (color_key & 0x00FFFFFFL),
607	SDC_GW_CTRL);	607	SDC_GW_CTRL);
608		608
609	sdc_conf \|= SDC_COM_KEY_COLOR_G;	609	sdc_conf \|= SDC_COM_KEY_COLOR_G;
610	} else {	610	} else {
611	sdc_conf &= ~SDC_COM_KEY_COLOR_G;	611	sdc_conf &= ~SDC_COM_KEY_COLOR_G;
612	}	612	}
613	mx3fb_write_reg(mx3fb, sdc_conf, SDC_COM_CONF);	613	mx3fb_write_reg(mx3fb, sdc_conf, SDC_COM_CONF);
614		614
615	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);	615	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);
616		616
617	return 0;	617	return 0;
618	}	618	}
619		619
620	/**	620	/**
621	* sdc_set_global_alpha() - set global alpha blending modes.	621	* sdc_set_global_alpha() - set global alpha blending modes.
622	* @mx3fb: mx3fb context.	622	* @mx3fb: mx3fb context.
623	* @enable: boolean to enable or disable global alpha blending. If disabled,	623	* @enable: boolean to enable or disable global alpha blending. If disabled,
624	* per pixel blending is used.	624	* per pixel blending is used.
625	* @alpha: global alpha value.	625	* @alpha: global alpha value.
626	* @return: 0 on success or negative error code on failure.	626	* @return: 0 on success or negative error code on failure.
627	*/	627	*/
628	static int sdc_set_global_alpha(struct mx3fb_data *mx3fb, bool enable, uint8_t alpha)	628	static int sdc_set_global_alpha(struct mx3fb_data *mx3fb, bool enable, uint8_t alpha)
629	{	629	{
630	uint32_t reg;	630	uint32_t reg;
631	unsigned long lock_flags;	631	unsigned long lock_flags;
632		632
633	spin_lock_irqsave(&mx3fb->lock, lock_flags);	633	spin_lock_irqsave(&mx3fb->lock, lock_flags);
634		634
635	if (enable) {	635	if (enable) {
636	reg = mx3fb_read_reg(mx3fb, SDC_GW_CTRL) & 0x00FFFFFFL;	636	reg = mx3fb_read_reg(mx3fb, SDC_GW_CTRL) & 0x00FFFFFFL;
637	mx3fb_write_reg(mx3fb, reg \| ((uint32_t) alpha << 24), SDC_GW_CTRL);	637	mx3fb_write_reg(mx3fb, reg \| ((uint32_t) alpha << 24), SDC_GW_CTRL);
638		638
639	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);	639	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);
640	mx3fb_write_reg(mx3fb, reg \| SDC_COM_GLB_A, SDC_COM_CONF);	640	mx3fb_write_reg(mx3fb, reg \| SDC_COM_GLB_A, SDC_COM_CONF);
641	} else {	641	} else {
642	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);	642	reg = mx3fb_read_reg(mx3fb, SDC_COM_CONF);
643	mx3fb_write_reg(mx3fb, reg & ~SDC_COM_GLB_A, SDC_COM_CONF);	643	mx3fb_write_reg(mx3fb, reg & ~SDC_COM_GLB_A, SDC_COM_CONF);
644	}	644	}
645		645
646	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);	646	spin_unlock_irqrestore(&mx3fb->lock, lock_flags);
647		647
648	return 0;	648	return 0;
649	}	649	}
650		650
651	static void sdc_set_brightness(struct mx3fb_data *mx3fb, uint8_t value)	651	static void sdc_set_brightness(struct mx3fb_data *mx3fb, uint8_t value)
652	{	652	{
653	dev_dbg(mx3fb->dev, "%s: value = %d\n", __func__, value);	653	dev_dbg(mx3fb->dev, "%s: value = %d\n", __func__, value);
654	/* This might be board-specific */	654	/* This might be board-specific */
655	mx3fb_write_reg(mx3fb, 0x03000000UL \| value << 16, SDC_PWM_CTRL);	655	mx3fb_write_reg(mx3fb, 0x03000000UL \| value << 16, SDC_PWM_CTRL);
656	return;	656	return;
657	}	657	}
658		658
659	static uint32_t bpp_to_pixfmt(int bpp)	659	static uint32_t bpp_to_pixfmt(int bpp)
660	{	660	{
661	uint32_t pixfmt = 0;	661	uint32_t pixfmt = 0;
662	switch (bpp) {	662	switch (bpp) {
663	case 24:	663	case 24:
664	pixfmt = IPU_PIX_FMT_BGR24;	664	pixfmt = IPU_PIX_FMT_BGR24;
665	break;	665	break;
666	case 32:	666	case 32:
667	pixfmt = IPU_PIX_FMT_BGR32;	667	pixfmt = IPU_PIX_FMT_BGR32;
668	break;	668	break;
669	case 16:	669	case 16:
670	pixfmt = IPU_PIX_FMT_RGB565;	670	pixfmt = IPU_PIX_FMT_RGB565;
671	break;	671	break;
672	}	672	}
673	return pixfmt;	673	return pixfmt;
674	}	674	}
675		675
676	static int mx3fb_blank(int blank, struct fb_info *fbi);	676	static int mx3fb_blank(int blank, struct fb_info *fbi);
677	static int mx3fb_map_video_memory(struct fb_info *fbi, unsigned int mem_len,	677	static int mx3fb_map_video_memory(struct fb_info *fbi, unsigned int mem_len,
678	bool lock);	678	bool lock);
679	static int mx3fb_unmap_video_memory(struct fb_info *fbi);	679	static int mx3fb_unmap_video_memory(struct fb_info *fbi);
680		680
681	/**	681	/**
682	* mx3fb_set_fix() - set fixed framebuffer parameters from variable settings.	682	* mx3fb_set_fix() - set fixed framebuffer parameters from variable settings.
683	* @info: framebuffer information pointer	683	* @info: framebuffer information pointer
684	* @return: 0 on success or negative error code on failure.	684	* @return: 0 on success or negative error code on failure.
685	*/	685	*/
686	static int mx3fb_set_fix(struct fb_info *fbi)	686	static int mx3fb_set_fix(struct fb_info *fbi)
687	{	687	{
688	struct fb_fix_screeninfo *fix = &fbi->fix;	688	struct fb_fix_screeninfo *fix = &fbi->fix;
689	struct fb_var_screeninfo *var = &fbi->var;	689	struct fb_var_screeninfo *var = &fbi->var;
690		690
691	strncpy(fix->id, "DISP3 BG", 8);	691	strncpy(fix->id, "DISP3 BG", 8);
692		692
693	fix->line_length = var->xres_virtual * var->bits_per_pixel / 8;	693	fix->line_length = var->xres_virtual * var->bits_per_pixel / 8;
694		694
695	fix->type = FB_TYPE_PACKED_PIXELS;	695	fix->type = FB_TYPE_PACKED_PIXELS;
696	fix->accel = FB_ACCEL_NONE;	696	fix->accel = FB_ACCEL_NONE;
697	fix->visual = FB_VISUAL_TRUECOLOR;	697	fix->visual = FB_VISUAL_TRUECOLOR;
698	fix->xpanstep = 1;	698	fix->xpanstep = 1;
699	fix->ypanstep = 1;	699	fix->ypanstep = 1;
700		700
701	return 0;	701	return 0;
702	}	702	}
703		703
704	static void mx3fb_dma_done(void *arg)	704	static void mx3fb_dma_done(void *arg)
705	{	705	{
706	struct idmac_tx_desc *tx_desc = to_tx_desc(arg);	706	struct idmac_tx_desc *tx_desc = to_tx_desc(arg);
707	struct dma_chan *chan = tx_desc->txd.chan;	707	struct dma_chan *chan = tx_desc->txd.chan;
708	struct idmac_channel *ichannel = to_idmac_chan(chan);	708	struct idmac_channel *ichannel = to_idmac_chan(chan);
709	struct mx3fb_data *mx3fb = ichannel->client;	709	struct mx3fb_data *mx3fb = ichannel->client;
710	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;	710	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;
711		711
712	dev_dbg(mx3fb->dev, "irq %d callback\n", ichannel->eof_irq);	712	dev_dbg(mx3fb->dev, "irq %d callback\n", ichannel->eof_irq);
713		713
714	/* We only need one interrupt, it will be re-enabled as needed */	714	/* We only need one interrupt, it will be re-enabled as needed */
715	disable_irq_nosync(ichannel->eof_irq);	715	disable_irq_nosync(ichannel->eof_irq);
716		716
717	complete(&mx3_fbi->flip_cmpl);	717	complete(&mx3_fbi->flip_cmpl);
718	}	718	}
719		719
720	static int __set_par(struct fb_info *fbi, bool lock)	720	static int __set_par(struct fb_info *fbi, bool lock)
721	{	721	{
722	u32 mem_len;	722	u32 mem_len;
723	struct ipu_di_signal_cfg sig_cfg;	723	struct ipu_di_signal_cfg sig_cfg;
724	enum ipu_panel mode = IPU_PANEL_TFT;	724	enum ipu_panel mode = IPU_PANEL_TFT;
725	struct mx3fb_info *mx3_fbi = fbi->par;	725	struct mx3fb_info *mx3_fbi = fbi->par;
726	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;	726	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;
727	struct idmac_channel *ichan = mx3_fbi->idmac_channel;	727	struct idmac_channel *ichan = mx3_fbi->idmac_channel;
728	struct idmac_video_param *video = &ichan->params.video;	728	struct idmac_video_param *video = &ichan->params.video;
729	struct scatterlist *sg = mx3_fbi->sg;	729	struct scatterlist *sg = mx3_fbi->sg;
730		730
731	/* Total cleanup */	731	/* Total cleanup */
732	if (mx3_fbi->txd)	732	if (mx3_fbi->txd)
733	sdc_disable_channel(mx3_fbi);	733	sdc_disable_channel(mx3_fbi);
734		734
735	mx3fb_set_fix(fbi);	735	mx3fb_set_fix(fbi);
736		736
737	mem_len = fbi->var.yres_virtual * fbi->fix.line_length;	737	mem_len = fbi->var.yres_virtual * fbi->fix.line_length;
738	if (mem_len > fbi->fix.smem_len) {	738	if (mem_len > fbi->fix.smem_len) {
739	if (fbi->fix.smem_start)	739	if (fbi->fix.smem_start)
740	mx3fb_unmap_video_memory(fbi);	740	mx3fb_unmap_video_memory(fbi);
741		741
742	if (mx3fb_map_video_memory(fbi, mem_len, lock) < 0)	742	if (mx3fb_map_video_memory(fbi, mem_len, lock) < 0)
743	return -ENOMEM;	743	return -ENOMEM;
744	}	744	}
745		745
746	sg_init_table(&sg[0], 1);	746	sg_init_table(&sg[0], 1);
747	sg_init_table(&sg[1], 1);	747	sg_init_table(&sg[1], 1);
748		748
749	sg_dma_address(&sg[0]) = fbi->fix.smem_start;	749	sg_dma_address(&sg[0]) = fbi->fix.smem_start;
750	sg_set_page(&sg[0], virt_to_page(fbi->screen_base),	750	sg_set_page(&sg[0], virt_to_page(fbi->screen_base),
751	fbi->fix.smem_len,	751	fbi->fix.smem_len,
752	offset_in_page(fbi->screen_base));	752	offset_in_page(fbi->screen_base));
753		753
754	if (mx3_fbi->ipu_ch == IDMAC_SDC_0) {	754	if (mx3_fbi->ipu_ch == IDMAC_SDC_0) {
755	memset(&sig_cfg, 0, sizeof(sig_cfg));	755	memset(&sig_cfg, 0, sizeof(sig_cfg));
756	if (fbi->var.sync & FB_SYNC_HOR_HIGH_ACT)	756	if (fbi->var.sync & FB_SYNC_HOR_HIGH_ACT)
757	sig_cfg.Hsync_pol = true;	757	sig_cfg.Hsync_pol = true;
758	if (fbi->var.sync & FB_SYNC_VERT_HIGH_ACT)	758	if (fbi->var.sync & FB_SYNC_VERT_HIGH_ACT)
759	sig_cfg.Vsync_pol = true;	759	sig_cfg.Vsync_pol = true;
760	if (fbi->var.sync & FB_SYNC_CLK_INVERT)	760	if (fbi->var.sync & FB_SYNC_CLK_INVERT)
761	sig_cfg.clk_pol = true;	761	sig_cfg.clk_pol = true;
762	if (fbi->var.sync & FB_SYNC_DATA_INVERT)	762	if (fbi->var.sync & FB_SYNC_DATA_INVERT)
763	sig_cfg.data_pol = true;	763	sig_cfg.data_pol = true;
764	if (fbi->var.sync & FB_SYNC_OE_ACT_HIGH)	764	if (fbi->var.sync & FB_SYNC_OE_ACT_HIGH)
765	sig_cfg.enable_pol = true;	765	sig_cfg.enable_pol = true;
766	if (fbi->var.sync & FB_SYNC_CLK_IDLE_EN)	766	if (fbi->var.sync & FB_SYNC_CLK_IDLE_EN)
767	sig_cfg.clkidle_en = true;	767	sig_cfg.clkidle_en = true;
768	if (fbi->var.sync & FB_SYNC_CLK_SEL_EN)	768	if (fbi->var.sync & FB_SYNC_CLK_SEL_EN)
769	sig_cfg.clksel_en = true;	769	sig_cfg.clksel_en = true;
770	if (fbi->var.sync & FB_SYNC_SHARP_MODE)	770	if (fbi->var.sync & FB_SYNC_SHARP_MODE)
771	mode = IPU_PANEL_SHARP_TFT;	771	mode = IPU_PANEL_SHARP_TFT;
772		772
773	dev_dbg(fbi->device, "pixclock = %ul Hz\n",	773	dev_dbg(fbi->device, "pixclock = %ul Hz\n",
774	(u32) (PICOS2KHZ(fbi->var.pixclock) * 1000UL));	774	(u32) (PICOS2KHZ(fbi->var.pixclock) * 1000UL));
775		775
776	if (sdc_init_panel(mx3fb, mode,	776	if (sdc_init_panel(mx3fb, mode,
777	(PICOS2KHZ(fbi->var.pixclock)) * 1000UL,	777	(PICOS2KHZ(fbi->var.pixclock)) * 1000UL,
778	fbi->var.xres, fbi->var.yres,	778	fbi->var.xres, fbi->var.yres,
779	(fbi->var.sync & FB_SYNC_SWAP_RGB) ?	779	(fbi->var.sync & FB_SYNC_SWAP_RGB) ?
780	IPU_PIX_FMT_BGR666 : IPU_PIX_FMT_RGB666,	780	IPU_PIX_FMT_BGR666 : IPU_PIX_FMT_RGB666,
781	fbi->var.left_margin,	781	fbi->var.left_margin,
782	fbi->var.hsync_len,	782	fbi->var.hsync_len,
783	fbi->var.right_margin +	783	fbi->var.right_margin +
784	fbi->var.hsync_len,	784	fbi->var.hsync_len,
785	fbi->var.upper_margin,	785	fbi->var.upper_margin,
786	fbi->var.vsync_len,	786	fbi->var.vsync_len,
787	fbi->var.lower_margin +	787	fbi->var.lower_margin +
788	fbi->var.vsync_len, sig_cfg) != 0) {	788	fbi->var.vsync_len, sig_cfg) != 0) {
789	dev_err(fbi->device,	789	dev_err(fbi->device,
790	"mx3fb: Error initializing panel.\n");	790	"mx3fb: Error initializing panel.\n");
791	return -EINVAL;	791	return -EINVAL;
792	}	792	}
793	}	793	}
794		794
795	sdc_set_window_pos(mx3fb, mx3_fbi->ipu_ch, 0, 0);	795	sdc_set_window_pos(mx3fb, mx3_fbi->ipu_ch, 0, 0);
796		796
797	mx3_fbi->cur_ipu_buf = 0;	797	mx3_fbi->cur_ipu_buf = 0;
798		798
799	video->out_pixel_fmt = bpp_to_pixfmt(fbi->var.bits_per_pixel);	799	video->out_pixel_fmt = bpp_to_pixfmt(fbi->var.bits_per_pixel);
800	video->out_width = fbi->var.xres;	800	video->out_width = fbi->var.xres;
801	video->out_height = fbi->var.yres;	801	video->out_height = fbi->var.yres;
802	video->out_stride = fbi->var.xres_virtual;	802	video->out_stride = fbi->var.xres_virtual;
803		803
804	if (mx3_fbi->blank == FB_BLANK_UNBLANK)	804	if (mx3_fbi->blank == FB_BLANK_UNBLANK)
805	sdc_enable_channel(mx3_fbi);	805	sdc_enable_channel(mx3_fbi);
806		806
807	return 0;	807	return 0;
808	}	808	}
809		809
810	/**	810	/**
811	* mx3fb_set_par() - set framebuffer parameters and change the operating mode.	811	* mx3fb_set_par() - set framebuffer parameters and change the operating mode.
812	* @fbi: framebuffer information pointer.	812	* @fbi: framebuffer information pointer.
813	* @return: 0 on success or negative error code on failure.	813	* @return: 0 on success or negative error code on failure.
814	*/	814	*/
815	static int mx3fb_set_par(struct fb_info *fbi)	815	static int mx3fb_set_par(struct fb_info *fbi)
816	{	816	{
817	struct mx3fb_info *mx3_fbi = fbi->par;	817	struct mx3fb_info *mx3_fbi = fbi->par;
818	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;	818	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;
819	struct idmac_channel *ichan = mx3_fbi->idmac_channel;	819	struct idmac_channel *ichan = mx3_fbi->idmac_channel;
820	int ret;	820	int ret;
821		821
822	dev_dbg(mx3fb->dev, "%s [%c]\n", __func__, list_empty(&ichan->queue) ? '-' : '+');	822	dev_dbg(mx3fb->dev, "%s [%c]\n", __func__, list_empty(&ichan->queue) ? '-' : '+');
823		823
824	mutex_lock(&mx3_fbi->mutex);	824	mutex_lock(&mx3_fbi->mutex);
825		825
826	ret = __set_par(fbi, true);	826	ret = __set_par(fbi, true);
827		827
828	mutex_unlock(&mx3_fbi->mutex);	828	mutex_unlock(&mx3_fbi->mutex);
829		829
830	return ret;	830	return ret;
831	}	831	}
832		832
833	/**	833	/**
834	* mx3fb_check_var() - check and adjust framebuffer variable parameters.	834	* mx3fb_check_var() - check and adjust framebuffer variable parameters.
835	* @var: framebuffer variable parameters	835	* @var: framebuffer variable parameters
836	* @fbi: framebuffer information pointer	836	* @fbi: framebuffer information pointer
837	*/	837	*/
838	static int mx3fb_check_var(struct fb_var_screeninfo var, struct fb_info fbi)	838	static int mx3fb_check_var(struct fb_var_screeninfo var, struct fb_info fbi)
839	{	839	{
840	struct mx3fb_info *mx3_fbi = fbi->par;	840	struct mx3fb_info *mx3_fbi = fbi->par;
841	u32 vtotal;	841	u32 vtotal;
842	u32 htotal;	842	u32 htotal;
843		843
844	dev_dbg(fbi->device, "%s\n", __func__);	844	dev_dbg(fbi->device, "%s\n", __func__);
845		845
846	if (var->xres_virtual < var->xres)	846	if (var->xres_virtual < var->xres)
847	var->xres_virtual = var->xres;	847	var->xres_virtual = var->xres;
848	if (var->yres_virtual < var->yres)	848	if (var->yres_virtual < var->yres)
849	var->yres_virtual = var->yres;	849	var->yres_virtual = var->yres;
850		850
851	if ((var->bits_per_pixel != 32) && (var->bits_per_pixel != 24) &&	851	if ((var->bits_per_pixel != 32) && (var->bits_per_pixel != 24) &&
852	(var->bits_per_pixel != 16))	852	(var->bits_per_pixel != 16))
853	var->bits_per_pixel = default_bpp;	853	var->bits_per_pixel = default_bpp;
854		854
855	switch (var->bits_per_pixel) {	855	switch (var->bits_per_pixel) {
856	case 16:	856	case 16:
857	var->red.length = 5;	857	var->red.length = 5;
858	var->red.offset = 11;	858	var->red.offset = 11;
859	var->red.msb_right = 0;	859	var->red.msb_right = 0;
860		860
861	var->green.length = 6;	861	var->green.length = 6;
862	var->green.offset = 5;	862	var->green.offset = 5;
863	var->green.msb_right = 0;	863	var->green.msb_right = 0;
864		864
865	var->blue.length = 5;	865	var->blue.length = 5;
866	var->blue.offset = 0;	866	var->blue.offset = 0;
867	var->blue.msb_right = 0;	867	var->blue.msb_right = 0;
868		868
869	var->transp.length = 0;	869	var->transp.length = 0;
870	var->transp.offset = 0;	870	var->transp.offset = 0;
871	var->transp.msb_right = 0;	871	var->transp.msb_right = 0;
872	break;	872	break;
873	case 24:	873	case 24:
874	var->red.length = 8;	874	var->red.length = 8;
875	var->red.offset = 16;	875	var->red.offset = 16;
876	var->red.msb_right = 0;	876	var->red.msb_right = 0;
877		877
878	var->green.length = 8;	878	var->green.length = 8;
879	var->green.offset = 8;	879	var->green.offset = 8;
880	var->green.msb_right = 0;	880	var->green.msb_right = 0;
881		881
882	var->blue.length = 8;	882	var->blue.length = 8;
883	var->blue.offset = 0;	883	var->blue.offset = 0;
884	var->blue.msb_right = 0;	884	var->blue.msb_right = 0;
885		885
886	var->transp.length = 0;	886	var->transp.length = 0;
887	var->transp.offset = 0;	887	var->transp.offset = 0;
888	var->transp.msb_right = 0;	888	var->transp.msb_right = 0;
889	break;	889	break;
890	case 32:	890	case 32:
891	var->red.length = 8;	891	var->red.length = 8;
892	var->red.offset = 16;	892	var->red.offset = 16;
893	var->red.msb_right = 0;	893	var->red.msb_right = 0;
894		894
895	var->green.length = 8;	895	var->green.length = 8;
896	var->green.offset = 8;	896	var->green.offset = 8;
897	var->green.msb_right = 0;	897	var->green.msb_right = 0;
898		898
899	var->blue.length = 8;	899	var->blue.length = 8;
900	var->blue.offset = 0;	900	var->blue.offset = 0;
901	var->blue.msb_right = 0;	901	var->blue.msb_right = 0;
902		902
903	var->transp.length = 8;	903	var->transp.length = 8;
904	var->transp.offset = 24;	904	var->transp.offset = 24;
905	var->transp.msb_right = 0;	905	var->transp.msb_right = 0;
906	break;	906	break;
907	}	907	}
908		908
909	if (var->pixclock < 1000) {	909	if (var->pixclock < 1000) {
910	htotal = var->xres + var->right_margin + var->hsync_len +	910	htotal = var->xres + var->right_margin + var->hsync_len +
911	var->left_margin;	911	var->left_margin;
912	vtotal = var->yres + var->lower_margin + var->vsync_len +	912	vtotal = var->yres + var->lower_margin + var->vsync_len +
913	var->upper_margin;	913	var->upper_margin;
914	var->pixclock = (vtotal * htotal * 6UL) / 100UL;	914	var->pixclock = (vtotal * htotal * 6UL) / 100UL;
915	var->pixclock = KHZ2PICOS(var->pixclock);	915	var->pixclock = KHZ2PICOS(var->pixclock);
916	dev_dbg(fbi->device, "pixclock set for 60Hz refresh = %u ps\n",	916	dev_dbg(fbi->device, "pixclock set for 60Hz refresh = %u ps\n",
917	var->pixclock);	917	var->pixclock);
918	}	918	}
919		919
920	var->height = -1;	920	var->height = -1;
921	var->width = -1;	921	var->width = -1;
922	var->grayscale = 0;	922	var->grayscale = 0;
923		923
924	/* Preserve sync flags */	924	/* Preserve sync flags */
925	var->sync \|= mx3_fbi->sync;	925	var->sync \|= mx3_fbi->sync;
926	mx3_fbi->sync \|= var->sync;	926	mx3_fbi->sync \|= var->sync;
927		927
928	return 0;	928	return 0;
929	}	929	}
930		930
931	static u32 chan_to_field(unsigned int chan, struct fb_bitfield *bf)	931	static u32 chan_to_field(unsigned int chan, struct fb_bitfield *bf)
932	{	932	{
933	chan &= 0xffff;	933	chan &= 0xffff;
934	chan >>= 16 - bf->length;	934	chan >>= 16 - bf->length;
935	return chan << bf->offset;	935	return chan << bf->offset;
936	}	936	}
937		937
938	static int mx3fb_setcolreg(unsigned int regno, unsigned int red,	938	static int mx3fb_setcolreg(unsigned int regno, unsigned int red,
939	unsigned int green, unsigned int blue,	939	unsigned int green, unsigned int blue,
940	unsigned int trans, struct fb_info *fbi)	940	unsigned int trans, struct fb_info *fbi)
941	{	941	{
942	struct mx3fb_info *mx3_fbi = fbi->par;	942	struct mx3fb_info *mx3_fbi = fbi->par;
943	u32 val;	943	u32 val;
944	int ret = 1;	944	int ret = 1;
945		945
946	dev_dbg(fbi->device, "%s, regno = %u\n", __func__, regno);	946	dev_dbg(fbi->device, "%s, regno = %u\n", __func__, regno);
947		947
948	mutex_lock(&mx3_fbi->mutex);	948	mutex_lock(&mx3_fbi->mutex);
949	/*	949	/*
950	* If greyscale is true, then we convert the RGB value	950	* If greyscale is true, then we convert the RGB value
951	* to greyscale no matter what visual we are using.	951	* to greyscale no matter what visual we are using.
952	*/	952	*/
953	if (fbi->var.grayscale)	953	if (fbi->var.grayscale)
954	red = green = blue = (19595 * red + 38470 * green +	954	red = green = blue = (19595 * red + 38470 * green +
955	7471 * blue) >> 16;	955	7471 * blue) >> 16;
956	switch (fbi->fix.visual) {	956	switch (fbi->fix.visual) {
957	case FB_VISUAL_TRUECOLOR:	957	case FB_VISUAL_TRUECOLOR:
958	/*	958	/*
959	* 16-bit True Colour. We encode the RGB value	959	* 16-bit True Colour. We encode the RGB value
960	* according to the RGB bitfield information.	960	* according to the RGB bitfield information.
961	*/	961	*/
962	if (regno < 16) {	962	if (regno < 16) {
963	u32 *pal = fbi->pseudo_palette;	963	u32 *pal = fbi->pseudo_palette;
964		964
965	val = chan_to_field(red, &fbi->var.red);	965	val = chan_to_field(red, &fbi->var.red);
966	val \|= chan_to_field(green, &fbi->var.green);	966	val \|= chan_to_field(green, &fbi->var.green);
967	val \|= chan_to_field(blue, &fbi->var.blue);	967	val \|= chan_to_field(blue, &fbi->var.blue);
968		968
969	pal[regno] = val;	969	pal[regno] = val;
970		970
971	ret = 0;	971	ret = 0;
972	}	972	}
973	break;	973	break;
974		974
975	case FB_VISUAL_STATIC_PSEUDOCOLOR:	975	case FB_VISUAL_STATIC_PSEUDOCOLOR:
976	case FB_VISUAL_PSEUDOCOLOR:	976	case FB_VISUAL_PSEUDOCOLOR:
977	break;	977	break;
978	}	978	}
979	mutex_unlock(&mx3_fbi->mutex);	979	mutex_unlock(&mx3_fbi->mutex);
980		980
981	return ret;	981	return ret;
982	}	982	}
983		983
984	static void __blank(int blank, struct fb_info *fbi)	984	static void __blank(int blank, struct fb_info *fbi)
985	{	985	{
986	struct mx3fb_info *mx3_fbi = fbi->par;	986	struct mx3fb_info *mx3_fbi = fbi->par;
987	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;	987	struct mx3fb_data *mx3fb = mx3_fbi->mx3fb;
988		988
989	mx3_fbi->blank = blank;	989	mx3_fbi->blank = blank;
990		990
991	switch (blank) {	991	switch (blank) {
992	case FB_BLANK_POWERDOWN:	992	case FB_BLANK_POWERDOWN:
993	case FB_BLANK_VSYNC_SUSPEND:	993	case FB_BLANK_VSYNC_SUSPEND:
994	case FB_BLANK_HSYNC_SUSPEND:	994	case FB_BLANK_HSYNC_SUSPEND:
995	case FB_BLANK_NORMAL:	995	case FB_BLANK_NORMAL:
996	sdc_set_brightness(mx3fb, 0);	996	sdc_set_brightness(mx3fb, 0);
997	memset((char *)fbi->screen_base, 0, fbi->fix.smem_len);	997	memset((char *)fbi->screen_base, 0, fbi->fix.smem_len);
998	/* Give LCD time to update - enough for 50 and 60 Hz */	998	/* Give LCD time to update - enough for 50 and 60 Hz */
999	msleep(25);	999	msleep(25);
1000	sdc_disable_channel(mx3_fbi);	1000	sdc_disable_channel(mx3_fbi);
1001	break;	1001	break;
1002	case FB_BLANK_UNBLANK:	1002	case FB_BLANK_UNBLANK:
1003	sdc_enable_channel(mx3_fbi);	1003	sdc_enable_channel(mx3_fbi);
1004	sdc_set_brightness(mx3fb, mx3fb->backlight_level);	1004	sdc_set_brightness(mx3fb, mx3fb->backlight_level);
1005	break;	1005	break;
1006	}	1006	}
1007	}	1007	}
1008		1008
1009	/**	1009	/**
1010	* mx3fb_blank() - blank the display.	1010	* mx3fb_blank() - blank the display.
1011	*/	1011	*/
1012	static int mx3fb_blank(int blank, struct fb_info *fbi)	1012	static int mx3fb_blank(int blank, struct fb_info *fbi)
1013	{	1013	{
1014	struct mx3fb_info *mx3_fbi = fbi->par;	1014	struct mx3fb_info *mx3_fbi = fbi->par;
1015		1015
1016	dev_dbg(fbi->device, "%s, blank = %d, base %p, len %u\n", __func__,	1016	dev_dbg(fbi->device, "%s, blank = %d, base %p, len %u\n", __func__,
1017	blank, fbi->screen_base, fbi->fix.smem_len);	1017	blank, fbi->screen_base, fbi->fix.smem_len);
1018		1018
1019	if (mx3_fbi->blank == blank)	1019	if (mx3_fbi->blank == blank)
1020	return 0;	1020	return 0;
1021		1021
1022	mutex_lock(&mx3_fbi->mutex);	1022	mutex_lock(&mx3_fbi->mutex);
1023	__blank(blank, fbi);	1023	__blank(blank, fbi);
1024	mutex_unlock(&mx3_fbi->mutex);	1024	mutex_unlock(&mx3_fbi->mutex);
1025		1025
1026	return 0;	1026	return 0;
1027	}	1027	}
1028		1028
1029	/**	1029	/**
1030	* mx3fb_pan_display() - pan or wrap the display	1030	* mx3fb_pan_display() - pan or wrap the display
1031	* @var: variable screen buffer information.	1031	* @var: variable screen buffer information.
1032	* @info: framebuffer information pointer.	1032	* @info: framebuffer information pointer.
1033	*	1033	*
1034	* We look only at xoffset, yoffset and the FB_VMODE_YWRAP flag	1034	* We look only at xoffset, yoffset and the FB_VMODE_YWRAP flag
1035	*/	1035	*/
1036	static int mx3fb_pan_display(struct fb_var_screeninfo *var,	1036	static int mx3fb_pan_display(struct fb_var_screeninfo *var,
1037	struct fb_info *fbi)	1037	struct fb_info *fbi)
1038	{	1038	{
1039	struct mx3fb_info *mx3_fbi = fbi->par;	1039	struct mx3fb_info *mx3_fbi = fbi->par;
1040	u32 y_bottom;	1040	u32 y_bottom;
1041	unsigned long base;	1041	unsigned long base;
1042	off_t offset;	1042	off_t offset;
1043	dma_cookie_t cookie;	1043	dma_cookie_t cookie;
1044	struct scatterlist *sg = mx3_fbi->sg;	1044	struct scatterlist *sg = mx3_fbi->sg;
1045	struct dma_chan *dma_chan = &mx3_fbi->idmac_channel->dma_chan;	1045	struct dma_chan *dma_chan = &mx3_fbi->idmac_channel->dma_chan;
1046	struct dma_async_tx_descriptor *txd;	1046	struct dma_async_tx_descriptor *txd;
1047	int ret;	1047	int ret;
1048		1048
1049	dev_dbg(fbi->device, "%s [%c]\n", __func__,	1049	dev_dbg(fbi->device, "%s [%c]\n", __func__,
1050	list_empty(&mx3_fbi->idmac_channel->queue) ? '-' : '+');	1050	list_empty(&mx3_fbi->idmac_channel->queue) ? '-' : '+');
1051		1051
1052	if (var->xoffset > 0) {	1052	if (var->xoffset > 0) {
1053	dev_dbg(fbi->device, "x panning not supported\n");	1053	dev_dbg(fbi->device, "x panning not supported\n");
1054	return -EINVAL;	1054	return -EINVAL;
1055	}	1055	}
1056		1056
1057	if (fbi->var.xoffset == var->xoffset &&	1057	if (fbi->var.xoffset == var->xoffset &&
1058	fbi->var.yoffset == var->yoffset)	1058	fbi->var.yoffset == var->yoffset)
1059	return 0; /* No change, do nothing */	1059	return 0; /* No change, do nothing */
1060		1060
1061	y_bottom = var->yoffset;	1061	y_bottom = var->yoffset;
1062		1062
1063	if (!(var->vmode & FB_VMODE_YWRAP))	1063	if (!(var->vmode & FB_VMODE_YWRAP))
1064	y_bottom += var->yres;	1064	y_bottom += var->yres;
1065		1065
1066	if (y_bottom > fbi->var.yres_virtual)	1066	if (y_bottom > fbi->var.yres_virtual)
1067	return -EINVAL;	1067	return -EINVAL;
1068		1068
1069	mutex_lock(&mx3_fbi->mutex);	1069	mutex_lock(&mx3_fbi->mutex);
1070		1070
1071	offset = (var->yoffset * var->xres_virtual + var->xoffset) *	1071	offset = (var->yoffset * var->xres_virtual + var->xoffset) *
1072	(var->bits_per_pixel / 8);	1072	(var->bits_per_pixel / 8);
1073	base = fbi->fix.smem_start + offset;	1073	base = fbi->fix.smem_start + offset;
1074		1074
1075	dev_dbg(fbi->device, "Updating SDC BG buf %d address=0x%08lX\n",	1075	dev_dbg(fbi->device, "Updating SDC BG buf %d address=0x%08lX\n",
1076	mx3_fbi->cur_ipu_buf, base);	1076	mx3_fbi->cur_ipu_buf, base);
1077		1077
1078	/*	1078	/*
1079	* We enable the End of Frame interrupt, which will free a tx-descriptor,	1079	* We enable the End of Frame interrupt, which will free a tx-descriptor,
1080	* which we will need for the next device_prep_slave_sg(). The	1080	* which we will need for the next device_prep_slave_sg(). The
1081	* IRQ-handler will disable the IRQ again.	1081	* IRQ-handler will disable the IRQ again.
1082	*/	1082	*/
1083	init_completion(&mx3_fbi->flip_cmpl);	1083	init_completion(&mx3_fbi->flip_cmpl);
1084	enable_irq(mx3_fbi->idmac_channel->eof_irq);	1084	enable_irq(mx3_fbi->idmac_channel->eof_irq);
1085		1085
1086	ret = wait_for_completion_timeout(&mx3_fbi->flip_cmpl, HZ / 10);	1086	ret = wait_for_completion_timeout(&mx3_fbi->flip_cmpl, HZ / 10);
1087	if (ret <= 0) {	1087	if (ret <= 0) {
1088	mutex_unlock(&mx3_fbi->mutex);	1088	mutex_unlock(&mx3_fbi->mutex);
1089	dev_info(fbi->device, "Panning failed due to %s\n", ret < 0 ?	1089	dev_info(fbi->device, "Panning failed due to %s\n", ret < 0 ?
1090	"user interrupt" : "timeout");	1090	"user interrupt" : "timeout");
1091	disable_irq(mx3_fbi->idmac_channel->eof_irq);	1091	disable_irq(mx3_fbi->idmac_channel->eof_irq);
1092	return ret ? : -ETIMEDOUT;	1092	return ret ? : -ETIMEDOUT;
1093	}	1093	}
1094		1094
1095	mx3_fbi->cur_ipu_buf = !mx3_fbi->cur_ipu_buf;	1095	mx3_fbi->cur_ipu_buf = !mx3_fbi->cur_ipu_buf;
1096		1096
1097	sg_dma_address(&sg[mx3_fbi->cur_ipu_buf]) = base;	1097	sg_dma_address(&sg[mx3_fbi->cur_ipu_buf]) = base;
1098	sg_set_page(&sg[mx3_fbi->cur_ipu_buf],	1098	sg_set_page(&sg[mx3_fbi->cur_ipu_buf],
1099	virt_to_page(fbi->screen_base + offset), fbi->fix.smem_len,	1099	virt_to_page(fbi->screen_base + offset), fbi->fix.smem_len,
1100	offset_in_page(fbi->screen_base + offset));	1100	offset_in_page(fbi->screen_base + offset));
1101		1101
1102	if (mx3_fbi->txd)	1102	if (mx3_fbi->txd)
1103	async_tx_ack(mx3_fbi->txd);	1103	async_tx_ack(mx3_fbi->txd);
1104		1104
1105	txd = dma_chan->device->device_prep_slave_sg(dma_chan, sg +	1105	txd = dma_chan->device->device_prep_slave_sg(dma_chan, sg +
1106	mx3_fbi->cur_ipu_buf, 1, DMA_TO_DEVICE, DMA_PREP_INTERRUPT);	1106	mx3_fbi->cur_ipu_buf, 1, DMA_TO_DEVICE, DMA_PREP_INTERRUPT);
1107	if (!txd) {	1107	if (!txd) {
1108	dev_err(fbi->device,	1108	dev_err(fbi->device,
1109	"Error preparing a DMA transaction descriptor.\n");	1109	"Error preparing a DMA transaction descriptor.\n");
1110	mutex_unlock(&mx3_fbi->mutex);	1110	mutex_unlock(&mx3_fbi->mutex);
1111	return -EIO;	1111	return -EIO;
1112	}	1112	}
1113		1113
1114	txd->callback_param = txd;	1114	txd->callback_param = txd;
1115	txd->callback = mx3fb_dma_done;	1115	txd->callback = mx3fb_dma_done;
1116		1116
1117	/*	1117	/*
1118	* Emulate original mx3fb behaviour: each new call to idmac_tx_submit()	1118	* Emulate original mx3fb behaviour: each new call to idmac_tx_submit()
1119	* should switch to another buffer	1119	* should switch to another buffer
1120	*/	1120	*/
1121	cookie = txd->tx_submit(txd);	1121	cookie = txd->tx_submit(txd);
1122	dev_dbg(fbi->device, "%d: Submit %p #%d\n", __LINE__, txd, cookie);	1122	dev_dbg(fbi->device, "%d: Submit %p #%d\n", __LINE__, txd, cookie);
1123	if (cookie < 0) {	1123	if (cookie < 0) {
1124	dev_err(fbi->device,	1124	dev_err(fbi->device,
1125	"Error updating SDC buf %d to address=0x%08lX\n",	1125	"Error updating SDC buf %d to address=0x%08lX\n",
1126	mx3_fbi->cur_ipu_buf, base);	1126	mx3_fbi->cur_ipu_buf, base);
1127	mutex_unlock(&mx3_fbi->mutex);	1127	mutex_unlock(&mx3_fbi->mutex);
1128	return -EIO;	1128	return -EIO;
1129	}	1129	}
1130		1130
1131	mx3_fbi->txd = txd;	1131	mx3_fbi->txd = txd;
1132		1132
1133	fbi->var.xoffset = var->xoffset;	1133	fbi->var.xoffset = var->xoffset;
1134	fbi->var.yoffset = var->yoffset;	1134	fbi->var.yoffset = var->yoffset;
1135		1135
1136	if (var->vmode & FB_VMODE_YWRAP)	1136	if (var->vmode & FB_VMODE_YWRAP)
1137	fbi->var.vmode \|= FB_VMODE_YWRAP;	1137	fbi->var.vmode \|= FB_VMODE_YWRAP;
1138	else	1138	else
1139	fbi->var.vmode &= ~FB_VMODE_YWRAP;	1139	fbi->var.vmode &= ~FB_VMODE_YWRAP;
1140		1140
1141	mutex_unlock(&mx3_fbi->mutex);	1141	mutex_unlock(&mx3_fbi->mutex);
1142		1142
1143	dev_dbg(fbi->device, "Update complete\n");	1143	dev_dbg(fbi->device, "Update complete\n");
1144		1144
1145	return 0;	1145	return 0;
1146	}	1146	}
1147		1147
1148	/*	1148	/*
1149	* This structure contains the pointers to the control functions that are	1149	* This structure contains the pointers to the control functions that are
1150	* invoked by the core framebuffer driver to perform operations like	1150	* invoked by the core framebuffer driver to perform operations like
1151	* blitting, rectangle filling, copy regions and cursor definition.	1151	* blitting, rectangle filling, copy regions and cursor definition.
1152	*/	1152	*/
1153	static struct fb_ops mx3fb_ops = {	1153	static struct fb_ops mx3fb_ops = {
1154	.owner = THIS_MODULE,	1154	.owner = THIS_MODULE,
1155	.fb_set_par = mx3fb_set_par,	1155	.fb_set_par = mx3fb_set_par,
1156	.fb_check_var = mx3fb_check_var,	1156	.fb_check_var = mx3fb_check_var,
1157	.fb_setcolreg = mx3fb_setcolreg,	1157	.fb_setcolreg = mx3fb_setcolreg,
1158	.fb_pan_display = mx3fb_pan_display,	1158	.fb_pan_display = mx3fb_pan_display,
1159	.fb_fillrect = cfb_fillrect,	1159	.fb_fillrect = cfb_fillrect,
1160	.fb_copyarea = cfb_copyarea,	1160	.fb_copyarea = cfb_copyarea,
1161	.fb_imageblit = cfb_imageblit,	1161	.fb_imageblit = cfb_imageblit,
1162	.fb_blank = mx3fb_blank,	1162	.fb_blank = mx3fb_blank,
1163	};	1163	};
1164		1164
1165	#ifdef CONFIG_PM	1165	#ifdef CONFIG_PM
1166	/*	1166	/*
1167	* Power management hooks. Note that we won't be called from IRQ context,	1167	* Power management hooks. Note that we won't be called from IRQ context,
1168	* unlike the blank functions above, so we may sleep.	1168	* unlike the blank functions above, so we may sleep.
1169	*/	1169	*/
1170		1170
1171	/*	1171	/*
1172	* Suspends the framebuffer and blanks the screen. Power management support	1172	* Suspends the framebuffer and blanks the screen. Power management support
1173	*/	1173	*/
1174	static int mx3fb_suspend(struct platform_device *pdev, pm_message_t state)	1174	static int mx3fb_suspend(struct platform_device *pdev, pm_message_t state)
1175	{	1175	{
1176	struct mx3fb_data *mx3fb = platform_get_drvdata(pdev);	1176	struct mx3fb_data *mx3fb = platform_get_drvdata(pdev);
1177	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;	1177	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;
1178		1178
1179	acquire_console_sem();	1179	acquire_console_sem();
1180	fb_set_suspend(mx3fb->fbi, 1);	1180	fb_set_suspend(mx3fb->fbi, 1);
1181	release_console_sem();	1181	release_console_sem();
1182		1182
1183	if (mx3_fbi->blank == FB_BLANK_UNBLANK) {	1183	if (mx3_fbi->blank == FB_BLANK_UNBLANK) {
1184	sdc_disable_channel(mx3_fbi);	1184	sdc_disable_channel(mx3_fbi);
1185	sdc_set_brightness(mx3fb, 0);	1185	sdc_set_brightness(mx3fb, 0);
1186		1186
1187	}	1187	}
1188	return 0;	1188	return 0;
1189	}	1189	}
1190		1190
1191	/*	1191	/*
1192	* Resumes the framebuffer and unblanks the screen. Power management support	1192	* Resumes the framebuffer and unblanks the screen. Power management support
1193	*/	1193	*/
1194	static int mx3fb_resume(struct platform_device *pdev)	1194	static int mx3fb_resume(struct platform_device *pdev)
1195	{	1195	{
1196	struct mx3fb_data *mx3fb = platform_get_drvdata(pdev);	1196	struct mx3fb_data *mx3fb = platform_get_drvdata(pdev);
1197	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;	1197	struct mx3fb_info *mx3_fbi = mx3fb->fbi->par;
1198		1198
1199	if (mx3_fbi->blank == FB_BLANK_UNBLANK) {	1199	if (mx3_fbi->blank == FB_BLANK_UNBLANK) {
1200	sdc_enable_channel(mx3_fbi);	1200	sdc_enable_channel(mx3_fbi);
1201	sdc_set_brightness(mx3fb, mx3fb->backlight_level);	1201	sdc_set_brightness(mx3fb, mx3fb->backlight_level);
1202	}	1202	}
1203		1203
1204	acquire_console_sem();	1204	acquire_console_sem();
1205	fb_set_suspend(mx3fb->fbi, 0);	1205	fb_set_suspend(mx3fb->fbi, 0);
1206	release_console_sem();	1206	release_console_sem();
1207		1207
1208	return 0;	1208	return 0;
1209	}	1209	}
1210	#else	1210	#else
1211	#define mx3fb_suspend NULL	1211	#define mx3fb_suspend NULL
1212	#define mx3fb_resume NULL	1212	#define mx3fb_resume NULL
1213	#endif	1213	#endif
1214		1214
1215	/*	1215	/*
1216	* Main framebuffer functions	1216	* Main framebuffer functions
1217	*/	1217	*/
1218		1218
1219	/**	1219	/**
1220	* mx3fb_map_video_memory() - allocates the DRAM memory for the frame buffer.	1220	* mx3fb_map_video_memory() - allocates the DRAM memory for the frame buffer.
1221	* @fbi: framebuffer information pointer	1221	* @fbi: framebuffer information pointer
1222	* @mem_len: length of mapped memory	1222	* @mem_len: length of mapped memory
1223	* @lock: do not lock during initialisation	1223	* @lock: do not lock during initialisation
1224	* @return: Error code indicating success or failure	1224	* @return: Error code indicating success or failure
1225	*	1225	*
1226	* This buffer is remapped into a non-cached, non-buffered, memory region to	1226	* This buffer is remapped into a non-cached, non-buffered, memory region to
1227	* allow palette and pixel writes to occur without flushing the cache. Once this	1227	* allow palette and pixel writes to occur without flushing the cache. Once this
1228	* area is remapped, all virtual memory access to the video memory should occur	1228	* area is remapped, all virtual memory access to the video memory should occur
1229	* at the new region.	1229	* at the new region.
1230	*/	1230	*/
1231	static int mx3fb_map_video_memory(struct fb_info *fbi, unsigned int mem_len,	1231	static int mx3fb_map_video_memory(struct fb_info *fbi, unsigned int mem_len,
1232	bool lock)	1232	bool lock)
1233	{	1233	{
1234	int retval = 0;	1234	int retval = 0;
1235	dma_addr_t addr;	1235	dma_addr_t addr;
1236		1236
1237	fbi->screen_base = dma_alloc_writecombine(fbi->device,	1237	fbi->screen_base = dma_alloc_writecombine(fbi->device,
1238	mem_len,	1238	mem_len,
1239	&addr, GFP_DMA);	1239	&addr, GFP_DMA);
1240		1240
1241	if (!fbi->screen_base) {	1241	if (!fbi->screen_base) {
1242	dev_err(fbi->device, "Cannot allocate %u bytes framebuffer memory\n",	1242	dev_err(fbi->device, "Cannot allocate %u bytes framebuffer memory\n",
1243	mem_len);	1243	mem_len);
1244	retval = -EBUSY;	1244	retval = -EBUSY;
1245	goto err0;	1245	goto err0;
1246	}	1246	}
1247		1247
1248	if (lock)	1248	if (lock)
1249	mutex_lock(&fbi->mm_lock);	1249	mutex_lock(&fbi->mm_lock);
1250	fbi->fix.smem_start = addr;	1250	fbi->fix.smem_start = addr;
1251	fbi->fix.smem_len = mem_len;	1251	fbi->fix.smem_len = mem_len;
1252	if (lock)	1252	if (lock)
1253	mutex_unlock(&fbi->mm_lock);	1253	mutex_unlock(&fbi->mm_lock);
1254		1254
1255	dev_dbg(fbi->device, "allocated fb @ p=0x%08x, v=0x%p, size=%d.\n",	1255	dev_dbg(fbi->device, "allocated fb @ p=0x%08x, v=0x%p, size=%d.\n",
1256	(uint32_t) fbi->fix.smem_start, fbi->screen_base, fbi->fix.smem_len);	1256	(uint32_t) fbi->fix.smem_start, fbi->screen_base, fbi->fix.smem_len);
1257		1257
1258	fbi->screen_size = fbi->fix.smem_len;	1258	fbi->screen_size = fbi->fix.smem_len;
1259		1259
1260	/* Clear the screen */	1260	/* Clear the screen */
1261	memset((char *)fbi->screen_base, 0, fbi->fix.smem_len);	1261	memset((char *)fbi->screen_base, 0, fbi->fix.smem_len);
1262		1262
1263	return 0;	1263	return 0;
1264		1264
1265	err0:	1265	err0:
1266	fbi->fix.smem_len = 0;	1266	fbi->fix.smem_len = 0;
1267	fbi->fix.smem_start = 0;	1267	fbi->fix.smem_start = 0;
1268	fbi->screen_base = NULL;	1268	fbi->screen_base = NULL;
1269	return retval;	1269	return retval;
1270	}	1270	}
1271		1271
1272	/**	1272	/**
1273	* mx3fb_unmap_video_memory() - de-allocate frame buffer memory.	1273	* mx3fb_unmap_video_memory() - de-allocate frame buffer memory.
1274	* @fbi: framebuffer information pointer	1274	* @fbi: framebuffer information pointer
1275	* @return: error code indicating success or failure	1275	* @return: error code indicating success or failure
1276	*/	1276	*/
1277	static int mx3fb_unmap_video_memory(struct fb_info *fbi)	1277	static int mx3fb_unmap_video_memory(struct fb_info *fbi)
1278	{	1278	{
1279	dma_free_writecombine(fbi->device, fbi->fix.smem_len,	1279	dma_free_writecombine(fbi->device, fbi->fix.smem_len,
1280	fbi->screen_base, fbi->fix.smem_start);	1280	fbi->screen_base, fbi->fix.smem_start);
1281		1281
1282	fbi->screen_base = 0;	1282	fbi->screen_base = 0;
1283	mutex_lock(&fbi->mm_lock);	1283	mutex_lock(&fbi->mm_lock);
1284	fbi->fix.smem_start = 0;	1284	fbi->fix.smem_start = 0;
1285	fbi->fix.smem_len = 0;	1285	fbi->fix.smem_len = 0;
1286	mutex_unlock(&fbi->mm_lock);	1286	mutex_unlock(&fbi->mm_lock);
1287	return 0;	1287	return 0;
1288	}	1288	}
1289		1289
1290	/**	1290	/**
1291	* mx3fb_init_fbinfo() - initialize framebuffer information object.	1291	* mx3fb_init_fbinfo() - initialize framebuffer information object.
1292	* @return: initialized framebuffer structure.	1292	* @return: initialized framebuffer structure.
1293	*/	1293	*/
1294	static struct fb_info mx3fb_init_fbinfo(struct device dev, struct fb_ops *ops)	1294	static struct fb_info mx3fb_init_fbinfo(struct device dev, struct fb_ops *ops)
1295	{	1295	{
1296	struct fb_info *fbi;	1296	struct fb_info *fbi;
1297	struct mx3fb_info *mx3fbi;	1297	struct mx3fb_info *mx3fbi;
1298	int ret;	1298	int ret;
1299		1299
1300	/* Allocate sufficient memory for the fb structure */	1300	/* Allocate sufficient memory for the fb structure */
1301	fbi = framebuffer_alloc(sizeof(struct mx3fb_info), dev);	1301	fbi = framebuffer_alloc(sizeof(struct mx3fb_info), dev);
1302	if (!fbi)	1302	if (!fbi)
1303	return NULL;	1303	return NULL;
1304		1304
1305	mx3fbi = fbi->par;	1305	mx3fbi = fbi->par;
1306	mx3fbi->cookie = -EINVAL;	1306	mx3fbi->cookie = -EINVAL;
1307	mx3fbi->cur_ipu_buf = 0;	1307	mx3fbi->cur_ipu_buf = 0;
1308		1308
1309	fbi->var.activate = FB_ACTIVATE_NOW;	1309	fbi->var.activate = FB_ACTIVATE_NOW;
1310		1310
1311	fbi->fbops = ops;	1311	fbi->fbops = ops;
1312	fbi->flags = FBINFO_FLAG_DEFAULT;	1312	fbi->flags = FBINFO_FLAG_DEFAULT;
1313	fbi->pseudo_palette = mx3fbi->pseudo_palette;	1313	fbi->pseudo_palette = mx3fbi->pseudo_palette;
1314		1314
1315	mutex_init(&mx3fbi->mutex);	1315	mutex_init(&mx3fbi->mutex);
1316		1316
1317	/* Allocate colormap */	1317	/* Allocate colormap */
1318	ret = fb_alloc_cmap(&fbi->cmap, 16, 0);	1318	ret = fb_alloc_cmap(&fbi->cmap, 16, 0);
1319	if (ret < 0) {	1319	if (ret < 0) {
1320	framebuffer_release(fbi);	1320	framebuffer_release(fbi);
1321	return NULL;	1321	return NULL;
1322	}	1322	}
1323		1323
1324	return fbi;	1324	return fbi;
1325	}	1325	}
1326		1326
1327	static int init_fb_chan(struct mx3fb_data mx3fb, struct idmac_channel ichan)	1327	static int init_fb_chan(struct mx3fb_data mx3fb, struct idmac_channel ichan)
1328	{	1328	{
1329	struct device *dev = mx3fb->dev;	1329	struct device *dev = mx3fb->dev;
1330	struct mx3fb_platform_data *mx3fb_pdata = dev->platform_data;	1330	struct mx3fb_platform_data *mx3fb_pdata = dev->platform_data;
1331	const char *name = mx3fb_pdata->name;	1331	const char *name = mx3fb_pdata->name;
1332	unsigned int irq;	1332	unsigned int irq;
1333	struct fb_info *fbi;	1333	struct fb_info *fbi;
1334	struct mx3fb_info *mx3fbi;	1334	struct mx3fb_info *mx3fbi;
1335	const struct fb_videomode *mode;	1335	const struct fb_videomode *mode;
1336	int ret, num_modes;	1336	int ret, num_modes;
1337		1337
1338	ichan->client = mx3fb;	1338	ichan->client = mx3fb;
1339	irq = ichan->eof_irq;	1339	irq = ichan->eof_irq;
1340		1340
1341	if (ichan->dma_chan.chan_id != IDMAC_SDC_0)	1341	if (ichan->dma_chan.chan_id != IDMAC_SDC_0)
1342	return -EINVAL;	1342	return -EINVAL;
1343		1343
1344	fbi = mx3fb_init_fbinfo(dev, &mx3fb_ops);	1344	fbi = mx3fb_init_fbinfo(dev, &mx3fb_ops);
1345	if (!fbi)	1345	if (!fbi)
1346	return -ENOMEM;	1346	return -ENOMEM;
1347		1347
1348	if (!fb_mode)	1348	if (!fb_mode)
1349	fb_mode = name;	1349	fb_mode = name;
1350		1350
1351	if (!fb_mode) {	1351	if (!fb_mode) {
1352	ret = -EINVAL;	1352	ret = -EINVAL;
1353	goto emode;	1353	goto emode;
1354	}	1354	}
1355		1355
1356	if (mx3fb_pdata->mode && mx3fb_pdata->num_modes) {	1356	if (mx3fb_pdata->mode && mx3fb_pdata->num_modes) {
1357	mode = mx3fb_pdata->mode;	1357	mode = mx3fb_pdata->mode;
1358	num_modes = mx3fb_pdata->num_modes;	1358	num_modes = mx3fb_pdata->num_modes;
1359	} else {	1359	} else {
1360	mode = mx3fb_modedb;	1360	mode = mx3fb_modedb;
1361	num_modes = ARRAY_SIZE(mx3fb_modedb);	1361	num_modes = ARRAY_SIZE(mx3fb_modedb);
1362	}	1362	}
1363		1363
1364	if (!fb_find_mode(&fbi->var, fbi, fb_mode, mode,	1364	if (!fb_find_mode(&fbi->var, fbi, fb_mode, mode,
1365	num_modes, NULL, default_bpp)) {	1365	num_modes, NULL, default_bpp)) {
1366	ret = -EBUSY;	1366	ret = -EBUSY;
1367	goto emode;	1367	goto emode;
1368	}	1368	}
1369		1369
1370	fb_videomode_to_modelist(mode, num_modes, &fbi->modelist);	1370	fb_videomode_to_modelist(mode, num_modes, &fbi->modelist);
1371		1371
1372	/* Default Y virtual size is 2x panel size */	1372	/* Default Y virtual size is 2x panel size */
1373	fbi->var.yres_virtual = fbi->var.yres * 2;	1373	fbi->var.yres_virtual = fbi->var.yres * 2;
1374		1374
1375	mx3fb->fbi = fbi;	1375	mx3fb->fbi = fbi;
1376		1376
1377	/* set Display Interface clock period */	1377	/* set Display Interface clock period */
1378	mx3fb_write_reg(mx3fb, 0x00100010L, DI_HSP_CLK_PER);	1378	mx3fb_write_reg(mx3fb, 0x00100010L, DI_HSP_CLK_PER);
1379	/* Might need to trigger HSP clock change - see 44.3.3.8.5 */	1379	/* Might need to trigger HSP clock change - see 44.3.3.8.5 */
1380		1380
1381	sdc_set_brightness(mx3fb, 255);	1381	sdc_set_brightness(mx3fb, 255);
1382	sdc_set_global_alpha(mx3fb, true, 0xFF);	1382	sdc_set_global_alpha(mx3fb, true, 0xFF);
1383	sdc_set_color_key(mx3fb, IDMAC_SDC_0, false, 0);	1383	sdc_set_color_key(mx3fb, IDMAC_SDC_0, false, 0);
1384		1384
1385	mx3fbi = fbi->par;	1385	mx3fbi = fbi->par;
1386	mx3fbi->idmac_channel = ichan;	1386	mx3fbi->idmac_channel = ichan;
1387	mx3fbi->ipu_ch = ichan->dma_chan.chan_id;	1387	mx3fbi->ipu_ch = ichan->dma_chan.chan_id;
1388	mx3fbi->mx3fb = mx3fb;	1388	mx3fbi->mx3fb = mx3fb;
1389	mx3fbi->blank = FB_BLANK_NORMAL;	1389	mx3fbi->blank = FB_BLANK_NORMAL;
1390		1390
1391	init_completion(&mx3fbi->flip_cmpl);	1391	init_completion(&mx3fbi->flip_cmpl);
1392	disable_irq(ichan->eof_irq);	1392	disable_irq(ichan->eof_irq);
1393	dev_dbg(mx3fb->dev, "disabling irq %d\n", ichan->eof_irq);	1393	dev_dbg(mx3fb->dev, "disabling irq %d\n", ichan->eof_irq);
1394	ret = __set_par(fbi, false);	1394	ret = __set_par(fbi, false);
1395	if (ret < 0)	1395	if (ret < 0)
1396	goto esetpar;	1396	goto esetpar;
1397		1397
1398	__blank(FB_BLANK_UNBLANK, fbi);	1398	__blank(FB_BLANK_UNBLANK, fbi);
1399		1399
1400	dev_info(dev, "registered, using mode %s\n", fb_mode);	1400	dev_info(dev, "registered, using mode %s\n", fb_mode);
1401		1401
1402	ret = register_framebuffer(fbi);	1402	ret = register_framebuffer(fbi);
1403	if (ret < 0)	1403	if (ret < 0)
1404	goto erfb;	1404	goto erfb;
1405		1405
1406	return 0;	1406	return 0;
1407		1407
1408	erfb:	1408	erfb:
1409	esetpar:	1409	esetpar:
1410	emode:	1410	emode:
1411	fb_dealloc_cmap(&fbi->cmap);	1411	fb_dealloc_cmap(&fbi->cmap);
1412	framebuffer_release(fbi);	1412	framebuffer_release(fbi);
1413		1413
1414	return ret;	1414	return ret;
1415	}	1415	}
1416		1416
1417	static bool chan_filter(struct dma_chan chan, void arg)	1417	static bool chan_filter(struct dma_chan chan, void arg)
1418	{	1418	{
1419	struct dma_chan_request *rq = arg;	1419	struct dma_chan_request *rq = arg;
1420	struct device *dev;	1420	struct device *dev;
1421	struct mx3fb_platform_data *mx3fb_pdata;	1421	struct mx3fb_platform_data *mx3fb_pdata;
1422		1422
1423	if (!rq)	1423	if (!rq)
1424	return false;	1424	return false;
1425		1425
1426	dev = rq->mx3fb->dev;	1426	dev = rq->mx3fb->dev;
1427	mx3fb_pdata = dev->platform_data;	1427	mx3fb_pdata = dev->platform_data;
1428		1428
1429	return rq->id == chan->chan_id &&	1429	return rq->id == chan->chan_id &&
1430	mx3fb_pdata->dma_dev == chan->device->dev;	1430	mx3fb_pdata->dma_dev == chan->device->dev;
1431	}	1431	}
1432		1432
1433	static void release_fbi(struct fb_info *fbi)	1433	static void release_fbi(struct fb_info *fbi)
1434	{	1434	{
1435	mx3fb_unmap_video_memory(fbi);	1435	mx3fb_unmap_video_memory(fbi);
1436		1436
1437	fb_dealloc_cmap(&fbi->cmap);	1437	fb_dealloc_cmap(&fbi->cmap);
1438		1438
1439	unregister_framebuffer(fbi);	1439	unregister_framebuffer(fbi);
1440	framebuffer_release(fbi);	1440	framebuffer_release(fbi);
1441	}	1441	}
1442		1442
1443	static int mx3fb_probe(struct platform_device *pdev)	1443	static int mx3fb_probe(struct platform_device *pdev)
1444	{	1444	{
1445	struct device *dev = &pdev->dev;	1445	struct device *dev = &pdev->dev;
1446	int ret;	1446	int ret;
1447	struct resource *sdc_reg;	1447	struct resource *sdc_reg;
1448	struct mx3fb_data *mx3fb;	1448	struct mx3fb_data *mx3fb;
1449	dma_cap_mask_t mask;	1449	dma_cap_mask_t mask;
1450	struct dma_chan *chan;	1450	struct dma_chan *chan;
1451	struct dma_chan_request rq;	1451	struct dma_chan_request rq;
1452		1452
1453	/*	1453	/*
1454	* Display Interface (DI) and Synchronous Display Controller (SDC)	1454	* Display Interface (DI) and Synchronous Display Controller (SDC)
1455	* registers	1455	* registers
1456	*/	1456	*/
1457	sdc_reg = platform_get_resource(pdev, IORESOURCE_MEM, 0);	1457	sdc_reg = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1458	if (!sdc_reg)	1458	if (!sdc_reg)
1459	return -EINVAL;	1459	return -EINVAL;
1460		1460
1461	mx3fb = kzalloc(sizeof(*mx3fb), GFP_KERNEL);	1461	mx3fb = kzalloc(sizeof(*mx3fb), GFP_KERNEL);
1462	if (!mx3fb)	1462	if (!mx3fb)
1463	return -ENOMEM;	1463	return -ENOMEM;
1464		1464
1465	spin_lock_init(&mx3fb->lock);	1465	spin_lock_init(&mx3fb->lock);
1466		1466
1467	mx3fb->reg_base = ioremap(sdc_reg->start, resource_size(sdc_reg));	1467	mx3fb->reg_base = ioremap(sdc_reg->start, resource_size(sdc_reg));
1468	if (!mx3fb->reg_base) {	1468	if (!mx3fb->reg_base) {
1469	ret = -ENOMEM;	1469	ret = -ENOMEM;
1470	goto eremap;	1470	goto eremap;
1471	}	1471	}
1472		1472
1473	pr_debug("Remapped %x to %x at %p\n", sdc_reg->start, sdc_reg->end,	1473	pr_debug("Remapped %x to %x at %p\n", sdc_reg->start, sdc_reg->end,
1474	mx3fb->reg_base);	1474	mx3fb->reg_base);
1475		1475
1476	/* IDMAC interface */	1476	/* IDMAC interface */
1477	dmaengine_get();	1477	dmaengine_get();
1478		1478
1479	mx3fb->dev = dev;	1479	mx3fb->dev = dev;
1480	platform_set_drvdata(pdev, mx3fb);	1480	platform_set_drvdata(pdev, mx3fb);
1481		1481
1482	rq.mx3fb = mx3fb;	1482	rq.mx3fb = mx3fb;
1483		1483
1484	dma_cap_zero(mask);	1484	dma_cap_zero(mask);
1485	dma_cap_set(DMA_SLAVE, mask);	1485	dma_cap_set(DMA_SLAVE, mask);
1486	dma_cap_set(DMA_PRIVATE, mask);	1486	dma_cap_set(DMA_PRIVATE, mask);
1487	rq.id = IDMAC_SDC_0;	1487	rq.id = IDMAC_SDC_0;
1488	chan = dma_request_channel(mask, chan_filter, &rq);	1488	chan = dma_request_channel(mask, chan_filter, &rq);
1489	if (!chan) {	1489	if (!chan) {
1490	ret = -EBUSY;	1490	ret = -EBUSY;
1491	goto ersdc0;	1491	goto ersdc0;
1492	}	1492	}
1493		1493
1494	mx3fb->backlight_level = 255;	1494	mx3fb->backlight_level = 255;
1495		1495
1496	ret = init_fb_chan(mx3fb, to_idmac_chan(chan));	1496	ret = init_fb_chan(mx3fb, to_idmac_chan(chan));
1497	if (ret < 0)	1497	if (ret < 0)
1498	goto eisdc0;	1498	goto eisdc0;
1499		1499
1500	return 0;	1500	return 0;
1501		1501
1502	eisdc0:	1502	eisdc0:
1503	dma_release_channel(chan);	1503	dma_release_channel(chan);
1504	ersdc0:	1504	ersdc0:
1505	dmaengine_put();	1505	dmaengine_put();
1506	iounmap(mx3fb->reg_base);	1506	iounmap(mx3fb->reg_base);
1507	eremap:	1507	eremap:
1508	kfree(mx3fb);	1508	kfree(mx3fb);
1509	dev_err(dev, "mx3fb: failed to register fb\n");	1509	dev_err(dev, "mx3fb: failed to register fb\n");
1510	return ret;	1510	return ret;
1511	}	1511	}
1512		1512
1513	static int mx3fb_remove(struct platform_device *dev)	1513	static int mx3fb_remove(struct platform_device *dev)
1514	{	1514	{
1515	struct mx3fb_data *mx3fb = platform_get_drvdata(dev);	1515	struct mx3fb_data *mx3fb = platform_get_drvdata(dev);
1516	struct fb_info *fbi = mx3fb->fbi;	1516	struct fb_info *fbi = mx3fb->fbi;
1517	struct mx3fb_info *mx3_fbi = fbi->par;	1517	struct mx3fb_info *mx3_fbi = fbi->par;
1518	struct dma_chan *chan;	1518	struct dma_chan *chan;
1519		1519
1520	chan = &mx3_fbi->idmac_channel->dma_chan;	1520	chan = &mx3_fbi->idmac_channel->dma_chan;
1521	release_fbi(fbi);	1521	release_fbi(fbi);
1522		1522
1523	dma_release_channel(chan);	1523	dma_release_channel(chan);
1524	dmaengine_put();	1524	dmaengine_put();
1525		1525
1526	iounmap(mx3fb->reg_base);	1526	iounmap(mx3fb->reg_base);
1527	kfree(mx3fb);	1527	kfree(mx3fb);
1528	return 0;	1528	return 0;
1529	}	1529	}
1530		1530
1531	static struct platform_driver mx3fb_driver = {	1531	static struct platform_driver mx3fb_driver = {
1532	.driver = {	1532	.driver = {
1533	.name = MX3FB_NAME,	1533	.name = MX3FB_NAME,
1534	},	1534	},
1535	.probe = mx3fb_probe,	1535	.probe = mx3fb_probe,
1536	.remove = mx3fb_remove,	1536	.remove = mx3fb_remove,
1537	.suspend = mx3fb_suspend,	1537	.suspend = mx3fb_suspend,
1538	.resume = mx3fb_resume,	1538	.resume = mx3fb_resume,
1539	};	1539	};
1540		1540
1541	/*	1541	/*
1542	* Parse user specified options (`video=mx3fb:')	1542	* Parse user specified options (`video=mx3fb:')
1543	* example:	1543	* example:
1544	* video=mx3fb:bpp=16	1544	* video=mx3fb:bpp=16
1545	*/	1545	*/
1546	static int __init mx3fb_setup(void)	1546	static int __init mx3fb_setup(void)
1547	{	1547	{
1548	#ifndef MODULE	1548	#ifndef MODULE
1549	char opt, options = NULL;	1549	char opt, options = NULL;
1550		1550
1551	if (fb_get_options("mx3fb", &options))	1551	if (fb_get_options("mx3fb", &options))
1552	return -ENODEV;	1552	return -ENODEV;
1553		1553
1554	if (!options \|\| !*options)	1554	if (!options \|\| !*options)
1555	return 0;	1555	return 0;
1556		1556
1557	while ((opt = strsep(&options, ",")) != NULL) {	1557	while ((opt = strsep(&options, ",")) != NULL) {
1558	if (!*opt)	1558	if (!*opt)
1559	continue;	1559	continue;
1560	if (!strncmp(opt, "bpp=", 4))	1560	if (!strncmp(opt, "bpp=", 4))
1561	default_bpp = simple_strtoul(opt + 4, NULL, 0);	1561	default_bpp = simple_strtoul(opt + 4, NULL, 0);
1562	else	1562	else
1563	fb_mode = opt;	1563	fb_mode = opt;
1564	}	1564	}
1565	#endif	1565	#endif
1566		1566
1567	return 0;	1567	return 0;
1568	}	1568	}
1569		1569
1570	static int __init mx3fb_init(void)	1570	static int __init mx3fb_init(void)
1571	{	1571	{
1572	int ret = mx3fb_setup();	1572	int ret = mx3fb_setup();
1573		1573
1574	if (ret < 0)	1574	if (ret < 0)
1575	return ret;	1575	return ret;
1576		1576
1577	ret = platform_driver_register(&mx3fb_driver);	1577	ret = platform_driver_register(&mx3fb_driver);
1578	return ret;	1578	return ret;
1579	}	1579	}
1580		1580
1581	static void __exit mx3fb_exit(void)	1581	static void __exit mx3fb_exit(void)
1582	{	1582	{
1583	platform_driver_unregister(&mx3fb_driver);	1583	platform_driver_unregister(&mx3fb_driver);
1584	}	1584	}
1585		1585
1586	module_init(mx3fb_init);	1586	module_init(mx3fb_init);
1587	module_exit(mx3fb_exit);	1587	module_exit(mx3fb_exit);
1588		1588
1589	MODULE_AUTHOR("Freescale Semiconductor, Inc.");	1589	MODULE_AUTHOR("Freescale Semiconductor, Inc.");
1590	MODULE_DESCRIPTION("MX3 framebuffer driver");	1590	MODULE_DESCRIPTION("MX3 framebuffer driver");
1591	MODULE_ALIAS("platform:" MX3FB_NAME);	1591	MODULE_ALIAS("platform:" MX3FB_NAME);
1592	MODULE_LICENSE("GPL v2");	1592	MODULE_LICENSE("GPL v2");
1593		1593

include/linux/dmaengine.h

Diff comments View file @ 0582763

 /*
  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the Free
  * Software Foundation; either version 2 of the License, or (at your option)
  * any later version.
  *
  * This program is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program; if not, write to the Free Software Foundation, Inc., 59
  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  *
  * The full GNU General Public License is included in this distribution in the
  * file called COPYING.
  */
 #ifndef DMAENGINE_H
 #define DMAENGINE_H
 #include <linux/device.h>
 #include <linux/uio.h>
 #include <linux/dma-mapping.h>
 /**
  * typedef dma_cookie_t - an opaque DMA cookie
  *
  * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code
  */
 typedef s32 dma_cookie_t;
 #define DMA_MIN_COOKIE	1
 #define DMA_MAX_COOKIE	INT_MAX
 #define dma_submit_error(cookie) ((cookie) < 0 ? 1 : 0)
 /**
  * enum dma_status - DMA transaction status
  * @DMA_SUCCESS: transaction completed successfully
  * @DMA_IN_PROGRESS: transaction not yet processed
  * @DMA_PAUSED: transaction is paused
  * @DMA_ERROR: transaction failed
  */
 enum dma_status {
 	DMA_SUCCESS,
 	DMA_IN_PROGRESS,
 	DMA_PAUSED,
 	DMA_ERROR,
 };
 /**
  * enum dma_transaction_type - DMA transaction types/indexes
  *
  * Note: The DMA_ASYNC_TX capability is not to be set by drivers.  It is
  * automatically set as dma devices are registered.
  */
 enum dma_transaction_type {
 	DMA_MEMCPY,
 	DMA_XOR,
 	DMA_PQ,
 	DMA_XOR_VAL,
 	DMA_PQ_VAL,
 	DMA_MEMSET,
 	DMA_INTERRUPT,
 	DMA_PRIVATE,
 	DMA_ASYNC_TX,
 	DMA_SLAVE,
 };
 /* last transaction type for creation of the capabilities mask */
 #define DMA_TX_TYPE_END (DMA_SLAVE + 1)
 /**
  * enum dma_ctrl_flags - DMA flags to augment operation preparation,
  *  control completion, and communicate status.
  * @DMA_PREP_INTERRUPT - trigger an interrupt (callback) upon completion of
  *  this transaction
  * @DMA_CTRL_ACK - if clear, the descriptor cannot be reused until the client
  *  acknowledges receipt, i.e. has has a chance to establish any dependency
  *  chains
  * @DMA_COMPL_SKIP_SRC_UNMAP - set to disable dma-unmapping the source buffer(s)
  * @DMA_COMPL_SKIP_DEST_UNMAP - set to disable dma-unmapping the destination(s)
  * @DMA_COMPL_SRC_UNMAP_SINGLE - set to do the source dma-unmapping as single
  * 	(if not set, do the source dma-unmapping as page)
  * @DMA_COMPL_DEST_UNMAP_SINGLE - set to do the destination dma-unmapping as single
  * 	(if not set, do the destination dma-unmapping as page)
  * @DMA_PREP_PQ_DISABLE_P - prevent generation of P while generating Q
  * @DMA_PREP_PQ_DISABLE_Q - prevent generation of Q while generating P
  * @DMA_PREP_CONTINUE - indicate to a driver that it is reusing buffers as
  *  sources that were the result of a previous operation, in the case of a PQ
  *  operation it continues the calculation with new sources
  * @DMA_PREP_FENCE - tell the driver that subsequent operations depend
  *  on the result of this operation
  */
 enum dma_ctrl_flags {
 	DMA_PREP_INTERRUPT = (1 << 0),
 	DMA_CTRL_ACK = (1 << 1),
 	DMA_COMPL_SKIP_SRC_UNMAP = (1 << 2),
 	DMA_COMPL_SKIP_DEST_UNMAP = (1 << 3),
 	DMA_COMPL_SRC_UNMAP_SINGLE = (1 << 4),
 	DMA_COMPL_DEST_UNMAP_SINGLE = (1 << 5),
 	DMA_PREP_PQ_DISABLE_P = (1 << 6),
 	DMA_PREP_PQ_DISABLE_Q = (1 << 7),
 	DMA_PREP_CONTINUE = (1 << 8),
 	DMA_PREP_FENCE = (1 << 9),
 };
 /**
  * enum dma_ctrl_cmd - DMA operations that can optionally be exercised
  * on a running channel.
  * @DMA_TERMINATE_ALL: terminate all ongoing transfers
  * @DMA_PAUSE: pause ongoing transfers
  * @DMA_RESUME: resume paused transfer
  */
 enum dma_ctrl_cmd {
 	DMA_TERMINATE_ALL,
 	DMA_PAUSE,
 	DMA_RESUME,
 };
 /**
  * enum sum_check_bits - bit position of pq_check_flags
  */
 enum sum_check_bits {
 	SUM_CHECK_P = 0,
 	SUM_CHECK_Q = 1,
 };
 /**
  * enum pq_check_flags - result of async_{xor,pq}_zero_sum operations
  * @SUM_CHECK_P_RESULT - 1 if xor zero sum error, 0 otherwise
  * @SUM_CHECK_Q_RESULT - 1 if reed-solomon zero sum error, 0 otherwise
  */
 enum sum_check_flags {
 	SUM_CHECK_P_RESULT = (1 << SUM_CHECK_P),
 	SUM_CHECK_Q_RESULT = (1 << SUM_CHECK_Q),
 };
 /**
  * dma_cap_mask_t - capabilities bitmap modeled after cpumask_t.
  * See linux/cpumask.h
  */
 typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t;
 /**
  * struct dma_chan_percpu - the per-CPU part of struct dma_chan
  * @memcpy_count: transaction counter
  * @bytes_transferred: byte counter
  */
 struct dma_chan_percpu {
 	/* stats */
 	unsigned long memcpy_count;
 	unsigned long bytes_transferred;
 };
 /**
  * struct dma_chan - devices supply DMA channels, clients use them
  * @device: ptr to the dma device who supplies this channel, always !%NULL
  * @cookie: last cookie value returned to client
  * @chan_id: channel ID for sysfs
  * @dev: class device for sysfs
  * @device_node: used to add this to the device chan list
  * @local: per-cpu pointer to a struct dma_chan_percpu
  * @client-count: how many clients are using this channel
  * @table_count: number of appearances in the mem-to-mem allocation table
  * @private: private data for certain client-channel associations
  */
 struct dma_chan {
 	struct dma_device *device;
 	dma_cookie_t cookie;
 	/* sysfs */
 	int chan_id;
 	struct dma_chan_dev *dev;
 	struct list_head device_node;
 	struct dma_chan_percpu __percpu *local;
 	int client_count;
 	int table_count;
 	void *private;
 };
 /**
  * struct dma_chan_dev - relate sysfs device node to backing channel device
  * @chan - driver channel device
  * @device - sysfs device
  * @dev_id - parent dma_device dev_id
  * @idr_ref - reference count to gate release of dma_device dev_id
  */
 struct dma_chan_dev {
 	struct dma_chan *chan;
 	struct device device;
 	int dev_id;
 	atomic_t *idr_ref;
 };
 static inline const char *dma_chan_name(struct dma_chan *chan)
 {
 	return dev_name(&chan->dev->device);
 }
 void dma_chan_cleanup(struct kref *kref);
 /**
  * typedef dma_filter_fn - callback filter for dma_request_channel
  * @chan: channel to be reviewed
  * @filter_param: opaque parameter passed through dma_request_channel
  *
  * When this optional parameter is specified in a call to dma_request_channel a
  * suitable channel is passed to this routine for further dispositioning before
  * being returned.  Where 'suitable' indicates a non-busy channel that
  * satisfies the given capability mask.  It returns 'true' to indicate that the
  * channel is suitable.
  */
 typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param);
 typedef void (*dma_async_tx_callback)(void *dma_async_param);
 /**
  * struct dma_async_tx_descriptor - async transaction descriptor
  * ---dma generic offload fields---
  * @cookie: tracking cookie for this transaction, set to -EBUSY if
  *	this tx is sitting on a dependency list
  * @flags: flags to augment operation preparation, control completion, and
  * 	communicate status
  * @phys: physical address of the descriptor
  * @chan: target channel for this operation
  * @tx_submit: set the prepared descriptor(s) to be executed by the engine
  * @callback: routine to call after this operation is complete
  * @callback_param: general parameter to pass to the callback routine
  * ---async_tx api specific fields---
  * @next: at completion submit this descriptor
  * @parent: pointer to the next level up in the dependency chain
  * @lock: protect the parent and next pointers
  */
 struct dma_async_tx_descriptor {
 	dma_cookie_t cookie;
 	enum dma_ctrl_flags flags; /* not a 'long' to pack with cookie */
 	dma_addr_t phys;
 	struct dma_chan *chan;
 	dma_cookie_t (*tx_submit)(struct dma_async_tx_descriptor *tx);
 	dma_async_tx_callback callback;
 	void *callback_param;
 	struct dma_async_tx_descriptor *next;
 	struct dma_async_tx_descriptor *parent;
 	spinlock_t lock;
 };
 /**
  * struct dma_tx_state - filled in to report the status of
  * a transfer.
  * @last: last completed DMA cookie
  * @used: last issued DMA cookie (i.e. the one in progress)
  * @residue: the remaining number of bytes left to transmit
  *	on the selected transfer for states DMA_IN_PROGRESS and
  *	DMA_PAUSED if this is implemented in the driver, else 0
  */
 struct dma_tx_state {
 	dma_cookie_t last;
 	dma_cookie_t used;
 	u32 residue;
 };
 /**
  * struct dma_device - info on the entity supplying DMA services
  * @chancnt: how many DMA channels are supported
  * @privatecnt: how many DMA channels are requested by dma_request_channel
  * @channels: the list of struct dma_chan
  * @global_node: list_head for global dma_device_list
  * @cap_mask: one or more dma_capability flags
  * @max_xor: maximum number of xor sources, 0 if no capability
  * @max_pq: maximum number of PQ sources and PQ-continue capability
  * @copy_align: alignment shift for memcpy operations
  * @xor_align: alignment shift for xor operations
  * @pq_align: alignment shift for pq operations
  * @fill_align: alignment shift for memset operations
  * @dev_id: unique device ID
  * @dev: struct device reference for dma mapping api
  * @device_alloc_chan_resources: allocate resources and return the
  *	number of allocated descriptors
  * @device_free_chan_resources: release DMA channel's resources
  * @device_prep_dma_memcpy: prepares a memcpy operation
  * @device_prep_dma_xor: prepares a xor operation
  * @device_prep_dma_xor_val: prepares a xor validation operation
  * @device_prep_dma_pq: prepares a pq operation
  * @device_prep_dma_pq_val: prepares a pqzero_sum operation
  * @device_prep_dma_memset: prepares a memset operation
  * @device_prep_dma_interrupt: prepares an end of chain interrupt operation
  * @device_prep_slave_sg: prepares a slave dma operation
  * @device_control: manipulate all pending operations on a channel, returns
  *	zero or error code
  * @device_tx_status: poll for transaction completion, the optional
  *	txstate parameter can be supplied with a pointer to get a
  *	struct with auxilary transfer status information, otherwise the call
  *	will just return a simple status code
  * @device_issue_pending: push pending transactions to hardware
  */
 struct dma_device {
 	unsigned int chancnt;
 	unsigned int privatecnt;
 	struct list_head channels;
 	struct list_head global_node;
 	dma_cap_mask_t  cap_mask;
 	unsigned short max_xor;
 	unsigned short max_pq;
 	u8 copy_align;
 	u8 xor_align;
 	u8 pq_align;
 	u8 fill_align;
 	#define DMA_HAS_PQ_CONTINUE (1 << 15)
 	int dev_id;
 	struct device *dev;
 	int (*device_alloc_chan_resources)(struct dma_chan *chan);
 	void (*device_free_chan_resources)(struct dma_chan *chan);
 	struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)(
 		struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 		size_t len, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_xor)(
 		struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
 		unsigned int src_cnt, size_t len, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_xor_val)(
 		struct dma_chan *chan, dma_addr_t *src,	unsigned int src_cnt,
 		size_t len, enum sum_check_flags *result, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_pq)(
 		struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
 		unsigned int src_cnt, const unsigned char *scf,
 		size_t len, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_pq_val)(
 		struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
 		unsigned int src_cnt, const unsigned char *scf, size_t len,
 		enum sum_check_flags *pqres, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_memset)(
 		struct dma_chan *chan, dma_addr_t dest, int value, size_t len,
 		unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_dma_interrupt)(
 		struct dma_chan *chan, unsigned long flags);
 	struct dma_async_tx_descriptor *(*device_prep_slave_sg)(
 		struct dma_chan *chan, struct scatterlist *sgl,
 		unsigned int sg_len, enum dma_data_direction direction,
 		unsigned long flags);
-	int (*device_control)(struct dma_chan *chan, enum dma_ctrl_cmd cmd);
+	int (*device_control)(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
+		unsigned long arg);
 	enum dma_status (*device_tx_status)(struct dma_chan *chan,
 					    dma_cookie_t cookie,
 					    struct dma_tx_state *txstate);
 	void (*device_issue_pending)(struct dma_chan *chan);
 };
 static inline bool dmaengine_check_align(u8 align, size_t off1, size_t off2, size_t len)
 {
 	size_t mask;
 	if (!align)
 		return true;
 	mask = (1 << align) - 1;
 	if (mask & (off1 | off2 | len))
 		return false;
 	return true;
 }
 static inline bool is_dma_copy_aligned(struct dma_device *dev, size_t off1,
 				       size_t off2, size_t len)
 {
 	return dmaengine_check_align(dev->copy_align, off1, off2, len);
 }
 static inline bool is_dma_xor_aligned(struct dma_device *dev, size_t off1,
 				      size_t off2, size_t len)
 {
 	return dmaengine_check_align(dev->xor_align, off1, off2, len);
 }
 static inline bool is_dma_pq_aligned(struct dma_device *dev, size_t off1,
 				     size_t off2, size_t len)
 {
 	return dmaengine_check_align(dev->pq_align, off1, off2, len);
 }
 static inline bool is_dma_fill_aligned(struct dma_device *dev, size_t off1,
 				       size_t off2, size_t len)
 {
 	return dmaengine_check_align(dev->fill_align, off1, off2, len);
 }
 static inline void
 dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue)
 {
 	dma->max_pq = maxpq;
 	if (has_pq_continue)
 		dma->max_pq |= DMA_HAS_PQ_CONTINUE;
 }
 static inline bool dmaf_continue(enum dma_ctrl_flags flags)
 {
 	return (flags & DMA_PREP_CONTINUE) == DMA_PREP_CONTINUE;
 }
 static inline bool dmaf_p_disabled_continue(enum dma_ctrl_flags flags)
 {
 	enum dma_ctrl_flags mask = DMA_PREP_CONTINUE | DMA_PREP_PQ_DISABLE_P;
 	return (flags & mask) == mask;
 }
 static inline bool dma_dev_has_pq_continue(struct dma_device *dma)
 {
 	return (dma->max_pq & DMA_HAS_PQ_CONTINUE) == DMA_HAS_PQ_CONTINUE;
 }
 static unsigned short dma_dev_to_maxpq(struct dma_device *dma)
 {
 	return dma->max_pq & ~DMA_HAS_PQ_CONTINUE;
 }
 /* dma_maxpq - reduce maxpq in the face of continued operations
  * @dma - dma device with PQ capability
  * @flags - to check if DMA_PREP_CONTINUE and DMA_PREP_PQ_DISABLE_P are set
  *
  * When an engine does not support native continuation we need 3 extra
  * source slots to reuse P and Q with the following coefficients:
  * 1/ {00} * P : remove P from Q', but use it as a source for P'
  * 2/ {01} * Q : use Q to continue Q' calculation
  * 3/ {00} * Q : subtract Q from P' to cancel (2)
  *
  * In the case where P is disabled we only need 1 extra source:
  * 1/ {01} * Q : use Q to continue Q' calculation
  */
 static inline int dma_maxpq(struct dma_device *dma, enum dma_ctrl_flags flags)
 {
 	if (dma_dev_has_pq_continue(dma) || !dmaf_continue(flags))
 		return dma_dev_to_maxpq(dma);
 	else if (dmaf_p_disabled_continue(flags))
 		return dma_dev_to_maxpq(dma) - 1;
 	else if (dmaf_continue(flags))
 		return dma_dev_to_maxpq(dma) - 3;
 	BUG();
 }
 /* --- public DMA engine API --- */
 #ifdef CONFIG_DMA_ENGINE
 void dmaengine_get(void);
 void dmaengine_put(void);
 #else
 static inline void dmaengine_get(void)
 {
 }
 static inline void dmaengine_put(void)
 {
 }
 #endif
 #ifdef CONFIG_NET_DMA
 #define net_dmaengine_get()	dmaengine_get()
 #define net_dmaengine_put()	dmaengine_put()
 #else
 static inline void net_dmaengine_get(void)
 {
 }
 static inline void net_dmaengine_put(void)
 {
 }
 #endif
 #ifdef CONFIG_ASYNC_TX_DMA
 #define async_dmaengine_get()	dmaengine_get()
 #define async_dmaengine_put()	dmaengine_put()
 #ifdef CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH
 #define async_dma_find_channel(type) dma_find_channel(DMA_ASYNC_TX)
 #else
 #define async_dma_find_channel(type) dma_find_channel(type)
 #endif /* CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH */
 #else
 static inline void async_dmaengine_get(void)
 {
 }
 static inline void async_dmaengine_put(void)
 {
 }
 static inline struct dma_chan *
 async_dma_find_channel(enum dma_transaction_type type)
 {
 	return NULL;
 }
 #endif /* CONFIG_ASYNC_TX_DMA */
 dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan,
 	void *dest, void *src, size_t len);
 dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan,
 	struct page *page, unsigned int offset, void *kdata, size_t len);
 dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan,
 	struct page *dest_pg, unsigned int dest_off, struct page *src_pg,
 	unsigned int src_off, size_t len);
 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
 	struct dma_chan *chan);
 static inline void async_tx_ack(struct dma_async_tx_descriptor *tx)
 {
 	tx->flags |= DMA_CTRL_ACK;
 }
 static inline void async_tx_clear_ack(struct dma_async_tx_descriptor *tx)
 {
 	tx->flags &= ~DMA_CTRL_ACK;
 }
 static inline bool async_tx_test_ack(struct dma_async_tx_descriptor *tx)
 {
 	return (tx->flags & DMA_CTRL_ACK) == DMA_CTRL_ACK;
 }
 #define first_dma_cap(mask) __first_dma_cap(&(mask))
 static inline int __first_dma_cap(const dma_cap_mask_t *srcp)
 {
 	return min_t(int, DMA_TX_TYPE_END,
 		find_first_bit(srcp->bits, DMA_TX_TYPE_END));
 }
 #define next_dma_cap(n, mask) __next_dma_cap((n), &(mask))
 static inline int __next_dma_cap(int n, const dma_cap_mask_t *srcp)
 {
 	return min_t(int, DMA_TX_TYPE_END,
 		find_next_bit(srcp->bits, DMA_TX_TYPE_END, n+1));
 }
 #define dma_cap_set(tx, mask) __dma_cap_set((tx), &(mask))
 static inline void
 __dma_cap_set(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
 {
 	set_bit(tx_type, dstp->bits);
 }
 #define dma_cap_clear(tx, mask) __dma_cap_clear((tx), &(mask))
 static inline void
 __dma_cap_clear(enum dma_transaction_type tx_type, dma_cap_mask_t *dstp)
 {
 	clear_bit(tx_type, dstp->bits);
 }
 #define dma_cap_zero(mask) __dma_cap_zero(&(mask))
 static inline void __dma_cap_zero(dma_cap_mask_t *dstp)
 {
 	bitmap_zero(dstp->bits, DMA_TX_TYPE_END);
 }
 #define dma_has_cap(tx, mask) __dma_has_cap((tx), &(mask))
 static inline int
 __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp)
 {
 	return test_bit(tx_type, srcp->bits);
 }
 #define for_each_dma_cap_mask(cap, mask) \
 	for ((cap) = first_dma_cap(mask);	\
 		(cap) < DMA_TX_TYPE_END;	\
 		(cap) = next_dma_cap((cap), (mask)))
 /**
  * dma_async_issue_pending - flush pending transactions to HW
  * @chan: target DMA channel
  *
  * This allows drivers to push copies to HW in batches,
  * reducing MMIO writes where possible.
  */
 static inline void dma_async_issue_pending(struct dma_chan *chan)
 {
 	chan->device->device_issue_pending(chan);
 }
 #define dma_async_memcpy_issue_pending(chan) dma_async_issue_pending(chan)
 /**
  * dma_async_is_tx_complete - poll for transaction completion
  * @chan: DMA channel
  * @cookie: transaction identifier to check status of
  * @last: returns last completed cookie, can be NULL
  * @used: returns last issued cookie, can be NULL
  *
  * If @last and @used are passed in, upon return they reflect the driver
  * internal state and can be used with dma_async_is_complete() to check
  * the status of multiple cookies without re-checking hardware state.
  */
 static inline enum dma_status dma_async_is_tx_complete(struct dma_chan *chan,
 	dma_cookie_t cookie, dma_cookie_t *last, dma_cookie_t *used)
 {
 	struct dma_tx_state state;
 	enum dma_status status;
 	status = chan->device->device_tx_status(chan, cookie, &state);
 	if (last)
 		*last = state.last;
 	if (used)
 		*used = state.used;
 	return status;
 }
 #define dma_async_memcpy_complete(chan, cookie, last, used)\
 	dma_async_is_tx_complete(chan, cookie, last, used)
 /**
  * dma_async_is_complete - test a cookie against chan state
  * @cookie: transaction identifier to test status of
  * @last_complete: last know completed transaction
  * @last_used: last cookie value handed out
  *
  * dma_async_is_complete() is used in dma_async_memcpy_complete()
  * the test logic is separated for lightweight testing of multiple cookies
  */
 static inline enum dma_status dma_async_is_complete(dma_cookie_t cookie,
 			dma_cookie_t last_complete, dma_cookie_t last_used)
 {
 	if (last_complete <= last_used) {
 		if ((cookie <= last_complete) || (cookie > last_used))
 			return DMA_SUCCESS;
 	} else {
 		if ((cookie <= last_complete) && (cookie > last_used))
 			return DMA_SUCCESS;
 	}
 	return DMA_IN_PROGRESS;
 }
 static inline void
 dma_set_tx_state(struct dma_tx_state *st, dma_cookie_t last, dma_cookie_t used, u32 residue)
 {
 	if (st) {
 		st->last = last;
 		st->used = used;
 		st->residue = residue;
 	}
 }
 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie);
 #ifdef CONFIG_DMA_ENGINE
 enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx);
 void dma_issue_pending_all(void);
 #else
 static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 {
 	return DMA_SUCCESS;
 }
 static inline void dma_issue_pending_all(void)
 {
 	do { } while (0);
 }
 #endif
 /* --- DMA device --- */
 int dma_async_device_register(struct dma_device *device);
 void dma_async_device_unregister(struct dma_device *device);
 void dma_run_dependencies(struct dma_async_tx_descriptor *tx);
 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type);
 #define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y)
 struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param);
 void dma_release_channel(struct dma_chan *chan);
 /* --- Helper iov-locking functions --- */
 struct dma_page_list {
 	char __user *base_address;
 	int nr_pages;
 	struct page **pages;
 };
 struct dma_pinned_list {
 	int nr_iovecs;
 	struct dma_page_list page_list[0];
 };
 struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len);
 void dma_unpin_iovec_pages(struct dma_pinned_list* pinned_list);
 dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
 	struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len);
 dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
 	struct dma_pinned_list *pinned_list, struct page *page,
 	unsigned int offset, size_t len);
 #endif /* DMAENGINE_H */

sound/soc/txx9/txx9aclc.c

Diff comments View file @ 0582763

 /*
  * Generic TXx9 ACLC platform driver
  *
  * Copyright (C) 2009 Atsushi Nemoto
  *
  * Based on RBTX49xx patch from CELF patch archive.
  * (C) Copyright TOSHIBA CORPORATION 2004-2006
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/pcm_params.h>
 #include <sound/soc.h>
 #include "txx9aclc.h"
 static const struct snd_pcm_hardware txx9aclc_pcm_hardware = {
 	/*
 	 * REVISIT: SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID
 	 * needs more works for noncoherent MIPS.
 	 */
 	.info		  = SNDRV_PCM_INFO_INTERLEAVED |
 			    SNDRV_PCM_INFO_BATCH |
 			    SNDRV_PCM_INFO_PAUSE,
 #ifdef __BIG_ENDIAN
 	.formats	  = SNDRV_PCM_FMTBIT_S16_BE,
 #else
 	.formats	  = SNDRV_PCM_FMTBIT_S16_LE,
 #endif
 	.period_bytes_min = 1024,
 	.period_bytes_max = 8 * 1024,
 	.periods_min	  = 2,
 	.periods_max	  = 4096,
 	.buffer_bytes_max = 32 * 1024,
 };
 static int txx9aclc_pcm_hw_params(struct snd_pcm_substream *substream,
 				  struct snd_pcm_hw_params *params)
 {
 	struct snd_soc_pcm_runtime *rtd = snd_pcm_substream_chip(substream);
 	struct snd_soc_device *socdev = rtd->socdev;
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct txx9aclc_dmadata *dmadata = runtime->private_data;
 	int ret;
 	ret = snd_pcm_lib_malloc_pages(substream, params_buffer_bytes(params));
 	if (ret < 0)
 		return ret;
 	dev_dbg(socdev->dev,
 		"runtime->dma_area = %#lx dma_addr = %#lx dma_bytes = %zd "
 		"runtime->min_align %ld\n",
 		(unsigned long)runtime->dma_area,
 		(unsigned long)runtime->dma_addr, runtime->dma_bytes,
 		runtime->min_align);
 	dev_dbg(socdev->dev,
 		"periods %d period_bytes %d stream %d\n",
 		params_periods(params), params_period_bytes(params),
 		substream->stream);
 	dmadata->substream = substream;
 	dmadata->pos = 0;
 	return 0;
 }
 static int txx9aclc_pcm_hw_free(struct snd_pcm_substream *substream)
 {
 	return snd_pcm_lib_free_pages(substream);
 }
 static int txx9aclc_pcm_prepare(struct snd_pcm_substream *substream)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct txx9aclc_dmadata *dmadata = runtime->private_data;
 	dmadata->dma_addr = runtime->dma_addr;
 	dmadata->buffer_bytes = snd_pcm_lib_buffer_bytes(substream);
 	dmadata->period_bytes = snd_pcm_lib_period_bytes(substream);
 	if (dmadata->buffer_bytes == dmadata->period_bytes) {
 		dmadata->frag_bytes = dmadata->period_bytes >> 1;
 		dmadata->frags = 2;
 	} else {
 		dmadata->frag_bytes = dmadata->period_bytes;
 		dmadata->frags = dmadata->buffer_bytes / dmadata->period_bytes;
 	}
 	dmadata->frag_count = 0;
 	dmadata->pos = 0;
 	return 0;
 }
 static void txx9aclc_dma_complete(void *arg)
 {
 	struct txx9aclc_dmadata *dmadata = arg;
 	unsigned long flags;
 	/* dma completion handler cannot submit new operations */
 	spin_lock_irqsave(&dmadata->dma_lock, flags);
 	if (dmadata->frag_count >= 0) {
 		dmadata->dmacount--;
 		BUG_ON(dmadata->dmacount < 0);
 		tasklet_schedule(&dmadata->tasklet);
 	}
 	spin_unlock_irqrestore(&dmadata->dma_lock, flags);
 }
 static struct dma_async_tx_descriptor *
 txx9aclc_dma_submit(struct txx9aclc_dmadata *dmadata, dma_addr_t buf_dma_addr)
 {
 	struct dma_chan *chan = dmadata->dma_chan;
 	struct dma_async_tx_descriptor *desc;
 	struct scatterlist sg;
 	sg_init_table(&sg, 1);
 	sg_set_page(&sg, pfn_to_page(PFN_DOWN(buf_dma_addr)),
 		    dmadata->frag_bytes, buf_dma_addr & (PAGE_SIZE - 1));
 	sg_dma_address(&sg) = buf_dma_addr;
 	desc = chan->device->device_prep_slave_sg(chan, &sg, 1,
 		dmadata->substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
 		DMA_TO_DEVICE : DMA_FROM_DEVICE,
 		DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!desc) {
 		dev_err(&chan->dev->device, "cannot prepare slave dma\n");
 		return NULL;
 	}
 	desc->callback = txx9aclc_dma_complete;
 	desc->callback_param = dmadata;
 	desc->tx_submit(desc);
 	return desc;
 }
 #define NR_DMA_CHAIN		2
 static void txx9aclc_dma_tasklet(unsigned long data)
 {
 	struct txx9aclc_dmadata *dmadata = (struct txx9aclc_dmadata *)data;
 	struct dma_chan *chan = dmadata->dma_chan;
 	struct dma_async_tx_descriptor *desc;
 	struct snd_pcm_substream *substream = dmadata->substream;
 	u32 ctlbit = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
 		ACCTL_AUDODMA : ACCTL_AUDIDMA;
 	int i;
 	unsigned long flags;
 	spin_lock_irqsave(&dmadata->dma_lock, flags);
 	if (dmadata->frag_count < 0) {
 		struct txx9aclc_soc_device *dev =
 			container_of(dmadata, struct txx9aclc_soc_device,
 				     dmadata[substream->stream]);
 		struct txx9aclc_plat_drvdata *drvdata =
 			txx9aclc_get_plat_drvdata(dev);
 		void __iomem *base = drvdata->base;
 		spin_unlock_irqrestore(&dmadata->dma_lock, flags);
-		chan->device->device_control(chan, DMA_TERMINATE_ALL);
+		chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
 		/* first time */
 		for (i = 0; i < NR_DMA_CHAIN; i++) {
 			desc = txx9aclc_dma_submit(dmadata,
 				dmadata->dma_addr + i * dmadata->frag_bytes);
 			if (!desc)
 				return;
 		}
 		dmadata->dmacount = NR_DMA_CHAIN;
 		chan->device->device_issue_pending(chan);
 		spin_lock_irqsave(&dmadata->dma_lock, flags);
 		__raw_writel(ctlbit, base + ACCTLEN);
 		dmadata->frag_count = NR_DMA_CHAIN % dmadata->frags;
 		spin_unlock_irqrestore(&dmadata->dma_lock, flags);
 		return;
 	}
 	BUG_ON(dmadata->dmacount >= NR_DMA_CHAIN);
 	while (dmadata->dmacount < NR_DMA_CHAIN) {
 		dmadata->dmacount++;
 		spin_unlock_irqrestore(&dmadata->dma_lock, flags);
 		desc = txx9aclc_dma_submit(dmadata,
 			dmadata->dma_addr +
 			dmadata->frag_count * dmadata->frag_bytes);
 		if (!desc)
 			return;
 		chan->device->device_issue_pending(chan);
 		spin_lock_irqsave(&dmadata->dma_lock, flags);
 		dmadata->frag_count++;
 		dmadata->frag_count %= dmadata->frags;
 		dmadata->pos += dmadata->frag_bytes;
 		dmadata->pos %= dmadata->buffer_bytes;
 		if ((dmadata->frag_count * dmadata->frag_bytes) %
 		    dmadata->period_bytes == 0)
 			snd_pcm_period_elapsed(substream);
 	}
 	spin_unlock_irqrestore(&dmadata->dma_lock, flags);
 }
 static int txx9aclc_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
 {
 	struct txx9aclc_dmadata *dmadata = substream->runtime->private_data;
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
 	struct txx9aclc_soc_device *dev =
 		container_of(rtd->socdev, struct txx9aclc_soc_device, soc_dev);
 	struct txx9aclc_plat_drvdata *drvdata = txx9aclc_get_plat_drvdata(dev);
 	void __iomem *base = drvdata->base;
 	unsigned long flags;
 	int ret = 0;
 	u32 ctlbit = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
 		ACCTL_AUDODMA : ACCTL_AUDIDMA;
 	spin_lock_irqsave(&dmadata->dma_lock, flags);
 	switch (cmd) {
 	case SNDRV_PCM_TRIGGER_START:
 		dmadata->frag_count = -1;
 		tasklet_schedule(&dmadata->tasklet);
 		break;
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 		__raw_writel(ctlbit, base + ACCTLDIS);
 		break;
 	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
 	case SNDRV_PCM_TRIGGER_RESUME:
 		__raw_writel(ctlbit, base + ACCTLEN);
 		break;
 	default:
 		ret = -EINVAL;
 	}
 	spin_unlock_irqrestore(&dmadata->dma_lock, flags);
 	return ret;
 }
 static snd_pcm_uframes_t
 txx9aclc_pcm_pointer(struct snd_pcm_substream *substream)
 {
 	struct txx9aclc_dmadata *dmadata = substream->runtime->private_data;
 	return bytes_to_frames(substream->runtime, dmadata->pos);
 }
 static int txx9aclc_pcm_open(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
 	struct txx9aclc_soc_device *dev =
 		container_of(rtd->socdev, struct txx9aclc_soc_device, soc_dev);
 	struct txx9aclc_dmadata *dmadata = &dev->dmadata[substream->stream];
 	int ret;
 	ret = snd_soc_set_runtime_hwparams(substream, &txx9aclc_pcm_hardware);
 	if (ret)
 		return ret;
 	/* ensure that buffer size is a multiple of period size */
 	ret = snd_pcm_hw_constraint_integer(substream->runtime,
 					    SNDRV_PCM_HW_PARAM_PERIODS);
 	if (ret < 0)
 		return ret;
 	substream->runtime->private_data = dmadata;
 	return 0;
 }
 static int txx9aclc_pcm_close(struct snd_pcm_substream *substream)
 {
 	struct txx9aclc_dmadata *dmadata = substream->runtime->private_data;
 	struct dma_chan *chan = dmadata->dma_chan;
 	dmadata->frag_count = -1;
-	chan->device->device_control(chan, DMA_TERMINATE_ALL);
+	chan->device->device_control(chan, DMA_TERMINATE_ALL, 0);
 	return 0;
 }
 static struct snd_pcm_ops txx9aclc_pcm_ops = {
 	.open		= txx9aclc_pcm_open,
 	.close		= txx9aclc_pcm_close,
 	.ioctl		= snd_pcm_lib_ioctl,
 	.hw_params	= txx9aclc_pcm_hw_params,
 	.hw_free	= txx9aclc_pcm_hw_free,
 	.prepare	= txx9aclc_pcm_prepare,
 	.trigger	= txx9aclc_pcm_trigger,
 	.pointer	= txx9aclc_pcm_pointer,
 };
 static void txx9aclc_pcm_free_dma_buffers(struct snd_pcm *pcm)
 {
 	snd_pcm_lib_preallocate_free_for_all(pcm);
 }
 static int txx9aclc_pcm_new(struct snd_card *card, struct snd_soc_dai *dai,
 			    struct snd_pcm *pcm)
 {
 	return snd_pcm_lib_preallocate_pages_for_all(pcm, SNDRV_DMA_TYPE_DEV,
 		card->dev, 64 * 1024, 4 * 1024 * 1024);
 }
 static bool filter(struct dma_chan *chan, void *param)
 {
 	struct txx9aclc_dmadata *dmadata = param;
 	char *devname;
 	bool found = false;
 	devname = kasprintf(GFP_KERNEL, "%s.%d", dmadata->dma_res->name,
 		(int)dmadata->dma_res->start);
 	if (strcmp(dev_name(chan->device->dev), devname) == 0) {
 		chan->private = &dmadata->dma_slave;
 		found = true;
 	}
 	kfree(devname);
 	return found;
 }
 static int txx9aclc_dma_init(struct txx9aclc_soc_device *dev,
 			     struct txx9aclc_dmadata *dmadata)
 {
 	struct txx9aclc_plat_drvdata *drvdata = txx9aclc_get_plat_drvdata(dev);
 	struct txx9dmac_slave *ds = &dmadata->dma_slave;
 	dma_cap_mask_t mask;
 	spin_lock_init(&dmadata->dma_lock);
 	ds->reg_width = sizeof(u32);
 	if (dmadata->stream == SNDRV_PCM_STREAM_PLAYBACK) {
 		ds->tx_reg = drvdata->physbase + ACAUDODAT;
 		ds->rx_reg = 0;
 	} else {
 		ds->tx_reg = 0;
 		ds->rx_reg = drvdata->physbase + ACAUDIDAT;
 	}
 	/* Try to grab a DMA channel */
 	dma_cap_zero(mask);
 	dma_cap_set(DMA_SLAVE, mask);
 	dmadata->dma_chan = dma_request_channel(mask, filter, dmadata);
 	if (!dmadata->dma_chan) {
 		dev_err(dev->soc_dev.dev,
 			"DMA channel for %s is not available\n",
 			dmadata->stream == SNDRV_PCM_STREAM_PLAYBACK ?
 			"playback" : "capture");
 		return -EBUSY;
 	}
 	tasklet_init(&dmadata->tasklet, txx9aclc_dma_tasklet,
 		     (unsigned long)dmadata);
 	return 0;
 }
 static int txx9aclc_pcm_probe(struct platform_device *pdev)
 {
 	struct snd_soc_device *socdev = platform_get_drvdata(pdev);
 	struct txx9aclc_soc_device *dev =
 		container_of(socdev, struct txx9aclc_soc_device, soc_dev);
 	struct resource *r;
 	int i;
 	int ret;
 	dev->dmadata[0].stream = SNDRV_PCM_STREAM_PLAYBACK;
 	dev->dmadata[1].stream = SNDRV_PCM_STREAM_CAPTURE;
 	for (i = 0; i < 2; i++) {
 		r = platform_get_resource(dev->aclc_pdev, IORESOURCE_DMA, i);
 		if (!r) {
 			ret = -EBUSY;
 			goto exit;
 		}
 		dev->dmadata[i].dma_res = r;
 		ret = txx9aclc_dma_init(dev, &dev->dmadata[i]);
 		if (ret)
 			goto exit;
 	}
 	return 0;
 exit:
 	for (i = 0; i < 2; i++) {
 		if (dev->dmadata[i].dma_chan)
 			dma_release_channel(dev->dmadata[i].dma_chan);
 		dev->dmadata[i].dma_chan = NULL;
 	}
 	return ret;
 }
 static int txx9aclc_pcm_remove(struct platform_device *pdev)
 {
 	struct snd_soc_device *socdev = platform_get_drvdata(pdev);
 	struct txx9aclc_soc_device *dev =
 		container_of(socdev, struct txx9aclc_soc_device, soc_dev);
 	struct txx9aclc_plat_drvdata *drvdata = txx9aclc_get_plat_drvdata(dev);
 	void __iomem *base = drvdata->base;
 	int i;
 	/* disable all FIFO DMAs */
 	__raw_writel(ACCTL_AUDODMA | ACCTL_AUDIDMA, base + ACCTLDIS);
 	/* dummy R/W to clear pending DMAREQ if any */
 	__raw_writel(__raw_readl(base + ACAUDIDAT), base + ACAUDODAT);
 	for (i = 0; i < 2; i++) {
 		struct txx9aclc_dmadata *dmadata = &dev->dmadata[i];
 		struct dma_chan *chan = dmadata->dma_chan;
 		if (chan) {
 			dmadata->frag_count = -1;
-			chan->device->device_control(chan, DMA_TERMINATE_ALL);
+			chan->device->device_control(chan,
+						     DMA_TERMINATE_ALL, 0);
 			dma_release_channel(chan);
 		}
 		dev->dmadata[i].dma_chan = NULL;
 	}
 	return 0;
 }
 struct snd_soc_platform txx9aclc_soc_platform = {
 	.name		= "txx9aclc-audio",
 	.probe		= txx9aclc_pcm_probe,
 	.remove		= txx9aclc_pcm_remove,
 	.pcm_ops 	= &txx9aclc_pcm_ops,
 	.pcm_new	= txx9aclc_pcm_new,
 	.pcm_free	= txx9aclc_pcm_free_dma_buffers,
 };
 EXPORT_SYMBOL_GPL(txx9aclc_soc_platform);
 static int __init txx9aclc_soc_platform_init(void)
 {
 	return snd_soc_register_platform(&txx9aclc_soc_platform);
 }
 static void __exit txx9aclc_soc_platform_exit(void)
 {
 	snd_soc_unregister_platform(&txx9aclc_soc_platform);
 }
 module_init(txx9aclc_soc_platform_init);
 module_exit(txx9aclc_soc_platform_exit);
 MODULE_AUTHOR("Atsushi Nemoto <anemo@mba.ocn.ne.jp>");
 MODULE_DESCRIPTION("TXx9 ACLC Audio DMA driver");
 MODULE_LICENSE("GPL");