// SPDX-License-Identifier: GPL-2.0+
  //
  // drivers/dma/imx-sdma.c
  //
  // This file contains a driver for the Freescale Smart DMA engine
  //
  // Copyright 2010 Sascha Hauer, Pengutronix <s.hauer@pengutronix.de>
  //
  // Based on code from Freescale:
  //

  // Copyright 2004-2016 Freescale Semiconductor, Inc. All Rights Reserved.
  // Copyright 2018 NXP.

  
  #include <linux/init.h>

  #include <linux/iopoll.h>

  #include <linux/module.h>

  #include <linux/types.h>

  #include <linux/bitops.h>

  #include <linux/mm.h>
  #include <linux/interrupt.h>
  #include <linux/clk.h>

  #include <linux/delay.h>

  #include <linux/sched.h>
  #include <linux/semaphore.h>
  #include <linux/spinlock.h>
  #include <linux/device.h>

  #include <linux/genalloc.h>

  #include <linux/dma-mapping.h>
  #include <linux/firmware.h>
  #include <linux/slab.h>
  #include <linux/platform_device.h>

  #include <linux/pm_runtime.h>

  #include <linux/dmaengine.h>

  #include <linux/of.h>

  #include <linux/of_address.h>

  #include <linux/of_device.h>

  #include <linux/of_dma.h>

  #include <linux/workqueue.h>

  
  #include <asm/irq.h>

  #include <linux/platform_data/dma-imx-sdma.h>
  #include <linux/platform_data/dma-imx.h>

  #include <linux/regmap.h>
  #include <linux/mfd/syscon.h>
  #include <linux/mfd/syscon/imx6q-iomuxc-gpr.h>

  #include "dmaengine.h"

  #include "virt-dma.h"

  /* SDMA registers */
  #define SDMA_H_C0PTR		0x000
  #define SDMA_H_INTR		0x004
  #define SDMA_H_STATSTOP		0x008
  #define SDMA_H_START		0x00c
  #define SDMA_H_EVTOVR		0x010
  #define SDMA_H_DSPOVR		0x014
  #define SDMA_H_HOSTOVR		0x018
  #define SDMA_H_EVTPEND		0x01c
  #define SDMA_H_DSPENBL		0x020
  #define SDMA_H_RESET		0x024
  #define SDMA_H_EVTERR		0x028
  #define SDMA_H_INTRMSK		0x02c
  #define SDMA_H_PSW		0x030
  #define SDMA_H_EVTERRDBG	0x034
  #define SDMA_H_CONFIG		0x038
  #define SDMA_ONCE_ENB		0x040
  #define SDMA_ONCE_DATA		0x044
  #define SDMA_ONCE_INSTR		0x048
  #define SDMA_ONCE_STAT		0x04c
  #define SDMA_ONCE_CMD		0x050
  #define SDMA_EVT_MIRROR		0x054
  #define SDMA_ILLINSTADDR	0x058
  #define SDMA_CHN0ADDR		0x05c
  #define SDMA_ONCE_RTB		0x060
  #define SDMA_XTRIG_CONF1	0x070
  #define SDMA_XTRIG_CONF2	0x074

  #define SDMA_CHNENBL0_IMX35	0x200
  #define SDMA_CHNENBL0_IMX31	0x080

  #define SDMA_CHNPRI_0		0x100

  #define SDMA_DONE0_CONFIG	0x1000
  #define SDMA_DONE0_CONFIG_DONE_SEL	0x7
  #define SDMA_DONE0_CONFIG_DONE_DIS	0x6

  
  /*
   * Buffer descriptor status values.
   */
  #define BD_DONE  0x01
  #define BD_WRAP  0x02
  #define BD_CONT  0x04
  #define BD_INTR  0x08
  #define BD_RROR  0x10
  #define BD_LAST  0x20
  #define BD_EXTD  0x80
  
  /*
   * Data Node descriptor status values.
   */
  #define DND_END_OF_FRAME  0x80
  #define DND_END_OF_XFER   0x40
  #define DND_DONE          0x20
  #define DND_UNUSED        0x01
  
  /*
   * IPCV2 descriptor status values.
   */
  #define BD_IPCV2_END_OF_FRAME  0x40
  
  #define IPCV2_MAX_NODES        50
  /*
   * Error bit set in the CCB status field by the SDMA,
   * in setbd routine, in case of a transfer error
   */
  #define DATA_ERROR  0x10000000
  
  /*
   * Buffer descriptor commands.
   */
  #define C0_ADDR             0x01
  #define C0_LOAD             0x02
  #define C0_DUMP             0x03
  #define C0_SETCTX           0x07
  #define C0_GETCTX           0x03
  #define C0_SETDM            0x01
  #define C0_SETPM            0x04
  #define C0_GETDM            0x02
  #define C0_GETPM            0x08
  /*
   * Change endianness indicator in the BD command field
   */
  #define CHANGE_ENDIANNESS   0x80
  
  /*

   *  p_2_p watermark_level description
   *	Bits		Name			Description
   *	0-7		Lower WML		Lower watermark level
   *	8		PS			1: Pad Swallowing
   *						0: No Pad Swallowing
   *	9		PA			1: Pad Adding
   *						0: No Pad Adding
   *	10		SPDIF			If this bit is set both source
   *						and destination are on SPBA
   *	11		Source Bit(SP)		1: Source on SPBA
   *						0: Source on AIPS
   *	12		Destination Bit(DP)	1: Destination on SPBA
   *						0: Destination on AIPS
   *	13-15		---------		MUST BE 0
   *	16-23		Higher WML		HWML
   *	24-27		N			Total number of samples after
   *						which Pad adding/Swallowing
   *						must be done. It must be odd.
   *	28		Lower WML Event(LWE)	SDMA events reg to check for
   *						LWML event mask
   *						0: LWE in EVENTS register
   *						1: LWE in EVENTS2 register
   *	29		Higher WML Event(HWE)	SDMA events reg to check for
   *						HWML event mask
   *						0: HWE in EVENTS register
   *						1: HWE in EVENTS2 register
   *	30		---------		MUST BE 0
   *	31		CONT			1: Amount of samples to be
   *						transferred is unknown and
   *						script will keep on
   *						transferring samples as long as
   *						both events are detected and
   *						script must be manually stopped
   *						by the application
   *						0: The amount of samples to be
   *						transferred is equal to the
   *						count field of mode word
   */
  #define SDMA_WATERMARK_LEVEL_LWML	0xFF
  #define SDMA_WATERMARK_LEVEL_PS		BIT(8)
  #define SDMA_WATERMARK_LEVEL_PA		BIT(9)
  #define SDMA_WATERMARK_LEVEL_SPDIF	BIT(10)
  #define SDMA_WATERMARK_LEVEL_SP		BIT(11)
  #define SDMA_WATERMARK_LEVEL_DP		BIT(12)

  #define SDMA_WATERMARK_LEVEL_SD		BIT(13)
  #define SDMA_WATERMARK_LEVEL_DD		BIT(14)

  #define SDMA_WATERMARK_LEVEL_HWML	(0xFF << 16)
  #define SDMA_WATERMARK_LEVEL_LWE	BIT(28)
  #define SDMA_WATERMARK_LEVEL_HWE	BIT(29)
  #define SDMA_WATERMARK_LEVEL_CONT	BIT(31)

  #define SDMA_DMA_BUSWIDTHS	(BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
  				 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \

  				 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \

  				 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
  
  #define SDMA_DMA_DIRECTIONS	(BIT(DMA_DEV_TO_MEM) | \
  				 BIT(DMA_MEM_TO_DEV) | \
  				 BIT(DMA_DEV_TO_DEV))

  #define SDMA_WATERMARK_LEVEL_FIFOS_OFF	12
  #define SDMA_WATERMARK_LEVEL_SW_DONE	BIT(23)
  #define SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF 24

  /*

   * Mode/Count of data node descriptors - IPCv2
   */
  struct sdma_mode_count {

  #define SDMA_BD_MAX_CNT	0xffff

  	u32 count   : 16; /* size of the buffer pointed by this BD */
  	u32 status  :  8; /* E,R,I,C,W,D status bits stored here */

  	u32 command :  8; /* command mostly used for channel 0 */

  };
  
  /*
   * Buffer descriptor
   */
  struct sdma_buffer_descriptor {
  	struct sdma_mode_count  mode;
  	u32 buffer_addr;	/* address of the buffer described */
  	u32 ext_buffer_addr;	/* extended buffer address */
  } __attribute__ ((packed));
  
  /**
   * struct sdma_channel_control - Channel control Block
   *

   * @current_bd_ptr:	current buffer descriptor processed
   * @base_bd_ptr:	first element of buffer descriptor array
   * @unused:		padding. The SDMA engine expects an array of 128 byte

   *			control blocks
   */
  struct sdma_channel_control {
  	u32 current_bd_ptr;
  	u32 base_bd_ptr;
  	u32 unused[2];
  } __attribute__ ((packed));
  
  /**
   * struct sdma_state_registers - SDMA context for a channel
   *
   * @pc:		program counter

   * @unused1:	unused

   * @t:		test bit: status of arithmetic & test instruction
   * @rpc:	return program counter

   * @unused0:	unused

   * @sf:		source fault while loading data
   * @spc:	loop start program counter

   * @unused2:	unused

   * @df:		destination fault while storing data
   * @epc:	loop end program counter
   * @lm:		loop mode
   */
  struct sdma_state_registers {
  	u32 pc     :14;
  	u32 unused1: 1;
  	u32 t      : 1;
  	u32 rpc    :14;
  	u32 unused0: 1;
  	u32 sf     : 1;
  	u32 spc    :14;
  	u32 unused2: 1;
  	u32 df     : 1;
  	u32 epc    :14;
  	u32 lm     : 2;
  } __attribute__ ((packed));
  
  /**
   * struct sdma_context_data - sdma context specific to a channel
   *
   * @channel_state:	channel state bits
   * @gReg:		general registers
   * @mda:		burst dma destination address register
   * @msa:		burst dma source address register
   * @ms:			burst dma status register
   * @md:			burst dma data register
   * @pda:		peripheral dma destination address register
   * @psa:		peripheral dma source address register
   * @ps:			peripheral dma status register
   * @pd:			peripheral dma data register
   * @ca:			CRC polynomial register
   * @cs:			CRC accumulator register
   * @dda:		dedicated core destination address register
   * @dsa:		dedicated core source address register
   * @ds:			dedicated core status register
   * @dd:			dedicated core data register

   * @scratch0:		1st word of dedicated ram for context switch
   * @scratch1:		2nd word of dedicated ram for context switch
   * @scratch2:		3rd word of dedicated ram for context switch
   * @scratch3:		4th word of dedicated ram for context switch
   * @scratch4:		5th word of dedicated ram for context switch
   * @scratch5:		6th word of dedicated ram for context switch
   * @scratch6:		7th word of dedicated ram for context switch
   * @scratch7:		8th word of dedicated ram for context switch

   */
  struct sdma_context_data {
  	struct sdma_state_registers  channel_state;
  	u32  gReg[8];
  	u32  mda;
  	u32  msa;
  	u32  ms;
  	u32  md;
  	u32  pda;
  	u32  psa;
  	u32  ps;
  	u32  pd;
  	u32  ca;
  	u32  cs;
  	u32  dda;
  	u32  dsa;
  	u32  ds;
  	u32  dd;
  	u32  scratch0;
  	u32  scratch1;
  	u32  scratch2;
  	u32  scratch3;
  	u32  scratch4;
  	u32  scratch5;
  	u32  scratch6;
  	u32  scratch7;
  } __attribute__ ((packed));

  
  struct sdma_engine;
  
  /**

   * struct sdma_desc - descriptor structor for one transfer

   * @vd:			descriptor for virt dma
   * @num_bd:		number of descriptors currently handling
   * @bd_phys:		physical address of bd
   * @buf_tail:		ID of the buffer that was processed
   * @buf_ptail:		ID of the previous buffer that was processed
   * @period_len:		period length, used in cyclic.
   * @chn_real_count:	the real count updated from bd->mode.count
   * @chn_count:		the transfer count set
   * @sdmac:		sdma_channel pointer
   * @bd:			pointer of allocate bd

   */
  struct sdma_desc {

  	struct virt_dma_desc	vd;

  	unsigned int		num_bd;
  	dma_addr_t		bd_phys;
  	unsigned int		buf_tail;
  	unsigned int		buf_ptail;
  	unsigned int		period_len;
  	unsigned int		chn_real_count;
  	unsigned int		chn_count;
  	struct sdma_channel	*sdmac;
  	struct sdma_buffer_descriptor *bd;
  };
  
  /**

   * struct sdma_channel - housekeeping for a SDMA channel
   *

   * @vc:			virt_dma base structure
   * @desc:		sdma description including vd and other special member
   * @sdma:		pointer to the SDMA engine for this channel
   * @channel:		the channel number, matches dmaengine chan_id + 1
   * @direction:		transfer type. Needed for setting SDMA script

   * @slave_config:	Slave configuration

   * @peripheral_type:	Peripheral type. Needed for setting SDMA script
   * @event_id0:		aka dma request line
   * @event_id1:		for channels that use 2 events
   * @word_size:		peripheral access size
   * @pc_from_device:	script address for those device_2_memory
   * @pc_to_device:	script address for those memory_2_device
   * @device_to_device:	script address for those device_2_device

   * @pc_to_pc:		script address for those memory_2_memory

   * @flags:		loop mode or not
   * @per_address:	peripheral source or destination address in common case
   *                      destination address in p_2_p case
   * @per_address2:	peripheral source address in p_2_p case
   * @event_mask:		event mask used in p_2_p script
   * @watermark_level:	value for gReg[7], some script will extend it from
   *			basic watermark such as p_2_p
   * @shp_addr:		value for gReg[6]
   * @per_addr:		value for gReg[2]
   * @status:		status of dma channel

   * @context_loaded:	ensure context is only loaded once

   * @data:		specific sdma interface structure
   * @bd_pool:		dma_pool for bd

   * @terminate_worker:	used to call back into terminate work function

   */
  struct sdma_channel {

  	struct virt_dma_chan		vc;

  	struct sdma_desc		*desc;

  	struct sdma_engine		*sdma;
  	unsigned int			channel;

  	enum dma_transfer_direction		direction;

  	struct dma_slave_config		slave_config;

  	enum sdma_peripheral_type	peripheral_type;
  	unsigned int			event_id0;
  	unsigned int			event_id1;
  	enum dma_slave_buswidth		word_size;

  	unsigned int			pc_from_device, pc_to_device;

  	unsigned int			device_to_device;

  	unsigned int                    pc_to_pc;

  	unsigned long			flags;

  	dma_addr_t			per_address, per_address2;

  	unsigned long			event_mask[2];
  	unsigned long			watermark_level;

  	u32				shp_addr, per_addr;

  	enum dma_status			status;

  	struct imx_dma_data		data;

  	struct work_struct		terminate_worker;

  	struct list_head                terminated;

  	bool				is_ram_script;

  	bool				src_dualfifo;
  	bool				dst_dualfifo;

  	unsigned int			fifo_num;
  	bool				sw_done;
  	u32				sw_done_sel;

  };

  #define IMX_DMA_SG_LOOP		BIT(0)

  
  #define MAX_DMA_CHANNELS 32
  #define MXC_SDMA_DEFAULT_PRIORITY 1
  #define MXC_SDMA_MIN_PRIORITY 1
  #define MXC_SDMA_MAX_PRIORITY 7

  /*
   * 0x78(SDMA_XTRIG_CONF2+4)~0x100(SDMA_CHNPRI_O) registers are reserved and
   * can't be accessed. Skip these register touch in suspend/resume. Also below
   * two macros are only used on i.mx6sx.
   */
  #define MXC_SDMA_RESERVED_REG (SDMA_CHNPRI_0 - SDMA_XTRIG_CONF2 - 4)
  #define MXC_SDMA_SAVED_REG_NUM (((SDMA_CHNENBL0_IMX35 + 4 * 48) - \
  				MXC_SDMA_RESERVED_REG) / 4)

  #define SDMA_FIRMWARE_MAGIC 0x414d4453
  
  /**
   * struct sdma_firmware_header - Layout of the firmware image
   *

   * @magic:		"SDMA"
   * @version_major:	increased whenever layout of struct
   *			sdma_script_start_addrs changes.
   * @version_minor:	firmware minor version (for binary compatible changes)
   * @script_addrs_start:	offset of struct sdma_script_start_addrs in this image
   * @num_script_addrs:	Number of script addresses in this image
   * @ram_code_start:	offset of SDMA ram image in this firmware image
   * @ram_code_size:	size of SDMA ram image
   * @script_addrs:	Stores the start address of the SDMA scripts

   *			(in SDMA memory space)
   */
  struct sdma_firmware_header {
  	u32	magic;
  	u32	version_major;
  	u32	version_minor;
  	u32	script_addrs_start;
  	u32	num_script_addrs;
  	u32	ram_code_start;
  	u32	ram_code_size;
  };

  struct sdma_driver_data {
  	int chnenbl0;
  	int num_events;

  	struct sdma_script_start_addrs	*script_addrs;

  	bool check_ratio;

  	/*
  	 * ecspi ERR009165 fixed should be done in sdma script
  	 * and it be fixed in soc from i.mx6ul.
  	 * please get more information from below link:
  	 * https://www.nxp.com/docs/en/errata/IMX6DQCE.pdf
  	 */
  	bool ecspi_fixed;

  	bool has_done0;

  	bool pm_runtime;

  };

  struct sdma_engine {
  	struct device			*dev;
  	struct sdma_channel		channel[MAX_DMA_CHANNELS];
  	struct sdma_channel_control	*channel_control;

  	u32				save_regs[MXC_SDMA_SAVED_REG_NUM];

  	u32				save_done0_regs[2];

  	const char			*fw_name;

  	void __iomem			*regs;

  	struct sdma_context_data	*context;
  	dma_addr_t			context_phys;

  	dma_addr_t			ccb_phys;

  	bool				is_on;

  	struct dma_device		dma_device;

  	struct clk			*clk_ipg;
  	struct clk			*clk_ahb;

  	spinlock_t			channel_0_lock;

  	u32				script_number;

  	struct sdma_script_start_addrs	*script_addrs;

  	const struct sdma_driver_data	*drvdata;

  	u32				spba_start_addr;
  	u32				spba_end_addr;

  	unsigned int			irq;

  	dma_addr_t			bd0_phys;
  	struct sdma_buffer_descriptor	*bd0;

  	/* clock ratio for AHB:SDMA core. 1:1 is 1, 2:1 is 0*/
  	bool				clk_ratio;

  	struct gen_pool			*iram_pool;

  	bool				fw_loaded;

  	u32				fw_fail;

  	unsigned short			ram_code_start;

  };

  static int sdma_config_write(struct dma_chan *chan,
  		       struct dma_slave_config *dmaengine_cfg,
  		       enum dma_transfer_direction direction);

  static int sdma_get_firmware(struct sdma_engine *sdma,
  		const char *fw_name);

  static int sdma_get_firmware_wait(struct sdma_engine *sdma,
  		const char *fw_name);

  static struct sdma_driver_data sdma_imx31 = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX31,
  	.num_events = 32,
  };

  static struct sdma_script_start_addrs sdma_script_imx25 = {
  	.ap_2_ap_addr = 729,
  	.uart_2_mcu_addr = 904,
  	.per_2_app_addr = 1255,
  	.mcu_2_app_addr = 834,
  	.uartsh_2_mcu_addr = 1120,
  	.per_2_shp_addr = 1329,
  	.mcu_2_shp_addr = 1048,
  	.ata_2_mcu_addr = 1560,
  	.mcu_2_ata_addr = 1479,
  	.app_2_per_addr = 1189,
  	.app_2_mcu_addr = 770,
  	.shp_2_per_addr = 1407,
  	.shp_2_mcu_addr = 979,
  };

  static struct sdma_driver_data sdma_imx25 = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx25,
  };

  static struct sdma_driver_data sdma_imx35 = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,

  };

  static struct sdma_script_start_addrs sdma_script_imx51 = {
  	.ap_2_ap_addr = 642,
  	.uart_2_mcu_addr = 817,
  	.mcu_2_app_addr = 747,
  	.mcu_2_shp_addr = 961,
  	.ata_2_mcu_addr = 1473,
  	.mcu_2_ata_addr = 1392,
  	.app_2_per_addr = 1033,
  	.app_2_mcu_addr = 683,
  	.shp_2_per_addr = 1251,
  	.shp_2_mcu_addr = 892,
  };

  static struct sdma_driver_data sdma_imx51 = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx51,
  };
  
  static struct sdma_script_start_addrs sdma_script_imx53 = {
  	.ap_2_ap_addr = 642,
  	.app_2_mcu_addr = 683,
  	.mcu_2_app_addr = 747,
  	.uart_2_mcu_addr = 817,
  	.shp_2_mcu_addr = 891,
  	.mcu_2_shp_addr = 960,
  	.uartsh_2_mcu_addr = 1032,
  	.spdif_2_mcu_addr = 1100,
  	.mcu_2_spdif_addr = 1134,
  	.firi_2_mcu_addr = 1193,
  	.mcu_2_firi_addr = 1290,
  };

  static struct sdma_driver_data sdma_imx53 = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx53,
  };
  
  static struct sdma_script_start_addrs sdma_script_imx6q = {
  	.ap_2_ap_addr = 642,
  	.uart_2_mcu_addr = 817,
  	.mcu_2_app_addr = 747,
  	.per_2_per_addr = 6331,
  	.uartsh_2_mcu_addr = 1032,
  	.mcu_2_shp_addr = 960,
  	.app_2_mcu_addr = 683,
  	.shp_2_mcu_addr = 891,
  	.spdif_2_mcu_addr = 1100,
  	.mcu_2_spdif_addr = 1134,
  };

  static struct sdma_driver_data sdma_imx6q = {

  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx6q,
  };

  static struct sdma_script_start_addrs sdma_script_imx6sx = {
  	.ap_2_ap_addr = 642,
  	.uart_2_mcu_addr = 817,
  	.mcu_2_app_addr = 747,
  	.uartsh_2_mcu_addr = 1032,
  	.mcu_2_shp_addr = 960,
  	.app_2_mcu_addr = 683,
  	.shp_2_mcu_addr = 891,
  	.spdif_2_mcu_addr = 1100,
  	.mcu_2_spdif_addr = 1134,
  };
  
  static struct sdma_driver_data sdma_imx6sx = {
  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx6sx,
  };
  
  static struct sdma_driver_data sdma_imx6ul = {
  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx6sx,
  	.ecspi_fixed = true,
  };

  static struct sdma_script_start_addrs sdma_script_imx7d = {
  	.ap_2_ap_addr = 644,
  	.uart_2_mcu_addr = 819,
  	.mcu_2_app_addr = 749,
  	.uartsh_2_mcu_addr = 1034,
  	.mcu_2_shp_addr = 962,
  	.app_2_mcu_addr = 685,
  	.shp_2_mcu_addr = 893,
  	.spdif_2_mcu_addr = 1102,
  	.mcu_2_spdif_addr = 1136,
  };
  
  static struct sdma_driver_data sdma_imx7d = {
  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx7d,
  };

  static struct sdma_driver_data sdma_imx8mq = {
  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx7d,
  	.check_ratio = 1,
  };

  static struct sdma_driver_data sdma_imx8mp = {
  	.chnenbl0 = SDMA_CHNENBL0_IMX35,
  	.num_events = 48,
  	.script_addrs = &sdma_script_imx7d,
  	.check_ratio = 1,
  	.ecspi_fixed = true,

  	.has_done0 = true,

  	.pm_runtime = true,

  };

  static const struct platform_device_id sdma_devtypes[] = {

  	{

  		.name = "imx25-sdma",
  		.driver_data = (unsigned long)&sdma_imx25,
  	}, {

  		.name = "imx31-sdma",

  		.driver_data = (unsigned long)&sdma_imx31,

  	}, {
  		.name = "imx35-sdma",

  		.driver_data = (unsigned long)&sdma_imx35,

  	}, {

  		.name = "imx51-sdma",
  		.driver_data = (unsigned long)&sdma_imx51,
  	}, {
  		.name = "imx53-sdma",
  		.driver_data = (unsigned long)&sdma_imx53,
  	}, {
  		.name = "imx6q-sdma",
  		.driver_data = (unsigned long)&sdma_imx6q,
  	}, {

  		.name = "imx6sx-sdma",
  		.driver_data = (unsigned long)&sdma_imx6sx,
  	}, {

  		.name = "imx7d-sdma",
  		.driver_data = (unsigned long)&sdma_imx7d,
  	}, {

  		.name = "imx6ul-sdma",
  		.driver_data = (unsigned long)&sdma_imx6ul,
  	}, {

  		.name = "imx8mq-sdma",
  		.driver_data = (unsigned long)&sdma_imx8mq,
  	}, {

  		.name = "imx8mp-sdma",
  		.driver_data = (unsigned long)&sdma_imx8mp,
  	}, {

  		/* sentinel */
  	}
  };
  MODULE_DEVICE_TABLE(platform, sdma_devtypes);

  static const struct of_device_id sdma_dt_ids[] = {

  	{ .compatible = "fsl,imx6q-sdma", .data = &sdma_imx6q, },
  	{ .compatible = "fsl,imx53-sdma", .data = &sdma_imx53, },
  	{ .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, },

  	{ .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, },

  	{ .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, },

  	{ .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, },

  	{ .compatible = "fsl,imx6sx-sdma", .data = &sdma_imx6sx, },

  	{ .compatible = "fsl,imx7d-sdma", .data = &sdma_imx7d, },

  	{ .compatible = "fsl,imx6ul-sdma", .data = &sdma_imx6ul, },

  	{ .compatible = "fsl,imx8mq-sdma", .data = &sdma_imx8mq, },

  	{ .compatible = "fsl,imx8mp-sdma", .data = &sdma_imx8mp, },

  	{ /* sentinel */ }
  };
  MODULE_DEVICE_TABLE(of, sdma_dt_ids);

  #define SDMA_H_CONFIG_DSPDMA	BIT(12) /* indicates if the DSPDMA is used */
  #define SDMA_H_CONFIG_RTD_PINS	BIT(11) /* indicates if Real-Time Debug pins are enabled */
  #define SDMA_H_CONFIG_ACR	BIT(4)  /* indicates if AHB freq /core freq = 2 or 1 */

  #define SDMA_H_CONFIG_CSM	(3)       /* indicates which context switch mode is selected*/
  
  static inline u32 chnenbl_ofs(struct sdma_engine *sdma, unsigned int event)
  {

  	u32 chnenbl0 = sdma->drvdata->chnenbl0;

  	return chnenbl0 + event * 4;
  }
  
  static int sdma_config_ownership(struct sdma_channel *sdmac,
  		bool event_override, bool mcu_override, bool dsp_override)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;

  	unsigned long evt, mcu, dsp;

  
  	if (event_override && mcu_override && dsp_override)
  		return -EINVAL;

  	evt = readl_relaxed(sdma->regs + SDMA_H_EVTOVR);
  	mcu = readl_relaxed(sdma->regs + SDMA_H_HOSTOVR);
  	dsp = readl_relaxed(sdma->regs + SDMA_H_DSPOVR);

  
  	if (dsp_override)

  		__clear_bit(channel, &dsp);

  	else

  		__set_bit(channel, &dsp);

  
  	if (event_override)

  		__clear_bit(channel, &evt);

  	else

  		__set_bit(channel, &evt);

  
  	if (mcu_override)

  		__clear_bit(channel, &mcu);

  	else

  		__set_bit(channel, &mcu);

  	writel_relaxed(evt, sdma->regs + SDMA_H_EVTOVR);
  	writel_relaxed(mcu, sdma->regs + SDMA_H_HOSTOVR);
  	writel_relaxed(dsp, sdma->regs + SDMA_H_DSPOVR);

  
  	return 0;
  }

  static void sdma_enable_channel(struct sdma_engine *sdma, int channel)
  {

  	writel(BIT(channel), sdma->regs + SDMA_H_START);

  }

  /*

   * sdma_run_channel0 - run a channel and wait till it's done

   */

  static int sdma_run_channel0(struct sdma_engine *sdma)

  {

  	int ret;

  	u32 reg;

  	sdma_enable_channel(sdma, 0);

  	ret = readl_relaxed_poll_timeout_atomic(sdma->regs + SDMA_H_STATSTOP,

  						reg, !(reg & 1), 1, 5000);

  	if (ret)

  		dev_err(sdma->dev, "Timeout waiting for CH0 ready
  ");

  	/* Set bits of CONFIG register with dynamic context switching */

  	reg = readl(sdma->regs + SDMA_H_CONFIG);
  	if ((reg & SDMA_H_CONFIG_CSM) == 0) {
  		reg |= SDMA_H_CONFIG_CSM;
  		writel_relaxed(reg, sdma->regs + SDMA_H_CONFIG);
  	}

  	return ret;

  }
  
  static int sdma_load_script(struct sdma_engine *sdma, void *buf, int size,
  		u32 address)
  {

  	struct sdma_buffer_descriptor *bd0 = sdma->bd0;

  	void *buf_virt;
  	dma_addr_t buf_phys;
  	int ret;

  	unsigned long flags;

  	buf_virt = dma_alloc_coherent(sdma->dev, size, &buf_phys,

  					      GFP_KERNEL);
  	if (!buf_virt)

  		return -ENOMEM;

  	spin_lock_irqsave(&sdma->channel_0_lock, flags);

  	bd0->mode.command = C0_SETPM;

  	bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;

  	bd0->mode.count = size / 2;
  	bd0->buffer_addr = buf_phys;
  	bd0->ext_buffer_addr = address;
  
  	memcpy(buf_virt, buf, size);

  	ret = sdma_run_channel0(sdma);

  	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);

  	dma_free_coherent(sdma->dev, size, buf_virt, buf_phys);

  	return ret;
  }
  
  static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;

  	unsigned long val;

  	u32 chnenbl = chnenbl_ofs(sdma, event);

  	val = readl_relaxed(sdma->regs + chnenbl);

  	__set_bit(channel, &val);

  	writel_relaxed(val, sdma->regs + chnenbl);

  
  	/* Set SDMA_DONEx_CONFIG is sw_done enabled */
  	if (sdmac->sw_done) {
  		u32 offset = SDMA_DONE0_CONFIG + sdmac->sw_done_sel / 4;
  		u32 done_sel = SDMA_DONE0_CONFIG_DONE_SEL +
  				((sdmac->sw_done_sel % 4) << 3);
  		u32 sw_done_dis = SDMA_DONE0_CONFIG_DONE_DIS +
  				((sdmac->sw_done_sel % 4) << 3);
  
  		val = readl_relaxed(sdma->regs + offset);
  		__set_bit(done_sel, &val);
  		__clear_bit(sw_done_dis, &val);
  		writel_relaxed(val, sdma->regs + offset);
  	}

  }
  
  static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;
  	u32 chnenbl = chnenbl_ofs(sdma, event);

  	unsigned long val;

  	val = readl_relaxed(sdma->regs + chnenbl);

  	__clear_bit(channel, &val);

  	writel_relaxed(val, sdma->regs + chnenbl);

  }

  static struct sdma_desc *to_sdma_desc(struct dma_async_tx_descriptor *t)
  {
  	return container_of(t, struct sdma_desc, vd.tx);
  }
  
  static void sdma_start_desc(struct sdma_channel *sdmac)
  {
  	struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);
  	struct sdma_desc *desc;
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;
  
  	if (!vd) {
  		sdmac->desc = NULL;
  		return;
  	}
  	sdmac->desc = desc = to_sdma_desc(&vd->tx);

  
  	list_del(&vd->node);

  
  	sdma->channel_control[channel].base_bd_ptr = desc->bd_phys;
  	sdma->channel_control[channel].current_bd_ptr = desc->bd_phys;
  	sdma_enable_channel(sdma, sdmac->channel);
  }

  static void sdma_update_channel_loop(struct sdma_channel *sdmac)
  {

  	struct sdma_buffer_descriptor *bd;

  	int error = 0;
  	enum dma_status	old_status = sdmac->status;

  
  	/*
  	 * loop mode. Iterate over descriptors, re-setup them and
  	 * call callback function.
  	 */

  	while (sdmac->desc) {

  		struct sdma_desc *desc = sdmac->desc;
  
  		bd = &desc->bd[desc->buf_tail];

  
  		if (bd->mode.status & BD_DONE)
  			break;

  		if (bd->mode.status & BD_RROR) {
  			bd->mode.status &= ~BD_RROR;

  			sdmac->status = DMA_ERROR;

  			error = -EIO;
  		}

  	       /*
  		* We use bd->mode.count to calculate the residue, since contains
  		* the number of bytes present in the current buffer descriptor.
  		*/

  		desc->chn_real_count = bd->mode.count;

  		bd->mode.status |= BD_DONE;

  		bd->mode.count = desc->period_len;
  		desc->buf_ptail = desc->buf_tail;
  		desc->buf_tail = (desc->buf_tail + 1) % desc->num_bd;

  
  		/*
  		 * The callback is called from the interrupt context in order
  		 * to reduce latency and to avoid the risk of altering the
  		 * SDMA transaction status by the time the client tasklet is
  		 * executed.
  		 */

  		spin_unlock(&sdmac->vc.lock);
  		dmaengine_desc_get_callback_invoke(&desc->vd.tx, NULL);
  		spin_lock(&sdmac->vc.lock);

  		if (error)
  			sdmac->status = old_status;

  	}
  }

  static void mxc_sdma_handle_channel_normal(struct sdma_channel *data)

  {

  	struct sdma_channel *sdmac = (struct sdma_channel *) data;

  	struct sdma_buffer_descriptor *bd;
  	int i, error = 0;

  	sdmac->desc->chn_real_count = 0;

  	/*
  	 * non loop mode. Iterate over all descriptors, collect
  	 * errors and call callback function
  	 */

  	for (i = 0; i < sdmac->desc->num_bd; i++) {
  		bd = &sdmac->desc->bd[i];

  
  		 if (bd->mode.status & (BD_DONE | BD_RROR))
  			error = -EIO;

  		 sdmac->desc->chn_real_count += bd->mode.count;

  	}
  
  	if (error)
  		sdmac->status = DMA_ERROR;
  	else

  		sdmac->status = DMA_COMPLETE;

  }

  static irqreturn_t sdma_int_handler(int irq, void *dev_id)
  {
  	struct sdma_engine *sdma = dev_id;

  	unsigned long stat;

  	clk_enable(sdma->clk_ipg);
  	clk_enable(sdma->clk_ahb);

  	stat = readl_relaxed(sdma->regs + SDMA_H_INTR);
  	writel_relaxed(stat, sdma->regs + SDMA_H_INTR);

  	/* channel 0 is special and not handled here, see run_channel0() */
  	stat &= ~1;

  
  	while (stat) {
  		int channel = fls(stat) - 1;
  		struct sdma_channel *sdmac = &sdma->channel[channel];

  		struct sdma_desc *desc;
  
  		spin_lock(&sdmac->vc.lock);
  		desc = sdmac->desc;
  		if (desc) {
  			if (sdmac->flags & IMX_DMA_SG_LOOP) {

  				if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
  					sdma_update_channel_loop(sdmac);
  				else
  					vchan_cyclic_callback(&desc->vd);

  			} else {
  				mxc_sdma_handle_channel_normal(sdmac);
  				vchan_cookie_complete(&desc->vd);
  				sdma_start_desc(sdmac);
  			}
  		}

  		spin_unlock(&sdmac->vc.lock);

  		__clear_bit(channel, &stat);

  	}

  	clk_disable(sdma->clk_ipg);
  	clk_disable(sdma->clk_ahb);

  	return IRQ_HANDLED;
  }
  
  /*
   * sets the pc of SDMA script according to the peripheral type
   */
  static void sdma_get_pc(struct sdma_channel *sdmac,
  		enum sdma_peripheral_type peripheral_type)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int per_2_emi = 0, emi_2_per = 0;
  	/*
  	 * These are needed once we start to support transfers between
  	 * two peripherals or memory-to-memory transfers
  	 */

  	int per_2_per = 0, emi_2_emi = 0;

  
  	sdmac->pc_from_device = 0;
  	sdmac->pc_to_device = 0;

  	sdmac->device_to_device = 0;

  	sdmac->pc_to_pc = 0;

  
  	switch (peripheral_type) {
  	case IMX_DMATYPE_MEMORY:

  		emi_2_emi = sdma->script_addrs->ap_2_ap_addr;

  		break;
  	case IMX_DMATYPE_DSP:
  		emi_2_per = sdma->script_addrs->bp_2_ap_addr;
  		per_2_emi = sdma->script_addrs->ap_2_bp_addr;
  		break;
  	case IMX_DMATYPE_FIRI:
  		per_2_emi = sdma->script_addrs->firi_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_firi_addr;
  		break;
  	case IMX_DMATYPE_UART:
  		per_2_emi = sdma->script_addrs->uart_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
  		break;
  	case IMX_DMATYPE_UART_SP:
  		per_2_emi = sdma->script_addrs->uartsh_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  		break;
  	case IMX_DMATYPE_ATA:
  		per_2_emi = sdma->script_addrs->ata_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_ata_addr;
  		break;
  	case IMX_DMATYPE_CSPI:

  		per_2_emi = sdma->script_addrs->app_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_ecspi_addr;

  		sdmac->is_ram_script = true;

  		break;

  	case IMX_DMATYPE_EXT:
  	case IMX_DMATYPE_SSI:

  	case IMX_DMATYPE_SAI:

  		per_2_emi = sdma->script_addrs->app_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_app_addr;
  		break;

  	case IMX_DMATYPE_SSI_DUAL:
  		per_2_emi = sdma->script_addrs->ssish_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_ssish_addr;

  		sdmac->is_ram_script = true;

  		break;

  	case IMX_DMATYPE_SSI_SP:
  	case IMX_DMATYPE_MMC:
  	case IMX_DMATYPE_SDHC:
  	case IMX_DMATYPE_CSPI_SP:
  	case IMX_DMATYPE_ESAI:
  	case IMX_DMATYPE_MSHC_SP:
  		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  		break;
  	case IMX_DMATYPE_ASRC:
  		per_2_emi = sdma->script_addrs->asrc_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->asrc_2_mcu_addr;
  		per_2_per = sdma->script_addrs->per_2_per_addr;

  		sdmac->is_ram_script = true;

  		break;

  	case IMX_DMATYPE_ASRC_SP:
  		per_2_emi = sdma->script_addrs->shp_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_shp_addr;
  		per_2_per = sdma->script_addrs->per_2_per_addr;

  		sdmac->is_ram_script = true;

  		break;

  	case IMX_DMATYPE_MSHC:
  		per_2_emi = sdma->script_addrs->mshc_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_mshc_addr;
  		break;
  	case IMX_DMATYPE_CCM:
  		per_2_emi = sdma->script_addrs->dptc_dvfs_addr;
  		break;
  	case IMX_DMATYPE_SPDIF:
  		per_2_emi = sdma->script_addrs->spdif_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_spdif_addr;
  		break;
  	case IMX_DMATYPE_IPU_MEMORY:
  		emi_2_per = sdma->script_addrs->ext_mem_2_ipu_addr;
  		break;

  	case IMX_DMATYPE_HDMI:
  		emi_2_per = sdma->script_addrs->hdmi_dma_addr;

  		sdmac->is_ram_script = true;

  		break;

  	case IMX_DMATYPE_MULTI_SAI:
  		per_2_emi = sdma->script_addrs->sai_2_mcu_addr;
  		emi_2_per = sdma->script_addrs->mcu_2_sai_addr;

  		sdmac->is_ram_script = true;

  	default:
  		break;
  	}
  
  	sdmac->pc_from_device = per_2_emi;
  	sdmac->pc_to_device = emi_2_per;

  	sdmac->device_to_device = per_2_per;

  	sdmac->pc_to_pc = emi_2_emi;

  }
  
  static int sdma_load_context(struct sdma_channel *sdmac)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;
  	int load_address;
  	struct sdma_context_data *context = sdma->context;

  	struct sdma_buffer_descriptor *bd0 = sdma->bd0;

  	int ret;

  	unsigned long flags;

  	if (sdmac->direction == DMA_DEV_TO_MEM)

  		load_address = sdmac->pc_from_device;

  	else if (sdmac->direction == DMA_DEV_TO_DEV)
  		load_address = sdmac->device_to_device;

  	else if (sdmac->direction == DMA_MEM_TO_MEM)
  		load_address = sdmac->pc_to_pc;

  	else

  		load_address = sdmac->pc_to_device;

  
  	if (load_address < 0)
  		return load_address;
  
  	dev_dbg(sdma->dev, "load_address = %d
  ", load_address);

  	dev_dbg(sdma->dev, "wml = 0x%08x
  ", (u32)sdmac->watermark_level);

  	dev_dbg(sdma->dev, "shp_addr = 0x%08x
  ", sdmac->shp_addr);
  	dev_dbg(sdma->dev, "per_addr = 0x%08x
  ", sdmac->per_addr);

  	dev_dbg(sdma->dev, "event_mask0 = 0x%08x
  ", (u32)sdmac->event_mask[0]);
  	dev_dbg(sdma->dev, "event_mask1 = 0x%08x
  ", (u32)sdmac->event_mask[1]);

  	spin_lock_irqsave(&sdma->channel_0_lock, flags);

  	memset(context, 0, sizeof(*context));
  	context->channel_state.pc = load_address;
  
  	/* Send by context the event mask,base address for peripheral
  	 * and watermark level
  	 */

  	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
  		context->gReg[4] = sdmac->per_addr;
  		context->gReg[6] = sdmac->shp_addr;
  	} else {
  		context->gReg[0] = sdmac->event_mask[1];
  		context->gReg[1] = sdmac->event_mask[0];
  		context->gReg[2] = sdmac->per_addr;
  		context->gReg[6] = sdmac->shp_addr;
  		context->gReg[7] = sdmac->watermark_level;
  	}

  
  	bd0->mode.command = C0_SETDM;

  	bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;

  	bd0->mode.count = sizeof(*context) / 4;
  	bd0->buffer_addr = sdma->context_phys;
  	bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * channel;

  	ret = sdma_run_channel0(sdma);

  	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);

  	return ret;
  }

  static int sdma_save_restore_context(struct sdma_engine *sdma, bool save)
  {
  	struct sdma_context_data *context = sdma->context;
  	struct sdma_buffer_descriptor *bd0 = sdma->bd0;
  	unsigned long flags;
  	int ret;
  
  	spin_lock_irqsave(&sdma->channel_0_lock, flags);
  
  	if (save)
  		bd0->mode.command = C0_GETDM;
  	else
  		bd0->mode.command = C0_SETDM;
  
  	bd0->mode.status = BD_DONE | BD_WRAP | BD_EXTD;
  	bd0->mode.count = MAX_DMA_CHANNELS * sizeof(*context) / 4;
  	bd0->buffer_addr = sdma->context_phys;
  	bd0->ext_buffer_addr = 2048;
  	ret = sdma_run_channel0(sdma);
  
  	spin_unlock_irqrestore(&sdma->channel_0_lock, flags);
  
  	return ret;
  }

  static struct sdma_channel *to_sdma_chan(struct dma_chan *chan)
  {

  	return container_of(chan, struct sdma_channel, vc.chan);

  }
  
  static int sdma_disable_channel(struct dma_chan *chan)

  {

  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;

  	writel_relaxed(BIT(channel), sdma->regs + SDMA_H_STATSTOP);

  	sdmac->status = DMA_ERROR;

  
  	return 0;

  }

  static void sdma_channel_terminate_work(struct work_struct *work)

  {

  	struct sdma_channel *sdmac = container_of(work, struct sdma_channel,
  						  terminate_worker);

  	/*
  	 * According to NXP R&D team a delay of one BD SDMA cost time
  	 * (maximum is 1ms) should be added after disable of the channel
  	 * bit, to ensure SDMA core has really been stopped after SDMA
  	 * clients call .device_terminate_all.
  	 */

  	usleep_range(1000, 2000);

  	vchan_dma_desc_free_list(&sdmac->vc, &sdmac->terminated);

  }

  static int sdma_terminate_all(struct dma_chan *chan)

  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	unsigned long flags;
  
  	spin_lock_irqsave(&sdmac->vc.lock, flags);

  
  	sdma_disable_channel(chan);

  	if (sdmac->desc) {
  		vchan_terminate_vdesc(&sdmac->desc->vd);

  		/*
  		 * move out current descriptor into terminated list so that
  		 * it could be free in sdma_channel_terminate_work alone
  		 * later without potential involving next descriptor raised
  		 * up before the last descriptor terminated.
  		 */
  		vchan_get_all_descriptors(&sdmac->vc, &sdmac->terminated);

  		sdmac->desc = NULL;

  		schedule_work(&sdmac->terminate_worker);

  	}
  
  	spin_unlock_irqrestore(&sdmac->vc.lock, flags);

  
  	return 0;
  }

  static void sdma_channel_synchronize(struct dma_chan *chan)
  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  
  	vchan_synchronize(&sdmac->vc);
  
  	flush_work(&sdmac->terminate_worker);
  }

  static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  
  	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;
  	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;
  
  	set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);
  	set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);
  
  	if (sdmac->event_id0 > 31)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;
  
  	if (sdmac->event_id1 > 31)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;
  
  	/*
  	 * If LWML(src_maxburst) > HWML(dst_maxburst), we need
  	 * swap LWML and HWML of INFO(A.3.2.5.1), also need swap
  	 * r0(event_mask[1]) and r1(event_mask[0]).
  	 */
  	if (lwml > hwml) {
  		sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |
  						SDMA_WATERMARK_LEVEL_HWML);
  		sdmac->watermark_level |= hwml;
  		sdmac->watermark_level |= lwml << 16;
  		swap(sdmac->event_mask[0], sdmac->event_mask[1]);
  	}
  
  	if (sdmac->per_address2 >= sdma->spba_start_addr &&
  			sdmac->per_address2 <= sdma->spba_end_addr)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;
  
  	if (sdmac->per_address >= sdma->spba_start_addr &&
  			sdmac->per_address <= sdma->spba_end_addr)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;
  
  	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;

  
  	if (sdmac->src_dualfifo)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SD;
  	if (sdmac->dst_dualfifo)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DD;

  }

  static void sdma_set_watermarklevel_for_sais(struct sdma_channel *sdmac)
  {
  	sdmac->watermark_level &= ~(0xFF << SDMA_WATERMARK_LEVEL_FIFOS_OFF |
  				    SDMA_WATERMARK_LEVEL_SW_DONE |
  				    0xf << SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF);
  
  	if (sdmac->sw_done)
  		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SW_DONE |
  			sdmac->sw_done_sel <<
  			SDMA_WATERMARK_LEVEL_SW_DONE_SEL_OFF;
  
  	/* For fifo_num
  	 * bit 12-15 is the fifo number;
  	 * bit 16-19 is the fifo offset,
  	 * so here only need to shift left fifo_num 12 bit for watermake_level
  	 */
  	sdmac->watermark_level |= sdmac->fifo_num<<
  				SDMA_WATERMARK_LEVEL_FIFOS_OFF;
  }

  static int sdma_config_channel(struct dma_chan *chan)

  {

  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	sdma_disable_channel(chan);

  	sdmac->event_mask[0] = 0;
  	sdmac->event_mask[1] = 0;

  	sdmac->shp_addr = 0;
  	sdmac->per_addr = 0;

  	switch (sdmac->peripheral_type) {
  	case IMX_DMATYPE_DSP:
  		sdma_config_ownership(sdmac, false, true, true);
  		break;
  	case IMX_DMATYPE_MEMORY:
  		sdma_config_ownership(sdmac, false, true, false);
  		break;
  	default:
  		sdma_config_ownership(sdmac, true, true, false);
  		break;
  	}
  
  	sdma_get_pc(sdmac, sdmac->peripheral_type);
  
  	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&
  			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {
  		/* Handle multiple event channels differently */
  		if (sdmac->event_id1) {

  			if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||
  			    sdmac->peripheral_type == IMX_DMATYPE_ASRC)
  				sdma_set_watermarklevel_for_p2p(sdmac);

  		} else {
  			/*
  			 * ERR009165 fixed from i.mx6ul, no errata need,
  			 * set bit31 to let sdma script skip the errata.
  			 */
  			if (sdmac->peripheral_type == IMX_DMATYPE_CSPI &&
  			    sdmac->direction == DMA_MEM_TO_DEV &&
  			    sdmac->sdma->drvdata->ecspi_fixed)
  				__set_bit(31, &sdmac->watermark_level);

  			else if (sdmac->peripheral_type ==
  					IMX_DMATYPE_MULTI_SAI)
  				sdma_set_watermarklevel_for_sais(sdmac);

  			__set_bit(sdmac->event_id0, sdmac->event_mask);

  		}

  		/* Address */
  		sdmac->shp_addr = sdmac->per_address;

  		sdmac->per_addr = sdmac->per_address2;

  	} else {
  		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */
  	}

  	return 0;

  }
  
  static int sdma_set_channel_priority(struct sdma_channel *sdmac,
  		unsigned int priority)
  {
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;
  
  	if (priority < MXC_SDMA_MIN_PRIORITY
  	    || priority > MXC_SDMA_MAX_PRIORITY) {
  		return -EINVAL;
  	}

  	writel_relaxed(priority, sdma->regs + SDMA_CHNPRI_0 + 4 * channel);

  
  	return 0;
  }

  static int sdma_request_channel0(struct sdma_engine *sdma)

  {

  	int ret = -EBUSY;

  	if (sdma->iram_pool)

  		sdma->bd0 = gen_pool_dma_alloc(sdma->iram_pool,
  					sizeof(struct sdma_buffer_descriptor),
  					&sdma->bd0_phys);

  	else

  		sdma->bd0 = dma_alloc_coherent(sdma->dev,
  					sizeof(struct sdma_buffer_descriptor),
  					&sdma->bd0_phys, GFP_NOWAIT);

  	if (!sdma->bd0) {

  		ret = -ENOMEM;
  		goto out;
  	}

  	sdma->channel_control[0].base_bd_ptr = sdma->bd0_phys;
  	sdma->channel_control[0].current_bd_ptr = sdma->bd0_phys;

  	sdma_set_channel_priority(&sdma->channel[0], MXC_SDMA_DEFAULT_PRIORITY);

  	return 0;
  out:
  
  	return ret;
  }

  
  static int sdma_alloc_bd(struct sdma_desc *desc)

  {

  	u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);

  	struct sdma_engine *sdma = desc->sdmac->sdma;

  	int ret = 0;

  	if (sdma->iram_pool)

  		desc->bd = gen_pool_dma_alloc(sdma->iram_pool, bd_size,

  					      &desc->bd_phys);
  	else
  		desc->bd = dma_alloc_coherent(sdma->dev, bd_size,
  					      &desc->bd_phys, GFP_NOWAIT);

  	if (!desc->bd) {
  		ret = -ENOMEM;
  		goto out;
  	}
  out:
  	return ret;
  }

  static void sdma_free_bd(struct sdma_desc *desc)
  {

  	u32 bd_size = desc->num_bd * sizeof(struct sdma_buffer_descriptor);

  	struct sdma_engine *sdma = desc->sdmac->sdma;

  	if (sdma->iram_pool)
  		gen_pool_free(sdma->iram_pool, (unsigned long)desc->bd,

  			      bd_size);

  	else
  		dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,
  				  desc->bd_phys);

  }

  static void sdma_desc_free(struct virt_dma_desc *vd)
  {
  	struct sdma_desc *desc = container_of(vd, struct sdma_desc, vd);
  
  	sdma_free_bd(desc);
  	kfree(desc);

  }

  static int sdma_runtime_suspend(struct device *dev)

  {

  	struct platform_device *pdev = to_platform_device(dev);
  	struct sdma_engine *sdma = platform_get_drvdata(pdev);
  
  	sdma->fw_loaded = false;
  	sdma->is_on = false;
  
  	clk_disable(sdma->clk_ipg);
  	clk_disable(sdma->clk_ahb);

  	/* free channel0 bd */
  	if (sdma->iram_pool)
  		gen_pool_free(sdma->iram_pool, (unsigned long)sdma->bd0,

  			      sizeof(struct sdma_buffer_descriptor));

  	else

  		dma_free_coherent(sdma->dev,
  				  sizeof(struct sdma_buffer_descriptor),
  				  sdma->bd0, sdma->bd0_phys);

  	return 0;
  }
  
  static int sdma_runtime_resume(struct device *dev)
  {
  	struct platform_device *pdev = to_platform_device(dev);
  	struct sdma_engine *sdma = platform_get_drvdata(pdev);

  	int i, ret;

  
  	ret = clk_enable(sdma->clk_ipg);
  	if (ret)
  		return ret;
  	ret = clk_enable(sdma->clk_ahb);
  	if (ret)
  		goto disable_clk_ipg;
  
  	/* Be sure SDMA has not started yet */
  	writel_relaxed(0, sdma->regs + SDMA_H_C0PTR);
  
  	/* disable all channels */
  	for (i = 0; i < sdma->drvdata->num_events; i++)
  		writel_relaxed(0, sdma->regs + chnenbl_ofs(sdma, i));
  
  	/* All channels have priority 0 */
  	for (i = 0; i < MAX_DMA_CHANNELS; i++)
  		writel_relaxed(0, sdma->regs + SDMA_CHNPRI_0 + i * 4);
  
  	ret = sdma_request_channel0(sdma);
  	if (ret)
  		return ret;
  
  	sdma_config_ownership(&sdma->channel[0], false, true, false);
  
  	/* Set Command Channel (Channel Zero) */
  	writel_relaxed(0x4050, sdma->regs + SDMA_CHN0ADDR);
  
  	/* Set bits of CONFIG register but with static context switching */
  	if (sdma->clk_ratio)
  		writel_relaxed(SDMA_H_CONFIG_ACR, sdma->regs + SDMA_H_CONFIG);
  	else
  		writel_relaxed(0, sdma->regs + SDMA_H_CONFIG);
  
  	writel_relaxed(sdma->ccb_phys, sdma->regs + SDMA_H_C0PTR);
  
  	/* Initializes channel's priorities */
  	sdma_set_channel_priority(&sdma->channel[0], 7);

  	if (sdma->drvdata->pm_runtime)
  		ret = sdma_get_firmware_wait(sdma, sdma->fw_name);
  	else
  		ret = sdma_get_firmware(sdma, sdma->fw_name);

  	if (ret)
  		dev_warn(sdma->dev, "failed to get firmware.
  ");

  	sdma->is_on = true;

  	return 0;
  
  disable_clk_ipg:
  	clk_disable(sdma->clk_ipg);
  	dev_err(sdma->dev, "initialisation failed with %d
  ", ret);
  
  	return ret;
  }

  static int sdma_alloc_chan_resources(struct dma_chan *chan)
  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  	struct imx_dma_data *data = chan->private;

  	struct imx_dma_data mem_data;

  	int prio, ret;

  	/*
  	 * MEMCPY may never setup chan->private by filter function such as
  	 * dmatest, thus create 'struct imx_dma_data mem_data' for this case.
  	 * Please note in any other slave case, you have to setup chan->private
  	 * with 'struct imx_dma_data' in your own filter function if you want to
  	 * request dma channel by dma_request_channel() rather than
  	 * dma_request_slave_channel(). Othwise, 'MEMCPY in case?' will appear
  	 * to warn you to correct your filter function.
  	 */
  	if (!data) {
  		dev_dbg(sdmac->sdma->dev, "MEMCPY in case?
  ");
  		mem_data.priority = 2;
  		mem_data.peripheral_type = IMX_DMATYPE_MEMORY;
  		mem_data.dma_request = 0;
  		mem_data.dma_request2 = 0;
  		data = &mem_data;
  
  		sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);
  	}

  
  	switch (data->priority) {
  	case DMA_PRIO_HIGH:
  		prio = 3;
  		break;
  	case DMA_PRIO_MEDIUM:
  		prio = 2;
  		break;
  	case DMA_PRIO_LOW:
  	default:
  		prio = 1;
  		break;
  	}
  
  	sdmac->peripheral_type = data->peripheral_type;
  	sdmac->event_id0 = data->dma_request;

  	sdmac->event_id1 = data->dma_request2;

  	sdmac->src_dualfifo = data->src_dualfifo;
  	sdmac->dst_dualfifo = data->dst_dualfifo;

  	/* Get software done selector if sw_done enabled */
  	if (data->done_sel & BIT(31)) {
  		sdmac->sw_done = true;
  		sdmac->sw_done_sel = (data->done_sel >> 8) & 0xff;
  	}

  	pm_runtime_get_sync(sdmac->sdma->dev);

  	ret = sdma_set_channel_priority(sdmac, prio);

  	if (ret)

  		return ret;

  	return 0;
  }
  
  static void sdma_free_chan_resources(struct dma_chan *chan)
  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  	struct sdma_engine *sdma = sdmac->sdma;

  	sdma_terminate_all(chan);

  
  	sdma_channel_synchronize(chan);

  	sdma_event_disable(sdmac, sdmac->event_id0);

  	if (sdmac->event_id1)
  		sdma_event_disable(sdmac, sdmac->event_id1);
  
  	sdmac->event_id0 = 0;
  	sdmac->event_id1 = 0;
  
  	sdma_set_channel_priority(sdmac, 0);

  	pm_runtime_put_sync(sdma->dev);

  }

  static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,
  				enum dma_transfer_direction direction, u32 bds)
  {
  	struct sdma_desc *desc;

  	if (!sdmac->sdma->fw_loaded && sdmac->is_ram_script) {
  		dev_err(sdmac->sdma->dev, "sdma firmware not ready!
  ");
  		goto err_out;
  	}

  	desc = kzalloc((sizeof(*desc)), GFP_NOWAIT);
  	if (!desc)
  		goto err_out;
  
  	sdmac->status = DMA_IN_PROGRESS;
  	sdmac->direction = direction;
  	sdmac->flags = 0;
  
  	desc->chn_count = 0;
  	desc->chn_real_count = 0;
  	desc->buf_tail = 0;
  	desc->buf_ptail = 0;
  	desc->sdmac = sdmac;
  	desc->num_bd = bds;

  	if (bds && sdma_alloc_bd(desc))

  		goto err_desc_out;

  	/* No slave_config called in MEMCPY case, so do here */
  	if (direction == DMA_MEM_TO_MEM)
  		sdma_config_ownership(sdmac, false, true, false);

  	if (sdma_load_context(sdmac))
  		goto err_desc_out;
  
  	return desc;
  
  err_desc_out:
  	kfree(desc);
  err_out:
  	return NULL;
  }

  static struct dma_async_tx_descriptor *sdma_prep_memcpy(
  		struct dma_chan *chan, dma_addr_t dma_dst,
  		dma_addr_t dma_src, size_t len, unsigned long flags)
  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  	struct sdma_engine *sdma = sdmac->sdma;
  	int channel = sdmac->channel;
  	size_t count;
  	int i = 0, param;
  	struct sdma_buffer_descriptor *bd;
  	struct sdma_desc *desc;
  
  	if (!chan || !len)
  		return NULL;
  
  	dev_dbg(sdma->dev, "memcpy: %pad->%pad, len=%zu, channel=%d.
  ",
  		&dma_src, &dma_dst, len, channel);
  
  	desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,
  					len / SDMA_BD_MAX_CNT + 1);
  	if (!desc)
  		return NULL;
  
  	do {
  		count = min_t(size_t, len, SDMA_BD_MAX_CNT);
  		bd = &desc->bd[i];
  		bd->buffer_addr = dma_src;
  		bd->ext_buffer_addr = dma_dst;
  		bd->mode.count = count;
  		desc->chn_count += count;
  		bd->mode.command = 0;
  
  		dma_src += count;
  		dma_dst += count;
  		len -= count;
  		i++;
  
  		param = BD_DONE | BD_EXTD | BD_CONT;
  		/* last bd */
  		if (!len) {
  			param |= BD_INTR;
  			param |= BD_LAST;
  			param &= ~BD_CONT;
  		}
  
  		dev_dbg(sdma->dev, "entry %d: count: %zd dma: 0x%x %s%s
  ",
  				i, count, bd->buffer_addr,
  				param & BD_WRAP ? "wrap" : "",
  				param & BD_INTR ? " intr" : "");
  
  		bd->mode.status = param;
  	} while (len);
  
  	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  }

  static struct dma_async_tx_descriptor *sdma_prep_slave_sg(
  		struct dma_chan *chan, struct scatterlist *sgl,

  		unsigned int sg_len, enum dma_transfer_direction direction,

  		unsigned long flags, void *context)

  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  	struct sdma_engine *sdma = sdmac->sdma;

  	int i, count;

  	int channel = sdmac->channel;

  	struct scatterlist *sg;

  	struct sdma_desc *desc;

  	sdma_config_write(chan, &sdmac->slave_config, direction);

  	desc = sdma_transfer_init(sdmac, direction, sg_len);

  	if (!desc)
  		goto err_out;

  	dev_dbg(sdma->dev, "setting up %d entries for channel %d.
  ",
  			sg_len, channel);

  	for_each_sg(sgl, sg, sg_len, i) {

  		struct sdma_buffer_descriptor *bd = &desc->bd[i];

  		int param;

  		bd->buffer_addr = sg->dma_address;

  		count = sg_dma_len(sg);

  		if (count > SDMA_BD_MAX_CNT) {

  			dev_err(sdma->dev, "SDMA channel %d: maximum bytes for sg entry exceeded: %d > %d
  ",

  					channel, count, SDMA_BD_MAX_CNT);

  			goto err_bd_out;

  		}
  
  		bd->mode.count = count;

  		desc->chn_count += count;

  		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)

  			goto err_bd_out;

  
  		switch (sdmac->word_size) {
  		case DMA_SLAVE_BUSWIDTH_4_BYTES:

  			bd->mode.command = 0;

  			if (count & 3 || sg->dma_address & 3)

  				goto err_bd_out;

  			break;

  		case DMA_SLAVE_BUSWIDTH_3_BYTES:
  			bd->mode.command = 3;
  			break;

  		case DMA_SLAVE_BUSWIDTH_2_BYTES:
  			bd->mode.command = 2;
  			if (count & 1 || sg->dma_address & 1)

  				goto err_bd_out;

  			break;
  		case DMA_SLAVE_BUSWIDTH_1_BYTE:
  			bd->mode.command = 1;
  			break;
  		default:

  			goto err_bd_out;

  		}

  
  		param = BD_DONE | BD_EXTD | BD_CONT;

  		if (i + 1 == sg_len) {

  			param |= BD_INTR;

  			param |= BD_LAST;
  			param &= ~BD_CONT;

  		}

  		dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s
  ",
  				i, count, (u64)sg->dma_address,

  				param & BD_WRAP ? "wrap" : "",
  				param & BD_INTR ? " intr" : "");
  
  		bd->mode.status = param;
  	}

  	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  err_bd_out:
  	sdma_free_bd(desc);
  	kfree(desc);

  err_out:

  	sdmac->status = DMA_ERROR;

  	return NULL;
  }
  
  static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic(
  		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,

  		size_t period_len, enum dma_transfer_direction direction,

  		unsigned long flags)

  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  	struct sdma_engine *sdma = sdmac->sdma;

  	int num_periods = 0;

  	int channel = sdmac->channel;

  	int i = 0, buf = 0;

  	struct sdma_desc *desc;

  
  	dev_dbg(sdma->dev, "%s channel: %d
  ", __func__, channel);

  	if (sdmac->peripheral_type != IMX_DMATYPE_HDMI)
  		num_periods = buf_len / period_len;

  	sdma_config_write(chan, &sdmac->slave_config, direction);

  	desc = sdma_transfer_init(sdmac, direction, num_periods);

  	if (!desc)
  		goto err_out;

  	desc->period_len = period_len;

  	sdmac->flags |= IMX_DMA_SG_LOOP;

  	if (period_len > SDMA_BD_MAX_CNT) {

  		dev_err(sdma->dev, "SDMA channel %d: maximum period size exceeded: %zu > %d
  ",

  				channel, period_len, SDMA_BD_MAX_CNT);

  		goto err_bd_out;

  	}

  	if (sdmac->peripheral_type == IMX_DMATYPE_HDMI)
  		return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);

  	while (buf < buf_len) {

  		struct sdma_buffer_descriptor *bd = &desc->bd[i];

  		int param;
  
  		bd->buffer_addr = dma_addr;
  
  		bd->mode.count = period_len;
  
  		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)

  			goto err_bd_out;

  		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)
  			bd->mode.command = 0;
  		else
  			bd->mode.command = sdmac->word_size;
  
  		param = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
  		if (i + 1 == num_periods)
  			param |= BD_WRAP;

  		dev_dbg(sdma->dev, "entry %d: count: %zu dma: %#llx %s%s
  ",

  				i, period_len, (u64)dma_addr,

  				param & BD_WRAP ? "wrap" : "",
  				param & BD_INTR ? " intr" : "");
  
  		bd->mode.status = param;
  
  		dma_addr += period_len;
  		buf += period_len;
  
  		i++;
  	}

  	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);
  err_bd_out:
  	sdma_free_bd(desc);
  	kfree(desc);

  err_out:
  	sdmac->status = DMA_ERROR;
  	return NULL;
  }

  static int sdma_config_write(struct dma_chan *chan,
  		       struct dma_slave_config *dmaengine_cfg,
  		       enum dma_transfer_direction direction)

  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	sdmac->watermark_level = 0;

  	sdmac->is_ram_script = false;

  	if (direction == DMA_DEV_TO_MEM) {

  		sdmac->per_address = dmaengine_cfg->src_addr;
  		sdmac->watermark_level = dmaengine_cfg->src_maxburst *
  			dmaengine_cfg->src_addr_width;
  		sdmac->word_size = dmaengine_cfg->src_addr_width;

  		sdmac->fifo_num =  dmaengine_cfg->src_fifo_num;

  	} else if (direction == DMA_DEV_TO_DEV) {

  		sdmac->per_address2 = dmaengine_cfg->src_addr;
  		sdmac->per_address = dmaengine_cfg->dst_addr;
  		sdmac->watermark_level = dmaengine_cfg->src_maxburst &
  			SDMA_WATERMARK_LEVEL_LWML;
  		sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &
  			SDMA_WATERMARK_LEVEL_HWML;
  		sdmac->word_size = dmaengine_cfg->dst_addr_width;

  	} else if (sdmac->peripheral_type == IMX_DMATYPE_HDMI) {
  			sdmac->per_address = dmaengine_cfg->dst_addr;
  			sdmac->per_address2 = dmaengine_cfg->src_addr;
  			sdmac->watermark_level = 0;

  	} else {
  		sdmac->per_address = dmaengine_cfg->dst_addr;
  		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *
  			dmaengine_cfg->dst_addr_width;
  		sdmac->word_size = dmaengine_cfg->dst_addr_width;

  		sdmac->fifo_num =  dmaengine_cfg->dst_fifo_num;

  	}

  	sdmac->direction = direction;

  	return sdma_config_channel(chan);

  }

  static int sdma_config(struct dma_chan *chan,
  		       struct dma_slave_config *dmaengine_cfg)
  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);
  
  	memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));
  
  	/* Set ENBLn earlier to make sure dma request triggered after that */

  	if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)
  		return -EINVAL;
  	sdma_event_enable(sdmac, sdmac->event_id0);

  
  	if (sdmac->event_id1) {
  		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)
  			return -EINVAL;
  		sdma_event_enable(sdmac, sdmac->event_id1);
  	}
  
  	return 0;
  }

  static enum dma_status sdma_tx_status(struct dma_chan *chan,

  				      dma_cookie_t cookie,
  				      struct dma_tx_state *txstate)

  {
  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	struct sdma_desc *desc = NULL;

  	u32 residue;

  	struct virt_dma_desc *vd;
  	enum dma_status ret;
  	unsigned long flags;

  	ret = dma_cookie_status(chan, cookie, txstate);
  	if (ret == DMA_COMPLETE || !txstate)
  		return ret;
  
  	spin_lock_irqsave(&sdmac->vc.lock, flags);

  	vd = vchan_find_desc(&sdmac->vc, cookie);

  	if (vd)

  		desc = to_sdma_desc(&vd->tx);

  	else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie)
  		desc = sdmac->desc;
  
  	if (desc) {

  		if (sdmac->flags & IMX_DMA_SG_LOOP)
  			residue = (desc->num_bd - desc->buf_ptail) *
  				desc->period_len - desc->chn_real_count;
  		else
  			residue = desc->chn_count - desc->chn_real_count;

  	} else {
  		residue = 0;
  	}

  	spin_unlock_irqrestore(&sdmac->vc.lock, flags);

  	dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie,

  			 residue);

  	return sdmac->status;

  }
  
  static void sdma_issue_pending(struct dma_chan *chan)
  {

  	struct sdma_channel *sdmac = to_sdma_chan(chan);

  	unsigned long flags;

  	spin_lock_irqsave(&sdmac->vc.lock, flags);
  	if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)
  		sdma_start_desc(sdmac);
  	spin_unlock_irqrestore(&sdmac->vc.lock, flags);

  }

  #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1	34

  #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2	38

  #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3	45
  #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4	46

  
  static void sdma_add_scripts(struct sdma_engine *sdma,
  		const struct sdma_script_start_addrs *addr)
  {
  	s32 *addr_arr = (u32 *)addr;
  	s32 *saddr_arr = (u32 *)sdma->script_addrs;
  	int i;

  	/* use the default firmware in ROM if missing external firmware */
  	if (!sdma->script_number)
  		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;

  	if (sdma->script_number > sizeof(struct sdma_script_start_addrs)
  				  / sizeof(s32)) {
  		dev_err(sdma->dev,
  			"SDMA script number %d not match with firmware.
  ",
  			sdma->script_number);
  		return;
  	}

  	for (i = 0; i < sdma->script_number; i++)

  		if (addr_arr[i] > 0)
  			saddr_arr[i] = addr_arr[i];
  }

  static void sdma_load_firmware(const struct firmware *fw, void *context)

  {

  	struct sdma_engine *sdma = context;

  	const struct sdma_firmware_header *header;

  	const struct sdma_script_start_addrs *addr;
  	unsigned short *ram_code;

  	if (!fw) {

  		/* Load firmware once more time if timeout */
  		if (sdma->fw_fail)
  			dev_info(sdma->dev, "external firmware not found, using ROM firmware
  ");
  		else {
  			request_firmware_nowait(THIS_MODULE,
  					FW_ACTION_HOTPLUG, sdma->fw_name,
  					sdma->dev, GFP_KERNEL, sdma,
  					sdma_load_firmware);
  			sdma->fw_fail++;
  		}

  		return;
  	}

  	if (fw->size < sizeof(*header) || sdma->fw_loaded)

  		goto err_firmware;
  
  	header = (struct sdma_firmware_header *)fw->data;
  
  	if (header->magic != SDMA_FIRMWARE_MAGIC)
  		goto err_firmware;
  	if (header->ram_code_start + header->ram_code_size > fw->size)
  		goto err_firmware;

  	switch (header->version_major) {

  	case 1:
  		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1;
  		break;
  	case 2:
  		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2;
  		break;

  	case 3:
  		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V3;
  		break;

  	case 4:
  		sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V4;
  		break;

  	default:
  		dev_err(sdma->dev, "unknown firmware version
  ");
  		goto err_firmware;

  	}

  
  	addr = (void *)header + header->script_addrs_start;
  	ram_code = (void *)header + header->ram_code_start;

  	sdma->ram_code_start = header->ram_code_start;

  	/* download the RAM image for SDMA */
  	sdma_load_script(sdma, ram_code,
  			header->ram_code_size,