Blame view
drivers/dma/ste_dma40.c
75 KB
8d318a50b DMAENGINE: Suppor... |
1 |
/* |
d49278e33 dmaengine: dma40:... |
2 3 |
* Copyright (C) Ericsson AB 2007-2008 * Copyright (C) ST-Ericsson SA 2008-2010 |
661385f9c DMAENGINE: Remove... |
4 |
* Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson |
767a9675c DMAENGINE: ste_dm... |
5 |
* Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson |
8d318a50b DMAENGINE: Suppor... |
6 |
* License terms: GNU General Public License (GPL) version 2 |
8d318a50b DMAENGINE: Suppor... |
7 |
*/ |
b7f080cfe net: remove mm.h ... |
8 |
#include <linux/dma-mapping.h> |
8d318a50b DMAENGINE: Suppor... |
9 10 |
#include <linux/kernel.h> #include <linux/slab.h> |
f492b2102 drivers/dma: Add ... |
11 |
#include <linux/export.h> |
8d318a50b DMAENGINE: Suppor... |
12 13 14 15 |
#include <linux/dmaengine.h> #include <linux/platform_device.h> #include <linux/clk.h> #include <linux/delay.h> |
698e4732e DMAENGINE: ste_dm... |
16 |
#include <linux/err.h> |
f4b89764c dmaengine/ste_dma... |
17 |
#include <linux/amba/bus.h> |
8d318a50b DMAENGINE: Suppor... |
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 |
#include <plat/ste_dma40.h> #include "ste_dma40_ll.h" #define D40_NAME "dma40" #define D40_PHY_CHAN -1 /* For masking out/in 2 bit channel positions */ #define D40_CHAN_POS(chan) (2 * (chan / 2)) #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan)) /* Maximum iterations taken before giving up suspending a channel */ #define D40_SUSPEND_MAX_IT 500 |
508849ade DMAENGINE: ste_dm... |
33 34 |
/* Hardware requirement on LCLA alignment */ #define LCLA_ALIGNMENT 0x40000 |
698e4732e DMAENGINE: ste_dm... |
35 36 37 38 |
/* Max number of links per event group */ #define D40_LCLA_LINK_PER_EVENT_GRP 128 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP |
508849ade DMAENGINE: ste_dm... |
39 40 41 42 |
/* Attempts before giving up to trying to get pages that are aligned */ #define MAX_LCLA_ALLOC_ATTEMPTS 256 /* Bit markings for allocation map */ |
8d318a50b DMAENGINE: Suppor... |
43 44 45 |
#define D40_ALLOC_FREE (1 << 31) #define D40_ALLOC_PHY (1 << 30) #define D40_ALLOC_LOG_FREE 0 |
8d318a50b DMAENGINE: Suppor... |
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
/** * enum 40_command - The different commands and/or statuses. * * @D40_DMA_STOP: DMA channel command STOP or status STOPPED, * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN. * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible. * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED. */ enum d40_command { D40_DMA_STOP = 0, D40_DMA_RUN = 1, D40_DMA_SUSPEND_REQ = 2, D40_DMA_SUSPENDED = 3 }; /** * struct d40_lli_pool - Structure for keeping LLIs in memory * * @base: Pointer to memory area when the pre_alloc_lli's are not large * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if * pre_alloc_lli is used. |
b00f938c8 dma40: fix DMA AP... |
67 |
* @dma_addr: DMA address, if mapped |
8d318a50b DMAENGINE: Suppor... |
68 69 70 71 72 73 |
* @size: The size in bytes of the memory at base or the size of pre_alloc_lli. * @pre_alloc_lli: Pre allocated area for the most common case of transfers, * one buffer to one buffer. */ struct d40_lli_pool { void *base; |
508849ade DMAENGINE: ste_dm... |
74 |
int size; |
b00f938c8 dma40: fix DMA AP... |
75 |
dma_addr_t dma_addr; |
8d318a50b DMAENGINE: Suppor... |
76 |
/* Space for dst and src, plus an extra for padding */ |
508849ade DMAENGINE: ste_dm... |
77 |
u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; |
8d318a50b DMAENGINE: Suppor... |
78 79 80 81 82 83 84 85 86 87 |
}; /** * struct d40_desc - A descriptor is one DMA job. * * @lli_phy: LLI settings for physical channel. Both src and dst= * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if * lli_len equals one. * @lli_log: Same as above but for logical channels. * @lli_pool: The pool with two entries pre-allocated. |
941b77a3b DMAENGINE: ste_dm... |
88 |
* @lli_len: Number of llis of current descriptor. |
25985edce Fix common misspe... |
89 |
* @lli_current: Number of transferred llis. |
698e4732e DMAENGINE: ste_dm... |
90 |
* @lcla_alloc: Number of LCLA entries allocated. |
8d318a50b DMAENGINE: Suppor... |
91 92 93 |
* @txd: DMA engine struct. Used for among other things for communication * during a transfer. * @node: List entry. |
8d318a50b DMAENGINE: Suppor... |
94 |
* @is_in_client_list: true if the client owns this descriptor. |
aa182ae26 DMAENGINE: ste_dm... |
95 |
* the previous one. |
8d318a50b DMAENGINE: Suppor... |
96 97 98 |
* * This descriptor is used for both logical and physical transfers. */ |
8d318a50b DMAENGINE: Suppor... |
99 100 101 102 103 104 105 |
struct d40_desc { /* LLI physical */ struct d40_phy_lli_bidir lli_phy; /* LLI logical */ struct d40_log_lli_bidir lli_log; struct d40_lli_pool lli_pool; |
941b77a3b DMAENGINE: ste_dm... |
106 |
int lli_len; |
698e4732e DMAENGINE: ste_dm... |
107 108 |
int lli_current; int lcla_alloc; |
8d318a50b DMAENGINE: Suppor... |
109 110 111 |
struct dma_async_tx_descriptor txd; struct list_head node; |
8d318a50b DMAENGINE: Suppor... |
112 |
bool is_in_client_list; |
0c842b551 dma40: cyclic xfe... |
113 |
bool cyclic; |
8d318a50b DMAENGINE: Suppor... |
114 115 116 117 118 |
}; /** * struct d40_lcla_pool - LCLA pool settings and data. * |
508849ade DMAENGINE: ste_dm... |
119 120 121 122 123 |
* @base: The virtual address of LCLA. 18 bit aligned. * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. * This pointer is only there for clean-up on error. * @pages: The number of pages needed for all physical channels. * Only used later for clean-up on error |
8d318a50b DMAENGINE: Suppor... |
124 |
* @lock: Lock to protect the content in this struct. |
698e4732e DMAENGINE: ste_dm... |
125 |
* @alloc_map: big map over which LCLA entry is own by which job. |
8d318a50b DMAENGINE: Suppor... |
126 127 128 |
*/ struct d40_lcla_pool { void *base; |
026cbc424 dma40: fix DMA AP... |
129 |
dma_addr_t dma_addr; |
508849ade DMAENGINE: ste_dm... |
130 131 |
void *base_unaligned; int pages; |
8d318a50b DMAENGINE: Suppor... |
132 |
spinlock_t lock; |
698e4732e DMAENGINE: ste_dm... |
133 |
struct d40_desc **alloc_map; |
8d318a50b DMAENGINE: Suppor... |
134 135 136 137 138 139 140 141 142 143 144 145 |
}; /** * struct d40_phy_res - struct for handling eventlines mapped to physical * channels. * * @lock: A lock protection this entity. * @num: The physical channel number of this entity. * @allocated_src: Bit mapped to show which src event line's are mapped to * this physical channel. Can also be free or physically allocated. * @allocated_dst: Same as for src but is dst. * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as |
767a9675c DMAENGINE: ste_dm... |
146 |
* event line number. |
8d318a50b DMAENGINE: Suppor... |
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 |
*/ struct d40_phy_res { spinlock_t lock; int num; u32 allocated_src; u32 allocated_dst; }; struct d40_base; /** * struct d40_chan - Struct that describes a channel. * * @lock: A spinlock to protect this struct. * @log_num: The logical number, if any of this channel. * @completed: Starts with 1, after first interrupt it is set to dma engine's * current cookie. * @pending_tx: The number of pending transfers. Used between interrupt handler * and tasklet. * @busy: Set to true when transfer is ongoing on this channel. |
2a6143407 DMAENGINE: ste_dm... |
167 168 |
* @phy_chan: Pointer to physical channel which this instance runs on. If this * point is NULL, then the channel is not allocated. |
8d318a50b DMAENGINE: Suppor... |
169 170 171 172 |
* @chan: DMA engine handle. * @tasklet: Tasklet that gets scheduled from interrupt context to complete a * transfer and call client callback. * @client: Cliented owned descriptor list. |
da063d260 dmaengine/ste_dma... |
173 |
* @pending_queue: Submitted jobs, to be issued by issue_pending() |
8d318a50b DMAENGINE: Suppor... |
174 175 |
* @active: Active descriptor. * @queue: Queued jobs. |
82babbb36 dmaengine/ste_dma... |
176 |
* @prepare_queue: Prepared jobs. |
8d318a50b DMAENGINE: Suppor... |
177 |
* @dma_cfg: The client configuration of this dma channel. |
ce2ca1252 ste_dma40: add va... |
178 |
* @configured: whether the dma_cfg configuration is valid |
8d318a50b DMAENGINE: Suppor... |
179 180 181 182 183 184 |
* @base: Pointer to the device instance struct. * @src_def_cfg: Default cfg register setting for src. * @dst_def_cfg: Default cfg register setting for dst. * @log_def: Default logical channel settings. * @lcla: Space for one dst src pair for logical channel transfers. * @lcpa: Pointer to dst and src lcpa settings. |
ae752bf4c dmaengine/ste_dma... |
185 186 |
* @runtime_addr: runtime configured address. * @runtime_direction: runtime configured direction. |
8d318a50b DMAENGINE: Suppor... |
187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
* * This struct can either "be" a logical or a physical channel. */ struct d40_chan { spinlock_t lock; int log_num; /* ID of the most recent completed transfer */ int completed; int pending_tx; bool busy; struct d40_phy_res *phy_chan; struct dma_chan chan; struct tasklet_struct tasklet; struct list_head client; |
a8f3067bc dmaengine/ste_dma... |
201 |
struct list_head pending_queue; |
8d318a50b DMAENGINE: Suppor... |
202 203 |
struct list_head active; struct list_head queue; |
82babbb36 dmaengine/ste_dma... |
204 |
struct list_head prepare_queue; |
8d318a50b DMAENGINE: Suppor... |
205 |
struct stedma40_chan_cfg dma_cfg; |
ce2ca1252 ste_dma40: add va... |
206 |
bool configured; |
8d318a50b DMAENGINE: Suppor... |
207 208 209 210 211 |
struct d40_base *base; /* Default register configurations */ u32 src_def_cfg; u32 dst_def_cfg; struct d40_def_lcsp log_def; |
8d318a50b DMAENGINE: Suppor... |
212 |
struct d40_log_lli_full *lcpa; |
95e1400fa DMAENGINE: add ru... |
213 214 215 |
/* Runtime reconfiguration */ dma_addr_t runtime_addr; enum dma_data_direction runtime_direction; |
8d318a50b DMAENGINE: Suppor... |
216 217 218 219 220 221 222 223 224 225 |
}; /** * struct d40_base - The big global struct, one for each probe'd instance. * * @interrupt_lock: Lock used to make sure one interrupt is handle a time. * @execmd_lock: Lock for execute command usage since several channels share * the same physical register. * @dev: The device structure. * @virtbase: The virtual base address of the DMA's register. |
f41855929 DMAENGINE: ste_dm... |
226 |
* @rev: silicon revision detected. |
8d318a50b DMAENGINE: Suppor... |
227 228 229 230 231 232 233 234 235 236 237 238 |
* @clk: Pointer to the DMA clock structure. * @phy_start: Physical memory start of the DMA registers. * @phy_size: Size of the DMA register map. * @irq: The IRQ number. * @num_phy_chans: The number of physical channels. Read from HW. This * is the number of available channels for this driver, not counting "Secure * mode" allocated physical channels. * @num_log_chans: The number of logical channels. Calculated from * num_phy_chans. * @dma_both: dma_device channels that can do both memcpy and slave transfers. * @dma_slave: dma_device channels that can do only do slave transfers. * @dma_memcpy: dma_device channels that can do only do memcpy transfers. |
8d318a50b DMAENGINE: Suppor... |
239 240 241 242 243 244 245 246 247 248 249 250 |
* @log_chans: Room for all possible logical channels in system. * @lookup_log_chans: Used to map interrupt number to logical channel. Points * to log_chans entries. * @lookup_phy_chans: Used to map interrupt number to physical channel. Points * to phy_chans entries. * @plat_data: Pointer to provided platform_data which is the driver * configuration. * @phy_res: Vector containing all physical channels. * @lcla_pool: lcla pool settings and data. * @lcpa_base: The virtual mapped address of LCPA. * @phy_lcpa: The physical address of the LCPA. * @lcpa_size: The size of the LCPA area. |
c675b1b42 DMAENGINE: ste_dm... |
251 |
* @desc_slab: cache for descriptors. |
8d318a50b DMAENGINE: Suppor... |
252 253 254 255 256 257 |
*/ struct d40_base { spinlock_t interrupt_lock; spinlock_t execmd_lock; struct device *dev; void __iomem *virtbase; |
f41855929 DMAENGINE: ste_dm... |
258 |
u8 rev:4; |
8d318a50b DMAENGINE: Suppor... |
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 |
struct clk *clk; phys_addr_t phy_start; resource_size_t phy_size; int irq; int num_phy_chans; int num_log_chans; struct dma_device dma_both; struct dma_device dma_slave; struct dma_device dma_memcpy; struct d40_chan *phy_chans; struct d40_chan *log_chans; struct d40_chan **lookup_log_chans; struct d40_chan **lookup_phy_chans; struct stedma40_platform_data *plat_data; /* Physical half channels */ struct d40_phy_res *phy_res; struct d40_lcla_pool lcla_pool; void *lcpa_base; dma_addr_t phy_lcpa; resource_size_t lcpa_size; |
c675b1b42 DMAENGINE: ste_dm... |
279 |
struct kmem_cache *desc_slab; |
8d318a50b DMAENGINE: Suppor... |
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 |
}; /** * struct d40_interrupt_lookup - lookup table for interrupt handler * * @src: Interrupt mask register. * @clr: Interrupt clear register. * @is_error: true if this is an error interrupt. * @offset: start delta in the lookup_log_chans in d40_base. If equals to * D40_PHY_CHAN, the lookup_phy_chans shall be used instead. */ struct d40_interrupt_lookup { u32 src; u32 clr; bool is_error; int offset; }; /** * struct d40_reg_val - simple lookup struct * * @reg: The register. * @val: The value that belongs to the register in reg. */ struct d40_reg_val { unsigned int reg; unsigned int val; }; |
262d2915d dma40: ensure eve... |
308 309 310 311 |
static struct device *chan2dev(struct d40_chan *d40c) { return &d40c->chan.dev->device; } |
724a8577d dma40: use helper... |
312 313 314 315 316 317 318 319 320 |
static bool chan_is_physical(struct d40_chan *chan) { return chan->log_num == D40_PHY_CHAN; } static bool chan_is_logical(struct d40_chan *chan) { return !chan_is_physical(chan); } |
8ca84687b dma40: use helper... |
321 322 323 324 325 |
static void __iomem *chan_base(struct d40_chan *chan) { return chan->base->virtbase + D40_DREG_PCBASE + chan->phy_chan->num * D40_DREG_PCDELTA; } |
6db5a8ba1 dma40: use helper... |
326 327 328 329 330 |
#define d40_err(dev, format, arg...) \ dev_err(dev, "[%s] " format, __func__, ## arg) #define chan_err(d40c, format, arg...) \ d40_err(chan2dev(d40c), format, ## arg) |
b00f938c8 dma40: fix DMA AP... |
331 |
static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d, |
dbd887880 dma40: combine du... |
332 |
int lli_len) |
8d318a50b DMAENGINE: Suppor... |
333 |
{ |
dbd887880 dma40: combine du... |
334 |
bool is_log = chan_is_logical(d40c); |
8d318a50b DMAENGINE: Suppor... |
335 336 337 338 339 340 341 342 343 344 345 346 347 |
u32 align; void *base; if (is_log) align = sizeof(struct d40_log_lli); else align = sizeof(struct d40_phy_lli); if (lli_len == 1) { base = d40d->lli_pool.pre_alloc_lli; d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli); d40d->lli_pool.base = NULL; } else { |
594ece4dc dma40: remove unn... |
348 |
d40d->lli_pool.size = lli_len * 2 * align; |
8d318a50b DMAENGINE: Suppor... |
349 350 351 352 353 354 355 356 357 |
base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT); d40d->lli_pool.base = base; if (d40d->lli_pool.base == NULL) return -ENOMEM; } if (is_log) { |
d924abad7 dma40: remove unn... |
358 |
d40d->lli_log.src = PTR_ALIGN(base, align); |
594ece4dc dma40: remove unn... |
359 |
d40d->lli_log.dst = d40d->lli_log.src + lli_len; |
b00f938c8 dma40: fix DMA AP... |
360 361 |
d40d->lli_pool.dma_addr = 0; |
8d318a50b DMAENGINE: Suppor... |
362 |
} else { |
d924abad7 dma40: remove unn... |
363 |
d40d->lli_phy.src = PTR_ALIGN(base, align); |
594ece4dc dma40: remove unn... |
364 |
d40d->lli_phy.dst = d40d->lli_phy.src + lli_len; |
b00f938c8 dma40: fix DMA AP... |
365 366 367 368 369 370 371 372 373 374 375 376 377 |
d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev, d40d->lli_phy.src, d40d->lli_pool.size, DMA_TO_DEVICE); if (dma_mapping_error(d40c->base->dev, d40d->lli_pool.dma_addr)) { kfree(d40d->lli_pool.base); d40d->lli_pool.base = NULL; d40d->lli_pool.dma_addr = 0; return -ENOMEM; } |
8d318a50b DMAENGINE: Suppor... |
378 379 380 381 |
} return 0; } |
b00f938c8 dma40: fix DMA AP... |
382 |
static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d) |
8d318a50b DMAENGINE: Suppor... |
383 |
{ |
b00f938c8 dma40: fix DMA AP... |
384 385 386 |
if (d40d->lli_pool.dma_addr) dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr, d40d->lli_pool.size, DMA_TO_DEVICE); |
8d318a50b DMAENGINE: Suppor... |
387 388 389 390 391 392 393 |
kfree(d40d->lli_pool.base); d40d->lli_pool.base = NULL; d40d->lli_pool.size = 0; d40d->lli_log.src = NULL; d40d->lli_log.dst = NULL; d40d->lli_phy.src = NULL; d40d->lli_phy.dst = NULL; |
8d318a50b DMAENGINE: Suppor... |
394 |
} |
698e4732e DMAENGINE: ste_dm... |
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 |
static int d40_lcla_alloc_one(struct d40_chan *d40c, struct d40_desc *d40d) { unsigned long flags; int i; int ret = -EINVAL; int p; spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP; /* * Allocate both src and dst at the same time, therefore the half * start on 1 since 0 can't be used since zero is used as end marker. */ for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { if (!d40c->base->lcla_pool.alloc_map[p + i]) { d40c->base->lcla_pool.alloc_map[p + i] = d40d; d40d->lcla_alloc++; ret = i; break; } } spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); return ret; } static int d40_lcla_free_all(struct d40_chan *d40c, struct d40_desc *d40d) { unsigned long flags; int i; int ret = -EINVAL; |
724a8577d dma40: use helper... |
431 |
if (chan_is_physical(d40c)) |
698e4732e DMAENGINE: ste_dm... |
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 |
return 0; spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) { d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL; d40d->lcla_alloc--; if (d40d->lcla_alloc == 0) { ret = 0; break; } } } spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); return ret; } |
8d318a50b DMAENGINE: Suppor... |
454 455 456 457 458 459 460 |
static void d40_desc_remove(struct d40_desc *d40d) { list_del(&d40d->node); } static struct d40_desc *d40_desc_get(struct d40_chan *d40c) { |
a2c15fa4c DMAENGINE: ste_dm... |
461 |
struct d40_desc *desc = NULL; |
8d318a50b DMAENGINE: Suppor... |
462 463 |
if (!list_empty(&d40c->client)) { |
a2c15fa4c DMAENGINE: ste_dm... |
464 465 |
struct d40_desc *d; struct d40_desc *_d; |
8d318a50b DMAENGINE: Suppor... |
466 467 |
list_for_each_entry_safe(d, _d, &d40c->client, node) if (async_tx_test_ack(&d->txd)) { |
8d318a50b DMAENGINE: Suppor... |
468 |
d40_desc_remove(d); |
a2c15fa4c DMAENGINE: ste_dm... |
469 470 |
desc = d; memset(desc, 0, sizeof(*desc)); |
c675b1b42 DMAENGINE: ste_dm... |
471 |
break; |
8d318a50b DMAENGINE: Suppor... |
472 |
} |
8d318a50b DMAENGINE: Suppor... |
473 |
} |
a2c15fa4c DMAENGINE: ste_dm... |
474 475 476 477 478 479 480 481 |
if (!desc) desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT); if (desc) INIT_LIST_HEAD(&desc->node); return desc; |
8d318a50b DMAENGINE: Suppor... |
482 483 484 485 |
} static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) { |
698e4732e DMAENGINE: ste_dm... |
486 |
|
b00f938c8 dma40: fix DMA AP... |
487 |
d40_pool_lli_free(d40c, d40d); |
698e4732e DMAENGINE: ste_dm... |
488 |
d40_lcla_free_all(d40c, d40d); |
c675b1b42 DMAENGINE: ste_dm... |
489 |
kmem_cache_free(d40c->base->desc_slab, d40d); |
8d318a50b DMAENGINE: Suppor... |
490 491 492 493 494 495 |
} static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) { list_add_tail(&desc->node, &d40c->active); } |
1c4b0927f dma40: move lli_l... |
496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 |
static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc) { struct d40_phy_lli *lli_dst = desc->lli_phy.dst; struct d40_phy_lli *lli_src = desc->lli_phy.src; void __iomem *base = chan_base(chan); writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG); writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT); writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR); writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK); writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG); writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT); writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR); writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK); } |
e65889c75 dma40: extract lc... |
512 |
static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc) |
698e4732e DMAENGINE: ste_dm... |
513 |
{ |
e65889c75 dma40: extract lc... |
514 515 516 517 |
struct d40_lcla_pool *pool = &chan->base->lcla_pool; struct d40_log_lli_bidir *lli = &desc->lli_log; int lli_current = desc->lli_current; int lli_len = desc->lli_len; |
0c842b551 dma40: cyclic xfe... |
518 |
bool cyclic = desc->cyclic; |
e65889c75 dma40: extract lc... |
519 |
int curr_lcla = -EINVAL; |
0c842b551 dma40: cyclic xfe... |
520 521 |
int first_lcla = 0; bool linkback; |
e65889c75 dma40: extract lc... |
522 |
|
0c842b551 dma40: cyclic xfe... |
523 524 525 526 527 528 529 530 531 532 533 |
/* * We may have partially running cyclic transfers, in case we did't get * enough LCLA entries. */ linkback = cyclic && lli_current == 0; /* * For linkback, we need one LCLA even with only one link, because we * can't link back to the one in LCPA space */ if (linkback || (lli_len - lli_current > 1)) { |
e65889c75 dma40: extract lc... |
534 |
curr_lcla = d40_lcla_alloc_one(chan, desc); |
0c842b551 dma40: cyclic xfe... |
535 536 537 538 539 540 541 542 543 544 545 |
first_lcla = curr_lcla; } /* * For linkback, we normally load the LCPA in the loop since we need to * link it to the second LCLA and not the first. However, if we * couldn't even get a first LCLA, then we have to run in LCPA and * reload manually. */ if (!linkback || curr_lcla == -EINVAL) { unsigned int flags = 0; |
e65889c75 dma40: extract lc... |
546 |
|
0c842b551 dma40: cyclic xfe... |
547 548 |
if (curr_lcla == -EINVAL) flags |= LLI_TERM_INT; |
e65889c75 dma40: extract lc... |
549 |
|
0c842b551 dma40: cyclic xfe... |
550 551 552 553 554 555 556 |
d40_log_lli_lcpa_write(chan->lcpa, &lli->dst[lli_current], &lli->src[lli_current], curr_lcla, flags); lli_current++; } |
6045f0bb2 dma40: handle fai... |
557 558 559 |
if (curr_lcla < 0) goto out; |
e65889c75 dma40: extract lc... |
560 561 562 563 |
for (; lli_current < lli_len; lli_current++) { unsigned int lcla_offset = chan->phy_chan->num * 1024 + 8 * curr_lcla * 2; struct d40_log_lli *lcla = pool->base + lcla_offset; |
0c842b551 dma40: cyclic xfe... |
564 |
unsigned int flags = 0; |
e65889c75 dma40: extract lc... |
565 566 567 568 569 |
int next_lcla; if (lli_current + 1 < lli_len) next_lcla = d40_lcla_alloc_one(chan, desc); else |
0c842b551 dma40: cyclic xfe... |
570 571 572 573 |
next_lcla = linkback ? first_lcla : -EINVAL; if (cyclic || next_lcla == -EINVAL) flags |= LLI_TERM_INT; |
e65889c75 dma40: extract lc... |
574 |
|
0c842b551 dma40: cyclic xfe... |
575 576 577 578 579 580 581 582 583 584 585 586 |
if (linkback && curr_lcla == first_lcla) { /* First link goes in both LCPA and LCLA */ d40_log_lli_lcpa_write(chan->lcpa, &lli->dst[lli_current], &lli->src[lli_current], next_lcla, flags); } /* * One unused LCLA in the cyclic case if the very first * next_lcla fails... */ |
e65889c75 dma40: extract lc... |
587 588 589 |
d40_log_lli_lcla_write(lcla, &lli->dst[lli_current], &lli->src[lli_current], |
0c842b551 dma40: cyclic xfe... |
590 |
next_lcla, flags); |
e65889c75 dma40: extract lc... |
591 592 593 594 595 596 597 |
dma_sync_single_range_for_device(chan->base->dev, pool->dma_addr, lcla_offset, 2 * sizeof(struct d40_log_lli), DMA_TO_DEVICE); curr_lcla = next_lcla; |
0c842b551 dma40: cyclic xfe... |
598 |
if (curr_lcla == -EINVAL || curr_lcla == first_lcla) { |
e65889c75 dma40: extract lc... |
599 600 601 602 |
lli_current++; break; } } |
6045f0bb2 dma40: handle fai... |
603 |
out: |
e65889c75 dma40: extract lc... |
604 605 |
desc->lli_current = lli_current; } |
698e4732e DMAENGINE: ste_dm... |
606 |
|
e65889c75 dma40: extract lc... |
607 608 |
static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) { |
724a8577d dma40: use helper... |
609 |
if (chan_is_physical(d40c)) { |
1c4b0927f dma40: move lli_l... |
610 |
d40_phy_lli_load(d40c, d40d); |
698e4732e DMAENGINE: ste_dm... |
611 |
d40d->lli_current = d40d->lli_len; |
e65889c75 dma40: extract lc... |
612 613 |
} else d40_log_lli_to_lcxa(d40c, d40d); |
698e4732e DMAENGINE: ste_dm... |
614 |
} |
8d318a50b DMAENGINE: Suppor... |
615 616 617 618 619 620 621 622 623 624 625 626 |
static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) { struct d40_desc *d; if (list_empty(&d40c->active)) return NULL; d = list_first_entry(&d40c->active, struct d40_desc, node); return d; } |
7404368c2 dmaengine/ste_dma... |
627 |
/* remove desc from current queue and add it to the pending_queue */ |
8d318a50b DMAENGINE: Suppor... |
628 629 |
static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc) { |
7404368c2 dmaengine/ste_dma... |
630 631 |
d40_desc_remove(desc); desc->is_in_client_list = false; |
a8f3067bc dmaengine/ste_dma... |
632 633 634 635 636 637 638 639 640 641 642 643 644 645 |
list_add_tail(&desc->node, &d40c->pending_queue); } static struct d40_desc *d40_first_pending(struct d40_chan *d40c) { struct d40_desc *d; if (list_empty(&d40c->pending_queue)) return NULL; d = list_first_entry(&d40c->pending_queue, struct d40_desc, node); return d; |
8d318a50b DMAENGINE: Suppor... |
646 647 648 649 650 651 652 653 654 655 656 657 658 659 |
} static struct d40_desc *d40_first_queued(struct d40_chan *d40c) { struct d40_desc *d; if (list_empty(&d40c->queue)) return NULL; d = list_first_entry(&d40c->queue, struct d40_desc, node); return d; } |
d49278e33 dmaengine: dma40:... |
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 |
static int d40_psize_2_burst_size(bool is_log, int psize) { if (is_log) { if (psize == STEDMA40_PSIZE_LOG_1) return 1; } else { if (psize == STEDMA40_PSIZE_PHY_1) return 1; } return 2 << psize; } /* * The dma only supports transmitting packages up to * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of * dma elements required to send the entire sg list */ static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2) { int dmalen; u32 max_w = max(data_width1, data_width2); u32 min_w = min(data_width1, data_width2); u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); if (seg_max > STEDMA40_MAX_SEG_SIZE) seg_max -= (1 << max_w); if (!IS_ALIGNED(size, 1 << max_w)) return -EINVAL; if (size <= seg_max) dmalen = 1; else { dmalen = size / seg_max; if (dmalen * seg_max < size) dmalen++; } return dmalen; } static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, u32 data_width1, u32 data_width2) { struct scatterlist *sg; int i; int len = 0; int ret; for_each_sg(sgl, sg, sg_len, i) { ret = d40_size_2_dmalen(sg_dma_len(sg), data_width1, data_width2); if (ret < 0) return ret; len += ret; } return len; } |
8d318a50b DMAENGINE: Suppor... |
718 |
|
d49278e33 dmaengine: dma40:... |
719 |
/* Support functions for logical channels */ |
8d318a50b DMAENGINE: Suppor... |
720 721 722 723 |
static int d40_channel_execute_command(struct d40_chan *d40c, enum d40_command command) { |
767a9675c DMAENGINE: ste_dm... |
724 725 |
u32 status; int i; |
8d318a50b DMAENGINE: Suppor... |
726 727 728 |
void __iomem *active_reg; int ret = 0; unsigned long flags; |
1d392a7ba DMAENGINE: ste_dm... |
729 |
u32 wmask; |
8d318a50b DMAENGINE: Suppor... |
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 |
spin_lock_irqsave(&d40c->base->execmd_lock, flags); if (d40c->phy_chan->num % 2 == 0) active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; else active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; if (command == D40_DMA_SUSPEND_REQ) { status = (readl(active_reg) & D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> D40_CHAN_POS(d40c->phy_chan->num); if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) goto done; } |
1d392a7ba DMAENGINE: ste_dm... |
746 747 748 |
wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num)); writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)), active_reg); |
8d318a50b DMAENGINE: Suppor... |
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 |
if (command == D40_DMA_SUSPEND_REQ) { for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) { status = (readl(active_reg) & D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> D40_CHAN_POS(d40c->phy_chan->num); cpu_relax(); /* * Reduce the number of bus accesses while * waiting for the DMA to suspend. */ udelay(3); if (status == D40_DMA_STOP || status == D40_DMA_SUSPENDED) break; } if (i == D40_SUSPEND_MAX_IT) { |
6db5a8ba1 dma40: use helper... |
770 771 772 773 |
chan_err(d40c, "unable to suspend the chl %d (log: %d) status %x ", d40c->phy_chan->num, d40c->log_num, |
8d318a50b DMAENGINE: Suppor... |
774 775 776 777 778 779 780 781 782 783 784 785 786 787 |
status); dump_stack(); ret = -EBUSY; } } done: spin_unlock_irqrestore(&d40c->base->execmd_lock, flags); return ret; } static void d40_term_all(struct d40_chan *d40c) { struct d40_desc *d40d; |
7404368c2 dmaengine/ste_dma... |
788 |
struct d40_desc *_d; |
8d318a50b DMAENGINE: Suppor... |
789 790 791 792 |
/* Release active descriptors */ while ((d40d = d40_first_active_get(d40c))) { d40_desc_remove(d40d); |
8d318a50b DMAENGINE: Suppor... |
793 794 795 796 797 798 |
d40_desc_free(d40c, d40d); } /* Release queued descriptors waiting for transfer */ while ((d40d = d40_first_queued(d40c))) { d40_desc_remove(d40d); |
8d318a50b DMAENGINE: Suppor... |
799 800 |
d40_desc_free(d40c, d40d); } |
a8f3067bc dmaengine/ste_dma... |
801 802 803 804 805 |
/* Release pending descriptors */ while ((d40d = d40_first_pending(d40c))) { d40_desc_remove(d40d); d40_desc_free(d40c, d40d); } |
8d318a50b DMAENGINE: Suppor... |
806 |
|
7404368c2 dmaengine/ste_dma... |
807 808 809 810 811 812 |
/* Release client owned descriptors */ if (!list_empty(&d40c->client)) list_for_each_entry_safe(d40d, _d, &d40c->client, node) { d40_desc_remove(d40d); d40_desc_free(d40c, d40d); } |
82babbb36 dmaengine/ste_dma... |
813 814 815 816 817 818 819 |
/* Release descriptors in prepare queue */ if (!list_empty(&d40c->prepare_queue)) list_for_each_entry_safe(d40d, _d, &d40c->prepare_queue, node) { d40_desc_remove(d40d); d40_desc_free(d40c, d40d); } |
7404368c2 dmaengine/ste_dma... |
820 |
|
8d318a50b DMAENGINE: Suppor... |
821 822 823 |
d40c->pending_tx = 0; d40c->busy = false; } |
262d2915d dma40: ensure eve... |
824 825 826 |
static void __d40_config_set_event(struct d40_chan *d40c, bool enable, u32 event, int reg) { |
8ca84687b dma40: use helper... |
827 |
void __iomem *addr = chan_base(d40c) + reg; |
262d2915d dma40: ensure eve... |
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 |
int tries; if (!enable) { writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) | ~D40_EVENTLINE_MASK(event), addr); return; } /* * The hardware sometimes doesn't register the enable when src and dst * event lines are active on the same logical channel. Retry to ensure * it does. Usually only one retry is sufficient. */ tries = 100; while (--tries) { writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) | ~D40_EVENTLINE_MASK(event), addr); if (readl(addr) & D40_EVENTLINE_MASK(event)) break; } if (tries != 99) dev_dbg(chan2dev(d40c), "[%s] workaround enable S%cLNK (%d tries) ", __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D', 100 - tries); WARN_ON(!tries); } |
8d318a50b DMAENGINE: Suppor... |
859 860 |
static void d40_config_set_event(struct d40_chan *d40c, bool do_enable) { |
8d318a50b DMAENGINE: Suppor... |
861 |
unsigned long flags; |
8d318a50b DMAENGINE: Suppor... |
862 863 864 865 866 867 |
spin_lock_irqsave(&d40c->phy_chan->lock, flags); /* Enable event line connected to device (or memcpy) */ if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) { u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); |
262d2915d dma40: ensure eve... |
868 869 |
__d40_config_set_event(d40c, do_enable, event, D40_CHAN_REG_SSLNK); |
8d318a50b DMAENGINE: Suppor... |
870 |
} |
262d2915d dma40: ensure eve... |
871 |
|
8d318a50b DMAENGINE: Suppor... |
872 873 |
if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) { u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); |
262d2915d dma40: ensure eve... |
874 875 |
__d40_config_set_event(d40c, do_enable, event, D40_CHAN_REG_SDLNK); |
8d318a50b DMAENGINE: Suppor... |
876 877 878 879 |
} spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); } |
a5ebca476 DMAENGINE: DMA40 ... |
880 |
static u32 d40_chan_has_events(struct d40_chan *d40c) |
8d318a50b DMAENGINE: Suppor... |
881 |
{ |
8ca84687b dma40: use helper... |
882 |
void __iomem *chanbase = chan_base(d40c); |
be8cb7dfd DMAENGINE: ste_dm... |
883 |
u32 val; |
8d318a50b DMAENGINE: Suppor... |
884 |
|
8ca84687b dma40: use helper... |
885 886 |
val = readl(chanbase + D40_CHAN_REG_SSLNK); val |= readl(chanbase + D40_CHAN_REG_SDLNK); |
be8cb7dfd DMAENGINE: ste_dm... |
887 |
|
a5ebca476 DMAENGINE: DMA40 ... |
888 |
return val; |
8d318a50b DMAENGINE: Suppor... |
889 |
} |
20a5b6d04 ste_dma40: move m... |
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 |
static u32 d40_get_prmo(struct d40_chan *d40c) { static const unsigned int phy_map[] = { [STEDMA40_PCHAN_BASIC_MODE] = D40_DREG_PRMO_PCHAN_BASIC, [STEDMA40_PCHAN_MODULO_MODE] = D40_DREG_PRMO_PCHAN_MODULO, [STEDMA40_PCHAN_DOUBLE_DST_MODE] = D40_DREG_PRMO_PCHAN_DOUBLE_DST, }; static const unsigned int log_map[] = { [STEDMA40_LCHAN_SRC_PHY_DST_LOG] = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG, [STEDMA40_LCHAN_SRC_LOG_DST_PHY] = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY, [STEDMA40_LCHAN_SRC_LOG_DST_LOG] = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG, }; |
724a8577d dma40: use helper... |
908 |
if (chan_is_physical(d40c)) |
20a5b6d04 ste_dma40: move m... |
909 910 911 912 |
return phy_map[d40c->dma_cfg.mode_opt]; else return log_map[d40c->dma_cfg.mode_opt]; } |
b55912c66 DMAENGINE: ste_dm... |
913 |
static void d40_config_write(struct d40_chan *d40c) |
8d318a50b DMAENGINE: Suppor... |
914 915 916 |
{ u32 addr_base; u32 var; |
8d318a50b DMAENGINE: Suppor... |
917 918 919 920 |
/* Odd addresses are even addresses + 4 */ addr_base = (d40c->phy_chan->num % 2) * 4; /* Setup channel mode to logical or physical */ |
724a8577d dma40: use helper... |
921 |
var = ((u32)(chan_is_logical(d40c)) + 1) << |
8d318a50b DMAENGINE: Suppor... |
922 923 924 925 |
D40_CHAN_POS(d40c->phy_chan->num); writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); /* Setup operational mode option register */ |
20a5b6d04 ste_dma40: move m... |
926 |
var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num); |
8d318a50b DMAENGINE: Suppor... |
927 928 |
writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); |
724a8577d dma40: use helper... |
929 |
if (chan_is_logical(d40c)) { |
8ca84687b dma40: use helper... |
930 931 932 |
int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) & D40_SREG_ELEM_LOG_LIDX_MASK; void __iomem *chanbase = chan_base(d40c); |
8d318a50b DMAENGINE: Suppor... |
933 |
/* Set default config for CFG reg */ |
8ca84687b dma40: use helper... |
934 935 |
writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG); writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG); |
8d318a50b DMAENGINE: Suppor... |
936 |
|
b55912c66 DMAENGINE: ste_dm... |
937 |
/* Set LIDX for lcla */ |
8ca84687b dma40: use helper... |
938 939 |
writel(lidx, chanbase + D40_CHAN_REG_SSELT); writel(lidx, chanbase + D40_CHAN_REG_SDELT); |
8d318a50b DMAENGINE: Suppor... |
940 |
} |
8d318a50b DMAENGINE: Suppor... |
941 |
} |
aa182ae26 DMAENGINE: ste_dm... |
942 943 944 |
static u32 d40_residue(struct d40_chan *d40c) { u32 num_elt; |
724a8577d dma40: use helper... |
945 |
if (chan_is_logical(d40c)) |
aa182ae26 DMAENGINE: ste_dm... |
946 947 |
num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) >> D40_MEM_LCSP2_ECNT_POS; |
8ca84687b dma40: use helper... |
948 949 950 951 952 |
else { u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT); num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK) >> D40_SREG_ELEM_PHY_ECNT_POS; } |
aa182ae26 DMAENGINE: ste_dm... |
953 954 955 956 957 958 |
return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); } static bool d40_tx_is_linked(struct d40_chan *d40c) { bool is_link; |
724a8577d dma40: use helper... |
959 |
if (chan_is_logical(d40c)) |
aa182ae26 DMAENGINE: ste_dm... |
960 961 |
is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; else |
8ca84687b dma40: use helper... |
962 963 |
is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK) & D40_SREG_LNK_PHYS_LNK_MASK; |
aa182ae26 DMAENGINE: ste_dm... |
964 965 |
return is_link; } |
86eb5fb61 dma40: stop ongoi... |
966 |
static int d40_pause(struct d40_chan *d40c) |
aa182ae26 DMAENGINE: ste_dm... |
967 |
{ |
aa182ae26 DMAENGINE: ste_dm... |
968 969 |
int res = 0; unsigned long flags; |
3ac012af3 DMAENGINE: ste_dm... |
970 971 |
if (!d40c->busy) return 0; |
aa182ae26 DMAENGINE: ste_dm... |
972 973 974 975 |
spin_lock_irqsave(&d40c->lock, flags); res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); if (res == 0) { |
724a8577d dma40: use helper... |
976 |
if (chan_is_logical(d40c)) { |
aa182ae26 DMAENGINE: ste_dm... |
977 978 979 980 981 982 983 984 985 986 987 |
d40_config_set_event(d40c, false); /* Resume the other logical channels if any */ if (d40_chan_has_events(d40c)) res = d40_channel_execute_command(d40c, D40_DMA_RUN); } } spin_unlock_irqrestore(&d40c->lock, flags); return res; } |
86eb5fb61 dma40: stop ongoi... |
988 |
static int d40_resume(struct d40_chan *d40c) |
aa182ae26 DMAENGINE: ste_dm... |
989 |
{ |
aa182ae26 DMAENGINE: ste_dm... |
990 991 |
int res = 0; unsigned long flags; |
3ac012af3 DMAENGINE: ste_dm... |
992 993 |
if (!d40c->busy) return 0; |
aa182ae26 DMAENGINE: ste_dm... |
994 995 996 |
spin_lock_irqsave(&d40c->lock, flags); if (d40c->base->rev == 0) |
724a8577d dma40: use helper... |
997 |
if (chan_is_logical(d40c)) { |
aa182ae26 DMAENGINE: ste_dm... |
998 999 1000 1001 1002 1003 1004 |
res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); goto no_suspend; } /* If bytes left to transfer or linked tx resume job */ if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { |
724a8577d dma40: use helper... |
1005 |
if (chan_is_logical(d40c)) |
aa182ae26 DMAENGINE: ste_dm... |
1006 1007 1008 1009 1010 1011 1012 1013 1014 |
d40_config_set_event(d40c, true); res = d40_channel_execute_command(d40c, D40_DMA_RUN); } no_suspend: spin_unlock_irqrestore(&d40c->lock, flags); return res; } |
86eb5fb61 dma40: stop ongoi... |
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 |
static int d40_terminate_all(struct d40_chan *chan) { unsigned long flags; int ret = 0; ret = d40_pause(chan); if (!ret && chan_is_physical(chan)) ret = d40_channel_execute_command(chan, D40_DMA_STOP); spin_lock_irqsave(&chan->lock, flags); d40_term_all(chan); spin_unlock_irqrestore(&chan->lock, flags); return ret; } |
8d318a50b DMAENGINE: Suppor... |
1030 1031 1032 1033 1034 1035 1036 1037 1038 |
static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) { struct d40_chan *d40c = container_of(tx->chan, struct d40_chan, chan); struct d40_desc *d40d = container_of(tx, struct d40_desc, txd); unsigned long flags; spin_lock_irqsave(&d40c->lock, flags); |
aa182ae26 DMAENGINE: ste_dm... |
1039 1040 1041 1042 1043 1044 |
d40c->chan.cookie++; if (d40c->chan.cookie < 0) d40c->chan.cookie = 1; d40d->txd.cookie = d40c->chan.cookie; |
8d318a50b DMAENGINE: Suppor... |
1045 1046 1047 1048 1049 1050 1051 1052 1053 |
d40_desc_queue(d40c, d40d); spin_unlock_irqrestore(&d40c->lock, flags); return tx->cookie; } static int d40_start(struct d40_chan *d40c) { |
f41855929 DMAENGINE: ste_dm... |
1054 1055 |
if (d40c->base->rev == 0) { int err; |
724a8577d dma40: use helper... |
1056 |
if (chan_is_logical(d40c)) { |
f41855929 DMAENGINE: ste_dm... |
1057 1058 1059 1060 1061 1062 |
err = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); if (err) return err; } } |
724a8577d dma40: use helper... |
1063 |
if (chan_is_logical(d40c)) |
8d318a50b DMAENGINE: Suppor... |
1064 |
d40_config_set_event(d40c, true); |
8d318a50b DMAENGINE: Suppor... |
1065 |
|
0c32269d8 DMAENGINE: ste_dm... |
1066 |
return d40_channel_execute_command(d40c, D40_DMA_RUN); |
8d318a50b DMAENGINE: Suppor... |
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 |
} static struct d40_desc *d40_queue_start(struct d40_chan *d40c) { struct d40_desc *d40d; int err; /* Start queued jobs, if any */ d40d = d40_first_queued(d40c); if (d40d != NULL) { d40c->busy = true; /* Remove from queue */ d40_desc_remove(d40d); /* Add to active queue */ d40_desc_submit(d40c, d40d); |
7d83a854a dma40: remove "ha... |
1085 1086 |
/* Initiate DMA job */ d40_desc_load(d40c, d40d); |
8d318a50b DMAENGINE: Suppor... |
1087 |
|
7d83a854a dma40: remove "ha... |
1088 1089 |
/* Start dma job */ err = d40_start(d40c); |
8d318a50b DMAENGINE: Suppor... |
1090 |
|
7d83a854a dma40: remove "ha... |
1091 1092 |
if (err) return NULL; |
8d318a50b DMAENGINE: Suppor... |
1093 1094 1095 1096 1097 1098 1099 1100 1101 |
} return d40d; } /* called from interrupt context */ static void dma_tc_handle(struct d40_chan *d40c) { struct d40_desc *d40d; |
8d318a50b DMAENGINE: Suppor... |
1102 1103 1104 1105 1106 |
/* Get first active entry from list */ d40d = d40_first_active_get(d40c); if (d40d == NULL) return; |
0c842b551 dma40: cyclic xfe... |
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 |
if (d40d->cyclic) { /* * If this was a paritially loaded list, we need to reloaded * it, and only when the list is completed. We need to check * for done because the interrupt will hit for every link, and * not just the last one. */ if (d40d->lli_current < d40d->lli_len && !d40_tx_is_linked(d40c) && !d40_residue(d40c)) { d40_lcla_free_all(d40c, d40d); d40_desc_load(d40c, d40d); (void) d40_start(d40c); |
8d318a50b DMAENGINE: Suppor... |
1120 |
|
0c842b551 dma40: cyclic xfe... |
1121 1122 1123 1124 1125 |
if (d40d->lli_current == d40d->lli_len) d40d->lli_current = 0; } } else { d40_lcla_free_all(d40c, d40d); |
8d318a50b DMAENGINE: Suppor... |
1126 |
|
0c842b551 dma40: cyclic xfe... |
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 |
if (d40d->lli_current < d40d->lli_len) { d40_desc_load(d40c, d40d); /* Start dma job */ (void) d40_start(d40c); return; } if (d40_queue_start(d40c) == NULL) d40c->busy = false; } |
8d318a50b DMAENGINE: Suppor... |
1137 1138 1139 1140 1141 1142 1143 1144 1145 |
d40c->pending_tx++; tasklet_schedule(&d40c->tasklet); } static void dma_tasklet(unsigned long data) { struct d40_chan *d40c = (struct d40_chan *) data; |
767a9675c DMAENGINE: ste_dm... |
1146 |
struct d40_desc *d40d; |
8d318a50b DMAENGINE: Suppor... |
1147 1148 1149 1150 1151 1152 1153 |
unsigned long flags; dma_async_tx_callback callback; void *callback_param; spin_lock_irqsave(&d40c->lock, flags); /* Get first active entry from list */ |
767a9675c DMAENGINE: ste_dm... |
1154 |
d40d = d40_first_active_get(d40c); |
767a9675c DMAENGINE: ste_dm... |
1155 |
if (d40d == NULL) |
8d318a50b DMAENGINE: Suppor... |
1156 |
goto err; |
0c842b551 dma40: cyclic xfe... |
1157 1158 |
if (!d40d->cyclic) d40c->completed = d40d->txd.cookie; |
8d318a50b DMAENGINE: Suppor... |
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 |
/* * If terminating a channel pending_tx is set to zero. * This prevents any finished active jobs to return to the client. */ if (d40c->pending_tx == 0) { spin_unlock_irqrestore(&d40c->lock, flags); return; } /* Callback to client */ |
767a9675c DMAENGINE: ste_dm... |
1170 1171 |
callback = d40d->txd.callback; callback_param = d40d->txd.callback_param; |
0c842b551 dma40: cyclic xfe... |
1172 1173 |
if (!d40d->cyclic) { if (async_tx_test_ack(&d40d->txd)) { |
767a9675c DMAENGINE: ste_dm... |
1174 |
d40_desc_remove(d40d); |
0c842b551 dma40: cyclic xfe... |
1175 1176 1177 1178 1179 1180 1181 1182 |
d40_desc_free(d40c, d40d); } else { if (!d40d->is_in_client_list) { d40_desc_remove(d40d); d40_lcla_free_all(d40c, d40d); list_add_tail(&d40d->node, &d40c->client); d40d->is_in_client_list = true; } |
8d318a50b DMAENGINE: Suppor... |
1183 1184 1185 1186 1187 1188 1189 1190 1191 |
} } d40c->pending_tx--; if (d40c->pending_tx) tasklet_schedule(&d40c->tasklet); spin_unlock_irqrestore(&d40c->lock, flags); |
767a9675c DMAENGINE: ste_dm... |
1192 |
if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT)) |
8d318a50b DMAENGINE: Suppor... |
1193 1194 1195 1196 1197 |
callback(callback_param); return; err: |
25985edce Fix common misspe... |
1198 |
/* Rescue manoeuvre if receiving double interrupts */ |
8d318a50b DMAENGINE: Suppor... |
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 |
if (d40c->pending_tx > 0) d40c->pending_tx--; spin_unlock_irqrestore(&d40c->lock, flags); } static irqreturn_t d40_handle_interrupt(int irq, void *data) { static const struct d40_interrupt_lookup il[] = { {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0}, {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32}, {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64}, {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96}, {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0}, {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32}, {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64}, {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96}, {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN}, {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN}, }; int i; u32 regs[ARRAY_SIZE(il)]; |
8d318a50b DMAENGINE: Suppor... |
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 |
u32 idx; u32 row; long chan = -1; struct d40_chan *d40c; unsigned long flags; struct d40_base *base = data; spin_lock_irqsave(&base->interrupt_lock, flags); /* Read interrupt status of both logical and physical channels */ for (i = 0; i < ARRAY_SIZE(il); i++) regs[i] = readl(base->virtbase + il[i].src); for (;;) { chan = find_next_bit((unsigned long *)regs, BITS_PER_LONG * ARRAY_SIZE(il), chan + 1); /* No more set bits found? */ if (chan == BITS_PER_LONG * ARRAY_SIZE(il)) break; row = chan / BITS_PER_LONG; idx = chan & (BITS_PER_LONG - 1); /* ACK interrupt */ |
1b00348d5 DMAENGINE: ste_dm... |
1247 |
writel(1 << idx, base->virtbase + il[row].clr); |
8d318a50b DMAENGINE: Suppor... |
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 |
if (il[row].offset == D40_PHY_CHAN) d40c = base->lookup_phy_chans[idx]; else d40c = base->lookup_log_chans[il[row].offset + idx]; spin_lock(&d40c->lock); if (!il[row].is_error) dma_tc_handle(d40c); else |
6db5a8ba1 dma40: use helper... |
1258 1259 1260 |
d40_err(base->dev, "IRQ chan: %ld offset %d idx %d ", chan, il[row].offset, idx); |
8d318a50b DMAENGINE: Suppor... |
1261 1262 1263 1264 1265 1266 1267 1268 |
spin_unlock(&d40c->lock); } spin_unlock_irqrestore(&base->interrupt_lock, flags); return IRQ_HANDLED; } |
8d318a50b DMAENGINE: Suppor... |
1269 1270 1271 1272 1273 1274 |
static int d40_validate_conf(struct d40_chan *d40c, struct stedma40_chan_cfg *conf) { int res = 0; u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type); u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type); |
38bdbf020 ste_dma40: move c... |
1275 |
bool is_log = conf->mode == STEDMA40_MODE_LOGICAL; |
8d318a50b DMAENGINE: Suppor... |
1276 |
|
0747c7bae DMAENGINE: ste_dm... |
1277 |
if (!conf->dir) { |
6db5a8ba1 dma40: use helper... |
1278 1279 |
chan_err(d40c, "Invalid direction. "); |
0747c7bae DMAENGINE: ste_dm... |
1280 1281 1282 1283 1284 1285 |
res = -EINVAL; } if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY && d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 && d40c->runtime_addr == 0) { |
6db5a8ba1 dma40: use helper... |
1286 1287 1288 |
chan_err(d40c, "Invalid TX channel address (%d) ", conf->dst_dev_type); |
0747c7bae DMAENGINE: ste_dm... |
1289 1290 1291 1292 1293 1294 |
res = -EINVAL; } if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY && d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 && d40c->runtime_addr == 0) { |
6db5a8ba1 dma40: use helper... |
1295 1296 1297 |
chan_err(d40c, "Invalid RX channel address (%d) ", conf->src_dev_type); |
0747c7bae DMAENGINE: ste_dm... |
1298 1299 1300 1301 |
res = -EINVAL; } if (conf->dir == STEDMA40_MEM_TO_PERIPH && |
8d318a50b DMAENGINE: Suppor... |
1302 |
dst_event_group == STEDMA40_DEV_DST_MEMORY) { |
6db5a8ba1 dma40: use helper... |
1303 1304 |
chan_err(d40c, "Invalid dst "); |
8d318a50b DMAENGINE: Suppor... |
1305 1306 |
res = -EINVAL; } |
0747c7bae DMAENGINE: ste_dm... |
1307 |
if (conf->dir == STEDMA40_PERIPH_TO_MEM && |
8d318a50b DMAENGINE: Suppor... |
1308 |
src_event_group == STEDMA40_DEV_SRC_MEMORY) { |
6db5a8ba1 dma40: use helper... |
1309 1310 |
chan_err(d40c, "Invalid src "); |
8d318a50b DMAENGINE: Suppor... |
1311 1312 1313 1314 1315 |
res = -EINVAL; } if (src_event_group == STEDMA40_DEV_SRC_MEMORY && dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) { |
6db5a8ba1 dma40: use helper... |
1316 1317 |
chan_err(d40c, "No event line "); |
8d318a50b DMAENGINE: Suppor... |
1318 1319 1320 1321 1322 |
res = -EINVAL; } if (conf->dir == STEDMA40_PERIPH_TO_PERIPH && (src_event_group != dst_event_group)) { |
6db5a8ba1 dma40: use helper... |
1323 1324 |
chan_err(d40c, "Invalid event group "); |
8d318a50b DMAENGINE: Suppor... |
1325 1326 1327 1328 1329 1330 1331 1332 |
res = -EINVAL; } if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) { /* * DMAC HW supports it. Will be added to this driver, * in case any dma client requires it. */ |
6db5a8ba1 dma40: use helper... |
1333 1334 |
chan_err(d40c, "periph to periph not supported "); |
8d318a50b DMAENGINE: Suppor... |
1335 1336 |
res = -EINVAL; } |
d49278e33 dmaengine: dma40:... |
1337 1338 1339 1340 1341 1342 1343 1344 |
if (d40_psize_2_burst_size(is_log, conf->src_info.psize) * (1 << conf->src_info.data_width) != d40_psize_2_burst_size(is_log, conf->dst_info.psize) * (1 << conf->dst_info.data_width)) { /* * The DMAC hardware only supports * src (burst x width) == dst (burst x width) */ |
6db5a8ba1 dma40: use helper... |
1345 1346 |
chan_err(d40c, "src (burst x width) != dst (burst x width) "); |
d49278e33 dmaengine: dma40:... |
1347 1348 |
res = -EINVAL; } |
8d318a50b DMAENGINE: Suppor... |
1349 1350 1351 1352 |
return res; } static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src, |
4aed79b28 DMAENGINE: DMA40 ... |
1353 |
int log_event_line, bool is_log) |
8d318a50b DMAENGINE: Suppor... |
1354 1355 1356 |
{ unsigned long flags; spin_lock_irqsave(&phy->lock, flags); |
4aed79b28 DMAENGINE: DMA40 ... |
1357 |
if (!is_log) { |
8d318a50b DMAENGINE: Suppor... |
1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 |
/* Physical interrupts are masked per physical full channel */ if (phy->allocated_src == D40_ALLOC_FREE && phy->allocated_dst == D40_ALLOC_FREE) { phy->allocated_dst = D40_ALLOC_PHY; phy->allocated_src = D40_ALLOC_PHY; goto found; } else goto not_found; } /* Logical channel */ if (is_src) { if (phy->allocated_src == D40_ALLOC_PHY) goto not_found; if (phy->allocated_src == D40_ALLOC_FREE) phy->allocated_src = D40_ALLOC_LOG_FREE; if (!(phy->allocated_src & (1 << log_event_line))) { phy->allocated_src |= 1 << log_event_line; goto found; } else goto not_found; } else { if (phy->allocated_dst == D40_ALLOC_PHY) goto not_found; if (phy->allocated_dst == D40_ALLOC_FREE) phy->allocated_dst = D40_ALLOC_LOG_FREE; if (!(phy->allocated_dst & (1 << log_event_line))) { phy->allocated_dst |= 1 << log_event_line; goto found; } else goto not_found; } not_found: spin_unlock_irqrestore(&phy->lock, flags); return false; found: spin_unlock_irqrestore(&phy->lock, flags); return true; } static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src, int log_event_line) { unsigned long flags; bool is_free = false; spin_lock_irqsave(&phy->lock, flags); if (!log_event_line) { |
8d318a50b DMAENGINE: Suppor... |
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 |
phy->allocated_dst = D40_ALLOC_FREE; phy->allocated_src = D40_ALLOC_FREE; is_free = true; goto out; } /* Logical channel */ if (is_src) { phy->allocated_src &= ~(1 << log_event_line); if (phy->allocated_src == D40_ALLOC_LOG_FREE) phy->allocated_src = D40_ALLOC_FREE; } else { phy->allocated_dst &= ~(1 << log_event_line); if (phy->allocated_dst == D40_ALLOC_LOG_FREE) phy->allocated_dst = D40_ALLOC_FREE; } is_free = ((phy->allocated_src | phy->allocated_dst) == D40_ALLOC_FREE); out: spin_unlock_irqrestore(&phy->lock, flags); return is_free; } static int d40_allocate_channel(struct d40_chan *d40c) { int dev_type; int event_group; int event_line; struct d40_phy_res *phys; int i; int j; int log_num; bool is_src; |
38bdbf020 ste_dma40: move c... |
1447 |
bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL; |
8d318a50b DMAENGINE: Suppor... |
1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 |
phys = d40c->base->phy_res; if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { dev_type = d40c->dma_cfg.src_dev_type; log_num = 2 * dev_type; is_src = true; } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { /* dst event lines are used for logical memcpy */ dev_type = d40c->dma_cfg.dst_dev_type; log_num = 2 * dev_type + 1; is_src = false; } else return -EINVAL; event_group = D40_TYPE_TO_GROUP(dev_type); event_line = D40_TYPE_TO_EVENT(dev_type); if (!is_log) { if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { /* Find physical half channel */ for (i = 0; i < d40c->base->num_phy_chans; i++) { |
4aed79b28 DMAENGINE: DMA40 ... |
1471 1472 |
if (d40_alloc_mask_set(&phys[i], is_src, 0, is_log)) |
8d318a50b DMAENGINE: Suppor... |
1473 1474 1475 1476 1477 1478 |
goto found_phy; } } else for (j = 0; j < d40c->base->num_phy_chans; j += 8) { int phy_num = j + event_group * 2; for (i = phy_num; i < phy_num + 2; i++) { |
508849ade DMAENGINE: ste_dm... |
1479 1480 1481 1482 |
if (d40_alloc_mask_set(&phys[i], is_src, 0, is_log)) |
8d318a50b DMAENGINE: Suppor... |
1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 |
goto found_phy; } } return -EINVAL; found_phy: d40c->phy_chan = &phys[i]; d40c->log_num = D40_PHY_CHAN; goto out; } if (dev_type == -1) return -EINVAL; /* Find logical channel */ for (j = 0; j < d40c->base->num_phy_chans; j += 8) { int phy_num = j + event_group * 2; /* * Spread logical channels across all available physical rather * than pack every logical channel at the first available phy * channels. */ if (is_src) { for (i = phy_num; i < phy_num + 2; i++) { if (d40_alloc_mask_set(&phys[i], is_src, |
4aed79b28 DMAENGINE: DMA40 ... |
1506 |
event_line, is_log)) |
8d318a50b DMAENGINE: Suppor... |
1507 1508 1509 1510 1511 |
goto found_log; } } else { for (i = phy_num + 1; i >= phy_num; i--) { if (d40_alloc_mask_set(&phys[i], is_src, |
4aed79b28 DMAENGINE: DMA40 ... |
1512 |
event_line, is_log)) |
8d318a50b DMAENGINE: Suppor... |
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 |
goto found_log; } } } return -EINVAL; found_log: d40c->phy_chan = &phys[i]; d40c->log_num = log_num; out: if (is_log) d40c->base->lookup_log_chans[d40c->log_num] = d40c; else d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c; return 0; } |
8d318a50b DMAENGINE: Suppor... |
1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 |
static int d40_config_memcpy(struct d40_chan *d40c) { dma_cap_mask_t cap = d40c->chan.device->cap_mask; if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) { d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log; d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY; d40c->dma_cfg.dst_dev_type = d40c->base->plat_data-> memcpy[d40c->chan.chan_id]; } else if (dma_has_cap(DMA_MEMCPY, cap) && dma_has_cap(DMA_SLAVE, cap)) { d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy; } else { |
6db5a8ba1 dma40: use helper... |
1546 1547 |
chan_err(d40c, "No memcpy "); |
8d318a50b DMAENGINE: Suppor... |
1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 |
return -EINVAL; } return 0; } static int d40_free_dma(struct d40_chan *d40c) { int res = 0; |
d181b3a8c DMAENGINE: ste_dm... |
1559 |
u32 event; |
8d318a50b DMAENGINE: Suppor... |
1560 1561 1562 1563 1564 1565 1566 |
struct d40_phy_res *phy = d40c->phy_chan; bool is_src; /* Terminate all queued and active transfers */ d40_term_all(d40c); if (phy == NULL) { |
6db5a8ba1 dma40: use helper... |
1567 1568 |
chan_err(d40c, "phy == null "); |
8d318a50b DMAENGINE: Suppor... |
1569 1570 1571 1572 1573 |
return -EINVAL; } if (phy->allocated_src == D40_ALLOC_FREE && phy->allocated_dst == D40_ALLOC_FREE) { |
6db5a8ba1 dma40: use helper... |
1574 1575 |
chan_err(d40c, "channel already free "); |
8d318a50b DMAENGINE: Suppor... |
1576 1577 |
return -EINVAL; } |
8d318a50b DMAENGINE: Suppor... |
1578 1579 1580 |
if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); |
8d318a50b DMAENGINE: Suppor... |
1581 1582 1583 |
is_src = false; } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); |
8d318a50b DMAENGINE: Suppor... |
1584 1585 |
is_src = true; } else { |
6db5a8ba1 dma40: use helper... |
1586 1587 |
chan_err(d40c, "Unknown direction "); |
8d318a50b DMAENGINE: Suppor... |
1588 1589 |
return -EINVAL; } |
d181b3a8c DMAENGINE: ste_dm... |
1590 1591 |
res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); if (res) { |
6db5a8ba1 dma40: use helper... |
1592 1593 |
chan_err(d40c, "suspend failed "); |
d181b3a8c DMAENGINE: ste_dm... |
1594 1595 |
return res; } |
724a8577d dma40: use helper... |
1596 |
if (chan_is_logical(d40c)) { |
d181b3a8c DMAENGINE: ste_dm... |
1597 |
/* Release logical channel, deactivate the event line */ |
8d318a50b DMAENGINE: Suppor... |
1598 |
|
d181b3a8c DMAENGINE: ste_dm... |
1599 |
d40_config_set_event(d40c, false); |
8d318a50b DMAENGINE: Suppor... |
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 |
d40c->base->lookup_log_chans[d40c->log_num] = NULL; /* * Check if there are more logical allocation * on this phy channel. */ if (!d40_alloc_mask_free(phy, is_src, event)) { /* Resume the other logical channels if any */ if (d40_chan_has_events(d40c)) { res = d40_channel_execute_command(d40c, D40_DMA_RUN); if (res) { |
6db5a8ba1 dma40: use helper... |
1612 1613 1614 |
chan_err(d40c, "Executing RUN command "); |
8d318a50b DMAENGINE: Suppor... |
1615 1616 1617 1618 1619 |
return res; } } return 0; } |
d181b3a8c DMAENGINE: ste_dm... |
1620 1621 1622 |
} else { (void) d40_alloc_mask_free(phy, is_src, 0); } |
8d318a50b DMAENGINE: Suppor... |
1623 1624 1625 1626 |
/* Release physical channel */ res = d40_channel_execute_command(d40c, D40_DMA_STOP); if (res) { |
6db5a8ba1 dma40: use helper... |
1627 1628 |
chan_err(d40c, "Failed to stop channel "); |
8d318a50b DMAENGINE: Suppor... |
1629 1630 1631 |
return res; } d40c->phy_chan = NULL; |
ce2ca1252 ste_dma40: add va... |
1632 |
d40c->configured = false; |
8d318a50b DMAENGINE: Suppor... |
1633 1634 1635 |
d40c->base->lookup_phy_chans[phy->num] = NULL; return 0; |
8d318a50b DMAENGINE: Suppor... |
1636 |
} |
a5ebca476 DMAENGINE: DMA40 ... |
1637 1638 |
static bool d40_is_paused(struct d40_chan *d40c) { |
8ca84687b dma40: use helper... |
1639 |
void __iomem *chanbase = chan_base(d40c); |
a5ebca476 DMAENGINE: DMA40 ... |
1640 1641 1642 1643 1644 |
bool is_paused = false; unsigned long flags; void __iomem *active_reg; u32 status; u32 event; |
a5ebca476 DMAENGINE: DMA40 ... |
1645 1646 |
spin_lock_irqsave(&d40c->lock, flags); |
724a8577d dma40: use helper... |
1647 |
if (chan_is_physical(d40c)) { |
a5ebca476 DMAENGINE: DMA40 ... |
1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 |
if (d40c->phy_chan->num % 2 == 0) active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; else active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; status = (readl(active_reg) & D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> D40_CHAN_POS(d40c->phy_chan->num); if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) is_paused = true; goto _exit; } |
a5ebca476 DMAENGINE: DMA40 ... |
1661 |
if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || |
9dbfbd35c DMAENGINE: ste_dm... |
1662 |
d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { |
a5ebca476 DMAENGINE: DMA40 ... |
1663 |
event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); |
8ca84687b dma40: use helper... |
1664 |
status = readl(chanbase + D40_CHAN_REG_SDLNK); |
9dbfbd35c DMAENGINE: ste_dm... |
1665 |
} else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { |
a5ebca476 DMAENGINE: DMA40 ... |
1666 |
event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); |
8ca84687b dma40: use helper... |
1667 |
status = readl(chanbase + D40_CHAN_REG_SSLNK); |
9dbfbd35c DMAENGINE: ste_dm... |
1668 |
} else { |
6db5a8ba1 dma40: use helper... |
1669 1670 |
chan_err(d40c, "Unknown direction "); |
a5ebca476 DMAENGINE: DMA40 ... |
1671 1672 |
goto _exit; } |
9dbfbd35c DMAENGINE: ste_dm... |
1673 |
|
a5ebca476 DMAENGINE: DMA40 ... |
1674 1675 1676 1677 1678 |
status = (status & D40_EVENTLINE_MASK(event)) >> D40_EVENTLINE_POS(event); if (status != D40_DMA_RUN) is_paused = true; |
a5ebca476 DMAENGINE: DMA40 ... |
1679 1680 1681 1682 1683 |
_exit: spin_unlock_irqrestore(&d40c->lock, flags); return is_paused; } |
8d318a50b DMAENGINE: Suppor... |
1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 |
static u32 stedma40_residue(struct dma_chan *chan) { struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); u32 bytes_left; unsigned long flags; spin_lock_irqsave(&d40c->lock, flags); bytes_left = d40_residue(d40c); spin_unlock_irqrestore(&d40c->lock, flags); return bytes_left; } |
3e3a0763e dma40: combine me... |
1697 1698 1699 |
static int d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc, struct scatterlist *sg_src, struct scatterlist *sg_dst, |
822c56763 dma40: unify src/... |
1700 1701 |
unsigned int sg_len, dma_addr_t src_dev_addr, dma_addr_t dst_dev_addr) |
3e3a0763e dma40: combine me... |
1702 1703 1704 1705 |
{ struct stedma40_chan_cfg *cfg = &chan->dma_cfg; struct stedma40_half_channel_info *src_info = &cfg->src_info; struct stedma40_half_channel_info *dst_info = &cfg->dst_info; |
5ed04b857 dma40: unify d40_... |
1706 |
int ret; |
3e3a0763e dma40: combine me... |
1707 |
|
5ed04b857 dma40: unify d40_... |
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 |
ret = d40_log_sg_to_lli(sg_src, sg_len, src_dev_addr, desc->lli_log.src, chan->log_def.lcsp1, src_info->data_width, dst_info->data_width); ret = d40_log_sg_to_lli(sg_dst, sg_len, dst_dev_addr, desc->lli_log.dst, chan->log_def.lcsp3, dst_info->data_width, src_info->data_width); return ret < 0 ? ret : 0; |
3e3a0763e dma40: combine me... |
1723 1724 1725 1726 1727 |
} static int d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc, struct scatterlist *sg_src, struct scatterlist *sg_dst, |
822c56763 dma40: unify src/... |
1728 1729 |
unsigned int sg_len, dma_addr_t src_dev_addr, dma_addr_t dst_dev_addr) |
3e3a0763e dma40: combine me... |
1730 |
{ |
3e3a0763e dma40: combine me... |
1731 1732 1733 |
struct stedma40_chan_cfg *cfg = &chan->dma_cfg; struct stedma40_half_channel_info *src_info = &cfg->src_info; struct stedma40_half_channel_info *dst_info = &cfg->dst_info; |
0c842b551 dma40: cyclic xfe... |
1734 |
unsigned long flags = 0; |
3e3a0763e dma40: combine me... |
1735 |
int ret; |
0c842b551 dma40: cyclic xfe... |
1736 1737 |
if (desc->cyclic) flags |= LLI_CYCLIC | LLI_TERM_INT; |
3e3a0763e dma40: combine me... |
1738 1739 1740 1741 |
ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr, desc->lli_phy.src, virt_to_phys(desc->lli_phy.src), chan->src_def_cfg, |
0c842b551 dma40: cyclic xfe... |
1742 |
src_info, dst_info, flags); |
3e3a0763e dma40: combine me... |
1743 1744 1745 1746 1747 |
ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr, desc->lli_phy.dst, virt_to_phys(desc->lli_phy.dst), chan->dst_def_cfg, |
0c842b551 dma40: cyclic xfe... |
1748 |
dst_info, src_info, flags); |
3e3a0763e dma40: combine me... |
1749 1750 1751 1752 1753 1754 |
dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr, desc->lli_pool.size, DMA_TO_DEVICE); return ret < 0 ? ret : 0; } |
5f81158f9 dma40: combine de... |
1755 1756 1757 1758 1759 1760 |
static struct d40_desc * d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg, unsigned int sg_len, unsigned long dma_flags) { struct stedma40_chan_cfg *cfg = &chan->dma_cfg; struct d40_desc *desc; |
dbd887880 dma40: combine du... |
1761 |
int ret; |
5f81158f9 dma40: combine de... |
1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 |
desc = d40_desc_get(chan); if (!desc) return NULL; desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width, cfg->dst_info.data_width); if (desc->lli_len < 0) { chan_err(chan, "Unaligned size "); |
dbd887880 dma40: combine du... |
1772 1773 |
goto err; } |
5f81158f9 dma40: combine de... |
1774 |
|
dbd887880 dma40: combine du... |
1775 1776 1777 1778 1779 |
ret = d40_pool_lli_alloc(chan, desc, desc->lli_len); if (ret < 0) { chan_err(chan, "Could not allocate lli "); goto err; |
5f81158f9 dma40: combine de... |
1780 |
} |
dbd887880 dma40: combine du... |
1781 |
|
5f81158f9 dma40: combine de... |
1782 1783 1784 1785 1786 1787 1788 |
desc->lli_current = 0; desc->txd.flags = dma_flags; desc->txd.tx_submit = d40_tx_submit; dma_async_tx_descriptor_init(&desc->txd, &chan->chan); return desc; |
dbd887880 dma40: combine du... |
1789 1790 1791 1792 |
err: d40_desc_free(chan, desc); return NULL; |
5f81158f9 dma40: combine de... |
1793 |
} |
cade1d30b dma40: combine me... |
1794 1795 |
static dma_addr_t d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction) |
8d318a50b DMAENGINE: Suppor... |
1796 |
{ |
cade1d30b dma40: combine me... |
1797 1798 |
struct stedma40_platform_data *plat = chan->base->plat_data; struct stedma40_chan_cfg *cfg = &chan->dma_cfg; |
711b9cea9 dmaengine/ste_dma... |
1799 |
dma_addr_t addr = 0; |
cade1d30b dma40: combine me... |
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 |
if (chan->runtime_addr) return chan->runtime_addr; if (direction == DMA_FROM_DEVICE) addr = plat->dev_rx[cfg->src_dev_type]; else if (direction == DMA_TO_DEVICE) addr = plat->dev_tx[cfg->dst_dev_type]; return addr; } static struct dma_async_tx_descriptor * d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, struct scatterlist *sg_dst, unsigned int sg_len, enum dma_data_direction direction, unsigned long dma_flags) { struct d40_chan *chan = container_of(dchan, struct d40_chan, chan); |
822c56763 dma40: unify src/... |
1818 1819 |
dma_addr_t src_dev_addr = 0; dma_addr_t dst_dev_addr = 0; |
cade1d30b dma40: combine me... |
1820 |
struct d40_desc *desc; |
2a6143407 DMAENGINE: ste_dm... |
1821 |
unsigned long flags; |
cade1d30b dma40: combine me... |
1822 |
int ret; |
8d318a50b DMAENGINE: Suppor... |
1823 |
|
cade1d30b dma40: combine me... |
1824 1825 1826 1827 |
if (!chan->phy_chan) { chan_err(chan, "Cannot prepare unallocated channel "); return NULL; |
0d0f6b8bb DMAENGINE: ste_dm... |
1828 |
} |
0c842b551 dma40: cyclic xfe... |
1829 |
|
cade1d30b dma40: combine me... |
1830 |
spin_lock_irqsave(&chan->lock, flags); |
8d318a50b DMAENGINE: Suppor... |
1831 |
|
cade1d30b dma40: combine me... |
1832 1833 |
desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags); if (desc == NULL) |
8d318a50b DMAENGINE: Suppor... |
1834 |
goto err; |
0c842b551 dma40: cyclic xfe... |
1835 1836 |
if (sg_next(&sg_src[sg_len - 1]) == sg_src) desc->cyclic = true; |
822c56763 dma40: unify src/... |
1837 1838 1839 1840 1841 1842 1843 1844 |
if (direction != DMA_NONE) { dma_addr_t dev_addr = d40_get_dev_addr(chan, direction); if (direction == DMA_FROM_DEVICE) src_dev_addr = dev_addr; else if (direction == DMA_TO_DEVICE) dst_dev_addr = dev_addr; } |
cade1d30b dma40: combine me... |
1845 1846 1847 |
if (chan_is_logical(chan)) ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst, |
822c56763 dma40: unify src/... |
1848 |
sg_len, src_dev_addr, dst_dev_addr); |
cade1d30b dma40: combine me... |
1849 1850 |
else ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst, |
822c56763 dma40: unify src/... |
1851 |
sg_len, src_dev_addr, dst_dev_addr); |
cade1d30b dma40: combine me... |
1852 1853 1854 1855 1856 1857 |
if (ret) { chan_err(chan, "Failed to prepare %s sg job: %d ", chan_is_logical(chan) ? "log" : "phy", ret); goto err; |
8d318a50b DMAENGINE: Suppor... |
1858 |
} |
82babbb36 dmaengine/ste_dma... |
1859 1860 1861 1862 1863 |
/* * add descriptor to the prepare queue in order to be able * to free them later in terminate_all */ list_add_tail(&desc->node, &chan->prepare_queue); |
cade1d30b dma40: combine me... |
1864 1865 1866 |
spin_unlock_irqrestore(&chan->lock, flags); return &desc->txd; |
8d318a50b DMAENGINE: Suppor... |
1867 |
|
8d318a50b DMAENGINE: Suppor... |
1868 |
err: |
cade1d30b dma40: combine me... |
1869 1870 1871 |
if (desc) d40_desc_free(chan, desc); spin_unlock_irqrestore(&chan->lock, flags); |
8d318a50b DMAENGINE: Suppor... |
1872 1873 |
return NULL; } |
8d318a50b DMAENGINE: Suppor... |
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 |
bool stedma40_filter(struct dma_chan *chan, void *data) { struct stedma40_chan_cfg *info = data; struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); int err; if (data) { err = d40_validate_conf(d40c, info); if (!err) d40c->dma_cfg = *info; } else err = d40_config_memcpy(d40c); |
ce2ca1252 ste_dma40: add va... |
1888 1889 |
if (!err) d40c->configured = true; |
8d318a50b DMAENGINE: Suppor... |
1890 1891 1892 |
return err == 0; } EXPORT_SYMBOL(stedma40_filter); |
ac2c0a387 dma40: allow real... |
1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 |
static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src) { bool realtime = d40c->dma_cfg.realtime; bool highprio = d40c->dma_cfg.high_priority; u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1; u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1; u32 event = D40_TYPE_TO_EVENT(dev_type); u32 group = D40_TYPE_TO_GROUP(dev_type); u32 bit = 1 << event; /* Destination event lines are stored in the upper halfword */ if (!src) bit <<= 16; writel(bit, d40c->base->virtbase + prioreg + group * 4); writel(bit, d40c->base->virtbase + rtreg + group * 4); } static void d40_set_prio_realtime(struct d40_chan *d40c) { if (d40c->base->rev < 3) return; if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true); if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) || (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false); } |
8d318a50b DMAENGINE: Suppor... |
1924 1925 1926 1927 1928 1929 1930 |
/* DMA ENGINE functions */ static int d40_alloc_chan_resources(struct dma_chan *chan) { int err; unsigned long flags; struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); |
ef1872ec6 DMAENGINE: ste_dm... |
1931 |
bool is_free_phy; |
8d318a50b DMAENGINE: Suppor... |
1932 1933 1934 |
spin_lock_irqsave(&d40c->lock, flags); d40c->completed = chan->cookie = 1; |
ce2ca1252 ste_dma40: add va... |
1935 1936 |
/* If no dma configuration is set use default configuration (memcpy) */ if (!d40c->configured) { |
8d318a50b DMAENGINE: Suppor... |
1937 |
err = d40_config_memcpy(d40c); |
ff0b12baa DMAENGINE: ste_dm... |
1938 |
if (err) { |
6db5a8ba1 dma40: use helper... |
1939 1940 |
chan_err(d40c, "Failed to configure memcpy channel "); |
ff0b12baa DMAENGINE: ste_dm... |
1941 1942 |
goto fail; } |
8d318a50b DMAENGINE: Suppor... |
1943 |
} |
ef1872ec6 DMAENGINE: ste_dm... |
1944 |
is_free_phy = (d40c->phy_chan == NULL); |
8d318a50b DMAENGINE: Suppor... |
1945 1946 1947 |
err = d40_allocate_channel(d40c); if (err) { |
6db5a8ba1 dma40: use helper... |
1948 1949 |
chan_err(d40c, "Failed to allocate channel "); |
ff0b12baa DMAENGINE: ste_dm... |
1950 |
goto fail; |
8d318a50b DMAENGINE: Suppor... |
1951 |
} |
ef1872ec6 DMAENGINE: ste_dm... |
1952 1953 |
/* Fill in basic CFG register values */ d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg, |
724a8577d dma40: use helper... |
1954 |
&d40c->dst_def_cfg, chan_is_logical(d40c)); |
ef1872ec6 DMAENGINE: ste_dm... |
1955 |
|
ac2c0a387 dma40: allow real... |
1956 |
d40_set_prio_realtime(d40c); |
724a8577d dma40: use helper... |
1957 |
if (chan_is_logical(d40c)) { |
ef1872ec6 DMAENGINE: ste_dm... |
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 |
d40_log_cfg(&d40c->dma_cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) d40c->lcpa = d40c->base->lcpa_base + d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE; else d40c->lcpa = d40c->base->lcpa_base + d40c->dma_cfg.dst_dev_type * D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA; } /* * Only write channel configuration to the DMA if the physical * resource is free. In case of multiple logical channels * on the same physical resource, only the first write is necessary. */ |
b55912c66 DMAENGINE: ste_dm... |
1975 1976 |
if (is_free_phy) d40_config_write(d40c); |
ff0b12baa DMAENGINE: ste_dm... |
1977 |
fail: |
8d318a50b DMAENGINE: Suppor... |
1978 |
spin_unlock_irqrestore(&d40c->lock, flags); |
ff0b12baa DMAENGINE: ste_dm... |
1979 |
return err; |
8d318a50b DMAENGINE: Suppor... |
1980 1981 1982 1983 1984 1985 1986 1987 |
} static void d40_free_chan_resources(struct dma_chan *chan) { struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); int err; unsigned long flags; |
0d0f6b8bb DMAENGINE: ste_dm... |
1988 |
if (d40c->phy_chan == NULL) { |
6db5a8ba1 dma40: use helper... |
1989 1990 |
chan_err(d40c, "Cannot free unallocated channel "); |
0d0f6b8bb DMAENGINE: ste_dm... |
1991 1992 |
return; } |
8d318a50b DMAENGINE: Suppor... |
1993 1994 1995 1996 1997 |
spin_lock_irqsave(&d40c->lock, flags); err = d40_free_dma(d40c); if (err) |
6db5a8ba1 dma40: use helper... |
1998 1999 |
chan_err(d40c, "Failed to free channel "); |
8d318a50b DMAENGINE: Suppor... |
2000 2001 2002 2003 2004 2005 2006 |
spin_unlock_irqrestore(&d40c->lock, flags); } static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, size_t size, |
2a6143407 DMAENGINE: ste_dm... |
2007 |
unsigned long dma_flags) |
8d318a50b DMAENGINE: Suppor... |
2008 |
{ |
95944c6ef dma40: implement ... |
2009 2010 |
struct scatterlist dst_sg; struct scatterlist src_sg; |
8d318a50b DMAENGINE: Suppor... |
2011 |
|
95944c6ef dma40: implement ... |
2012 2013 |
sg_init_table(&dst_sg, 1); sg_init_table(&src_sg, 1); |
8d318a50b DMAENGINE: Suppor... |
2014 |
|
95944c6ef dma40: implement ... |
2015 2016 |
sg_dma_address(&dst_sg) = dst; sg_dma_address(&src_sg) = src; |
8d318a50b DMAENGINE: Suppor... |
2017 |
|
95944c6ef dma40: implement ... |
2018 2019 |
sg_dma_len(&dst_sg) = size; sg_dma_len(&src_sg) = size; |
8d318a50b DMAENGINE: Suppor... |
2020 |
|
cade1d30b dma40: combine me... |
2021 |
return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags); |
8d318a50b DMAENGINE: Suppor... |
2022 |
} |
0d688662a ste_dma40: implem... |
2023 |
static struct dma_async_tx_descriptor * |
cade1d30b dma40: combine me... |
2024 2025 2026 2027 |
d40_prep_memcpy_sg(struct dma_chan *chan, struct scatterlist *dst_sg, unsigned int dst_nents, struct scatterlist *src_sg, unsigned int src_nents, unsigned long dma_flags) |
0d688662a ste_dma40: implem... |
2028 2029 2030 |
{ if (dst_nents != src_nents) return NULL; |
cade1d30b dma40: combine me... |
2031 |
return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags); |
00ac03414 dma40: remove dup... |
2032 |
} |
8d318a50b DMAENGINE: Suppor... |
2033 2034 2035 2036 |
static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, |
2a6143407 DMAENGINE: ste_dm... |
2037 |
unsigned long dma_flags) |
8d318a50b DMAENGINE: Suppor... |
2038 |
{ |
00ac03414 dma40: remove dup... |
2039 2040 |
if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) return NULL; |
cade1d30b dma40: combine me... |
2041 |
return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags); |
8d318a50b DMAENGINE: Suppor... |
2042 |
} |
0c842b551 dma40: cyclic xfe... |
2043 2044 2045 2046 2047 2048 2049 2050 2051 |
static struct dma_async_tx_descriptor * dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, size_t period_len, enum dma_data_direction direction) { unsigned int periods = buf_len / period_len; struct dma_async_tx_descriptor *txd; struct scatterlist *sg; int i; |
79ca7ec3d dmaengine/ste_dma... |
2052 |
sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_NOWAIT); |
0c842b551 dma40: cyclic xfe... |
2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 |
for (i = 0; i < periods; i++) { sg_dma_address(&sg[i]) = dma_addr; sg_dma_len(&sg[i]) = period_len; dma_addr += period_len; } sg[periods].offset = 0; sg[periods].length = 0; sg[periods].page_link = ((unsigned long)sg | 0x01) & ~0x02; txd = d40_prep_sg(chan, sg, sg, periods, direction, DMA_PREP_INTERRUPT); kfree(sg); return txd; } |
8d318a50b DMAENGINE: Suppor... |
2071 2072 2073 2074 2075 2076 2077 2078 |
static enum dma_status d40_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); dma_cookie_t last_used; dma_cookie_t last_complete; int ret; |
0d0f6b8bb DMAENGINE: ste_dm... |
2079 |
if (d40c->phy_chan == NULL) { |
6db5a8ba1 dma40: use helper... |
2080 2081 |
chan_err(d40c, "Cannot read status of unallocated channel "); |
0d0f6b8bb DMAENGINE: ste_dm... |
2082 2083 |
return -EINVAL; } |
8d318a50b DMAENGINE: Suppor... |
2084 2085 |
last_complete = d40c->completed; last_used = chan->cookie; |
a5ebca476 DMAENGINE: DMA40 ... |
2086 2087 2088 2089 |
if (d40_is_paused(d40c)) ret = DMA_PAUSED; else ret = dma_async_is_complete(cookie, last_complete, last_used); |
8d318a50b DMAENGINE: Suppor... |
2090 |
|
a5ebca476 DMAENGINE: DMA40 ... |
2091 2092 |
dma_set_tx_state(txstate, last_complete, last_used, stedma40_residue(chan)); |
8d318a50b DMAENGINE: Suppor... |
2093 2094 2095 2096 2097 2098 2099 2100 |
return ret; } static void d40_issue_pending(struct dma_chan *chan) { struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); unsigned long flags; |
0d0f6b8bb DMAENGINE: ste_dm... |
2101 |
if (d40c->phy_chan == NULL) { |
6db5a8ba1 dma40: use helper... |
2102 2103 |
chan_err(d40c, "Channel is not allocated! "); |
0d0f6b8bb DMAENGINE: ste_dm... |
2104 2105 |
return; } |
8d318a50b DMAENGINE: Suppor... |
2106 |
spin_lock_irqsave(&d40c->lock, flags); |
a8f3067bc dmaengine/ste_dma... |
2107 2108 2109 |
list_splice_tail_init(&d40c->pending_queue, &d40c->queue); /* Busy means that queued jobs are already being processed */ |
8d318a50b DMAENGINE: Suppor... |
2110 2111 2112 2113 2114 |
if (!d40c->busy) (void) d40_queue_start(d40c); spin_unlock_irqrestore(&d40c->lock, flags); } |
98ca52891 dmaengine/ste_dma... |
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 |
static int dma40_config_to_halfchannel(struct d40_chan *d40c, struct stedma40_half_channel_info *info, enum dma_slave_buswidth width, u32 maxburst) { enum stedma40_periph_data_width addr_width; int psize; switch (width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: addr_width = STEDMA40_BYTE_WIDTH; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: addr_width = STEDMA40_HALFWORD_WIDTH; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: addr_width = STEDMA40_WORD_WIDTH; break; case DMA_SLAVE_BUSWIDTH_8_BYTES: addr_width = STEDMA40_DOUBLEWORD_WIDTH; break; default: dev_err(d40c->base->dev, "illegal peripheral address width " "requested (%d) ", width); return -EINVAL; } if (chan_is_logical(d40c)) { if (maxburst >= 16) psize = STEDMA40_PSIZE_LOG_16; else if (maxburst >= 8) psize = STEDMA40_PSIZE_LOG_8; else if (maxburst >= 4) psize = STEDMA40_PSIZE_LOG_4; else psize = STEDMA40_PSIZE_LOG_1; } else { if (maxburst >= 16) psize = STEDMA40_PSIZE_PHY_16; else if (maxburst >= 8) psize = STEDMA40_PSIZE_PHY_8; else if (maxburst >= 4) psize = STEDMA40_PSIZE_PHY_4; else psize = STEDMA40_PSIZE_PHY_1; } info->data_width = addr_width; info->psize = psize; info->flow_ctrl = STEDMA40_NO_FLOW_CTRL; return 0; } |
95e1400fa DMAENGINE: add ru... |
2172 |
/* Runtime reconfiguration extension */ |
98ca52891 dmaengine/ste_dma... |
2173 2174 |
static int d40_set_runtime_config(struct dma_chan *chan, struct dma_slave_config *config) |
95e1400fa DMAENGINE: add ru... |
2175 2176 2177 |
{ struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); struct stedma40_chan_cfg *cfg = &d40c->dma_cfg; |
98ca52891 dmaengine/ste_dma... |
2178 |
enum dma_slave_buswidth src_addr_width, dst_addr_width; |
95e1400fa DMAENGINE: add ru... |
2179 |
dma_addr_t config_addr; |
98ca52891 dmaengine/ste_dma... |
2180 2181 2182 2183 2184 2185 2186 |
u32 src_maxburst, dst_maxburst; int ret; src_addr_width = config->src_addr_width; src_maxburst = config->src_maxburst; dst_addr_width = config->dst_addr_width; dst_maxburst = config->dst_maxburst; |
95e1400fa DMAENGINE: add ru... |
2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 |
if (config->direction == DMA_FROM_DEVICE) { dma_addr_t dev_addr_rx = d40c->base->plat_data->dev_rx[cfg->src_dev_type]; config_addr = config->src_addr; if (dev_addr_rx) dev_dbg(d40c->base->dev, "channel has a pre-wired RX address %08x " "overriding with %08x ", dev_addr_rx, config_addr); if (cfg->dir != STEDMA40_PERIPH_TO_MEM) dev_dbg(d40c->base->dev, "channel was not configured for peripheral " "to memory transfer (%d) overriding ", cfg->dir); cfg->dir = STEDMA40_PERIPH_TO_MEM; |
98ca52891 dmaengine/ste_dma... |
2206 2207 2208 2209 2210 |
/* Configure the memory side */ if (dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) dst_addr_width = src_addr_width; if (dst_maxburst == 0) dst_maxburst = src_maxburst; |
95e1400fa DMAENGINE: add ru... |
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 |
} else if (config->direction == DMA_TO_DEVICE) { dma_addr_t dev_addr_tx = d40c->base->plat_data->dev_tx[cfg->dst_dev_type]; config_addr = config->dst_addr; if (dev_addr_tx) dev_dbg(d40c->base->dev, "channel has a pre-wired TX address %08x " "overriding with %08x ", dev_addr_tx, config_addr); if (cfg->dir != STEDMA40_MEM_TO_PERIPH) dev_dbg(d40c->base->dev, "channel was not configured for memory " "to peripheral transfer (%d) overriding ", cfg->dir); cfg->dir = STEDMA40_MEM_TO_PERIPH; |
98ca52891 dmaengine/ste_dma... |
2230 2231 2232 2233 2234 |
/* Configure the memory side */ if (src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) src_addr_width = dst_addr_width; if (src_maxburst == 0) src_maxburst = dst_maxburst; |
95e1400fa DMAENGINE: add ru... |
2235 2236 2237 2238 2239 |
} else { dev_err(d40c->base->dev, "unrecognized channel direction %d ", config->direction); |
98ca52891 dmaengine/ste_dma... |
2240 |
return -EINVAL; |
95e1400fa DMAENGINE: add ru... |
2241 |
} |
98ca52891 dmaengine/ste_dma... |
2242 |
if (src_maxburst * src_addr_width != dst_maxburst * dst_addr_width) { |
95e1400fa DMAENGINE: add ru... |
2243 |
dev_err(d40c->base->dev, |
98ca52891 dmaengine/ste_dma... |
2244 2245 2246 2247 2248 2249 2250 |
"src/dst width/maxburst mismatch: %d*%d != %d*%d ", src_maxburst, src_addr_width, dst_maxburst, dst_addr_width); return -EINVAL; |
95e1400fa DMAENGINE: add ru... |
2251 |
} |
98ca52891 dmaengine/ste_dma... |
2252 2253 2254 2255 2256 |
ret = dma40_config_to_halfchannel(d40c, &cfg->src_info, src_addr_width, src_maxburst); if (ret) return ret; |
95e1400fa DMAENGINE: add ru... |
2257 |
|
98ca52891 dmaengine/ste_dma... |
2258 2259 2260 2261 2262 |
ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info, dst_addr_width, dst_maxburst); if (ret) return ret; |
95e1400fa DMAENGINE: add ru... |
2263 |
|
a59670a40 DMAENGINE: Set bu... |
2264 |
/* Fill in register values */ |
724a8577d dma40: use helper... |
2265 |
if (chan_is_logical(d40c)) |
a59670a40 DMAENGINE: Set bu... |
2266 2267 2268 2269 |
d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); else d40_phy_cfg(cfg, &d40c->src_def_cfg, &d40c->dst_def_cfg, false); |
95e1400fa DMAENGINE: add ru... |
2270 2271 2272 2273 |
/* These settings will take precedence later */ d40c->runtime_addr = config_addr; d40c->runtime_direction = config->direction; dev_dbg(d40c->base->dev, |
98ca52891 dmaengine/ste_dma... |
2274 2275 2276 |
"configured channel %s for %s, data width %d/%d, " "maxburst %d/%d elements, LE, no flow control ", |
95e1400fa DMAENGINE: add ru... |
2277 2278 |
dma_chan_name(chan), (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", |
98ca52891 dmaengine/ste_dma... |
2279 2280 2281 2282 |
src_addr_width, dst_addr_width, src_maxburst, dst_maxburst); return 0; |
95e1400fa DMAENGINE: add ru... |
2283 |
} |
058276303 DMAENGINE: extend... |
2284 2285 |
static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) |
8d318a50b DMAENGINE: Suppor... |
2286 |
{ |
8d318a50b DMAENGINE: Suppor... |
2287 |
struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); |
0d0f6b8bb DMAENGINE: ste_dm... |
2288 |
if (d40c->phy_chan == NULL) { |
6db5a8ba1 dma40: use helper... |
2289 2290 |
chan_err(d40c, "Channel is not allocated! "); |
0d0f6b8bb DMAENGINE: ste_dm... |
2291 2292 |
return -EINVAL; } |
8d318a50b DMAENGINE: Suppor... |
2293 2294 |
switch (cmd) { case DMA_TERMINATE_ALL: |
86eb5fb61 dma40: stop ongoi... |
2295 |
return d40_terminate_all(d40c); |
8d318a50b DMAENGINE: Suppor... |
2296 |
case DMA_PAUSE: |
86eb5fb61 dma40: stop ongoi... |
2297 |
return d40_pause(d40c); |
8d318a50b DMAENGINE: Suppor... |
2298 |
case DMA_RESUME: |
86eb5fb61 dma40: stop ongoi... |
2299 |
return d40_resume(d40c); |
95e1400fa DMAENGINE: add ru... |
2300 |
case DMA_SLAVE_CONFIG: |
98ca52891 dmaengine/ste_dma... |
2301 |
return d40_set_runtime_config(chan, |
95e1400fa DMAENGINE: add ru... |
2302 |
(struct dma_slave_config *) arg); |
95e1400fa DMAENGINE: add ru... |
2303 2304 |
default: break; |
8d318a50b DMAENGINE: Suppor... |
2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 |
} /* Other commands are unimplemented */ return -ENXIO; } /* Initialization functions */ static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, struct d40_chan *chans, int offset, int num_chans) { int i = 0; struct d40_chan *d40c; INIT_LIST_HEAD(&dma->channels); for (i = offset; i < offset + num_chans; i++) { d40c = &chans[i]; d40c->base = base; d40c->chan.device = dma; |
8d318a50b DMAENGINE: Suppor... |
2326 2327 2328 |
spin_lock_init(&d40c->lock); d40c->log_num = D40_PHY_CHAN; |
8d318a50b DMAENGINE: Suppor... |
2329 2330 |
INIT_LIST_HEAD(&d40c->active); INIT_LIST_HEAD(&d40c->queue); |
a8f3067bc dmaengine/ste_dma... |
2331 |
INIT_LIST_HEAD(&d40c->pending_queue); |
8d318a50b DMAENGINE: Suppor... |
2332 |
INIT_LIST_HEAD(&d40c->client); |
82babbb36 dmaengine/ste_dma... |
2333 |
INIT_LIST_HEAD(&d40c->prepare_queue); |
8d318a50b DMAENGINE: Suppor... |
2334 |
|
8d318a50b DMAENGINE: Suppor... |
2335 2336 2337 2338 2339 2340 2341 |
tasklet_init(&d40c->tasklet, dma_tasklet, (unsigned long) d40c); list_add_tail(&d40c->chan.device_node, &dma->channels); } } |
7ad74a7cf dma40: fix DMA_SG... |
2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 |
static void d40_ops_init(struct d40_base *base, struct dma_device *dev) { if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) dev->device_prep_slave_sg = d40_prep_slave_sg; if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) { dev->device_prep_dma_memcpy = d40_prep_memcpy; /* * This controller can only access address at even * 32bit boundaries, i.e. 2^2 */ dev->copy_align = 2; } if (dma_has_cap(DMA_SG, dev->cap_mask)) dev->device_prep_dma_sg = d40_prep_memcpy_sg; |
0c842b551 dma40: cyclic xfe... |
2359 2360 |
if (dma_has_cap(DMA_CYCLIC, dev->cap_mask)) dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic; |
7ad74a7cf dma40: fix DMA_SG... |
2361 2362 2363 2364 2365 2366 2367 |
dev->device_alloc_chan_resources = d40_alloc_chan_resources; dev->device_free_chan_resources = d40_free_chan_resources; dev->device_issue_pending = d40_issue_pending; dev->device_tx_status = d40_tx_status; dev->device_control = d40_control; dev->dev = base->dev; } |
8d318a50b DMAENGINE: Suppor... |
2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 |
static int __init d40_dmaengine_init(struct d40_base *base, int num_reserved_chans) { int err ; d40_chan_init(base, &base->dma_slave, base->log_chans, 0, base->num_log_chans); dma_cap_zero(base->dma_slave.cap_mask); dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); |
0c842b551 dma40: cyclic xfe... |
2378 |
dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); |
8d318a50b DMAENGINE: Suppor... |
2379 |
|
7ad74a7cf dma40: fix DMA_SG... |
2380 |
d40_ops_init(base, &base->dma_slave); |
8d318a50b DMAENGINE: Suppor... |
2381 2382 2383 2384 |
err = dma_async_device_register(&base->dma_slave); if (err) { |
6db5a8ba1 dma40: use helper... |
2385 2386 |
d40_err(base->dev, "Failed to register slave channels "); |
8d318a50b DMAENGINE: Suppor... |
2387 2388 2389 2390 2391 2392 2393 2394 |
goto failure1; } d40_chan_init(base, &base->dma_memcpy, base->log_chans, base->num_log_chans, base->plat_data->memcpy_len); dma_cap_zero(base->dma_memcpy.cap_mask); dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); |
7ad74a7cf dma40: fix DMA_SG... |
2395 2396 2397 |
dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask); d40_ops_init(base, &base->dma_memcpy); |
8d318a50b DMAENGINE: Suppor... |
2398 2399 2400 2401 |
err = dma_async_device_register(&base->dma_memcpy); if (err) { |
6db5a8ba1 dma40: use helper... |
2402 2403 2404 |
d40_err(base->dev, "Failed to regsiter memcpy only channels "); |
8d318a50b DMAENGINE: Suppor... |
2405 2406 2407 2408 2409 2410 2411 2412 2413 |
goto failure2; } d40_chan_init(base, &base->dma_both, base->phy_chans, 0, num_reserved_chans); dma_cap_zero(base->dma_both.cap_mask); dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); |
7ad74a7cf dma40: fix DMA_SG... |
2414 |
dma_cap_set(DMA_SG, base->dma_both.cap_mask); |
0c842b551 dma40: cyclic xfe... |
2415 |
dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); |
7ad74a7cf dma40: fix DMA_SG... |
2416 2417 |
d40_ops_init(base, &base->dma_both); |
8d318a50b DMAENGINE: Suppor... |
2418 2419 2420 |
err = dma_async_device_register(&base->dma_both); if (err) { |
6db5a8ba1 dma40: use helper... |
2421 2422 2423 |
d40_err(base->dev, "Failed to register logical and physical capable channels "); |
8d318a50b DMAENGINE: Suppor... |
2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 |
goto failure3; } return 0; failure3: dma_async_device_unregister(&base->dma_memcpy); failure2: dma_async_device_unregister(&base->dma_slave); failure1: return err; } /* Initialization functions. */ static int __init d40_phy_res_init(struct d40_base *base) { int i; int num_phy_chans_avail = 0; u32 val[2]; int odd_even_bit = -2; val[0] = readl(base->virtbase + D40_DREG_PRSME); val[1] = readl(base->virtbase + D40_DREG_PRSMO); for (i = 0; i < base->num_phy_chans; i++) { base->phy_res[i].num = i; odd_even_bit += 2 * ((i % 2) == 0); if (((val[i % 2] >> odd_even_bit) & 3) == 1) { /* Mark security only channels as occupied */ base->phy_res[i].allocated_src = D40_ALLOC_PHY; base->phy_res[i].allocated_dst = D40_ALLOC_PHY; } else { base->phy_res[i].allocated_src = D40_ALLOC_FREE; base->phy_res[i].allocated_dst = D40_ALLOC_FREE; num_phy_chans_avail++; } spin_lock_init(&base->phy_res[i].lock); } |
6b7acd844 DMAENGINE: ste_dm... |
2461 2462 2463 |
/* Mark disabled channels as occupied */ for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) { |
f57b407cf DMAENGINE: ste_dm... |
2464 2465 2466 2467 2468 |
int chan = base->plat_data->disabled_channels[i]; base->phy_res[chan].allocated_src = D40_ALLOC_PHY; base->phy_res[chan].allocated_dst = D40_ALLOC_PHY; num_phy_chans_avail--; |
6b7acd844 DMAENGINE: ste_dm... |
2469 |
} |
8d318a50b DMAENGINE: Suppor... |
2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 |
dev_info(base->dev, "%d of %d physical DMA channels available ", num_phy_chans_avail, base->num_phy_chans); /* Verify settings extended vs standard */ val[0] = readl(base->virtbase + D40_DREG_PRTYP); for (i = 0; i < base->num_phy_chans; i++) { if (base->phy_res[i].allocated_src == D40_ALLOC_FREE && (val[0] & 0x3) != 1) dev_info(base->dev, "[%s] INFO: channel %d is misconfigured (%d) ", __func__, i, val[0] & 0x3); val[0] = val[0] >> 2; } return num_phy_chans_avail; } static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) { |
8d318a50b DMAENGINE: Suppor... |
2494 2495 2496 2497 2498 2499 2500 2501 |
struct stedma40_platform_data *plat_data; struct clk *clk = NULL; void __iomem *virtbase = NULL; struct resource *res = NULL; struct d40_base *base = NULL; int num_log_chans = 0; int num_phy_chans; int i; |
f4b89764c dmaengine/ste_dma... |
2502 2503 2504 |
u32 pid; u32 cid; u8 rev; |
8d318a50b DMAENGINE: Suppor... |
2505 2506 2507 2508 |
clk = clk_get(&pdev->dev, NULL); if (IS_ERR(clk)) { |
6db5a8ba1 dma40: use helper... |
2509 2510 |
d40_err(&pdev->dev, "No matching clock found "); |
8d318a50b DMAENGINE: Suppor... |
2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 |
goto failure; } clk_enable(clk); /* Get IO for DMAC base address */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); if (!res) goto failure; if (request_mem_region(res->start, resource_size(res), D40_NAME " I/O base") == NULL) goto failure; virtbase = ioremap(res->start, resource_size(res)); if (!virtbase) goto failure; |
f4b89764c dmaengine/ste_dma... |
2528 2529 2530 2531 2532 2533 2534 |
/* This is just a regular AMBA PrimeCell ID actually */ for (pid = 0, i = 0; i < 4; i++) pid |= (readl(virtbase + resource_size(res) - 0x20 + 4 * i) & 255) << (i * 8); for (cid = 0, i = 0; i < 4; i++) cid |= (readl(virtbase + resource_size(res) - 0x10 + 4 * i) & 255) << (i * 8); |
8d318a50b DMAENGINE: Suppor... |
2535 |
|
f4b89764c dmaengine/ste_dma... |
2536 2537 2538 2539 2540 2541 |
if (cid != AMBA_CID) { d40_err(&pdev->dev, "Unknown hardware! No PrimeCell ID "); goto failure; } if (AMBA_MANF_BITS(pid) != AMBA_VENDOR_ST) { |
6db5a8ba1 dma40: use helper... |
2542 2543 |
d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x ", |
f4b89764c dmaengine/ste_dma... |
2544 2545 |
AMBA_MANF_BITS(pid), AMBA_VENDOR_ST); |
8d318a50b DMAENGINE: Suppor... |
2546 2547 |
goto failure; } |
f4b89764c dmaengine/ste_dma... |
2548 2549 2550 2551 2552 2553 2554 2555 |
/* * HW revision: * DB8500ed has revision 0 * ? has revision 1 * DB8500v1 has revision 2 * DB8500v2 has revision 3 */ rev = AMBA_REV_BITS(pid); |
3ae0267fd DMAENGINE: ste_dm... |
2556 |
|
8d318a50b DMAENGINE: Suppor... |
2557 2558 2559 2560 2561 |
/* The number of physical channels on this HW */ num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; dev_info(&pdev->dev, "hardware revision: %d @ 0x%x ", |
3ae0267fd DMAENGINE: ste_dm... |
2562 |
rev, res->start); |
8d318a50b DMAENGINE: Suppor... |
2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 |
plat_data = pdev->dev.platform_data; /* Count the number of logical channels in use */ for (i = 0; i < plat_data->dev_len; i++) if (plat_data->dev_rx[i] != 0) num_log_chans++; for (i = 0; i < plat_data->dev_len; i++) if (plat_data->dev_tx[i] != 0) num_log_chans++; base = kzalloc(ALIGN(sizeof(struct d40_base), 4) + (num_phy_chans + num_log_chans + plat_data->memcpy_len) * sizeof(struct d40_chan), GFP_KERNEL); if (base == NULL) { |
6db5a8ba1 dma40: use helper... |
2580 2581 |
d40_err(&pdev->dev, "Out of memory "); |
8d318a50b DMAENGINE: Suppor... |
2582 2583 |
goto failure; } |
3ae0267fd DMAENGINE: ste_dm... |
2584 |
base->rev = rev; |
8d318a50b DMAENGINE: Suppor... |
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 |
base->clk = clk; base->num_phy_chans = num_phy_chans; base->num_log_chans = num_log_chans; base->phy_start = res->start; base->phy_size = resource_size(res); base->virtbase = virtbase; base->plat_data = plat_data; base->dev = &pdev->dev; base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); base->log_chans = &base->phy_chans[num_phy_chans]; base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res), GFP_KERNEL); if (!base->phy_res) goto failure; base->lookup_phy_chans = kzalloc(num_phy_chans * sizeof(struct d40_chan *), GFP_KERNEL); if (!base->lookup_phy_chans) goto failure; if (num_log_chans + plat_data->memcpy_len) { /* * The max number of logical channels are event lines for all * src devices and dst devices */ base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 * sizeof(struct d40_chan *), GFP_KERNEL); if (!base->lookup_log_chans) goto failure; } |
698e4732e DMAENGINE: ste_dm... |
2618 2619 2620 2621 |
base->lcla_pool.alloc_map = kzalloc(num_phy_chans * sizeof(struct d40_desc *) * D40_LCLA_LINK_PER_EVENT_GRP, |
8d318a50b DMAENGINE: Suppor... |
2622 2623 2624 |
GFP_KERNEL); if (!base->lcla_pool.alloc_map) goto failure; |
c675b1b42 DMAENGINE: ste_dm... |
2625 2626 2627 2628 2629 |
base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc), 0, SLAB_HWCACHE_ALIGN, NULL); if (base->desc_slab == NULL) goto failure; |
8d318a50b DMAENGINE: Suppor... |
2630 2631 2632 |
return base; failure: |
c6134c967 DMAENGINE: ste_dm... |
2633 |
if (!IS_ERR(clk)) { |
8d318a50b DMAENGINE: Suppor... |
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 |
clk_disable(clk); clk_put(clk); } if (virtbase) iounmap(virtbase); if (res) release_mem_region(res->start, resource_size(res)); if (virtbase) iounmap(virtbase); if (base) { kfree(base->lcla_pool.alloc_map); kfree(base->lookup_log_chans); kfree(base->lookup_phy_chans); kfree(base->phy_res); kfree(base); } return NULL; } static void __init d40_hw_init(struct d40_base *base) { static const struct d40_reg_val dma_init_reg[] = { /* Clock every part of the DMA block from start */ { .reg = D40_DREG_GCC, .val = 0x0000ff01}, /* Interrupts on all logical channels */ { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF}, { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF} }; int i; u32 prmseo[2] = {0, 0}; u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF}; u32 pcmis = 0; u32 pcicr = 0; for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++) writel(dma_init_reg[i].val, base->virtbase + dma_init_reg[i].reg); /* Configure all our dma channels to default settings */ for (i = 0; i < base->num_phy_chans; i++) { activeo[i % 2] = activeo[i % 2] << 2; if (base->phy_res[base->num_phy_chans - i - 1].allocated_src == D40_ALLOC_PHY) { activeo[i % 2] |= 3; continue; } /* Enable interrupt # */ pcmis = (pcmis << 1) | 1; /* Clear interrupt # */ pcicr = (pcicr << 1) | 1; /* Set channel to physical mode */ prmseo[i % 2] = prmseo[i % 2] << 2; prmseo[i % 2] |= 1; } writel(prmseo[1], base->virtbase + D40_DREG_PRMSE); writel(prmseo[0], base->virtbase + D40_DREG_PRMSO); writel(activeo[1], base->virtbase + D40_DREG_ACTIVE); writel(activeo[0], base->virtbase + D40_DREG_ACTIVO); /* Write which interrupt to enable */ writel(pcmis, base->virtbase + D40_DREG_PCMIS); /* Write which interrupt to clear */ writel(pcicr, base->virtbase + D40_DREG_PCICR); } |
508849ade DMAENGINE: ste_dm... |
2722 2723 |
static int __init d40_lcla_allocate(struct d40_base *base) { |
026cbc424 dma40: fix DMA AP... |
2724 |
struct d40_lcla_pool *pool = &base->lcla_pool; |
508849ade DMAENGINE: ste_dm... |
2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 |
unsigned long *page_list; int i, j; int ret = 0; /* * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned, * To full fill this hardware requirement without wasting 256 kb * we allocate pages until we get an aligned one. */ page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS, GFP_KERNEL); if (!page_list) { ret = -ENOMEM; goto failure; } /* Calculating how many pages that are required */ base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE; for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) { page_list[i] = __get_free_pages(GFP_KERNEL, base->lcla_pool.pages); if (!page_list[i]) { |
6db5a8ba1 dma40: use helper... |
2749 2750 2751 |
d40_err(base->dev, "Failed to allocate %d pages. ", base->lcla_pool.pages); |
508849ade DMAENGINE: ste_dm... |
2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 |
for (j = 0; j < i; j++) free_pages(page_list[j], base->lcla_pool.pages); goto failure; } if ((virt_to_phys((void *)page_list[i]) & (LCLA_ALIGNMENT - 1)) == 0) break; } for (j = 0; j < i; j++) free_pages(page_list[j], base->lcla_pool.pages); if (i < MAX_LCLA_ALLOC_ATTEMPTS) { base->lcla_pool.base = (void *)page_list[i]; } else { |
767a9675c DMAENGINE: ste_dm... |
2769 2770 2771 2772 |
/* * After many attempts and no succees with finding the correct * alignment, try with allocating a big buffer. */ |
508849ade DMAENGINE: ste_dm... |
2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 |
dev_warn(base->dev, "[%s] Failed to get %d pages @ 18 bit align. ", __func__, base->lcla_pool.pages); base->lcla_pool.base_unaligned = kmalloc(SZ_1K * base->num_phy_chans + LCLA_ALIGNMENT, GFP_KERNEL); if (!base->lcla_pool.base_unaligned) { ret = -ENOMEM; goto failure; } base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned, LCLA_ALIGNMENT); } |
026cbc424 dma40: fix DMA AP... |
2789 2790 2791 2792 2793 2794 2795 2796 |
pool->dma_addr = dma_map_single(base->dev, pool->base, SZ_1K * base->num_phy_chans, DMA_TO_DEVICE); if (dma_mapping_error(base->dev, pool->dma_addr)) { pool->dma_addr = 0; ret = -ENOMEM; goto failure; } |
508849ade DMAENGINE: ste_dm... |
2797 2798 2799 2800 2801 2802 |
writel(virt_to_phys(base->lcla_pool.base), base->virtbase + D40_DREG_LCLA); failure: kfree(page_list); return ret; } |
8d318a50b DMAENGINE: Suppor... |
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 |
static int __init d40_probe(struct platform_device *pdev) { int err; int ret = -ENOENT; struct d40_base *base; struct resource *res = NULL; int num_reserved_chans; u32 val; base = d40_hw_detect_init(pdev); if (!base) goto failure; num_reserved_chans = d40_phy_res_init(base); platform_set_drvdata(pdev, base); spin_lock_init(&base->interrupt_lock); spin_lock_init(&base->execmd_lock); /* Get IO for logical channel parameter address */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); if (!res) { ret = -ENOENT; |
6db5a8ba1 dma40: use helper... |
2828 2829 |
d40_err(&pdev->dev, "No \"lcpa\" memory resource "); |
8d318a50b DMAENGINE: Suppor... |
2830 2831 2832 2833 2834 2835 2836 2837 |
goto failure; } base->lcpa_size = resource_size(res); base->phy_lcpa = res->start; if (request_mem_region(res->start, resource_size(res), D40_NAME " I/O lcpa") == NULL) { ret = -EBUSY; |
6db5a8ba1 dma40: use helper... |
2838 2839 2840 2841 |
d40_err(&pdev->dev, "Failed to request LCPA region 0x%x-0x%x ", res->start, res->end); |
8d318a50b DMAENGINE: Suppor... |
2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 |
goto failure; } /* We make use of ESRAM memory for this. */ val = readl(base->virtbase + D40_DREG_LCPA); if (res->start != val && val != 0) { dev_warn(&pdev->dev, "[%s] Mismatch LCPA dma 0x%x, def 0x%x ", __func__, val, res->start); } else writel(res->start, base->virtbase + D40_DREG_LCPA); base->lcpa_base = ioremap(res->start, resource_size(res)); if (!base->lcpa_base) { ret = -ENOMEM; |
6db5a8ba1 dma40: use helper... |
2858 2859 |
d40_err(&pdev->dev, "Failed to ioremap LCPA region "); |
8d318a50b DMAENGINE: Suppor... |
2860 2861 |
goto failure; } |
8d318a50b DMAENGINE: Suppor... |
2862 |
|
508849ade DMAENGINE: ste_dm... |
2863 2864 |
ret = d40_lcla_allocate(base); if (ret) { |
6db5a8ba1 dma40: use helper... |
2865 2866 |
d40_err(&pdev->dev, "Failed to allocate LCLA area "); |
8d318a50b DMAENGINE: Suppor... |
2867 2868 2869 2870 |
goto failure; } spin_lock_init(&base->lcla_pool.lock); |
8d318a50b DMAENGINE: Suppor... |
2871 2872 2873 |
base->irq = platform_get_irq(pdev, 0); ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); |
8d318a50b DMAENGINE: Suppor... |
2874 |
if (ret) { |
6db5a8ba1 dma40: use helper... |
2875 2876 |
d40_err(&pdev->dev, "No IRQ defined "); |
8d318a50b DMAENGINE: Suppor... |
2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 |
goto failure; } err = d40_dmaengine_init(base, num_reserved_chans); if (err) goto failure; d40_hw_init(base); dev_info(base->dev, "initialized "); return 0; failure: if (base) { |
c675b1b42 DMAENGINE: ste_dm... |
2892 2893 |
if (base->desc_slab) kmem_cache_destroy(base->desc_slab); |
8d318a50b DMAENGINE: Suppor... |
2894 2895 |
if (base->virtbase) iounmap(base->virtbase); |
026cbc424 dma40: fix DMA AP... |
2896 2897 2898 2899 2900 |
if (base->lcla_pool.dma_addr) dma_unmap_single(base->dev, base->lcla_pool.dma_addr, SZ_1K * base->num_phy_chans, DMA_TO_DEVICE); |
508849ade DMAENGINE: ste_dm... |
2901 2902 2903 |
if (!base->lcla_pool.base_unaligned && base->lcla_pool.base) free_pages((unsigned long)base->lcla_pool.base, base->lcla_pool.pages); |
767a9675c DMAENGINE: ste_dm... |
2904 2905 |
kfree(base->lcla_pool.base_unaligned); |
8d318a50b DMAENGINE: Suppor... |
2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 |
if (base->phy_lcpa) release_mem_region(base->phy_lcpa, base->lcpa_size); if (base->phy_start) release_mem_region(base->phy_start, base->phy_size); if (base->clk) { clk_disable(base->clk); clk_put(base->clk); } kfree(base->lcla_pool.alloc_map); kfree(base->lookup_log_chans); kfree(base->lookup_phy_chans); kfree(base->phy_res); kfree(base); } |
6db5a8ba1 dma40: use helper... |
2923 2924 |
d40_err(&pdev->dev, "probe failed "); |
8d318a50b DMAENGINE: Suppor... |
2925 2926 2927 2928 2929 2930 2931 2932 2933 |
return ret; } static struct platform_driver d40_driver = { .driver = { .owner = THIS_MODULE, .name = D40_NAME, }, }; |
cb9ab2d8e dma40: make init ... |
2934 |
static int __init stedma40_init(void) |
8d318a50b DMAENGINE: Suppor... |
2935 2936 2937 |
{ return platform_driver_probe(&d40_driver, d40_probe); } |
a0eb221a4 dmaengine: move l... |
2938 |
subsys_initcall(stedma40_init); |