2 * bfin_dma_5xx.c - Blackfin DMA implementation
4 * Copyright 2004-2006 Analog Devices Inc.
5 * Licensed under the GPL-2 or later.
8 #include <linux/errno.h>
9 #include <linux/interrupt.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/param.h>
13 #include <linux/proc_fs.h>
14 #include <linux/sched.h>
15 #include <linux/seq_file.h>
16 #include <linux/spinlock.h>
18 #include <asm/blackfin.h>
19 #include <asm/cacheflush.h>
21 #include <asm/uaccess.h>
23 /**************************************************************************
25 ***************************************************************************/
27 static struct dma_channel dma_ch[MAX_DMA_CHANNELS];
29 /*------------------------------------------------------------------------------
30 * Set the Buffer Clear bit in the Configuration register of specific DMA
31 * channel. This will stop the descriptor based DMA operation.
32 *-----------------------------------------------------------------------------*/
33 static void clear_dma_buffer(unsigned int channel)
35 dma_ch[channel].regs->cfg |= RESTART;
37 dma_ch[channel].regs->cfg &= ~RESTART;
40 static int __init blackfin_dma_init(void)
44 printk(KERN_INFO "Blackfin DMA Controller\n");
46 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
47 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
48 dma_ch[i].regs = dma_io_base_addr[i];
49 mutex_init(&(dma_ch[i].dmalock));
51 /* Mark MEMDMA Channel 0 as requested since we're using it internally */
52 request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
53 request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
55 #if defined(CONFIG_DEB_DMA_URGENT)
56 bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
57 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
62 arch_initcall(blackfin_dma_init);
65 static int proc_dma_show(struct seq_file *m, void *v)
69 for (i = 0; i < MAX_DMA_CHANNELS; ++i)
70 if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
71 seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
76 static int proc_dma_open(struct inode *inode, struct file *file)
78 return single_open(file, proc_dma_show, NULL);
81 static const struct file_operations proc_dma_operations = {
82 .open = proc_dma_open,
85 .release = single_release,
88 static int __init proc_dma_init(void)
90 return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
92 late_initcall(proc_dma_init);
95 /*------------------------------------------------------------------------------
96 * Request the specific DMA channel from the system.
97 *-----------------------------------------------------------------------------*/
98 int request_dma(unsigned int channel, const char *device_id)
100 pr_debug("request_dma() : BEGIN \n");
102 if (device_id == NULL)
103 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
105 #if defined(CONFIG_BF561) && ANOMALY_05000182
106 if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
107 if (get_cclk() > 500000000) {
109 "Request IMDMA failed due to ANOMALY 05000182\n");
115 mutex_lock(&(dma_ch[channel].dmalock));
117 if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
118 || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
119 mutex_unlock(&(dma_ch[channel].dmalock));
120 pr_debug("DMA CHANNEL IN USE \n");
123 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
124 pr_debug("DMA CHANNEL IS ALLOCATED \n");
127 mutex_unlock(&(dma_ch[channel].dmalock));
130 if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
131 unsigned int per_map;
132 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
133 if (strncmp(device_id, "BFIN_UART", 9) == 0)
134 dma_ch[channel].regs->peripheral_map = per_map |
135 ((channel - CH_UART2_RX + 0xC)<<12);
137 dma_ch[channel].regs->peripheral_map = per_map |
138 ((channel - CH_UART2_RX + 0x6)<<12);
142 dma_ch[channel].device_id = device_id;
143 dma_ch[channel].irq_callback = NULL;
145 /* This is to be enabled by putting a restriction -
146 * you have to request DMA, before doing any operations on
149 pr_debug("request_dma() : END \n");
152 EXPORT_SYMBOL(request_dma);
154 int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
156 BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
157 && channel < MAX_DMA_CHANNELS));
159 if (callback != NULL) {
161 dma_ch[channel].irq = channel2irq(channel);
162 dma_ch[channel].data = data;
165 request_irq(dma_ch[channel].irq, callback, IRQF_DISABLED,
166 dma_ch[channel].device_id, data);
169 "Request irq in DMA engine failed.\n");
172 dma_ch[channel].irq_callback = callback;
176 EXPORT_SYMBOL(set_dma_callback);
178 void free_dma(unsigned int channel)
180 pr_debug("freedma() : BEGIN \n");
181 BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
182 && channel < MAX_DMA_CHANNELS));
185 disable_dma(channel);
186 clear_dma_buffer(channel);
188 if (dma_ch[channel].irq_callback != NULL)
189 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
191 /* Clear the DMA Variable in the Channel */
192 mutex_lock(&(dma_ch[channel].dmalock));
193 dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
194 mutex_unlock(&(dma_ch[channel].dmalock));
196 pr_debug("freedma() : END \n");
198 EXPORT_SYMBOL(free_dma);
200 void dma_enable_irq(unsigned int channel)
202 pr_debug("dma_enable_irq() : BEGIN \n");
203 enable_irq(dma_ch[channel].irq);
205 EXPORT_SYMBOL(dma_enable_irq);
207 void dma_disable_irq(unsigned int channel)
209 pr_debug("dma_disable_irq() : BEGIN \n");
210 disable_irq(dma_ch[channel].irq);
212 EXPORT_SYMBOL(dma_disable_irq);
214 int dma_channel_active(unsigned int channel)
216 if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
222 EXPORT_SYMBOL(dma_channel_active);
224 /*------------------------------------------------------------------------------
225 * stop the specific DMA channel.
226 *-----------------------------------------------------------------------------*/
227 void disable_dma(unsigned int channel)
229 pr_debug("stop_dma() : BEGIN \n");
230 dma_ch[channel].regs->cfg &= ~DMAEN; /* Clean the enable bit */
232 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
233 /* Needs to be enabled Later */
234 pr_debug("stop_dma() : END \n");
237 EXPORT_SYMBOL(disable_dma);
239 void enable_dma(unsigned int channel)
241 pr_debug("enable_dma() : BEGIN \n");
242 dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
243 dma_ch[channel].regs->curr_x_count = 0;
244 dma_ch[channel].regs->curr_y_count = 0;
246 dma_ch[channel].regs->cfg |= DMAEN; /* Set the enable bit */
247 pr_debug("enable_dma() : END \n");
250 EXPORT_SYMBOL(enable_dma);
252 /*------------------------------------------------------------------------------
253 * Set the Start Address register for the specific DMA channel
254 * This function can be used for register based DMA,
255 * to setup the start address
256 * addr: Starting address of the DMA Data to be transferred.
257 *-----------------------------------------------------------------------------*/
258 void set_dma_start_addr(unsigned int channel, unsigned long addr)
260 pr_debug("set_dma_start_addr() : BEGIN \n");
261 dma_ch[channel].regs->start_addr = addr;
262 pr_debug("set_dma_start_addr() : END\n");
264 EXPORT_SYMBOL(set_dma_start_addr);
266 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
268 pr_debug("set_dma_next_desc_addr() : BEGIN \n");
269 dma_ch[channel].regs->next_desc_ptr = addr;
270 pr_debug("set_dma_next_desc_addr() : END\n");
272 EXPORT_SYMBOL(set_dma_next_desc_addr);
274 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
276 pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
277 dma_ch[channel].regs->curr_desc_ptr = addr;
278 pr_debug("set_dma_curr_desc_addr() : END\n");
280 EXPORT_SYMBOL(set_dma_curr_desc_addr);
282 void set_dma_x_count(unsigned int channel, unsigned short x_count)
284 dma_ch[channel].regs->x_count = x_count;
286 EXPORT_SYMBOL(set_dma_x_count);
288 void set_dma_y_count(unsigned int channel, unsigned short y_count)
290 dma_ch[channel].regs->y_count = y_count;
292 EXPORT_SYMBOL(set_dma_y_count);
294 void set_dma_x_modify(unsigned int channel, short x_modify)
296 dma_ch[channel].regs->x_modify = x_modify;
298 EXPORT_SYMBOL(set_dma_x_modify);
300 void set_dma_y_modify(unsigned int channel, short y_modify)
302 dma_ch[channel].regs->y_modify = y_modify;
304 EXPORT_SYMBOL(set_dma_y_modify);
306 void set_dma_config(unsigned int channel, unsigned short config)
308 dma_ch[channel].regs->cfg = config;
310 EXPORT_SYMBOL(set_dma_config);
313 set_bfin_dma_config(char direction, char flow_mode,
314 char intr_mode, char dma_mode, char width, char syncmode)
316 unsigned short config;
319 ((direction << 1) | (width << 2) | (dma_mode << 4) |
320 (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
323 EXPORT_SYMBOL(set_bfin_dma_config);
325 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
327 dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
328 dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
330 EXPORT_SYMBOL(set_dma_sg);
332 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
334 dma_ch[channel].regs->curr_addr_ptr = addr;
336 EXPORT_SYMBOL(set_dma_curr_addr);
338 /*------------------------------------------------------------------------------
339 * Get the DMA status of a specific DMA channel from the system.
340 *-----------------------------------------------------------------------------*/
341 unsigned short get_dma_curr_irqstat(unsigned int channel)
343 return dma_ch[channel].regs->irq_status;
345 EXPORT_SYMBOL(get_dma_curr_irqstat);
347 /*------------------------------------------------------------------------------
348 * Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
349 *-----------------------------------------------------------------------------*/
350 void clear_dma_irqstat(unsigned int channel)
352 dma_ch[channel].regs->irq_status |= 3;
354 EXPORT_SYMBOL(clear_dma_irqstat);
356 /*------------------------------------------------------------------------------
357 * Get current DMA xcount of a specific DMA channel from the system.
358 *-----------------------------------------------------------------------------*/
359 unsigned short get_dma_curr_xcount(unsigned int channel)
361 return dma_ch[channel].regs->curr_x_count;
363 EXPORT_SYMBOL(get_dma_curr_xcount);
365 /*------------------------------------------------------------------------------
366 * Get current DMA ycount of a specific DMA channel from the system.
367 *-----------------------------------------------------------------------------*/
368 unsigned short get_dma_curr_ycount(unsigned int channel)
370 return dma_ch[channel].regs->curr_y_count;
372 EXPORT_SYMBOL(get_dma_curr_ycount);
374 unsigned long get_dma_next_desc_ptr(unsigned int channel)
376 return dma_ch[channel].regs->next_desc_ptr;
378 EXPORT_SYMBOL(get_dma_next_desc_ptr);
380 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
382 return dma_ch[channel].regs->curr_desc_ptr;
384 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
386 unsigned long get_dma_curr_addr(unsigned int channel)
388 return dma_ch[channel].regs->curr_addr_ptr;
390 EXPORT_SYMBOL(get_dma_curr_addr);
393 # ifndef MAX_DMA_SUSPEND_CHANNELS
394 # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
396 int blackfin_dma_suspend(void)
400 for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) {
401 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
402 printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
406 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
412 void blackfin_dma_resume(void)
415 for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i)
416 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
421 * blackfin_dma_early_init - minimal DMA init
423 * Setup a few DMA registers so we can safely do DMA transfers early on in
424 * the kernel booting process. Really this just means using dma_memcpy().
426 void __init blackfin_dma_early_init(void)
428 bfin_write_MDMA_S0_CONFIG(0);
432 * __dma_memcpy - program the MDMA registers
434 * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
435 * while programming registers so that everything is fully configured. Wait
436 * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
437 * check will make sure we don't clobber any existing transfer.
439 static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
441 static DEFINE_SPINLOCK(mdma_lock);
444 spin_lock_irqsave(&mdma_lock, flags);
446 if (bfin_read_MDMA_S0_CONFIG())
447 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
451 /* For larger bit sizes, we've already divided down cnt so it
452 * is no longer a multiple of 64k. So we have to break down
453 * the limit here so it is a multiple of the incoming size.
454 * There is no limitation here in terms of total size other
455 * than the hardware though as the bits lost in the shift are
456 * made up by MODIFY (== we can hit the whole address space).
457 * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
459 u32 shift = abs(dmod) >> 1;
460 size_t ycnt = cnt >> (16 - shift);
461 cnt = 1 << (16 - shift);
462 bfin_write_MDMA_D0_Y_COUNT(ycnt);
463 bfin_write_MDMA_S0_Y_COUNT(ycnt);
464 bfin_write_MDMA_D0_Y_MODIFY(dmod);
465 bfin_write_MDMA_S0_Y_MODIFY(smod);
468 bfin_write_MDMA_D0_START_ADDR(daddr);
469 bfin_write_MDMA_D0_X_COUNT(cnt);
470 bfin_write_MDMA_D0_X_MODIFY(dmod);
471 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
473 bfin_write_MDMA_S0_START_ADDR(saddr);
474 bfin_write_MDMA_S0_X_COUNT(cnt);
475 bfin_write_MDMA_S0_X_MODIFY(smod);
476 bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
478 bfin_write_MDMA_S0_CONFIG(DMAEN | conf);
479 bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | conf);
481 spin_unlock_irqrestore(&mdma_lock, flags);
485 while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
486 if (bfin_read_MDMA_S0_CONFIG())
491 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
493 bfin_write_MDMA_S0_CONFIG(0);
494 bfin_write_MDMA_D0_CONFIG(0);
498 * _dma_memcpy - translate C memcpy settings into MDMA settings
500 * Handle all the high level steps before we touch the MDMA registers. So
501 * handle caching, tweaking of sizes, and formatting of addresses.
503 static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
507 unsigned long dst = (unsigned long)pdst;
508 unsigned long src = (unsigned long)psrc;
513 if (bfin_addr_dcachable(src))
514 blackfin_dcache_flush_range(src, src + size);
516 if (bfin_addr_dcachable(dst))
517 blackfin_dcache_invalidate_range(dst, dst + size);
519 if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
522 } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
530 /* If the two memory regions have a chance of overlapping, make
531 * sure the memcpy still works as expected. Do this by having the
532 * copy run backwards instead.
545 __dma_memcpy(dst, mod, src, mod, size, conf);
551 * dma_memcpy - DMA memcpy under mutex lock
553 * Do not check arguments before starting the DMA memcpy. Break the transfer
554 * up into two pieces. The first transfer is in multiples of 64k and the
555 * second transfer is the piece smaller than 64k.
557 void *dma_memcpy(void *dst, const void *src, size_t size)
560 bulk = size & ~0xffff;
563 _dma_memcpy(dst, src, bulk);
564 _dma_memcpy(dst + bulk, src + bulk, rest);
567 EXPORT_SYMBOL(dma_memcpy);
570 * safe_dma_memcpy - DMA memcpy w/argument checking
572 * Verify arguments are safe before heading to dma_memcpy().
574 void *safe_dma_memcpy(void *dst, const void *src, size_t size)
576 if (!access_ok(VERIFY_WRITE, dst, size))
578 if (!access_ok(VERIFY_READ, src, size))
580 return dma_memcpy(dst, src, size);
582 EXPORT_SYMBOL(safe_dma_memcpy);
584 static void _dma_out(unsigned long addr, unsigned long buf, unsigned short len,
585 u16 size, u16 dma_size)
587 blackfin_dcache_flush_range(buf, buf + len * size);
588 __dma_memcpy(addr, 0, buf, size, len, dma_size);
591 static void _dma_in(unsigned long addr, unsigned long buf, unsigned short len,
592 u16 size, u16 dma_size)
594 blackfin_dcache_invalidate_range(buf, buf + len * size);
595 __dma_memcpy(buf, size, addr, 0, len, dma_size);
598 #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
599 void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned short len) \
601 _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
603 EXPORT_SYMBOL(dma_##io##s##bwl)
604 MAKE_DMA_IO(out, b, 1, 8, const);
605 MAKE_DMA_IO(in, b, 1, 8, );
606 MAKE_DMA_IO(out, w, 2, 16, const);
607 MAKE_DMA_IO(in, w, 2, 16, );
608 MAKE_DMA_IO(out, l, 4, 32, const);
609 MAKE_DMA_IO(in, l, 4, 32, );