arch/blackfin/kernel/bfin_dma_5xx.c

   1 /*
   2  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
   3  * Based on:
   4  * Author:
   5  *
   6  * Created:
   7  * Description:  This file contains the simple DMA Implementation for Blackfin
   8  *
   9  * Modified:
  10  *               Copyright 2004-2006 Analog Devices Inc.
  11  *
  12  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License
  25  * along with this program; if not, see the file COPYING, or write
  26  * to the Free Software Foundation, Inc.,
  27  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  28  */
  29
  30 #include <linux/errno.h>
  31 #include <linux/module.h>
  32 #include <linux/proc_fs.h>
  33 #include <linux/sched.h>
  34 #include <linux/seq_file.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/kernel.h>
  37 #include <linux/param.h>
  38
  39 #include <asm/blackfin.h>
  40 #include <asm/dma.h>
  41 #include <asm/cacheflush.h>
  42
  43 /**************************************************************************
  44  * Global Variables
  45 ***************************************************************************/
  46
  47 static struct dma_channel dma_ch[MAX_DMA_CHANNELS];
  48
  49 /*------------------------------------------------------------------------------
  50  *       Set the Buffer Clear bit in the Configuration register of specific DMA
  51  *       channel. This will stop the descriptor based DMA operation.
  52  *-----------------------------------------------------------------------------*/
  53 static void clear_dma_buffer(unsigned int channel)
  54 {
  55         dma_ch[channel].regs->cfg |= RESTART;
  56         SSYNC();
  57         dma_ch[channel].regs->cfg &= ~RESTART;
  58 }
  59
  60 static int __init blackfin_dma_init(void)
  61 {
  62         int i;
  63
  64         printk(KERN_INFO "Blackfin DMA Controller\n");
  65
  66         for (i = 0; i < MAX_DMA_CHANNELS; i++) {
  67                 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
  68                 dma_ch[i].regs = dma_io_base_addr[i];
  69                 mutex_init(&(dma_ch[i].dmalock));
  70         }
  71         /* Mark MEMDMA Channel 0 as requested since we're using it internally */
  72         request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
  73         request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
  74
  75 #if defined(CONFIG_DEB_DMA_URGENT)
  76         bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
  77                          | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
  78 #endif
  79
  80         return 0;
  81 }
  82 arch_initcall(blackfin_dma_init);
  83
  84 #ifdef CONFIG_PROC_FS
  85
  86 static int proc_dma_show(struct seq_file *m, void *v)
  87 {
  88         int i;
  89
  90         for (i = 0 ; i < MAX_DMA_CHANNELS; ++i)
  91                 if (dma_ch[i].chan_status != DMA_CHANNEL_FREE)
  92                         seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
  93
  94         return 0;
  95 }
  96
  97 static int proc_dma_open(struct inode *inode, struct file *file)
  98 {
  99         return single_open(file, proc_dma_show, NULL);
 100 }
 101
 102 static const struct file_operations proc_dma_operations = {
 103         .open           = proc_dma_open,
 104         .read           = seq_read,
 105         .llseek         = seq_lseek,
 106         .release        = single_release,
 107 };
 108
 109 static int __init proc_dma_init(void)
 110 {
 111         return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
 112 }
 113 late_initcall(proc_dma_init);
 114 #endif
 115
 116 /*------------------------------------------------------------------------------
 117  *      Request the specific DMA channel from the system.
 118  *-----------------------------------------------------------------------------*/
 119 int request_dma(unsigned int channel, const char *device_id)
 120 {
 121         pr_debug("request_dma() : BEGIN \n");
 122
 123         if (device_id == NULL)
 124                 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
 125
 126 #if defined(CONFIG_BF561) && ANOMALY_05000182
 127         if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
 128                 if (get_cclk() > 500000000) {
 129                         printk(KERN_WARNING
 130                                "Request IMDMA failed due to ANOMALY 05000182\n");
 131                         return -EFAULT;
 132                 }
 133         }
 134 #endif
 135
 136         mutex_lock(&(dma_ch[channel].dmalock));
 137
 138         if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
 139             || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
 140                 mutex_unlock(&(dma_ch[channel].dmalock));
 141                 pr_debug("DMA CHANNEL IN USE  \n");
 142                 return -EBUSY;
 143         } else {
 144                 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 145                 pr_debug("DMA CHANNEL IS ALLOCATED  \n");
 146         }
 147
 148         mutex_unlock(&(dma_ch[channel].dmalock));
 149
 150 #ifdef CONFIG_BF54x
 151         if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
 152                 unsigned int per_map;
 153                 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
 154                 if (strncmp(device_id, "BFIN_UART", 9) == 0)
 155                         dma_ch[channel].regs->peripheral_map = per_map |
 156                                 ((channel - CH_UART2_RX + 0xC)<<12);
 157                 else
 158                         dma_ch[channel].regs->peripheral_map = per_map |
 159                                 ((channel - CH_UART2_RX + 0x6)<<12);
 160         }
 161 #endif
 162
 163         dma_ch[channel].device_id = device_id;
 164         dma_ch[channel].irq_callback = NULL;
 165
 166         /* This is to be enabled by putting a restriction -
 167          * you have to request DMA, before doing any operations on
 168          * descriptor/channel
 169          */
 170         pr_debug("request_dma() : END  \n");
 171         return channel;
 172 }
 173 EXPORT_SYMBOL(request_dma);
 174
 175 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
 176 {
 177         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 178                && channel < MAX_DMA_CHANNELS));
 179
 180         if (callback != NULL) {
 181                 int ret_val;
 182                 dma_ch[channel].irq = channel2irq(channel);
 183                 dma_ch[channel].data = data;
 184
 185                 ret_val =
 186                     request_irq(dma_ch[channel].irq, callback, IRQF_DISABLED,
 187                                 dma_ch[channel].device_id, data);
 188                 if (ret_val) {
 189                         printk(KERN_NOTICE
 190                                "Request irq in DMA engine failed.\n");
 191                         return -EPERM;
 192                 }
 193                 dma_ch[channel].irq_callback = callback;
 194         }
 195         return 0;
 196 }
 197 EXPORT_SYMBOL(set_dma_callback);
 198
 199 void free_dma(unsigned int channel)
 200 {
 201         pr_debug("freedma() : BEGIN \n");
 202         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 203                && channel < MAX_DMA_CHANNELS));
 204
 205         /* Halt the DMA */
 206         disable_dma(channel);
 207         clear_dma_buffer(channel);
 208
 209         if (dma_ch[channel].irq_callback != NULL)
 210                 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
 211
 212         /* Clear the DMA Variable in the Channel */
 213         mutex_lock(&(dma_ch[channel].dmalock));
 214         dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
 215         mutex_unlock(&(dma_ch[channel].dmalock));
 216
 217         pr_debug("freedma() : END \n");
 218 }
 219 EXPORT_SYMBOL(free_dma);
 220
 221 void dma_enable_irq(unsigned int channel)
 222 {
 223         pr_debug("dma_enable_irq() : BEGIN \n");
 224         enable_irq(dma_ch[channel].irq);
 225 }
 226 EXPORT_SYMBOL(dma_enable_irq);
 227
 228 void dma_disable_irq(unsigned int channel)
 229 {
 230         pr_debug("dma_disable_irq() : BEGIN \n");
 231         disable_irq(dma_ch[channel].irq);
 232 }
 233 EXPORT_SYMBOL(dma_disable_irq);
 234
 235 int dma_channel_active(unsigned int channel)
 236 {
 237         if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
 238                 return 0;
 239         } else {
 240                 return 1;
 241         }
 242 }
 243 EXPORT_SYMBOL(dma_channel_active);
 244
 245 /*------------------------------------------------------------------------------
 246 *       stop the specific DMA channel.
 247 *-----------------------------------------------------------------------------*/
 248 void disable_dma(unsigned int channel)
 249 {
 250         pr_debug("stop_dma() : BEGIN \n");
 251         dma_ch[channel].regs->cfg &= ~DMAEN;    /* Clean the enable bit */
 252         SSYNC();
 253         dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 254         /* Needs to be enabled Later */
 255         pr_debug("stop_dma() : END \n");
 256         return;
 257 }
 258 EXPORT_SYMBOL(disable_dma);
 259
 260 void enable_dma(unsigned int channel)
 261 {
 262         pr_debug("enable_dma() : BEGIN \n");
 263         dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
 264         dma_ch[channel].regs->curr_x_count = 0;
 265         dma_ch[channel].regs->curr_y_count = 0;
 266
 267         dma_ch[channel].regs->cfg |= DMAEN;     /* Set the enable bit */
 268         pr_debug("enable_dma() : END \n");
 269         return;
 270 }
 271 EXPORT_SYMBOL(enable_dma);
 272
 273 /*------------------------------------------------------------------------------
 274 *               Set the Start Address register for the specific DMA channel
 275 *               This function can be used for register based DMA,
 276 *               to setup the start address
 277 *               addr:           Starting address of the DMA Data to be transferred.
 278 *-----------------------------------------------------------------------------*/
 279 void set_dma_start_addr(unsigned int channel, unsigned long addr)
 280 {
 281         pr_debug("set_dma_start_addr() : BEGIN \n");
 282         dma_ch[channel].regs->start_addr = addr;
 283         pr_debug("set_dma_start_addr() : END\n");
 284 }
 285 EXPORT_SYMBOL(set_dma_start_addr);
 286
 287 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
 288 {
 289         pr_debug("set_dma_next_desc_addr() : BEGIN \n");
 290         dma_ch[channel].regs->next_desc_ptr = addr;
 291         pr_debug("set_dma_next_desc_addr() : END\n");
 292 }
 293 EXPORT_SYMBOL(set_dma_next_desc_addr);
 294
 295 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
 296 {
 297         pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
 298         dma_ch[channel].regs->curr_desc_ptr = addr;
 299         pr_debug("set_dma_curr_desc_addr() : END\n");
 300 }
 301 EXPORT_SYMBOL(set_dma_curr_desc_addr);
 302
 303 void set_dma_x_count(unsigned int channel, unsigned short x_count)
 304 {
 305         dma_ch[channel].regs->x_count = x_count;
 306 }
 307 EXPORT_SYMBOL(set_dma_x_count);
 308
 309 void set_dma_y_count(unsigned int channel, unsigned short y_count)
 310 {
 311         dma_ch[channel].regs->y_count = y_count;
 312 }
 313 EXPORT_SYMBOL(set_dma_y_count);
 314
 315 void set_dma_x_modify(unsigned int channel, short x_modify)
 316 {
 317         dma_ch[channel].regs->x_modify = x_modify;
 318 }
 319 EXPORT_SYMBOL(set_dma_x_modify);
 320
 321 void set_dma_y_modify(unsigned int channel, short y_modify)
 322 {
 323         dma_ch[channel].regs->y_modify = y_modify;
 324 }
 325 EXPORT_SYMBOL(set_dma_y_modify);
 326
 327 void set_dma_config(unsigned int channel, unsigned short config)
 328 {
 329         dma_ch[channel].regs->cfg = config;
 330 }
 331 EXPORT_SYMBOL(set_dma_config);
 332
 333 unsigned short
 334 set_bfin_dma_config(char direction, char flow_mode,
 335                     char intr_mode, char dma_mode, char width, char syncmode)
 336 {
 337         unsigned short config;
 338
 339         config =
 340             ((direction << 1) | (width << 2) | (dma_mode << 4) |
 341              (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
 342         return config;
 343 }
 344 EXPORT_SYMBOL(set_bfin_dma_config);
 345
 346 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
 347 {
 348         dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
 349         dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
 350 }
 351 EXPORT_SYMBOL(set_dma_sg);
 352
 353 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
 354 {
 355         dma_ch[channel].regs->curr_addr_ptr = addr;
 356 }
 357 EXPORT_SYMBOL(set_dma_curr_addr);
 358
 359 /*------------------------------------------------------------------------------
 360  *      Get the DMA status of a specific DMA channel from the system.
 361  *-----------------------------------------------------------------------------*/
 362 unsigned short get_dma_curr_irqstat(unsigned int channel)
 363 {
 364         return dma_ch[channel].regs->irq_status;
 365 }
 366 EXPORT_SYMBOL(get_dma_curr_irqstat);
 367
 368 /*------------------------------------------------------------------------------
 369  *      Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
 370  *-----------------------------------------------------------------------------*/
 371 void clear_dma_irqstat(unsigned int channel)
 372 {
 373         dma_ch[channel].regs->irq_status |= 3;
 374 }
 375 EXPORT_SYMBOL(clear_dma_irqstat);
 376
 377 /*------------------------------------------------------------------------------
 378  *      Get current DMA xcount of a specific DMA channel from the system.
 379  *-----------------------------------------------------------------------------*/
 380 unsigned short get_dma_curr_xcount(unsigned int channel)
 381 {
 382         return dma_ch[channel].regs->curr_x_count;
 383 }
 384 EXPORT_SYMBOL(get_dma_curr_xcount);
 385
 386 /*------------------------------------------------------------------------------
 387  *      Get current DMA ycount of a specific DMA channel from the system.
 388  *-----------------------------------------------------------------------------*/
 389 unsigned short get_dma_curr_ycount(unsigned int channel)
 390 {
 391         return dma_ch[channel].regs->curr_y_count;
 392 }
 393 EXPORT_SYMBOL(get_dma_curr_ycount);
 394
 395 unsigned long get_dma_next_desc_ptr(unsigned int channel)
 396 {
 397         return dma_ch[channel].regs->next_desc_ptr;
 398 }
 399 EXPORT_SYMBOL(get_dma_next_desc_ptr);
 400
 401 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
 402 {
 403         return dma_ch[channel].regs->curr_desc_ptr;
 404 }
 405 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
 406
 407 unsigned long get_dma_curr_addr(unsigned int channel)
 408 {
 409         return dma_ch[channel].regs->curr_addr_ptr;
 410 }
 411 EXPORT_SYMBOL(get_dma_curr_addr);
 412
 413 #ifdef CONFIG_PM
 414 # ifndef MAX_DMA_SUSPEND_CHANNELS
 415 #  define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
 416 # endif
 417 int blackfin_dma_suspend(void)
 418 {
 419         int i;
 420
 421         for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i) {
 422                 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
 423                         printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
 424                         return -EBUSY;
 425                 }
 426
 427                 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
 428         }
 429
 430         return 0;
 431 }
 432
 433 void blackfin_dma_resume(void)
 434 {
 435         int i;
 436         for (i = 0; i < MAX_DMA_SUSPEND_CHANNELS; ++i)
 437                 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
 438 }
 439 #endif
 440
 441 static void *__dma_memcpy(void *dest, const void *src, size_t size)
 442 {
 443         int direction;  /* 1 - address decrease, 0 - address increase */
 444         int flag_align; /* 1 - address aligned,  0 - address unaligned */
 445         int flag_2D;    /* 1 - 2D DMA needed,    0 - 1D DMA needed */
 446         unsigned long flags;
 447
 448         if (size <= 0)
 449                 return NULL;
 450
 451         local_irq_save(flags);
 452
 453         if ((unsigned long)src < memory_end)
 454                 blackfin_dcache_flush_range((unsigned int)src,
 455                                             (unsigned int)(src + size));
 456
 457         if ((unsigned long)dest < memory_end)
 458                 blackfin_dcache_invalidate_range((unsigned int)dest,
 459                                                  (unsigned int)(dest + size));
 460
 461         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 462
 463         if ((unsigned long)src < (unsigned long)dest)
 464                 direction = 1;
 465         else
 466                 direction = 0;
 467
 468         if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
 469             && ((size % 2) == 0))
 470                 flag_align = 1;
 471         else
 472                 flag_align = 0;
 473
 474         if (size > 0x10000)     /* size > 64K */
 475                 flag_2D = 1;
 476         else
 477                 flag_2D = 0;
 478
 479         /* Setup destination and source start address */
 480         if (direction) {
 481                 if (flag_align) {
 482                         bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
 483                         bfin_write_MDMA_S0_START_ADDR(src + size - 2);
 484                 } else {
 485                         bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
 486                         bfin_write_MDMA_S0_START_ADDR(src + size - 1);
 487                 }
 488         } else {
 489                 bfin_write_MDMA_D0_START_ADDR(dest);
 490                 bfin_write_MDMA_S0_START_ADDR(src);
 491         }
 492
 493         /* Setup destination and source xcount */
 494         if (flag_2D) {
 495                 if (flag_align) {
 496                         bfin_write_MDMA_D0_X_COUNT(1024 / 2);
 497                         bfin_write_MDMA_S0_X_COUNT(1024 / 2);
 498                 } else {
 499                         bfin_write_MDMA_D0_X_COUNT(1024);
 500                         bfin_write_MDMA_S0_X_COUNT(1024);
 501                 }
 502                 bfin_write_MDMA_D0_Y_COUNT(size >> 10);
 503                 bfin_write_MDMA_S0_Y_COUNT(size >> 10);
 504         } else {
 505                 if (flag_align) {
 506                         bfin_write_MDMA_D0_X_COUNT(size / 2);
 507                         bfin_write_MDMA_S0_X_COUNT(size / 2);
 508                 } else {
 509                         bfin_write_MDMA_D0_X_COUNT(size);
 510                         bfin_write_MDMA_S0_X_COUNT(size);
 511                 }
 512         }
 513
 514         /* Setup destination and source xmodify and ymodify */
 515         if (direction) {
 516                 if (flag_align) {
 517                         bfin_write_MDMA_D0_X_MODIFY(-2);
 518                         bfin_write_MDMA_S0_X_MODIFY(-2);
 519                         if (flag_2D) {
 520                                 bfin_write_MDMA_D0_Y_MODIFY(-2);
 521                                 bfin_write_MDMA_S0_Y_MODIFY(-2);
 522                         }
 523                 } else {
 524                         bfin_write_MDMA_D0_X_MODIFY(-1);
 525                         bfin_write_MDMA_S0_X_MODIFY(-1);
 526                         if (flag_2D) {
 527                                 bfin_write_MDMA_D0_Y_MODIFY(-1);
 528                                 bfin_write_MDMA_S0_Y_MODIFY(-1);
 529                         }
 530                 }
 531         } else {
 532                 if (flag_align) {
 533                         bfin_write_MDMA_D0_X_MODIFY(2);
 534                         bfin_write_MDMA_S0_X_MODIFY(2);
 535                         if (flag_2D) {
 536                                 bfin_write_MDMA_D0_Y_MODIFY(2);
 537                                 bfin_write_MDMA_S0_Y_MODIFY(2);
 538                         }
 539                 } else {
 540                         bfin_write_MDMA_D0_X_MODIFY(1);
 541                         bfin_write_MDMA_S0_X_MODIFY(1);
 542                         if (flag_2D) {
 543                                 bfin_write_MDMA_D0_Y_MODIFY(1);
 544                                 bfin_write_MDMA_S0_Y_MODIFY(1);
 545                         }
 546                 }
 547         }
 548
 549         /* Enable source DMA */
 550         if (flag_2D) {
 551                 if (flag_align) {
 552                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
 553                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
 554                 } else {
 555                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
 556                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
 557                 }
 558         } else {
 559                 if (flag_align) {
 560                         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 561                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 562                 } else {
 563                         bfin_write_MDMA_S0_CONFIG(DMAEN);
 564                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
 565                 }
 566         }
 567
 568         SSYNC();
 569
 570         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
 571                 ;
 572
 573         bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
 574                                       (DMA_DONE | DMA_ERR));
 575
 576         bfin_write_MDMA_S0_CONFIG(0);
 577         bfin_write_MDMA_D0_CONFIG(0);
 578
 579         local_irq_restore(flags);
 580
 581         return dest;
 582 }
 583
 584 void *dma_memcpy(void *dest, const void *src, size_t size)
 585 {
 586         size_t bulk;
 587         size_t rest;
 588         void * addr;
 589
 590         bulk = (size >> 16) << 16;
 591         rest = size - bulk;
 592         if (bulk)
 593                 __dma_memcpy(dest, src, bulk);
 594         __dma_memcpy(dest+bulk, src+bulk, rest);
 595         return dest;
 596 }
 597 EXPORT_SYMBOL(dma_memcpy);
 598
 599 /**
 600  *      safe_dma_memcpy - DMA memcpy w/argument checking
 601  *
 602  * Verify arguments are safe before heading to dma_memcpy().
 603  */
 604 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
 605 {
 606         if (!access_ok(VERIFY_WRITE, dst, size))
 607                 return NULL;
 608         if (!access_ok(VERIFY_READ, src, size))
 609                 return NULL;
 610         return dma_memcpy(dst, src, size);
 611 }
 612 EXPORT_SYMBOL(safe_dma_memcpy);
 613
 614 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
 615 {
 616         unsigned long flags;
 617
 618         local_irq_save(flags);
 619
 620         blackfin_dcache_flush_range((unsigned int)buf,
 621                          (unsigned int)(buf) + len);
 622
 623         bfin_write_MDMA_D0_START_ADDR(addr);
 624         bfin_write_MDMA_D0_X_COUNT(len);
 625         bfin_write_MDMA_D0_X_MODIFY(0);
 626         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 627
 628         bfin_write_MDMA_S0_START_ADDR(buf);
 629         bfin_write_MDMA_S0_X_COUNT(len);
 630         bfin_write_MDMA_S0_X_MODIFY(1);
 631         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 632
 633         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 634         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 635
 636         SSYNC();
 637
 638         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 639
 640         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 641
 642         bfin_write_MDMA_S0_CONFIG(0);
 643         bfin_write_MDMA_D0_CONFIG(0);
 644         local_irq_restore(flags);
 645
 646 }
 647 EXPORT_SYMBOL(dma_outsb);
 648
 649
 650 void dma_insb(unsigned long addr, void *buf, unsigned short len)
 651 {
 652         unsigned long flags;
 653
 654         blackfin_dcache_invalidate_range((unsigned int)buf,
 655                          (unsigned int)(buf) + len);
 656
 657         local_irq_save(flags);
 658         bfin_write_MDMA_D0_START_ADDR(buf);
 659         bfin_write_MDMA_D0_X_COUNT(len);
 660         bfin_write_MDMA_D0_X_MODIFY(1);
 661         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 662
 663         bfin_write_MDMA_S0_START_ADDR(addr);
 664         bfin_write_MDMA_S0_X_COUNT(len);
 665         bfin_write_MDMA_S0_X_MODIFY(0);
 666         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 667
 668         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 669         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 670
 671         SSYNC();
 672
 673         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 674
 675         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 676
 677         bfin_write_MDMA_S0_CONFIG(0);
 678         bfin_write_MDMA_D0_CONFIG(0);
 679         local_irq_restore(flags);
 680
 681 }
 682 EXPORT_SYMBOL(dma_insb);
 683
 684 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
 685 {
 686         unsigned long flags;
 687
 688         local_irq_save(flags);
 689
 690         blackfin_dcache_flush_range((unsigned int)buf,
 691                          (unsigned int)(buf) + len * sizeof(short));
 692
 693         bfin_write_MDMA_D0_START_ADDR(addr);
 694         bfin_write_MDMA_D0_X_COUNT(len);
 695         bfin_write_MDMA_D0_X_MODIFY(0);
 696         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 697
 698         bfin_write_MDMA_S0_START_ADDR(buf);
 699         bfin_write_MDMA_S0_X_COUNT(len);
 700         bfin_write_MDMA_S0_X_MODIFY(2);
 701         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 702
 703         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 704         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 705
 706         SSYNC();
 707
 708         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 709
 710         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 711
 712         bfin_write_MDMA_S0_CONFIG(0);
 713         bfin_write_MDMA_D0_CONFIG(0);
 714         local_irq_restore(flags);
 715
 716 }
 717 EXPORT_SYMBOL(dma_outsw);
 718
 719 void dma_insw(unsigned long addr, void *buf, unsigned short len)
 720 {
 721         unsigned long flags;
 722
 723         blackfin_dcache_invalidate_range((unsigned int)buf,
 724                          (unsigned int)(buf) + len * sizeof(short));
 725
 726         local_irq_save(flags);
 727
 728         bfin_write_MDMA_D0_START_ADDR(buf);
 729         bfin_write_MDMA_D0_X_COUNT(len);
 730         bfin_write_MDMA_D0_X_MODIFY(2);
 731         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 732
 733         bfin_write_MDMA_S0_START_ADDR(addr);
 734         bfin_write_MDMA_S0_X_COUNT(len);
 735         bfin_write_MDMA_S0_X_MODIFY(0);
 736         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 737
 738         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 739         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 740
 741         SSYNC();
 742
 743         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 744
 745         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 746
 747         bfin_write_MDMA_S0_CONFIG(0);
 748         bfin_write_MDMA_D0_CONFIG(0);
 749         local_irq_restore(flags);
 750
 751 }
 752 EXPORT_SYMBOL(dma_insw);
 753
 754 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
 755 {
 756         unsigned long flags;
 757
 758         local_irq_save(flags);
 759
 760         blackfin_dcache_flush_range((unsigned int)buf,
 761                          (unsigned int)(buf) + len * sizeof(long));
 762
 763         bfin_write_MDMA_D0_START_ADDR(addr);
 764         bfin_write_MDMA_D0_X_COUNT(len);
 765         bfin_write_MDMA_D0_X_MODIFY(0);
 766         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 767
 768         bfin_write_MDMA_S0_START_ADDR(buf);
 769         bfin_write_MDMA_S0_X_COUNT(len);
 770         bfin_write_MDMA_S0_X_MODIFY(4);
 771         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 772
 773         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 774         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 775
 776         SSYNC();
 777
 778         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 779
 780         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 781
 782         bfin_write_MDMA_S0_CONFIG(0);
 783         bfin_write_MDMA_D0_CONFIG(0);
 784         local_irq_restore(flags);
 785
 786 }
 787 EXPORT_SYMBOL(dma_outsl);
 788
 789 void dma_insl(unsigned long addr, void *buf, unsigned short len)
 790 {
 791         unsigned long flags;
 792
 793         blackfin_dcache_invalidate_range((unsigned int)buf,
 794                          (unsigned int)(buf) + len * sizeof(long));
 795
 796         local_irq_save(flags);
 797
 798         bfin_write_MDMA_D0_START_ADDR(buf);
 799         bfin_write_MDMA_D0_X_COUNT(len);
 800         bfin_write_MDMA_D0_X_MODIFY(4);
 801         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 802
 803         bfin_write_MDMA_S0_START_ADDR(addr);
 804         bfin_write_MDMA_S0_X_COUNT(len);
 805         bfin_write_MDMA_S0_X_MODIFY(0);
 806         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 807
 808         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 809         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 810
 811         SSYNC();
 812
 813         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 814
 815         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 816
 817         bfin_write_MDMA_S0_CONFIG(0);
 818         bfin_write_MDMA_D0_CONFIG(0);
 819         local_irq_restore(flags);
 820
 821 }
 822 EXPORT_SYMBOL(dma_insl);