arch/blackfin/kernel/bfin_dma_5xx.c

   1 /*
   2  * File:         arch/blackfin/kernel/bfin_dma_5xx.c
   3  * Based on:
   4  * Author:
   5  *
   6  * Created:
   7  * Description:  This file contains the simple DMA Implementation for Blackfin
   8  *
   9  * Modified:
  10  *               Copyright 2004-2006 Analog Devices Inc.
  11  *
  12  * Bugs:         Enter bugs at http://blackfin.uclinux.org/
  13  *
  14  * This program is free software; you can redistribute it and/or modify
  15  * it under the terms of the GNU General Public License as published by
  16  * the Free Software Foundation; either version 2 of the License, or
  17  * (at your option) any later version.
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * You should have received a copy of the GNU General Public License
  25  * along with this program; if not, see the file COPYING, or write
  26  * to the Free Software Foundation, Inc.,
  27  * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  28  */
  29
  30 #include <linux/errno.h>
  31 #include <linux/module.h>
  32 #include <linux/sched.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/kernel.h>
  35 #include <linux/param.h>
  36
  37 #include <asm/blackfin.h>
  38 #include <asm/dma.h>
  39 #include <asm/cacheflush.h>
  40
  41 /* Remove unused code not exported by symbol or internally called */
  42 #define REMOVE_DEAD_CODE
  43
  44 /**************************************************************************
  45  * Global Variables
  46 ***************************************************************************/
  47
  48 static struct dma_channel dma_ch[MAX_BLACKFIN_DMA_CHANNEL];
  49
  50 /*------------------------------------------------------------------------------
  51  *       Set the Buffer Clear bit in the Configuration register of specific DMA
  52  *       channel. This will stop the descriptor based DMA operation.
  53  *-----------------------------------------------------------------------------*/
  54 static void clear_dma_buffer(unsigned int channel)
  55 {
  56         dma_ch[channel].regs->cfg |= RESTART;
  57         SSYNC();
  58         dma_ch[channel].regs->cfg &= ~RESTART;
  59         SSYNC();
  60 }
  61
  62 static int __init blackfin_dma_init(void)
  63 {
  64         int i;
  65
  66         printk(KERN_INFO "Blackfin DMA Controller\n");
  67
  68         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
  69                 dma_ch[i].chan_status = DMA_CHANNEL_FREE;
  70                 dma_ch[i].regs = dma_io_base_addr[i];
  71                 mutex_init(&(dma_ch[i].dmalock));
  72         }
  73         /* Mark MEMDMA Channel 0 as requested since we're using it internally */
  74         dma_ch[CH_MEM_STREAM0_DEST].chan_status = DMA_CHANNEL_REQUESTED;
  75         dma_ch[CH_MEM_STREAM0_SRC].chan_status = DMA_CHANNEL_REQUESTED;
  76
  77 #if defined(CONFIG_DEB_DMA_URGENT)
  78         bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
  79                          | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
  80 #endif
  81         return 0;
  82 }
  83
  84 arch_initcall(blackfin_dma_init);
  85
  86 /*------------------------------------------------------------------------------
  87  *      Request the specific DMA channel from the system.
  88  *-----------------------------------------------------------------------------*/
  89 int request_dma(unsigned int channel, char *device_id)
  90 {
  91
  92         pr_debug("request_dma() : BEGIN \n");
  93
  94 #if defined(CONFIG_BF561) && ANOMALY_05000182
  95         if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
  96                 if (get_cclk() > 500000000) {
  97                         printk(KERN_WARNING
  98                                "Request IMDMA failed due to ANOMALY 05000182\n");
  99                         return -EFAULT;
 100                 }
 101         }
 102 #endif
 103
 104         mutex_lock(&(dma_ch[channel].dmalock));
 105
 106         if ((dma_ch[channel].chan_status == DMA_CHANNEL_REQUESTED)
 107             || (dma_ch[channel].chan_status == DMA_CHANNEL_ENABLED)) {
 108                 mutex_unlock(&(dma_ch[channel].dmalock));
 109                 pr_debug("DMA CHANNEL IN USE  \n");
 110                 return -EBUSY;
 111         } else {
 112                 dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 113                 pr_debug("DMA CHANNEL IS ALLOCATED  \n");
 114         }
 115
 116         mutex_unlock(&(dma_ch[channel].dmalock));
 117
 118 #ifdef CONFIG_BF54x
 119         if (channel >= CH_UART2_RX && channel <= CH_UART3_TX) {
 120                 unsigned int per_map;
 121                 per_map = dma_ch[channel].regs->peripheral_map & 0xFFF;
 122                 if (strncmp(device_id, "BFIN_UART", 9) == 0)
 123                         dma_ch[channel].regs->peripheral_map = per_map |
 124                                 ((channel - CH_UART2_RX + 0xC)<<12);
 125                 else
 126                         dma_ch[channel].regs->peripheral_map = per_map |
 127                                 ((channel - CH_UART2_RX + 0x6)<<12);
 128         }
 129 #endif
 130
 131         dma_ch[channel].device_id = device_id;
 132         dma_ch[channel].irq_callback = NULL;
 133
 134         /* This is to be enabled by putting a restriction -
 135          * you have to request DMA, before doing any operations on
 136          * descriptor/channel
 137          */
 138         pr_debug("request_dma() : END  \n");
 139         return channel;
 140 }
 141 EXPORT_SYMBOL(request_dma);
 142
 143 int set_dma_callback(unsigned int channel, dma_interrupt_t callback, void *data)
 144 {
 145         int ret_irq = 0;
 146
 147         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 148                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 149
 150         if (callback != NULL) {
 151                 int ret_val;
 152                 ret_irq = channel2irq(channel);
 153
 154                 dma_ch[channel].data = data;
 155
 156                 ret_val =
 157                     request_irq(ret_irq, (void *)callback, IRQF_DISABLED,
 158                                 dma_ch[channel].device_id, data);
 159                 if (ret_val) {
 160                         printk(KERN_NOTICE
 161                                "Request irq in DMA engine failed.\n");
 162                         return -EPERM;
 163                 }
 164                 dma_ch[channel].irq_callback = callback;
 165         }
 166         return 0;
 167 }
 168 EXPORT_SYMBOL(set_dma_callback);
 169
 170 void free_dma(unsigned int channel)
 171 {
 172         int ret_irq;
 173
 174         pr_debug("freedma() : BEGIN \n");
 175         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 176                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 177
 178         /* Halt the DMA */
 179         disable_dma(channel);
 180         clear_dma_buffer(channel);
 181
 182         if (dma_ch[channel].irq_callback != NULL) {
 183                 ret_irq = channel2irq(channel);
 184                 free_irq(ret_irq, dma_ch[channel].data);
 185         }
 186
 187         /* Clear the DMA Variable in the Channel */
 188         mutex_lock(&(dma_ch[channel].dmalock));
 189         dma_ch[channel].chan_status = DMA_CHANNEL_FREE;
 190         mutex_unlock(&(dma_ch[channel].dmalock));
 191
 192         pr_debug("freedma() : END \n");
 193 }
 194 EXPORT_SYMBOL(free_dma);
 195
 196 void dma_enable_irq(unsigned int channel)
 197 {
 198         int ret_irq;
 199
 200         pr_debug("dma_enable_irq() : BEGIN \n");
 201         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 202                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 203
 204         ret_irq = channel2irq(channel);
 205         enable_irq(ret_irq);
 206 }
 207 EXPORT_SYMBOL(dma_enable_irq);
 208
 209 void dma_disable_irq(unsigned int channel)
 210 {
 211         int ret_irq;
 212
 213         pr_debug("dma_disable_irq() : BEGIN \n");
 214         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 215                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 216
 217         ret_irq = channel2irq(channel);
 218         disable_irq(ret_irq);
 219 }
 220 EXPORT_SYMBOL(dma_disable_irq);
 221
 222 int dma_channel_active(unsigned int channel)
 223 {
 224         if (dma_ch[channel].chan_status == DMA_CHANNEL_FREE) {
 225                 return 0;
 226         } else {
 227                 return 1;
 228         }
 229 }
 230 EXPORT_SYMBOL(dma_channel_active);
 231
 232 /*------------------------------------------------------------------------------
 233 *       stop the specific DMA channel.
 234 *-----------------------------------------------------------------------------*/
 235 void disable_dma(unsigned int channel)
 236 {
 237         pr_debug("stop_dma() : BEGIN \n");
 238
 239         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 240                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 241
 242         dma_ch[channel].regs->cfg &= ~DMAEN;    /* Clean the enable bit */
 243         SSYNC();
 244         dma_ch[channel].chan_status = DMA_CHANNEL_REQUESTED;
 245         /* Needs to be enabled Later */
 246         pr_debug("stop_dma() : END \n");
 247         return;
 248 }
 249 EXPORT_SYMBOL(disable_dma);
 250
 251 void enable_dma(unsigned int channel)
 252 {
 253         pr_debug("enable_dma() : BEGIN \n");
 254
 255         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 256                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 257
 258         dma_ch[channel].chan_status = DMA_CHANNEL_ENABLED;
 259         dma_ch[channel].regs->curr_x_count = 0;
 260         dma_ch[channel].regs->curr_y_count = 0;
 261
 262         dma_ch[channel].regs->cfg |= DMAEN;     /* Set the enable bit */
 263         SSYNC();
 264         pr_debug("enable_dma() : END \n");
 265         return;
 266 }
 267 EXPORT_SYMBOL(enable_dma);
 268
 269 /*------------------------------------------------------------------------------
 270 *               Set the Start Address register for the specific DMA channel
 271 *               This function can be used for register based DMA,
 272 *               to setup the start address
 273 *               addr:           Starting address of the DMA Data to be transferred.
 274 *-----------------------------------------------------------------------------*/
 275 void set_dma_start_addr(unsigned int channel, unsigned long addr)
 276 {
 277         pr_debug("set_dma_start_addr() : BEGIN \n");
 278
 279         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 280                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 281
 282         dma_ch[channel].regs->start_addr = addr;
 283         SSYNC();
 284         pr_debug("set_dma_start_addr() : END\n");
 285 }
 286 EXPORT_SYMBOL(set_dma_start_addr);
 287
 288 void set_dma_next_desc_addr(unsigned int channel, unsigned long addr)
 289 {
 290         pr_debug("set_dma_next_desc_addr() : BEGIN \n");
 291
 292         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 293                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 294
 295         dma_ch[channel].regs->next_desc_ptr = addr;
 296         SSYNC();
 297         pr_debug("set_dma_next_desc_addr() : END\n");
 298 }
 299 EXPORT_SYMBOL(set_dma_next_desc_addr);
 300
 301 void set_dma_curr_desc_addr(unsigned int channel, unsigned long addr)
 302 {
 303         pr_debug("set_dma_curr_desc_addr() : BEGIN \n");
 304
 305         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 306                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 307
 308         dma_ch[channel].regs->curr_desc_ptr = addr;
 309         SSYNC();
 310         pr_debug("set_dma_curr_desc_addr() : END\n");
 311 }
 312 EXPORT_SYMBOL(set_dma_curr_desc_addr);
 313
 314 void set_dma_x_count(unsigned int channel, unsigned short x_count)
 315 {
 316         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 317                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 318
 319         dma_ch[channel].regs->x_count = x_count;
 320         SSYNC();
 321 }
 322 EXPORT_SYMBOL(set_dma_x_count);
 323
 324 void set_dma_y_count(unsigned int channel, unsigned short y_count)
 325 {
 326         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 327                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 328
 329         dma_ch[channel].regs->y_count = y_count;
 330         SSYNC();
 331 }
 332 EXPORT_SYMBOL(set_dma_y_count);
 333
 334 void set_dma_x_modify(unsigned int channel, short x_modify)
 335 {
 336         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 337                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 338
 339         dma_ch[channel].regs->x_modify = x_modify;
 340         SSYNC();
 341 }
 342 EXPORT_SYMBOL(set_dma_x_modify);
 343
 344 void set_dma_y_modify(unsigned int channel, short y_modify)
 345 {
 346         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 347                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 348
 349         dma_ch[channel].regs->y_modify = y_modify;
 350         SSYNC();
 351 }
 352 EXPORT_SYMBOL(set_dma_y_modify);
 353
 354 void set_dma_config(unsigned int channel, unsigned short config)
 355 {
 356         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 357                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 358
 359         dma_ch[channel].regs->cfg = config;
 360         SSYNC();
 361 }
 362 EXPORT_SYMBOL(set_dma_config);
 363
 364 unsigned short
 365 set_bfin_dma_config(char direction, char flow_mode,
 366                     char intr_mode, char dma_mode, char width, char syncmode)
 367 {
 368         unsigned short config;
 369
 370         config =
 371             ((direction << 1) | (width << 2) | (dma_mode << 4) |
 372              (intr_mode << 6) | (flow_mode << 12) | (syncmode << 5));
 373         return config;
 374 }
 375 EXPORT_SYMBOL(set_bfin_dma_config);
 376
 377 void set_dma_sg(unsigned int channel, struct dmasg *sg, int nr_sg)
 378 {
 379         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 380                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 381
 382         dma_ch[channel].regs->cfg |= ((nr_sg & 0x0F) << 8);
 383
 384         dma_ch[channel].regs->next_desc_ptr = (unsigned int)sg;
 385
 386         SSYNC();
 387 }
 388 EXPORT_SYMBOL(set_dma_sg);
 389
 390 void set_dma_curr_addr(unsigned int channel, unsigned long addr)
 391 {
 392         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 393                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 394
 395         dma_ch[channel].regs->curr_addr_ptr = addr;
 396         SSYNC();
 397 }
 398 EXPORT_SYMBOL(set_dma_curr_addr);
 399
 400 /*------------------------------------------------------------------------------
 401  *      Get the DMA status of a specific DMA channel from the system.
 402  *-----------------------------------------------------------------------------*/
 403 unsigned short get_dma_curr_irqstat(unsigned int channel)
 404 {
 405         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 406                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 407
 408         return dma_ch[channel].regs->irq_status;
 409 }
 410 EXPORT_SYMBOL(get_dma_curr_irqstat);
 411
 412 /*------------------------------------------------------------------------------
 413  *      Clear the DMA_DONE bit in DMA status. Stop the DMA completion interrupt.
 414  *-----------------------------------------------------------------------------*/
 415 void clear_dma_irqstat(unsigned int channel)
 416 {
 417         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 418                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 419         dma_ch[channel].regs->irq_status |= 3;
 420 }
 421 EXPORT_SYMBOL(clear_dma_irqstat);
 422
 423 /*------------------------------------------------------------------------------
 424  *      Get current DMA xcount of a specific DMA channel from the system.
 425  *-----------------------------------------------------------------------------*/
 426 unsigned short get_dma_curr_xcount(unsigned int channel)
 427 {
 428         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 429                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 430
 431         return dma_ch[channel].regs->curr_x_count;
 432 }
 433 EXPORT_SYMBOL(get_dma_curr_xcount);
 434
 435 /*------------------------------------------------------------------------------
 436  *      Get current DMA ycount of a specific DMA channel from the system.
 437  *-----------------------------------------------------------------------------*/
 438 unsigned short get_dma_curr_ycount(unsigned int channel)
 439 {
 440         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 441                && channel < MAX_BLACKFIN_DMA_CHANNEL));
 442
 443         return dma_ch[channel].regs->curr_y_count;
 444 }
 445 EXPORT_SYMBOL(get_dma_curr_ycount);
 446
 447 unsigned long get_dma_next_desc_ptr(unsigned int channel)
 448 {
 449         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 450               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 451
 452         return dma_ch[channel].regs->next_desc_ptr;
 453 }
 454 EXPORT_SYMBOL(get_dma_next_desc_ptr);
 455
 456 unsigned long get_dma_curr_desc_ptr(unsigned int channel)
 457 {
 458         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 459               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 460
 461         return dma_ch[channel].regs->curr_desc_ptr;
 462 }
 463 EXPORT_SYMBOL(get_dma_curr_desc_ptr);
 464
 465 unsigned long get_dma_curr_addr(unsigned int channel)
 466 {
 467         BUG_ON(!(dma_ch[channel].chan_status != DMA_CHANNEL_FREE
 468               && channel < MAX_BLACKFIN_DMA_CHANNEL));
 469
 470         return dma_ch[channel].regs->curr_addr_ptr;
 471 }
 472 EXPORT_SYMBOL(get_dma_curr_addr);
 473
 474 #ifdef CONFIG_PM
 475 int blackfin_dma_suspend(void)
 476 {
 477         int i;
 478
 479 #ifdef CONFIG_BF561     /* IMDMA channels doesn't have a PERIPHERAL_MAP */
 480         for (i = 0; i <= CH_MEM_STREAM3_SRC; i++) {
 481 #else
 482         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++) {
 483 #endif
 484                 if (dma_ch[i].chan_status == DMA_CHANNEL_ENABLED) {
 485                         printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
 486                         return -EBUSY;
 487                 }
 488
 489                 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
 490         }
 491
 492         return 0;
 493 }
 494
 495 void blackfin_dma_resume(void)
 496 {
 497         int i;
 498
 499 #ifdef CONFIG_BF561     /* IMDMA channels doesn't have a PERIPHERAL_MAP */
 500         for (i = 0; i <= CH_MEM_STREAM3_SRC; i++)
 501 #else
 502         for (i = 0; i < MAX_BLACKFIN_DMA_CHANNEL; i++)
 503 #endif
 504                 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
 505 }
 506 #endif
 507
 508 static void *__dma_memcpy(void *dest, const void *src, size_t size)
 509 {
 510         int direction;  /* 1 - address decrease, 0 - address increase */
 511         int flag_align; /* 1 - address aligned,  0 - address unaligned */
 512         int flag_2D;    /* 1 - 2D DMA needed,    0 - 1D DMA needed */
 513         unsigned long flags;
 514
 515         if (size <= 0)
 516                 return NULL;
 517
 518         local_irq_save(flags);
 519
 520         if ((unsigned long)src < memory_end)
 521                 blackfin_dcache_flush_range((unsigned int)src,
 522                                             (unsigned int)(src + size));
 523
 524         if ((unsigned long)dest < memory_end)
 525                 blackfin_dcache_invalidate_range((unsigned int)dest,
 526                                                  (unsigned int)(dest + size));
 527
 528         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 529
 530         if ((unsigned long)src < (unsigned long)dest)
 531                 direction = 1;
 532         else
 533                 direction = 0;
 534
 535         if ((((unsigned long)dest % 2) == 0) && (((unsigned long)src % 2) == 0)
 536             && ((size % 2) == 0))
 537                 flag_align = 1;
 538         else
 539                 flag_align = 0;
 540
 541         if (size > 0x10000)     /* size > 64K */
 542                 flag_2D = 1;
 543         else
 544                 flag_2D = 0;
 545
 546         /* Setup destination and source start address */
 547         if (direction) {
 548                 if (flag_align) {
 549                         bfin_write_MDMA_D0_START_ADDR(dest + size - 2);
 550                         bfin_write_MDMA_S0_START_ADDR(src + size - 2);
 551                 } else {
 552                         bfin_write_MDMA_D0_START_ADDR(dest + size - 1);
 553                         bfin_write_MDMA_S0_START_ADDR(src + size - 1);
 554                 }
 555         } else {
 556                 bfin_write_MDMA_D0_START_ADDR(dest);
 557                 bfin_write_MDMA_S0_START_ADDR(src);
 558         }
 559
 560         /* Setup destination and source xcount */
 561         if (flag_2D) {
 562                 if (flag_align) {
 563                         bfin_write_MDMA_D0_X_COUNT(1024 / 2);
 564                         bfin_write_MDMA_S0_X_COUNT(1024 / 2);
 565                 } else {
 566                         bfin_write_MDMA_D0_X_COUNT(1024);
 567                         bfin_write_MDMA_S0_X_COUNT(1024);
 568                 }
 569                 bfin_write_MDMA_D0_Y_COUNT(size >> 10);
 570                 bfin_write_MDMA_S0_Y_COUNT(size >> 10);
 571         } else {
 572                 if (flag_align) {
 573                         bfin_write_MDMA_D0_X_COUNT(size / 2);
 574                         bfin_write_MDMA_S0_X_COUNT(size / 2);
 575                 } else {
 576                         bfin_write_MDMA_D0_X_COUNT(size);
 577                         bfin_write_MDMA_S0_X_COUNT(size);
 578                 }
 579         }
 580
 581         /* Setup destination and source xmodify and ymodify */
 582         if (direction) {
 583                 if (flag_align) {
 584                         bfin_write_MDMA_D0_X_MODIFY(-2);
 585                         bfin_write_MDMA_S0_X_MODIFY(-2);
 586                         if (flag_2D) {
 587                                 bfin_write_MDMA_D0_Y_MODIFY(-2);
 588                                 bfin_write_MDMA_S0_Y_MODIFY(-2);
 589                         }
 590                 } else {
 591                         bfin_write_MDMA_D0_X_MODIFY(-1);
 592                         bfin_write_MDMA_S0_X_MODIFY(-1);
 593                         if (flag_2D) {
 594                                 bfin_write_MDMA_D0_Y_MODIFY(-1);
 595                                 bfin_write_MDMA_S0_Y_MODIFY(-1);
 596                         }
 597                 }
 598         } else {
 599                 if (flag_align) {
 600                         bfin_write_MDMA_D0_X_MODIFY(2);
 601                         bfin_write_MDMA_S0_X_MODIFY(2);
 602                         if (flag_2D) {
 603                                 bfin_write_MDMA_D0_Y_MODIFY(2);
 604                                 bfin_write_MDMA_S0_Y_MODIFY(2);
 605                         }
 606                 } else {
 607                         bfin_write_MDMA_D0_X_MODIFY(1);
 608                         bfin_write_MDMA_S0_X_MODIFY(1);
 609                         if (flag_2D) {
 610                                 bfin_write_MDMA_D0_Y_MODIFY(1);
 611                                 bfin_write_MDMA_S0_Y_MODIFY(1);
 612                         }
 613                 }
 614         }
 615
 616         /* Enable source DMA */
 617         if (flag_2D) {
 618                 if (flag_align) {
 619                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D | WDSIZE_16);
 620                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D | WDSIZE_16);
 621                 } else {
 622                         bfin_write_MDMA_S0_CONFIG(DMAEN | DMA2D);
 623                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | DMA2D);
 624                 }
 625         } else {
 626                 if (flag_align) {
 627                         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 628                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 629                 } else {
 630                         bfin_write_MDMA_S0_CONFIG(DMAEN);
 631                         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN);
 632                 }
 633         }
 634
 635         SSYNC();
 636
 637         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE))
 638                 ;
 639
 640         bfin_write_MDMA_D0_IRQ_STATUS(bfin_read_MDMA_D0_IRQ_STATUS() |
 641                                       (DMA_DONE | DMA_ERR));
 642
 643         bfin_write_MDMA_S0_CONFIG(0);
 644         bfin_write_MDMA_D0_CONFIG(0);
 645
 646         local_irq_restore(flags);
 647
 648         return dest;
 649 }
 650
 651 void *dma_memcpy(void *dest, const void *src, size_t size)
 652 {
 653         size_t bulk;
 654         size_t rest;
 655         void * addr;
 656
 657         bulk = (size >> 16) << 16;
 658         rest = size - bulk;
 659         if (bulk)
 660                 __dma_memcpy(dest, src, bulk);
 661         addr = __dma_memcpy(dest+bulk, src+bulk, rest);
 662         return addr;
 663 }
 664 EXPORT_SYMBOL(dma_memcpy);
 665
 666 void *safe_dma_memcpy(void *dest, const void *src, size_t size)
 667 {
 668         void *addr;
 669         addr = dma_memcpy(dest, src, size);
 670         return addr;
 671 }
 672 EXPORT_SYMBOL(safe_dma_memcpy);
 673
 674 void dma_outsb(unsigned long addr, const void *buf, unsigned short len)
 675 {
 676         unsigned long flags;
 677
 678         local_irq_save(flags);
 679
 680         blackfin_dcache_flush_range((unsigned int)buf,
 681                          (unsigned int)(buf) + len);
 682
 683         bfin_write_MDMA_D0_START_ADDR(addr);
 684         bfin_write_MDMA_D0_X_COUNT(len);
 685         bfin_write_MDMA_D0_X_MODIFY(0);
 686         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 687
 688         bfin_write_MDMA_S0_START_ADDR(buf);
 689         bfin_write_MDMA_S0_X_COUNT(len);
 690         bfin_write_MDMA_S0_X_MODIFY(1);
 691         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 692
 693         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 694         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 695
 696         SSYNC();
 697
 698         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 699
 700         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 701
 702         bfin_write_MDMA_S0_CONFIG(0);
 703         bfin_write_MDMA_D0_CONFIG(0);
 704         local_irq_restore(flags);
 705
 706 }
 707 EXPORT_SYMBOL(dma_outsb);
 708
 709
 710 void dma_insb(unsigned long addr, void *buf, unsigned short len)
 711 {
 712         unsigned long flags;
 713
 714         blackfin_dcache_invalidate_range((unsigned int)buf,
 715                          (unsigned int)(buf) + len);
 716
 717         local_irq_save(flags);
 718         bfin_write_MDMA_D0_START_ADDR(buf);
 719         bfin_write_MDMA_D0_X_COUNT(len);
 720         bfin_write_MDMA_D0_X_MODIFY(1);
 721         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 722
 723         bfin_write_MDMA_S0_START_ADDR(addr);
 724         bfin_write_MDMA_S0_X_COUNT(len);
 725         bfin_write_MDMA_S0_X_MODIFY(0);
 726         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 727
 728         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_8);
 729         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_8);
 730
 731         SSYNC();
 732
 733         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 734
 735         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 736
 737         bfin_write_MDMA_S0_CONFIG(0);
 738         bfin_write_MDMA_D0_CONFIG(0);
 739         local_irq_restore(flags);
 740
 741 }
 742 EXPORT_SYMBOL(dma_insb);
 743
 744 void dma_outsw(unsigned long addr, const void  *buf, unsigned short len)
 745 {
 746         unsigned long flags;
 747
 748         local_irq_save(flags);
 749
 750         blackfin_dcache_flush_range((unsigned int)buf,
 751                          (unsigned int)(buf) + len * sizeof(short));
 752
 753         bfin_write_MDMA_D0_START_ADDR(addr);
 754         bfin_write_MDMA_D0_X_COUNT(len);
 755         bfin_write_MDMA_D0_X_MODIFY(0);
 756         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 757
 758         bfin_write_MDMA_S0_START_ADDR(buf);
 759         bfin_write_MDMA_S0_X_COUNT(len);
 760         bfin_write_MDMA_S0_X_MODIFY(2);
 761         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 762
 763         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 764         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 765
 766         SSYNC();
 767
 768         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 769
 770         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 771
 772         bfin_write_MDMA_S0_CONFIG(0);
 773         bfin_write_MDMA_D0_CONFIG(0);
 774         local_irq_restore(flags);
 775
 776 }
 777 EXPORT_SYMBOL(dma_outsw);
 778
 779 void dma_insw(unsigned long addr, void *buf, unsigned short len)
 780 {
 781         unsigned long flags;
 782
 783         blackfin_dcache_invalidate_range((unsigned int)buf,
 784                          (unsigned int)(buf) + len * sizeof(short));
 785
 786         local_irq_save(flags);
 787
 788         bfin_write_MDMA_D0_START_ADDR(buf);
 789         bfin_write_MDMA_D0_X_COUNT(len);
 790         bfin_write_MDMA_D0_X_MODIFY(2);
 791         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 792
 793         bfin_write_MDMA_S0_START_ADDR(addr);
 794         bfin_write_MDMA_S0_X_COUNT(len);
 795         bfin_write_MDMA_S0_X_MODIFY(0);
 796         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 797
 798         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_16);
 799         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_16);
 800
 801         SSYNC();
 802
 803         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 804
 805         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 806
 807         bfin_write_MDMA_S0_CONFIG(0);
 808         bfin_write_MDMA_D0_CONFIG(0);
 809         local_irq_restore(flags);
 810
 811 }
 812 EXPORT_SYMBOL(dma_insw);
 813
 814 void dma_outsl(unsigned long addr, const void *buf, unsigned short len)
 815 {
 816         unsigned long flags;
 817
 818         local_irq_save(flags);
 819
 820         blackfin_dcache_flush_range((unsigned int)buf,
 821                          (unsigned int)(buf) + len * sizeof(long));
 822
 823         bfin_write_MDMA_D0_START_ADDR(addr);
 824         bfin_write_MDMA_D0_X_COUNT(len);
 825         bfin_write_MDMA_D0_X_MODIFY(0);
 826         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 827
 828         bfin_write_MDMA_S0_START_ADDR(buf);
 829         bfin_write_MDMA_S0_X_COUNT(len);
 830         bfin_write_MDMA_S0_X_MODIFY(4);
 831         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 832
 833         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 834         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 835
 836         SSYNC();
 837
 838         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 839
 840         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 841
 842         bfin_write_MDMA_S0_CONFIG(0);
 843         bfin_write_MDMA_D0_CONFIG(0);
 844         local_irq_restore(flags);
 845
 846 }
 847 EXPORT_SYMBOL(dma_outsl);
 848
 849 void dma_insl(unsigned long addr, void *buf, unsigned short len)
 850 {
 851         unsigned long flags;
 852
 853         blackfin_dcache_invalidate_range((unsigned int)buf,
 854                          (unsigned int)(buf) + len * sizeof(long));
 855
 856         local_irq_save(flags);
 857
 858         bfin_write_MDMA_D0_START_ADDR(buf);
 859         bfin_write_MDMA_D0_X_COUNT(len);
 860         bfin_write_MDMA_D0_X_MODIFY(4);
 861         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 862
 863         bfin_write_MDMA_S0_START_ADDR(addr);
 864         bfin_write_MDMA_S0_X_COUNT(len);
 865         bfin_write_MDMA_S0_X_MODIFY(0);
 866         bfin_write_MDMA_S0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 867
 868         bfin_write_MDMA_S0_CONFIG(DMAEN | WDSIZE_32);
 869         bfin_write_MDMA_D0_CONFIG(WNR | DI_EN | DMAEN | WDSIZE_32);
 870
 871         SSYNC();
 872
 873         while (!(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE));
 874
 875         bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
 876
 877         bfin_write_MDMA_S0_CONFIG(0);
 878         bfin_write_MDMA_D0_CONFIG(0);
 879         local_irq_restore(flags);
 880
 881 }
 882 EXPORT_SYMBOL(dma_insl);