drivers/block/elevator.c

   1 /*
   2  *  linux/drivers/block/elevator.c
   3  *
   4  *  Block device elevator/IO-scheduler.
   5  *
   6  *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
   7  *
   8  * 30042000 Jens Axboe <axboe@suse.de> :
   9  *
  10  * Split the elevator a bit so that it is possible to choose a different
  11  * one or even write a new "plug in". There are three pieces:
  12  * - elevator_fn, inserts a new request in the queue list
  13  * - elevator_merge_fn, decides whether a new buffer can be merged with
  14  *   an existing request
  15  * - elevator_dequeue_fn, called when a request is taken off the active list
  16  *
  17  * 20082000 Dave Jones <davej@suse.de> :
  18  * Removed tests for max-bomb-segments, which was breaking elvtune
  19  *  when run without -bN
  20  *
  21  * Jens:
  22  * - Rework again to work with bio instead of buffer_heads
  23  * - loose bi_dev comparisons, partition handling is right now
  24  * - completely modularize elevator setup and teardown
  25  *
  26  */
  27 #include <linux/kernel.h>
  28 #include <linux/fs.h>
  29 #include <linux/blkdev.h>
  30 #include <linux/elevator.h>
  31 #include <linux/bio.h>
  32 #include <linux/config.h>
  33 #include <linux/module.h>
  34 #include <linux/slab.h>
  35 #include <linux/init.h>
  36 #include <linux/compiler.h>
  37
  38 #include <asm/uaccess.h>
  39
  40 static DEFINE_SPINLOCK(elv_list_lock);
  41 static LIST_HEAD(elv_list);
  42
  43 static inline sector_t rq_last_sector(struct request *rq)
  44 {
  45         return rq->sector + rq->nr_sectors;
  46 }
  47
  48 /*
  49  * can we safely merge with this request?
  50  */
  51 inline int elv_rq_merge_ok(struct request *rq, struct bio *bio)
  52 {
  53         if (!rq_mergeable(rq))
  54                 return 0;
  55
  56         /*
  57          * different data direction or already started, don't merge
  58          */
  59         if (bio_data_dir(bio) != rq_data_dir(rq))
  60                 return 0;
  61
  62         /*
  63          * same device and no special stuff set, merge is ok
  64          */
  65         if (rq->rq_disk == bio->bi_bdev->bd_disk &&
  66             !rq->waiting && !rq->special)
  67                 return 1;
  68
  69         return 0;
  70 }
  71 EXPORT_SYMBOL(elv_rq_merge_ok);
  72
  73 inline int elv_try_merge(struct request *__rq, struct bio *bio)
  74 {
  75         int ret = ELEVATOR_NO_MERGE;
  76
  77         /*
  78          * we can merge and sequence is ok, check if it's possible
  79          */
  80         if (elv_rq_merge_ok(__rq, bio)) {
  81                 if (__rq->sector + __rq->nr_sectors == bio->bi_sector)
  82                         ret = ELEVATOR_BACK_MERGE;
  83                 else if (__rq->sector - bio_sectors(bio) == bio->bi_sector)
  84                         ret = ELEVATOR_FRONT_MERGE;
  85         }
  86
  87         return ret;
  88 }
  89 EXPORT_SYMBOL(elv_try_merge);
  90
  91 inline int elv_try_last_merge(request_queue_t *q, struct bio *bio)
  92 {
  93         if (q->last_merge)
  94                 return elv_try_merge(q->last_merge, bio);
  95
  96         return ELEVATOR_NO_MERGE;
  97 }
  98 EXPORT_SYMBOL(elv_try_last_merge);
  99
 100 static struct elevator_type *elevator_find(const char *name)
 101 {
 102         struct elevator_type *e = NULL;
 103         struct list_head *entry;
 104
 105         list_for_each(entry, &elv_list) {
 106                 struct elevator_type *__e;
 107
 108                 __e = list_entry(entry, struct elevator_type, list);
 109
 110                 if (!strcmp(__e->elevator_name, name)) {
 111                         e = __e;
 112                         break;
 113                 }
 114         }
 115
 116         return e;
 117 }
 118
 119 static void elevator_put(struct elevator_type *e)
 120 {
 121         module_put(e->elevator_owner);
 122 }
 123
 124 static struct elevator_type *elevator_get(const char *name)
 125 {
 126         struct elevator_type *e;
 127
 128         spin_lock_irq(&elv_list_lock);
 129
 130         e = elevator_find(name);
 131         if (e && !try_module_get(e->elevator_owner))
 132                 e = NULL;
 133
 134         spin_unlock_irq(&elv_list_lock);
 135
 136         return e;
 137 }
 138
 139 static int elevator_attach(request_queue_t *q, struct elevator_type *e,
 140                            struct elevator_queue *eq)
 141 {
 142         int ret = 0;
 143
 144         memset(eq, 0, sizeof(*eq));
 145         eq->ops = &e->ops;
 146         eq->elevator_type = e;
 147
 148         INIT_LIST_HEAD(&q->queue_head);
 149         q->last_merge = NULL;
 150         q->elevator = eq;
 151         q->last_sector = 0;
 152         q->boundary_rq = NULL;
 153         q->max_back_kb = 0;
 154
 155         if (eq->ops->elevator_init_fn)
 156                 ret = eq->ops->elevator_init_fn(q, eq);
 157
 158         return ret;
 159 }
 160
 161 static char chosen_elevator[16];
 162
 163 static void elevator_setup_default(void)
 164 {
 165         struct elevator_type *e;
 166
 167         /*
 168          * check if default is set and exists
 169          */
 170         if (chosen_elevator[0] && (e = elevator_get(chosen_elevator))) {
 171                 elevator_put(e);
 172                 return;
 173         }
 174
 175 #if defined(CONFIG_IOSCHED_AS)
 176         strcpy(chosen_elevator, "anticipatory");
 177 #elif defined(CONFIG_IOSCHED_DEADLINE)
 178         strcpy(chosen_elevator, "deadline");
 179 #elif defined(CONFIG_IOSCHED_CFQ)
 180         strcpy(chosen_elevator, "cfq");
 181 #elif defined(CONFIG_IOSCHED_NOOP)
 182         strcpy(chosen_elevator, "noop");
 183 #else
 184 #error "You must build at least 1 IO scheduler into the kernel"
 185 #endif
 186 }
 187
 188 static int __init elevator_setup(char *str)
 189 {
 190         strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
 191         return 0;
 192 }
 193
 194 __setup("elevator=", elevator_setup);
 195
 196 int elevator_init(request_queue_t *q, char *name)
 197 {
 198         struct elevator_type *e = NULL;
 199         struct elevator_queue *eq;
 200         int ret = 0;
 201
 202         elevator_setup_default();
 203
 204         if (!name)
 205                 name = chosen_elevator;
 206
 207         e = elevator_get(name);
 208         if (!e)
 209                 return -EINVAL;
 210
 211         eq = kmalloc(sizeof(struct elevator_queue), GFP_KERNEL);
 212         if (!eq) {
 213                 elevator_put(e->elevator_type);
 214                 return -ENOMEM;
 215         }
 216
 217         ret = elevator_attach(q, e, eq);
 218         if (ret) {
 219                 kfree(eq);
 220                 elevator_put(e->elevator_type);
 221         }
 222
 223         return ret;
 224 }
 225
 226 void elevator_exit(elevator_t *e)
 227 {
 228         if (e->ops->elevator_exit_fn)
 229                 e->ops->elevator_exit_fn(e);
 230
 231         elevator_put(e->elevator_type);
 232         e->elevator_type = NULL;
 233         kfree(e);
 234 }
 235
 236 /*
 237  * Insert rq into dispatch queue of q.  Queue lock must be held on
 238  * entry.  If sort != 0, rq is sort-inserted; otherwise, rq will be
 239  * appended to the dispatch queue.  To be used by specific elevators.
 240  */
 241 void elv_dispatch_insert(request_queue_t *q, struct request *rq, int sort)
 242 {
 243         sector_t boundary;
 244         unsigned max_back;
 245         struct list_head *entry;
 246
 247         if (!sort) {
 248                 /* Specific elevator is performing sort.  Step away. */
 249                 q->last_sector = rq_last_sector(rq);
 250                 q->boundary_rq = rq;
 251                 list_add_tail(&rq->queuelist, &q->queue_head);
 252                 return;
 253         }
 254
 255         boundary = q->last_sector;
 256         max_back = q->max_back_kb * 2;
 257         boundary = boundary > max_back ? boundary - max_back : 0;
 258
 259         list_for_each_prev(entry, &q->queue_head) {
 260                 struct request *pos = list_entry_rq(entry);
 261
 262                 if (pos->flags & (REQ_SOFTBARRIER|REQ_HARDBARRIER|REQ_STARTED))
 263                         break;
 264                 if (rq->sector >= boundary) {
 265                         if (pos->sector < boundary)
 266                                 continue;
 267                 } else {
 268                         if (pos->sector >= boundary)
 269                                 break;
 270                 }
 271                 if (rq->sector >= pos->sector)
 272                         break;
 273         }
 274
 275         list_add(&rq->queuelist, entry);
 276 }
 277
 278 int elv_merge(request_queue_t *q, struct request **req, struct bio *bio)
 279 {
 280         elevator_t *e = q->elevator;
 281
 282         if (e->ops->elevator_merge_fn)
 283                 return e->ops->elevator_merge_fn(q, req, bio);
 284
 285         return ELEVATOR_NO_MERGE;
 286 }
 287
 288 void elv_merged_request(request_queue_t *q, struct request *rq)
 289 {
 290         elevator_t *e = q->elevator;
 291
 292         if (e->ops->elevator_merged_fn)
 293                 e->ops->elevator_merged_fn(q, rq);
 294 }
 295
 296 void elv_merge_requests(request_queue_t *q, struct request *rq,
 297                              struct request *next)
 298 {
 299         elevator_t *e = q->elevator;
 300
 301         if (q->last_merge == next)
 302                 q->last_merge = NULL;
 303
 304         if (e->ops->elevator_merge_req_fn)
 305                 e->ops->elevator_merge_req_fn(q, rq, next);
 306 }
 307
 308 void elv_requeue_request(request_queue_t *q, struct request *rq)
 309 {
 310         elevator_t *e = q->elevator;
 311
 312         /*
 313          * it already went through dequeue, we need to decrement the
 314          * in_flight count again
 315          */
 316         if (blk_account_rq(rq)) {
 317                 q->in_flight--;
 318                 if (blk_sorted_rq(rq) && e->ops->elevator_deactivate_req_fn)
 319                         e->ops->elevator_deactivate_req_fn(q, rq);
 320         }
 321
 322         rq->flags &= ~REQ_STARTED;
 323
 324         /*
 325          * if this is the flush, requeue the original instead and drop the flush
 326          */
 327         if (rq->flags & REQ_BAR_FLUSH) {
 328                 clear_bit(QUEUE_FLAG_FLUSH, &q->queue_flags);
 329                 rq = rq->end_io_data;
 330         }
 331
 332         __elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 0);
 333 }
 334
 335 void __elv_add_request(request_queue_t *q, struct request *rq, int where,
 336                        int plug)
 337 {
 338         if (rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)) {
 339                 /*
 340                  * barriers implicitly indicate back insertion
 341                  */
 342                 if (where == ELEVATOR_INSERT_SORT)
 343                         where = ELEVATOR_INSERT_BACK;
 344
 345                 /*
 346                  * this request is scheduling boundary, update last_sector
 347                  */
 348                 if (blk_fs_request(rq)) {
 349                         q->last_sector = rq_last_sector(rq);
 350                         q->boundary_rq = rq;
 351                 }
 352         }
 353
 354         if (plug)
 355                 blk_plug_device(q);
 356
 357         rq->q = q;
 358
 359         if (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))) {
 360                 /*
 361                  * if drain is set, store the request "locally". when the drain
 362                  * is finished, the requests will be handed ordered to the io
 363                  * scheduler
 364                  */
 365                 list_add_tail(&rq->queuelist, &q->drain_list);
 366                 return;
 367         }
 368
 369         switch (where) {
 370         case ELEVATOR_INSERT_FRONT:
 371                 rq->flags |= REQ_SOFTBARRIER;
 372
 373                 list_add(&rq->queuelist, &q->queue_head);
 374                 break;
 375
 376         case ELEVATOR_INSERT_BACK:
 377                 rq->flags |= REQ_SOFTBARRIER;
 378
 379                 while (q->elevator->ops->elevator_dispatch_fn(q, 1))
 380                         ;
 381                 list_add_tail(&rq->queuelist, &q->queue_head);
 382                 /*
 383                  * We kick the queue here for the following reasons.
 384                  * - The elevator might have returned NULL previously
 385                  *   to delay requests and returned them now.  As the
 386                  *   queue wasn't empty before this request, ll_rw_blk
 387                  *   won't run the queue on return, resulting in hang.
 388                  * - Usually, back inserted requests won't be merged
 389                  *   with anything.  There's no point in delaying queue
 390                  *   processing.
 391                  */
 392                 blk_remove_plug(q);
 393                 q->request_fn(q);
 394                 break;
 395
 396         case ELEVATOR_INSERT_SORT:
 397                 BUG_ON(!blk_fs_request(rq));
 398                 rq->flags |= REQ_SORTED;
 399                 q->elevator->ops->elevator_add_req_fn(q, rq);
 400                 break;
 401
 402         default:
 403                 printk(KERN_ERR "%s: bad insertion point %d\n",
 404                        __FUNCTION__, where);
 405                 BUG();
 406         }
 407
 408         if (blk_queue_plugged(q)) {
 409                 int nrq = q->rq.count[READ] + q->rq.count[WRITE]
 410                         - q->in_flight;
 411
 412                 if (nrq >= q->unplug_thresh)
 413                         __generic_unplug_device(q);
 414         }
 415 }
 416
 417 void elv_add_request(request_queue_t *q, struct request *rq, int where,
 418                      int plug)
 419 {
 420         unsigned long flags;
 421
 422         spin_lock_irqsave(q->queue_lock, flags);
 423         __elv_add_request(q, rq, where, plug);
 424         spin_unlock_irqrestore(q->queue_lock, flags);
 425 }
 426
 427 static inline struct request *__elv_next_request(request_queue_t *q)
 428 {
 429         struct request *rq;
 430
 431         if (unlikely(list_empty(&q->queue_head) &&
 432                      !q->elevator->ops->elevator_dispatch_fn(q, 0)))
 433                 return NULL;
 434
 435         rq = list_entry_rq(q->queue_head.next);
 436
 437         /*
 438          * if this is a barrier write and the device has to issue a
 439          * flush sequence to support it, check how far we are
 440          */
 441         if (blk_fs_request(rq) && blk_barrier_rq(rq)) {
 442                 BUG_ON(q->ordered == QUEUE_ORDERED_NONE);
 443
 444                 if (q->ordered == QUEUE_ORDERED_FLUSH &&
 445                     !blk_barrier_preflush(rq))
 446                         rq = blk_start_pre_flush(q, rq);
 447         }
 448
 449         return rq;
 450 }
 451
 452 struct request *elv_next_request(request_queue_t *q)
 453 {
 454         struct request *rq;
 455         int ret;
 456
 457         while ((rq = __elv_next_request(q)) != NULL) {
 458                 if (!(rq->flags & REQ_STARTED)) {
 459                         elevator_t *e = q->elevator;
 460
 461                         /*
 462                          * This is the first time the device driver
 463                          * sees this request (possibly after
 464                          * requeueing).  Notify IO scheduler.
 465                          */
 466                         if (blk_sorted_rq(rq) &&
 467                             e->ops->elevator_activate_req_fn)
 468                                 e->ops->elevator_activate_req_fn(q, rq);
 469
 470                         /*
 471                          * just mark as started even if we don't start
 472                          * it, a request that has been delayed should
 473                          * not be passed by new incoming requests
 474                          */
 475                         rq->flags |= REQ_STARTED;
 476                 }
 477
 478                 if (rq == q->last_merge)
 479                         q->last_merge = NULL;
 480
 481                 if (!q->boundary_rq || q->boundary_rq == rq) {
 482                         q->last_sector = rq_last_sector(rq);
 483                         q->boundary_rq = NULL;
 484                 }
 485
 486                 if ((rq->flags & REQ_DONTPREP) || !q->prep_rq_fn)
 487                         break;
 488
 489                 ret = q->prep_rq_fn(q, rq);
 490                 if (ret == BLKPREP_OK) {
 491                         break;
 492                 } else if (ret == BLKPREP_DEFER) {
 493                         /*
 494                          * the request may have been (partially) prepped.
 495                          * we need to keep this request in the front to
 496                          * avoid resource deadlock.  REQ_STARTED will
 497                          * prevent other fs requests from passing this one.
 498                          */
 499                         rq = NULL;
 500                         break;
 501                 } else if (ret == BLKPREP_KILL) {
 502                         int nr_bytes = rq->hard_nr_sectors << 9;
 503
 504                         if (!nr_bytes)
 505                                 nr_bytes = rq->data_len;
 506
 507                         blkdev_dequeue_request(rq);
 508                         rq->flags |= REQ_QUIET;
 509                         end_that_request_chunk(rq, 0, nr_bytes);
 510                         end_that_request_last(rq);
 511                 } else {
 512                         printk(KERN_ERR "%s: bad return=%d\n", __FUNCTION__,
 513                                                                 ret);
 514                         break;
 515                 }
 516         }
 517
 518         return rq;
 519 }
 520
 521 void elv_dequeue_request(request_queue_t *q, struct request *rq)
 522 {
 523         BUG_ON(list_empty(&rq->queuelist));
 524
 525         list_del_init(&rq->queuelist);
 526
 527         /*
 528          * the time frame between a request being removed from the lists
 529          * and to it is freed is accounted as io that is in progress at
 530          * the driver side.
 531          */
 532         if (blk_account_rq(rq))
 533                 q->in_flight++;
 534
 535         /*
 536          * the main clearing point for q->last_merge is on retrieval of
 537          * request by driver (it calls elv_next_request()), but it _can_
 538          * also happen here if a request is added to the queue but later
 539          * deleted without ever being given to driver (merged with another
 540          * request).
 541          */
 542         if (rq == q->last_merge)
 543                 q->last_merge = NULL;
 544 }
 545
 546 int elv_queue_empty(request_queue_t *q)
 547 {
 548         elevator_t *e = q->elevator;
 549
 550         if (!list_empty(&q->queue_head))
 551                 return 0;
 552
 553         if (e->ops->elevator_queue_empty_fn)
 554                 return e->ops->elevator_queue_empty_fn(q);
 555
 556         return 1;
 557 }
 558
 559 struct request *elv_latter_request(request_queue_t *q, struct request *rq)
 560 {
 561         struct list_head *next;
 562
 563         elevator_t *e = q->elevator;
 564
 565         if (e->ops->elevator_latter_req_fn)
 566                 return e->ops->elevator_latter_req_fn(q, rq);
 567
 568         next = rq->queuelist.next;
 569         if (next != &q->queue_head && next != &rq->queuelist)
 570                 return list_entry_rq(next);
 571
 572         return NULL;
 573 }
 574
 575 struct request *elv_former_request(request_queue_t *q, struct request *rq)
 576 {
 577         struct list_head *prev;
 578
 579         elevator_t *e = q->elevator;
 580
 581         if (e->ops->elevator_former_req_fn)
 582                 return e->ops->elevator_former_req_fn(q, rq);
 583
 584         prev = rq->queuelist.prev;
 585         if (prev != &q->queue_head && prev != &rq->queuelist)
 586                 return list_entry_rq(prev);
 587
 588         return NULL;
 589 }
 590
 591 int elv_set_request(request_queue_t *q, struct request *rq, struct bio *bio,
 592                     int gfp_mask)
 593 {
 594         elevator_t *e = q->elevator;
 595
 596         if (e->ops->elevator_set_req_fn)
 597                 return e->ops->elevator_set_req_fn(q, rq, bio, gfp_mask);
 598
 599         rq->elevator_private = NULL;
 600         return 0;
 601 }
 602
 603 void elv_put_request(request_queue_t *q, struct request *rq)
 604 {
 605         elevator_t *e = q->elevator;
 606
 607         if (e->ops->elevator_put_req_fn)
 608                 e->ops->elevator_put_req_fn(q, rq);
 609 }
 610
 611 int elv_may_queue(request_queue_t *q, int rw, struct bio *bio)
 612 {
 613         elevator_t *e = q->elevator;
 614
 615         if (e->ops->elevator_may_queue_fn)
 616                 return e->ops->elevator_may_queue_fn(q, rw, bio);
 617
 618         return ELV_MQUEUE_MAY;
 619 }
 620
 621 void elv_completed_request(request_queue_t *q, struct request *rq)
 622 {
 623         elevator_t *e = q->elevator;
 624
 625         /*
 626          * request is released from the driver, io must be done
 627          */
 628         if (blk_account_rq(rq)) {
 629                 q->in_flight--;
 630                 if (blk_sorted_rq(rq) && e->ops->elevator_completed_req_fn)
 631                         e->ops->elevator_completed_req_fn(q, rq);
 632         }
 633 }
 634
 635 int elv_register_queue(struct request_queue *q)
 636 {
 637         elevator_t *e = q->elevator;
 638
 639         e->kobj.parent = kobject_get(&q->kobj);
 640         if (!e->kobj.parent)
 641                 return -EBUSY;
 642
 643         snprintf(e->kobj.name, KOBJ_NAME_LEN, "%s", "iosched");
 644         e->kobj.ktype = e->elevator_type->elevator_ktype;
 645
 646         return kobject_register(&e->kobj);
 647 }
 648
 649 void elv_unregister_queue(struct request_queue *q)
 650 {
 651         if (q) {
 652                 elevator_t *e = q->elevator;
 653                 kobject_unregister(&e->kobj);
 654                 kobject_put(&q->kobj);
 655         }
 656 }
 657
 658 int elv_register(struct elevator_type *e)
 659 {
 660         spin_lock_irq(&elv_list_lock);
 661         if (elevator_find(e->elevator_name))
 662                 BUG();
 663         list_add_tail(&e->list, &elv_list);
 664         spin_unlock_irq(&elv_list_lock);
 665
 666         printk(KERN_INFO "io scheduler %s registered", e->elevator_name);
 667         if (!strcmp(e->elevator_name, chosen_elevator))
 668                 printk(" (default)");
 669         printk("\n");
 670         return 0;
 671 }
 672 EXPORT_SYMBOL_GPL(elv_register);
 673
 674 void elv_unregister(struct elevator_type *e)
 675 {
 676         spin_lock_irq(&elv_list_lock);
 677         list_del_init(&e->list);
 678         spin_unlock_irq(&elv_list_lock);
 679 }
 680 EXPORT_SYMBOL_GPL(elv_unregister);
 681
 682 /*
 683  * switch to new_e io scheduler. be careful not to introduce deadlocks -
 684  * we don't free the old io scheduler, before we have allocated what we
 685  * need for the new one. this way we have a chance of going back to the old
 686  * one, if the new one fails init for some reason. we also do an intermediate
 687  * switch to noop to ensure safety with stack-allocated requests, since they
 688  * don't originate from the block layer allocator. noop is safe here, because
 689  * it never needs to touch the elevator itself for completion events. DRAIN
 690  * flags will make sure we don't touch it for additions either.
 691  */
 692 static void elevator_switch(request_queue_t *q, struct elevator_type *new_e)
 693 {
 694         elevator_t *e = kmalloc(sizeof(elevator_t), GFP_KERNEL);
 695         struct elevator_type *noop_elevator = NULL;
 696         elevator_t *old_elevator;
 697
 698         if (!e)
 699                 goto error;
 700
 701         /*
 702          * first step, drain requests from the block freelist
 703          */
 704         blk_wait_queue_drained(q, 0);
 705
 706         /*
 707          * unregister old elevator data
 708          */
 709         elv_unregister_queue(q);
 710         old_elevator = q->elevator;
 711
 712         /*
 713          * next step, switch to noop since it uses no private rq structures
 714          * and doesn't allocate any memory for anything. then wait for any
 715          * non-fs requests in-flight
 716          */
 717         noop_elevator = elevator_get("noop");
 718         spin_lock_irq(q->queue_lock);
 719         elevator_attach(q, noop_elevator, e);
 720         spin_unlock_irq(q->queue_lock);
 721
 722         blk_wait_queue_drained(q, 1);
 723
 724         /*
 725          * attach and start new elevator
 726          */
 727         if (elevator_attach(q, new_e, e))
 728                 goto fail;
 729
 730         if (elv_register_queue(q))
 731                 goto fail_register;
 732
 733         /*
 734          * finally exit old elevator and start queue again
 735          */
 736         elevator_exit(old_elevator);
 737         blk_finish_queue_drain(q);
 738         elevator_put(noop_elevator);
 739         return;
 740
 741 fail_register:
 742         /*
 743          * switch failed, exit the new io scheduler and reattach the old
 744          * one again (along with re-adding the sysfs dir)
 745          */
 746         elevator_exit(e);
 747 fail:
 748         q->elevator = old_elevator;
 749         elv_register_queue(q);
 750         blk_finish_queue_drain(q);
 751 error:
 752         if (noop_elevator)
 753                 elevator_put(noop_elevator);
 754         elevator_put(new_e);
 755         printk(KERN_ERR "elevator: switch to %s failed\n",new_e->elevator_name);
 756 }
 757
 758 ssize_t elv_iosched_store(request_queue_t *q, const char *name, size_t count)
 759 {
 760         char elevator_name[ELV_NAME_MAX];
 761         struct elevator_type *e;
 762
 763         memset(elevator_name, 0, sizeof(elevator_name));
 764         strncpy(elevator_name, name, sizeof(elevator_name));
 765
 766         if (elevator_name[strlen(elevator_name) - 1] == '\n')
 767                 elevator_name[strlen(elevator_name) - 1] = '\0';
 768
 769         e = elevator_get(elevator_name);
 770         if (!e) {
 771                 printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
 772                 return -EINVAL;
 773         }
 774
 775         if (!strcmp(elevator_name, q->elevator->elevator_type->elevator_name))
 776                 return count;
 777
 778         elevator_switch(q, e);
 779         return count;
 780 }
 781
 782 ssize_t elv_iosched_show(request_queue_t *q, char *name)
 783 {
 784         elevator_t *e = q->elevator;
 785         struct elevator_type *elv = e->elevator_type;
 786         struct list_head *entry;
 787         int len = 0;
 788
 789         spin_lock_irq(q->queue_lock);
 790         list_for_each(entry, &elv_list) {
 791                 struct elevator_type *__e;
 792
 793                 __e = list_entry(entry, struct elevator_type, list);
 794                 if (!strcmp(elv->elevator_name, __e->elevator_name))
 795                         len += sprintf(name+len, "[%s] ", elv->elevator_name);
 796                 else
 797                         len += sprintf(name+len, "%s ", __e->elevator_name);
 798         }
 799         spin_unlock_irq(q->queue_lock);
 800
 801         len += sprintf(len+name, "\n");
 802         return len;
 803 }
 804
 805 EXPORT_SYMBOL(elv_dispatch_insert);
 806 EXPORT_SYMBOL(elv_add_request);
 807 EXPORT_SYMBOL(__elv_add_request);
 808 EXPORT_SYMBOL(elv_requeue_request);
 809 EXPORT_SYMBOL(elv_next_request);
 810 EXPORT_SYMBOL(elv_dequeue_request);
 811 EXPORT_SYMBOL(elv_queue_empty);
 812 EXPORT_SYMBOL(elv_completed_request);
 813 EXPORT_SYMBOL(elevator_exit);
 814 EXPORT_SYMBOL(elevator_init);