fs/btrfs/async-thread.c

   1 /*
   2  * Copyright (C) 2007 Oracle.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/kthread.h>
  20 #include <linux/list.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/freezer.h>
  23 #include <linux/ftrace.h>
  24 #include "async-thread.h"
  25
  26 #define WORK_QUEUED_BIT 0
  27 #define WORK_DONE_BIT 1
  28 #define WORK_ORDER_DONE_BIT 2
  29
  30 /*
  31  * container for the kthread task pointer and the list of pending work
  32  * One of these is allocated per thread.
  33  */
  34 struct btrfs_worker_thread {
  35         /* pool we belong to */
  36         struct btrfs_workers *workers;
  37
  38         /* list of struct btrfs_work that are waiting for service */
  39         struct list_head pending;
  40
  41         /* list of worker threads from struct btrfs_workers */
  42         struct list_head worker_list;
  43
  44         /* kthread */
  45         struct task_struct *task;
  46
  47         /* number of things on the pending list */
  48         atomic_t num_pending;
  49
  50         unsigned long sequence;
  51
  52         /* protects the pending list. */
  53         spinlock_t lock;
  54
  55         /* set to non-zero when this thread is already awake and kicking */
  56         int working;
  57
  58         /* are we currently idle */
  59         int idle;
  60 };
  61
  62 /*
  63  * helper function to move a thread onto the idle list after it
  64  * has finished some requests.
  65  */
  66 static void check_idle_worker(struct btrfs_worker_thread *worker)
  67 {
  68         if (!worker->idle && atomic_read(&worker->num_pending) <
  69             worker->workers->idle_thresh / 2) {
  70                 unsigned long flags;
  71                 spin_lock_irqsave(&worker->workers->lock, flags);
  72                 worker->idle = 1;
  73                 list_move(&worker->worker_list, &worker->workers->idle_list);
  74                 spin_unlock_irqrestore(&worker->workers->lock, flags);
  75         }
  76 }
  77
  78 /*
  79  * helper function to move a thread off the idle list after new
  80  * pending work is added.
  81  */
  82 static void check_busy_worker(struct btrfs_worker_thread *worker)
  83 {
  84         if (worker->idle && atomic_read(&worker->num_pending) >=
  85             worker->workers->idle_thresh) {
  86                 unsigned long flags;
  87                 spin_lock_irqsave(&worker->workers->lock, flags);
  88                 worker->idle = 0;
  89                 list_move_tail(&worker->worker_list,
  90                                &worker->workers->worker_list);
  91                 spin_unlock_irqrestore(&worker->workers->lock, flags);
  92         }
  93 }
  94
  95 static noinline int run_ordered_completions(struct btrfs_workers *workers,
  96                                             struct btrfs_work *work)
  97 {
  98         unsigned long flags;
  99
 100         if (!workers->ordered)
 101                 return 0;
 102
 103         set_bit(WORK_DONE_BIT, &work->flags);
 104
 105         spin_lock_irqsave(&workers->lock, flags);
 106
 107         while (!list_empty(&workers->order_list)) {
 108                 work = list_entry(workers->order_list.next,
 109                                   struct btrfs_work, order_list);
 110
 111                 if (!test_bit(WORK_DONE_BIT, &work->flags))
 112                         break;
 113
 114                 /* we are going to call the ordered done function, but
 115                  * we leave the work item on the list as a barrier so
 116                  * that later work items that are done don't have their
 117                  * functions called before this one returns
 118                  */
 119                 if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags))
 120                         break;
 121
 122                 spin_unlock_irqrestore(&workers->lock, flags);
 123
 124                 work->ordered_func(work);
 125
 126                 /* now take the lock again and call the freeing code */
 127                 spin_lock_irqsave(&workers->lock, flags);
 128                 list_del(&work->order_list);
 129                 work->ordered_free(work);
 130         }
 131
 132         spin_unlock_irqrestore(&workers->lock, flags);
 133         return 0;
 134 }
 135
 136 /*
 137  * main loop for servicing work items
 138  */
 139 static int worker_loop(void *arg)
 140 {
 141         struct btrfs_worker_thread *worker = arg;
 142         struct list_head *cur;
 143         struct btrfs_work *work;
 144         do {
 145                 spin_lock_irq(&worker->lock);
 146 again_locked:
 147                 while (!list_empty(&worker->pending)) {
 148                         cur = worker->pending.next;
 149                         work = list_entry(cur, struct btrfs_work, list);
 150                         list_del(&work->list);
 151                         clear_bit(WORK_QUEUED_BIT, &work->flags);
 152
 153                         work->worker = worker;
 154                         spin_unlock_irq(&worker->lock);
 155
 156                         work->func(work);
 157
 158                         atomic_dec(&worker->num_pending);
 159                         /*
 160                          * unless this is an ordered work queue,
 161                          * 'work' was probably freed by func above.
 162                          */
 163                         run_ordered_completions(worker->workers, work);
 164
 165                         spin_lock_irq(&worker->lock);
 166                         check_idle_worker(worker);
 167
 168                 }
 169                 if (freezing(current)) {
 170                         worker->working = 0;
 171                         spin_unlock_irq(&worker->lock);
 172                         refrigerator();
 173                 } else {
 174                         spin_unlock_irq(&worker->lock);
 175                         if (!kthread_should_stop()) {
 176                                 cpu_relax();
 177                                 /*
 178                                  * we've dropped the lock, did someone else
 179                                  * jump_in?
 180                                  */
 181                                 smp_mb();
 182                                 if (!list_empty(&worker->pending))
 183                                         continue;
 184
 185                                 /*
 186                                  * this short schedule allows more work to
 187                                  * come in without the queue functions
 188                                  * needing to go through wake_up_process()
 189                                  *
 190                                  * worker->working is still 1, so nobody
 191                                  * is going to try and wake us up
 192                                  */
 193                                 schedule_timeout(1);
 194                                 smp_mb();
 195                                 if (!list_empty(&worker->pending))
 196                                         continue;
 197
 198                                 if (kthread_should_stop())
 199                                         break;
 200
 201                                 /* still no more work?, sleep for real */
 202                                 spin_lock_irq(&worker->lock);
 203                                 set_current_state(TASK_INTERRUPTIBLE);
 204                                 if (!list_empty(&worker->pending))
 205                                         goto again_locked;
 206
 207                                 /*
 208                                  * this makes sure we get a wakeup when someone
 209                                  * adds something new to the queue
 210                                  */
 211                                 worker->working = 0;
 212                                 spin_unlock_irq(&worker->lock);
 213
 214                                 if (!kthread_should_stop())
 215                                         schedule();
 216                         }
 217                         __set_current_state(TASK_RUNNING);
 218                 }
 219         } while (!kthread_should_stop());
 220         return 0;
 221 }
 222
 223 /*
 224  * this will wait for all the worker threads to shutdown
 225  */
 226 int btrfs_stop_workers(struct btrfs_workers *workers)
 227 {
 228         struct list_head *cur;
 229         struct btrfs_worker_thread *worker;
 230
 231         list_splice_init(&workers->idle_list, &workers->worker_list);
 232         while (!list_empty(&workers->worker_list)) {
 233                 cur = workers->worker_list.next;
 234                 worker = list_entry(cur, struct btrfs_worker_thread,
 235                                     worker_list);
 236                 kthread_stop(worker->task);
 237                 list_del(&worker->worker_list);
 238                 kfree(worker);
 239         }
 240         return 0;
 241 }
 242
 243 /*
 244  * simple init on struct btrfs_workers
 245  */
 246 void btrfs_init_workers(struct btrfs_workers *workers, char *name, int max)
 247 {
 248         workers->num_workers = 0;
 249         INIT_LIST_HEAD(&workers->worker_list);
 250         INIT_LIST_HEAD(&workers->idle_list);
 251         INIT_LIST_HEAD(&workers->order_list);
 252         spin_lock_init(&workers->lock);
 253         workers->max_workers = max;
 254         workers->idle_thresh = 32;
 255         workers->name = name;
 256         workers->ordered = 0;
 257 }
 258
 259 /*
 260  * starts new worker threads.  This does not enforce the max worker
 261  * count in case you need to temporarily go past it.
 262  */
 263 int btrfs_start_workers(struct btrfs_workers *workers, int num_workers)
 264 {
 265         struct btrfs_worker_thread *worker;
 266         int ret = 0;
 267         int i;
 268
 269         for (i = 0; i < num_workers; i++) {
 270                 worker = kzalloc(sizeof(*worker), GFP_NOFS);
 271                 if (!worker) {
 272                         ret = -ENOMEM;
 273                         goto fail;
 274                 }
 275
 276                 INIT_LIST_HEAD(&worker->pending);
 277                 INIT_LIST_HEAD(&worker->worker_list);
 278                 spin_lock_init(&worker->lock);
 279                 atomic_set(&worker->num_pending, 0);
 280                 worker->task = kthread_run(worker_loop, worker,
 281                                            "btrfs-%s-%d", workers->name,
 282                                            workers->num_workers + i);
 283                 worker->workers = workers;
 284                 if (IS_ERR(worker->task)) {
 285                         kfree(worker);
 286                         ret = PTR_ERR(worker->task);
 287                         goto fail;
 288                 }
 289
 290                 spin_lock_irq(&workers->lock);
 291                 list_add_tail(&worker->worker_list, &workers->idle_list);
 292                 worker->idle = 1;
 293                 workers->num_workers++;
 294                 spin_unlock_irq(&workers->lock);
 295         }
 296         return 0;
 297 fail:
 298         btrfs_stop_workers(workers);
 299         return ret;
 300 }
 301
 302 /*
 303  * run through the list and find a worker thread that doesn't have a lot
 304  * to do right now.  This can return null if we aren't yet at the thread
 305  * count limit and all of the threads are busy.
 306  */
 307 static struct btrfs_worker_thread *next_worker(struct btrfs_workers *workers)
 308 {
 309         struct btrfs_worker_thread *worker;
 310         struct list_head *next;
 311         int enforce_min = workers->num_workers < workers->max_workers;
 312
 313         /*
 314          * if we find an idle thread, don't move it to the end of the
 315          * idle list.  This improves the chance that the next submission
 316          * will reuse the same thread, and maybe catch it while it is still
 317          * working
 318          */
 319         if (!list_empty(&workers->idle_list)) {
 320                 next = workers->idle_list.next;
 321                 worker = list_entry(next, struct btrfs_worker_thread,
 322                                     worker_list);
 323                 return worker;
 324         }
 325         if (enforce_min || list_empty(&workers->worker_list))
 326                 return NULL;
 327
 328         /*
 329          * if we pick a busy task, move the task to the end of the list.
 330          * hopefully this will keep things somewhat evenly balanced.
 331          * Do the move in batches based on the sequence number.  This groups
 332          * requests submitted at roughly the same time onto the same worker.
 333          */
 334         next = workers->worker_list.next;
 335         worker = list_entry(next, struct btrfs_worker_thread, worker_list);
 336         atomic_inc(&worker->num_pending);
 337         worker->sequence++;
 338
 339         if (worker->sequence % workers->idle_thresh == 0)
 340                 list_move_tail(next, &workers->worker_list);
 341         return worker;
 342 }
 343
 344 /*
 345  * selects a worker thread to take the next job.  This will either find
 346  * an idle worker, start a new worker up to the max count, or just return
 347  * one of the existing busy workers.
 348  */
 349 static struct btrfs_worker_thread *find_worker(struct btrfs_workers *workers)
 350 {
 351         struct btrfs_worker_thread *worker;
 352         unsigned long flags;
 353
 354 again:
 355         spin_lock_irqsave(&workers->lock, flags);
 356         worker = next_worker(workers);
 357         spin_unlock_irqrestore(&workers->lock, flags);
 358
 359         if (!worker) {
 360                 spin_lock_irqsave(&workers->lock, flags);
 361                 if (workers->num_workers >= workers->max_workers) {
 362                         struct list_head *fallback = NULL;
 363                         /*
 364                          * we have failed to find any workers, just
 365                          * return the force one
 366                          */
 367                         if (!list_empty(&workers->worker_list))
 368                                 fallback = workers->worker_list.next;
 369                         if (!list_empty(&workers->idle_list))
 370                                 fallback = workers->idle_list.next;
 371                         BUG_ON(!fallback);
 372                         worker = list_entry(fallback,
 373                                   struct btrfs_worker_thread, worker_list);
 374                         spin_unlock_irqrestore(&workers->lock, flags);
 375                 } else {
 376                         spin_unlock_irqrestore(&workers->lock, flags);
 377                         /* we're below the limit, start another worker */
 378                         btrfs_start_workers(workers, 1);
 379                         goto again;
 380                 }
 381         }
 382         return worker;
 383 }
 384
 385 /*
 386  * btrfs_requeue_work just puts the work item back on the tail of the list
 387  * it was taken from.  It is intended for use with long running work functions
 388  * that make some progress and want to give the cpu up for others.
 389  */
 390 int btrfs_requeue_work(struct btrfs_work *work)
 391 {
 392         struct btrfs_worker_thread *worker = work->worker;
 393         unsigned long flags;
 394         int wake = 0;
 395
 396         if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
 397                 goto out;
 398
 399         spin_lock_irqsave(&worker->lock, flags);
 400         list_add_tail(&work->list, &worker->pending);
 401         atomic_inc(&worker->num_pending);
 402
 403         /* by definition we're busy, take ourselves off the idle
 404          * list
 405          */
 406         if (worker->idle) {
 407                 spin_lock_irqsave(&worker->workers->lock, flags);
 408                 worker->idle = 0;
 409                 list_move_tail(&worker->worker_list,
 410                                &worker->workers->worker_list);
 411                 spin_unlock_irqrestore(&worker->workers->lock, flags);
 412         }
 413         if (!worker->working) {
 414                 wake = 1;
 415                 worker->working = 1;
 416         }
 417
 418         spin_unlock_irqrestore(&worker->lock, flags);
 419         if (wake)
 420                 wake_up_process(worker->task);
 421 out:
 422
 423         return 0;
 424 }
 425
 426 /*
 427  * places a struct btrfs_work into the pending queue of one of the kthreads
 428  */
 429 int btrfs_queue_worker(struct btrfs_workers *workers, struct btrfs_work *work)
 430 {
 431         struct btrfs_worker_thread *worker;
 432         unsigned long flags;
 433         int wake = 0;
 434
 435         /* don't requeue something already on a list */
 436         if (test_and_set_bit(WORK_QUEUED_BIT, &work->flags))
 437                 goto out;
 438
 439         worker = find_worker(workers);
 440         if (workers->ordered) {
 441                 spin_lock_irqsave(&workers->lock, flags);
 442                 list_add_tail(&work->order_list, &workers->order_list);
 443                 spin_unlock_irqrestore(&workers->lock, flags);
 444         } else {
 445                 INIT_LIST_HEAD(&work->order_list);
 446         }
 447
 448         spin_lock_irqsave(&worker->lock, flags);
 449
 450         list_add_tail(&work->list, &worker->pending);
 451         atomic_inc(&worker->num_pending);
 452         check_busy_worker(worker);
 453
 454         /*
 455          * avoid calling into wake_up_process if this thread has already
 456          * been kicked
 457          */
 458         if (!worker->working)
 459                 wake = 1;
 460         worker->working = 1;
 461
 462         spin_unlock_irqrestore(&worker->lock, flags);
 463
 464         if (wake)
 465                 wake_up_process(worker->task);
 466 out:
 467         return 0;
 468 }