2 * net/sched/sch_api.c Packet scheduler API.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
13 * Rani Assaf <rani@magic.metawire.com> :980802: JIFFIES and CPU clock sources are repaired.
14 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support
15 * Jamal Hadi Salim <hadi@nortelnetworks.com>: 990601: ingress support
18 #include <linux/module.h>
19 #include <linux/types.h>
20 #include <linux/kernel.h>
21 #include <linux/string.h>
22 #include <linux/errno.h>
23 #include <linux/skbuff.h>
24 #include <linux/init.h>
25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h>
27 #include <linux/kmod.h>
28 #include <linux/list.h>
29 #include <linux/hrtimer.h>
31 #include <net/net_namespace.h>
33 #include <net/netlink.h>
34 #include <net/pkt_sched.h>
36 static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n, u32 clid,
37 struct Qdisc *old, struct Qdisc *new);
38 static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
39 struct Qdisc *q, unsigned long cl, int event);
46 This file consists of two interrelated parts:
48 1. queueing disciplines manager frontend.
49 2. traffic classes manager frontend.
51 Generally, queueing discipline ("qdisc") is a black box,
52 which is able to enqueue packets and to dequeue them (when
53 device is ready to send something) in order and at times
54 determined by algorithm hidden in it.
56 qdisc's are divided to two categories:
57 - "queues", which have no internal structure visible from outside.
58 - "schedulers", which split all the packets to "traffic classes",
59 using "packet classifiers" (look at cls_api.c)
61 In turn, classes may have child qdiscs (as rule, queues)
62 attached to them etc. etc. etc.
64 The goal of the routines in this file is to translate
65 information supplied by user in the form of handles
66 to more intelligible for kernel form, to make some sanity
67 checks and part of work, which is common to all qdiscs
68 and to provide rtnetlink notifications.
70 All real intelligent work is done inside qdisc modules.
74 Every discipline has two major routines: enqueue and dequeue.
78 dequeue usually returns a skb to send. It is allowed to return NULL,
79 but it does not mean that queue is empty, it just means that
80 discipline does not want to send anything this time.
81 Queue is really empty if q->q.qlen == 0.
82 For complicated disciplines with multiple queues q->q is not
83 real packet queue, but however q->q.qlen must be valid.
87 enqueue returns 0, if packet was enqueued successfully.
88 If packet (this one or another one) was dropped, it returns
90 NET_XMIT_DROP - this packet dropped
91 Expected action: do not backoff, but wait until queue will clear.
92 NET_XMIT_CN - probably this packet enqueued, but another one dropped.
93 Expected action: backoff or ignore
94 NET_XMIT_POLICED - dropped by police.
95 Expected action: backoff or error to real-time apps.
101 requeues once dequeued packet. It is used for non-standard or
102 just buggy devices, which can defer output even if netif_queue_stopped()=0.
106 returns qdisc to initial state: purge all buffers, clear all
107 timers, counters (except for statistics) etc.
111 initializes newly created qdisc.
115 destroys resources allocated by init and during lifetime of qdisc.
119 changes qdisc parameters.
122 /* Protects list of registered TC modules. It is pure SMP lock. */
123 static DEFINE_RWLOCK(qdisc_mod_lock);
126 /************************************************
127 * Queueing disciplines manipulation. *
128 ************************************************/
131 /* The list of all installed queueing disciplines. */
133 static struct Qdisc_ops *qdisc_base;
135 /* Register/uregister queueing discipline */
137 int register_qdisc(struct Qdisc_ops *qops)
139 struct Qdisc_ops *q, **qp;
142 write_lock(&qdisc_mod_lock);
143 for (qp = &qdisc_base; (q = *qp) != NULL; qp = &q->next)
144 if (!strcmp(qops->id, q->id))
147 if (qops->enqueue == NULL)
148 qops->enqueue = noop_qdisc_ops.enqueue;
149 if (qops->requeue == NULL)
150 qops->requeue = noop_qdisc_ops.requeue;
151 if (qops->dequeue == NULL)
152 qops->dequeue = noop_qdisc_ops.dequeue;
158 write_unlock(&qdisc_mod_lock);
161 EXPORT_SYMBOL(register_qdisc);
163 int unregister_qdisc(struct Qdisc_ops *qops)
165 struct Qdisc_ops *q, **qp;
168 write_lock(&qdisc_mod_lock);
169 for (qp = &qdisc_base; (q=*qp)!=NULL; qp = &q->next)
177 write_unlock(&qdisc_mod_lock);
180 EXPORT_SYMBOL(unregister_qdisc);
182 /* We know handle. Find qdisc among all qdisc's attached to device
183 (root qdisc, all its children, children of children etc.)
186 static struct Qdisc *__qdisc_lookup(struct netdev_queue *dev_queue, u32 handle)
190 list_for_each_entry(q, &dev_queue->qdisc_list, list) {
191 if (q->handle == handle)
197 struct Qdisc *qdisc_lookup(struct net_device *dev, u32 handle)
201 for (i = 0; i < dev->num_tx_queues; i++) {
202 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
203 struct Qdisc *q = __qdisc_lookup(txq, handle);
210 static struct Qdisc *qdisc_leaf(struct Qdisc *p, u32 classid)
214 const struct Qdisc_class_ops *cops = p->ops->cl_ops;
218 cl = cops->get(p, classid);
222 leaf = cops->leaf(p, cl);
227 /* Find queueing discipline by name */
229 static struct Qdisc_ops *qdisc_lookup_ops(struct nlattr *kind)
231 struct Qdisc_ops *q = NULL;
234 read_lock(&qdisc_mod_lock);
235 for (q = qdisc_base; q; q = q->next) {
236 if (nla_strcmp(kind, q->id) == 0) {
237 if (!try_module_get(q->owner))
242 read_unlock(&qdisc_mod_lock);
247 static struct qdisc_rate_table *qdisc_rtab_list;
249 struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab)
251 struct qdisc_rate_table *rtab;
253 for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) {
254 if (memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) == 0) {
260 if (tab == NULL || r->rate == 0 || r->cell_log == 0 ||
261 nla_len(tab) != TC_RTAB_SIZE)
264 rtab = kmalloc(sizeof(*rtab), GFP_KERNEL);
268 memcpy(rtab->data, nla_data(tab), 1024);
269 rtab->next = qdisc_rtab_list;
270 qdisc_rtab_list = rtab;
274 EXPORT_SYMBOL(qdisc_get_rtab);
276 void qdisc_put_rtab(struct qdisc_rate_table *tab)
278 struct qdisc_rate_table *rtab, **rtabp;
280 if (!tab || --tab->refcnt)
283 for (rtabp = &qdisc_rtab_list; (rtab=*rtabp) != NULL; rtabp = &rtab->next) {
291 EXPORT_SYMBOL(qdisc_put_rtab);
293 static enum hrtimer_restart qdisc_watchdog(struct hrtimer *timer)
295 struct qdisc_watchdog *wd = container_of(timer, struct qdisc_watchdog,
298 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
300 __netif_schedule(wd->qdisc);
302 return HRTIMER_NORESTART;
305 void qdisc_watchdog_init(struct qdisc_watchdog *wd, struct Qdisc *qdisc)
307 hrtimer_init(&wd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
308 wd->timer.function = qdisc_watchdog;
311 EXPORT_SYMBOL(qdisc_watchdog_init);
313 void qdisc_watchdog_schedule(struct qdisc_watchdog *wd, psched_time_t expires)
317 wd->qdisc->flags |= TCQ_F_THROTTLED;
318 time = ktime_set(0, 0);
319 time = ktime_add_ns(time, PSCHED_US2NS(expires));
320 hrtimer_start(&wd->timer, time, HRTIMER_MODE_ABS);
322 EXPORT_SYMBOL(qdisc_watchdog_schedule);
324 void qdisc_watchdog_cancel(struct qdisc_watchdog *wd)
326 hrtimer_cancel(&wd->timer);
327 wd->qdisc->flags &= ~TCQ_F_THROTTLED;
329 EXPORT_SYMBOL(qdisc_watchdog_cancel);
331 struct hlist_head *qdisc_class_hash_alloc(unsigned int n)
333 unsigned int size = n * sizeof(struct hlist_head), i;
334 struct hlist_head *h;
336 if (size <= PAGE_SIZE)
337 h = kmalloc(size, GFP_KERNEL);
339 h = (struct hlist_head *)
340 __get_free_pages(GFP_KERNEL, get_order(size));
343 for (i = 0; i < n; i++)
344 INIT_HLIST_HEAD(&h[i]);
349 static void qdisc_class_hash_free(struct hlist_head *h, unsigned int n)
351 unsigned int size = n * sizeof(struct hlist_head);
353 if (size <= PAGE_SIZE)
356 free_pages((unsigned long)h, get_order(size));
359 void qdisc_class_hash_grow(struct Qdisc *sch, struct Qdisc_class_hash *clhash)
361 struct Qdisc_class_common *cl;
362 struct hlist_node *n, *next;
363 struct hlist_head *nhash, *ohash;
364 unsigned int nsize, nmask, osize;
367 /* Rehash when load factor exceeds 0.75 */
368 if (clhash->hashelems * 4 <= clhash->hashsize * 3)
370 nsize = clhash->hashsize * 2;
372 nhash = qdisc_class_hash_alloc(nsize);
376 ohash = clhash->hash;
377 osize = clhash->hashsize;
380 for (i = 0; i < osize; i++) {
381 hlist_for_each_entry_safe(cl, n, next, &ohash[i], hnode) {
382 h = qdisc_class_hash(cl->classid, nmask);
383 hlist_add_head(&cl->hnode, &nhash[h]);
386 clhash->hash = nhash;
387 clhash->hashsize = nsize;
388 clhash->hashmask = nmask;
389 sch_tree_unlock(sch);
391 qdisc_class_hash_free(ohash, osize);
393 EXPORT_SYMBOL(qdisc_class_hash_grow);
395 int qdisc_class_hash_init(struct Qdisc_class_hash *clhash)
397 unsigned int size = 4;
399 clhash->hash = qdisc_class_hash_alloc(size);
400 if (clhash->hash == NULL)
402 clhash->hashsize = size;
403 clhash->hashmask = size - 1;
404 clhash->hashelems = 0;
407 EXPORT_SYMBOL(qdisc_class_hash_init);
409 void qdisc_class_hash_destroy(struct Qdisc_class_hash *clhash)
411 qdisc_class_hash_free(clhash->hash, clhash->hashsize);
413 EXPORT_SYMBOL(qdisc_class_hash_destroy);
415 void qdisc_class_hash_insert(struct Qdisc_class_hash *clhash,
416 struct Qdisc_class_common *cl)
420 INIT_HLIST_NODE(&cl->hnode);
421 h = qdisc_class_hash(cl->classid, clhash->hashmask);
422 hlist_add_head(&cl->hnode, &clhash->hash[h]);
425 EXPORT_SYMBOL(qdisc_class_hash_insert);
427 void qdisc_class_hash_remove(struct Qdisc_class_hash *clhash,
428 struct Qdisc_class_common *cl)
430 hlist_del(&cl->hnode);
433 EXPORT_SYMBOL(qdisc_class_hash_remove);
435 /* Allocate an unique handle from space managed by kernel */
437 static u32 qdisc_alloc_handle(struct net_device *dev)
440 static u32 autohandle = TC_H_MAKE(0x80000000U, 0);
443 autohandle += TC_H_MAKE(0x10000U, 0);
444 if (autohandle == TC_H_MAKE(TC_H_ROOT, 0))
445 autohandle = TC_H_MAKE(0x80000000U, 0);
446 } while (qdisc_lookup(dev, autohandle) && --i > 0);
448 return i>0 ? autohandle : 0;
451 /* Attach toplevel qdisc to device dev */
453 static struct Qdisc *
454 dev_graft_qdisc(struct net_device *dev, struct Qdisc *qdisc)
456 struct netdev_queue *dev_queue;
457 struct Qdisc *oqdisc;
459 if (dev->flags & IFF_UP)
462 qdisc_lock_tree(dev);
463 if (qdisc && qdisc->flags&TCQ_F_INGRESS) {
464 dev_queue = &dev->rx_queue;
465 oqdisc = dev_queue->qdisc;
466 /* Prune old scheduler */
467 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1) {
470 dev_queue->qdisc = NULL;
472 dev_queue->qdisc = qdisc;
476 dev_queue = netdev_get_tx_queue(dev, 0);
477 oqdisc = dev_queue->qdisc_sleeping;
479 /* Prune old scheduler */
480 if (oqdisc && atomic_read(&oqdisc->refcnt) <= 1)
483 /* ... and graft new one */
486 dev_queue->qdisc_sleeping = qdisc;
487 dev_queue->qdisc = &noop_qdisc;
490 qdisc_unlock_tree(dev);
492 if (dev->flags & IFF_UP)
498 void qdisc_tree_decrease_qlen(struct Qdisc *sch, unsigned int n)
500 const struct Qdisc_class_ops *cops;
506 while ((parentid = sch->parent)) {
507 if (TC_H_MAJ(parentid) == TC_H_MAJ(TC_H_INGRESS))
510 sch = qdisc_lookup(qdisc_dev(sch), TC_H_MAJ(parentid));
512 WARN_ON(parentid != TC_H_ROOT);
515 cops = sch->ops->cl_ops;
516 if (cops->qlen_notify) {
517 cl = cops->get(sch, parentid);
518 cops->qlen_notify(sch, cl);
524 EXPORT_SYMBOL(qdisc_tree_decrease_qlen);
526 /* Graft qdisc "new" to class "classid" of qdisc "parent" or
529 Old qdisc is not destroyed but returned in *old.
532 static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
534 struct Qdisc *new, struct Qdisc **old)
537 struct Qdisc *q = *old;
540 if (parent == NULL) {
541 if (q && q->flags&TCQ_F_INGRESS) {
542 *old = dev_graft_qdisc(dev, q);
544 *old = dev_graft_qdisc(dev, new);
547 const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
552 unsigned long cl = cops->get(parent, classid);
554 err = cops->graft(parent, cl, new, old);
555 cops->put(parent, cl);
563 Allocate and initialize new qdisc.
565 Parameters are passed via opt.
568 static struct Qdisc *
569 qdisc_create(struct net_device *dev, struct netdev_queue *dev_queue,
570 u32 parent, u32 handle, struct nlattr **tca, int *errp)
573 struct nlattr *kind = tca[TCA_KIND];
575 struct Qdisc_ops *ops;
577 ops = qdisc_lookup_ops(kind);
579 if (ops == NULL && kind != NULL) {
581 if (nla_strlcpy(name, kind, IFNAMSIZ) < IFNAMSIZ) {
582 /* We dropped the RTNL semaphore in order to
583 * perform the module load. So, even if we
584 * succeeded in loading the module we have to
585 * tell the caller to replay the request. We
586 * indicate this using -EAGAIN.
587 * We replay the request because the device may
588 * go away in the mean time.
591 request_module("sch_%s", name);
593 ops = qdisc_lookup_ops(kind);
595 /* We will try again qdisc_lookup_ops,
596 * so don't keep a reference.
598 module_put(ops->owner);
610 sch = qdisc_alloc(dev_queue, ops);
616 sch->parent = parent;
618 if (handle == TC_H_INGRESS) {
619 sch->flags |= TCQ_F_INGRESS;
620 handle = TC_H_MAKE(TC_H_INGRESS, 0);
623 handle = qdisc_alloc_handle(dev);
630 sch->handle = handle;
632 if (!ops->init || (err = ops->init(sch, tca[TCA_OPTIONS])) == 0) {
634 err = gen_new_estimator(&sch->bstats, &sch->rate_est,
635 qdisc_root_lock(sch),
639 * Any broken qdiscs that would require
640 * a ops->reset() here? The qdisc was never
641 * in action so it shouldn't be necessary.
648 qdisc_lock_tree(dev);
649 list_add_tail(&sch->list, &dev_queue->qdisc_list);
650 qdisc_unlock_tree(dev);
656 kfree((char *) sch - sch->padded);
658 module_put(ops->owner);
664 static int qdisc_change(struct Qdisc *sch, struct nlattr **tca)
666 if (tca[TCA_OPTIONS]) {
669 if (sch->ops->change == NULL)
671 err = sch->ops->change(sch, tca[TCA_OPTIONS]);
676 gen_replace_estimator(&sch->bstats, &sch->rate_est,
677 qdisc_root_lock(sch), tca[TCA_RATE]);
681 struct check_loop_arg
683 struct qdisc_walker w;
688 static int check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w);
690 static int check_loop(struct Qdisc *q, struct Qdisc *p, int depth)
692 struct check_loop_arg arg;
694 if (q->ops->cl_ops == NULL)
697 arg.w.stop = arg.w.skip = arg.w.count = 0;
698 arg.w.fn = check_loop_fn;
701 q->ops->cl_ops->walk(q, &arg.w);
702 return arg.w.stop ? -ELOOP : 0;
706 check_loop_fn(struct Qdisc *q, unsigned long cl, struct qdisc_walker *w)
709 const struct Qdisc_class_ops *cops = q->ops->cl_ops;
710 struct check_loop_arg *arg = (struct check_loop_arg *)w;
712 leaf = cops->leaf(q, cl);
714 if (leaf == arg->p || arg->depth > 7)
716 return check_loop(leaf, arg->p, arg->depth + 1);
725 static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
727 struct net *net = sock_net(skb->sk);
728 struct tcmsg *tcm = NLMSG_DATA(n);
729 struct nlattr *tca[TCA_MAX + 1];
730 struct net_device *dev;
731 u32 clid = tcm->tcm_parent;
732 struct Qdisc *q = NULL;
733 struct Qdisc *p = NULL;
736 if (net != &init_net)
739 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
742 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
747 if (clid != TC_H_ROOT) {
748 if (TC_H_MAJ(clid) != TC_H_MAJ(TC_H_INGRESS)) {
749 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
751 q = qdisc_leaf(p, clid);
752 } else { /* ingress */
753 q = dev->rx_queue.qdisc;
756 struct netdev_queue *dev_queue;
757 dev_queue = netdev_get_tx_queue(dev, 0);
758 q = dev_queue->qdisc_sleeping;
763 if (tcm->tcm_handle && q->handle != tcm->tcm_handle)
766 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
770 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
773 if (n->nlmsg_type == RTM_DELQDISC) {
778 if ((err = qdisc_graft(dev, p, clid, NULL, &q)) != 0)
781 qdisc_notify(skb, n, clid, q, NULL);
782 qdisc_lock_tree(dev);
784 qdisc_unlock_tree(dev);
787 qdisc_notify(skb, n, clid, NULL, q);
796 static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
798 struct net *net = sock_net(skb->sk);
800 struct nlattr *tca[TCA_MAX + 1];
801 struct net_device *dev;
806 if (net != &init_net)
810 /* Reinit, just in case something touches this. */
812 clid = tcm->tcm_parent;
815 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
818 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
823 if (clid != TC_H_ROOT) {
824 if (clid != TC_H_INGRESS) {
825 if ((p = qdisc_lookup(dev, TC_H_MAJ(clid))) == NULL)
827 q = qdisc_leaf(p, clid);
828 } else { /*ingress */
829 q = dev->rx_queue.qdisc;
832 struct netdev_queue *dev_queue;
833 dev_queue = netdev_get_tx_queue(dev, 0);
834 q = dev_queue->qdisc_sleeping;
837 /* It may be default qdisc, ignore it */
838 if (q && q->handle == 0)
841 if (!q || !tcm->tcm_handle || q->handle != tcm->tcm_handle) {
842 if (tcm->tcm_handle) {
843 if (q && !(n->nlmsg_flags&NLM_F_REPLACE))
845 if (TC_H_MIN(tcm->tcm_handle))
847 if ((q = qdisc_lookup(dev, tcm->tcm_handle)) == NULL)
849 if (n->nlmsg_flags&NLM_F_EXCL)
851 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
854 (p && check_loop(q, p, 0)))
856 atomic_inc(&q->refcnt);
862 /* This magic test requires explanation.
864 * We know, that some child q is already
865 * attached to this parent and have choice:
866 * either to change it or to create/graft new one.
868 * 1. We are allowed to create/graft only
869 * if CREATE and REPLACE flags are set.
871 * 2. If EXCL is set, requestor wanted to say,
872 * that qdisc tcm_handle is not expected
873 * to exist, so that we choose create/graft too.
875 * 3. The last case is when no flags are set.
876 * Alas, it is sort of hole in API, we
877 * cannot decide what to do unambiguously.
878 * For now we select create/graft, if
879 * user gave KIND, which does not match existing.
881 if ((n->nlmsg_flags&NLM_F_CREATE) &&
882 (n->nlmsg_flags&NLM_F_REPLACE) &&
883 ((n->nlmsg_flags&NLM_F_EXCL) ||
885 nla_strcmp(tca[TCA_KIND], q->ops->id))))
890 if (!tcm->tcm_handle)
892 q = qdisc_lookup(dev, tcm->tcm_handle);
895 /* Change qdisc parameters */
898 if (n->nlmsg_flags&NLM_F_EXCL)
900 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], q->ops->id))
902 err = qdisc_change(q, tca);
904 qdisc_notify(skb, n, clid, NULL, q);
908 if (!(n->nlmsg_flags&NLM_F_CREATE))
910 if (clid == TC_H_INGRESS)
911 q = qdisc_create(dev, &dev->rx_queue,
912 tcm->tcm_parent, tcm->tcm_parent,
915 q = qdisc_create(dev, netdev_get_tx_queue(dev, 0),
916 tcm->tcm_parent, tcm->tcm_handle,
926 struct Qdisc *old_q = NULL;
927 err = qdisc_graft(dev, p, clid, q, &old_q);
930 qdisc_lock_tree(dev);
932 qdisc_unlock_tree(dev);
936 qdisc_notify(skb, n, clid, old_q, q);
938 qdisc_lock_tree(dev);
939 qdisc_destroy(old_q);
940 qdisc_unlock_tree(dev);
946 static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
947 u32 pid, u32 seq, u16 flags, int event)
950 struct nlmsghdr *nlh;
951 unsigned char *b = skb_tail_pointer(skb);
954 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
955 tcm = NLMSG_DATA(nlh);
956 tcm->tcm_family = AF_UNSPEC;
959 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
960 tcm->tcm_parent = clid;
961 tcm->tcm_handle = q->handle;
962 tcm->tcm_info = atomic_read(&q->refcnt);
963 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
964 if (q->ops->dump && q->ops->dump(q, skb) < 0)
965 goto nla_put_failure;
966 q->qstats.qlen = q->q.qlen;
968 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
969 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
970 goto nla_put_failure;
972 if (q->ops->dump_stats && q->ops->dump_stats(q, &d) < 0)
973 goto nla_put_failure;
975 if (gnet_stats_copy_basic(&d, &q->bstats) < 0 ||
976 gnet_stats_copy_rate_est(&d, &q->rate_est) < 0 ||
977 gnet_stats_copy_queue(&d, &q->qstats) < 0)
978 goto nla_put_failure;
980 if (gnet_stats_finish_copy(&d) < 0)
981 goto nla_put_failure;
983 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
992 static int qdisc_notify(struct sk_buff *oskb, struct nlmsghdr *n,
993 u32 clid, struct Qdisc *old, struct Qdisc *new)
996 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
998 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1002 if (old && old->handle) {
1003 if (tc_fill_qdisc(skb, old, clid, pid, n->nlmsg_seq, 0, RTM_DELQDISC) < 0)
1007 if (tc_fill_qdisc(skb, new, clid, pid, n->nlmsg_seq, old ? NLM_F_REPLACE : 0, RTM_NEWQDISC) < 0)
1012 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1019 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
1021 struct net *net = sock_net(skb->sk);
1024 struct net_device *dev;
1027 if (net != &init_net)
1030 s_idx = cb->args[0];
1031 s_q_idx = q_idx = cb->args[1];
1032 read_lock(&dev_base_lock);
1034 for_each_netdev(&init_net, dev) {
1035 struct netdev_queue *dev_queue;
1041 dev_queue = netdev_get_tx_queue(dev, 0);
1042 list_for_each_entry(q, &dev_queue->qdisc_list, list) {
1043 if (q_idx < s_q_idx) {
1047 if (tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).pid,
1048 cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWQDISC) <= 0)
1057 read_unlock(&dev_base_lock);
1060 cb->args[1] = q_idx;
1067 /************************************************
1068 * Traffic classes manipulation. *
1069 ************************************************/
1073 static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n, void *arg)
1075 struct net *net = sock_net(skb->sk);
1076 struct netdev_queue *dev_queue;
1077 struct tcmsg *tcm = NLMSG_DATA(n);
1078 struct nlattr *tca[TCA_MAX + 1];
1079 struct net_device *dev;
1080 struct Qdisc *q = NULL;
1081 const struct Qdisc_class_ops *cops;
1082 unsigned long cl = 0;
1083 unsigned long new_cl;
1084 u32 pid = tcm->tcm_parent;
1085 u32 clid = tcm->tcm_handle;
1086 u32 qid = TC_H_MAJ(clid);
1089 if (net != &init_net)
1092 if ((dev = __dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
1095 err = nlmsg_parse(n, sizeof(*tcm), tca, TCA_MAX, NULL);
1100 parent == TC_H_UNSPEC - unspecified parent.
1101 parent == TC_H_ROOT - class is root, which has no parent.
1102 parent == X:0 - parent is root class.
1103 parent == X:Y - parent is a node in hierarchy.
1104 parent == 0:Y - parent is X:Y, where X:0 is qdisc.
1106 handle == 0:0 - generate handle from kernel pool.
1107 handle == 0:Y - class is X:Y, where X:0 is qdisc.
1108 handle == X:Y - clear.
1109 handle == X:0 - root class.
1112 /* Step 1. Determine qdisc handle X:0 */
1114 dev_queue = netdev_get_tx_queue(dev, 0);
1115 if (pid != TC_H_ROOT) {
1116 u32 qid1 = TC_H_MAJ(pid);
1119 /* If both majors are known, they must be identical. */
1124 } else if (qid == 0)
1125 qid = dev_queue->qdisc_sleeping->handle;
1127 /* Now qid is genuine qdisc handle consistent
1128 both with parent and child.
1130 TC_H_MAJ(pid) still may be unspecified, complete it now.
1133 pid = TC_H_MAKE(qid, pid);
1136 qid = dev_queue->qdisc_sleeping->handle;
1139 /* OK. Locate qdisc */
1140 if ((q = qdisc_lookup(dev, qid)) == NULL)
1143 /* An check that it supports classes */
1144 cops = q->ops->cl_ops;
1148 /* Now try to get class */
1150 if (pid == TC_H_ROOT)
1153 clid = TC_H_MAKE(qid, clid);
1156 cl = cops->get(q, clid);
1160 if (n->nlmsg_type != RTM_NEWTCLASS || !(n->nlmsg_flags&NLM_F_CREATE))
1163 switch (n->nlmsg_type) {
1166 if (n->nlmsg_flags&NLM_F_EXCL)
1170 err = cops->delete(q, cl);
1172 tclass_notify(skb, n, q, cl, RTM_DELTCLASS);
1175 err = tclass_notify(skb, n, q, cl, RTM_NEWTCLASS);
1184 err = cops->change(q, clid, pid, tca, &new_cl);
1186 tclass_notify(skb, n, q, new_cl, RTM_NEWTCLASS);
1196 static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
1198 u32 pid, u32 seq, u16 flags, int event)
1201 struct nlmsghdr *nlh;
1202 unsigned char *b = skb_tail_pointer(skb);
1204 const struct Qdisc_class_ops *cl_ops = q->ops->cl_ops;
1206 nlh = NLMSG_NEW(skb, pid, seq, event, sizeof(*tcm), flags);
1207 tcm = NLMSG_DATA(nlh);
1208 tcm->tcm_family = AF_UNSPEC;
1209 tcm->tcm_ifindex = qdisc_dev(q)->ifindex;
1210 tcm->tcm_parent = q->handle;
1211 tcm->tcm_handle = q->handle;
1213 NLA_PUT_STRING(skb, TCA_KIND, q->ops->id);
1214 if (cl_ops->dump && cl_ops->dump(q, cl, skb, tcm) < 0)
1215 goto nla_put_failure;
1217 if (gnet_stats_start_copy_compat(skb, TCA_STATS2, TCA_STATS,
1218 TCA_XSTATS, qdisc_root_lock(q), &d) < 0)
1219 goto nla_put_failure;
1221 if (cl_ops->dump_stats && cl_ops->dump_stats(q, cl, &d) < 0)
1222 goto nla_put_failure;
1224 if (gnet_stats_finish_copy(&d) < 0)
1225 goto nla_put_failure;
1227 nlh->nlmsg_len = skb_tail_pointer(skb) - b;
1236 static int tclass_notify(struct sk_buff *oskb, struct nlmsghdr *n,
1237 struct Qdisc *q, unsigned long cl, int event)
1239 struct sk_buff *skb;
1240 u32 pid = oskb ? NETLINK_CB(oskb).pid : 0;
1242 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
1246 if (tc_fill_tclass(skb, q, cl, pid, n->nlmsg_seq, 0, event) < 0) {
1251 return rtnetlink_send(skb, &init_net, pid, RTNLGRP_TC, n->nlmsg_flags&NLM_F_ECHO);
1254 struct qdisc_dump_args
1256 struct qdisc_walker w;
1257 struct sk_buff *skb;
1258 struct netlink_callback *cb;
1261 static int qdisc_class_dump(struct Qdisc *q, unsigned long cl, struct qdisc_walker *arg)
1263 struct qdisc_dump_args *a = (struct qdisc_dump_args *)arg;
1265 return tc_fill_tclass(a->skb, q, cl, NETLINK_CB(a->cb->skb).pid,
1266 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, RTM_NEWTCLASS);
1269 static int tc_dump_tclass(struct sk_buff *skb, struct netlink_callback *cb)
1271 struct net *net = sock_net(skb->sk);
1272 struct netdev_queue *dev_queue;
1275 struct net_device *dev;
1277 struct tcmsg *tcm = (struct tcmsg*)NLMSG_DATA(cb->nlh);
1278 struct qdisc_dump_args arg;
1280 if (net != &init_net)
1283 if (cb->nlh->nlmsg_len < NLMSG_LENGTH(sizeof(*tcm)))
1285 if ((dev = dev_get_by_index(&init_net, tcm->tcm_ifindex)) == NULL)
1291 dev_queue = netdev_get_tx_queue(dev, 0);
1292 list_for_each_entry(q, &dev_queue->qdisc_list, list) {
1293 if (t < s_t || !q->ops->cl_ops ||
1295 TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
1300 memset(&cb->args[1], 0, sizeof(cb->args)-sizeof(cb->args[0]));
1301 arg.w.fn = qdisc_class_dump;
1305 arg.w.skip = cb->args[1];
1307 q->ops->cl_ops->walk(q, &arg.w);
1308 cb->args[1] = arg.w.count;
1320 /* Main classifier routine: scans classifier chain attached
1321 to this qdisc, (optionally) tests for protocol and asks
1322 specific classifiers.
1324 int tc_classify_compat(struct sk_buff *skb, struct tcf_proto *tp,
1325 struct tcf_result *res)
1327 __be16 protocol = skb->protocol;
1330 for (; tp; tp = tp->next) {
1331 if ((tp->protocol == protocol ||
1332 tp->protocol == htons(ETH_P_ALL)) &&
1333 (err = tp->classify(skb, tp, res)) >= 0) {
1334 #ifdef CONFIG_NET_CLS_ACT
1335 if (err != TC_ACT_RECLASSIFY && skb->tc_verd)
1336 skb->tc_verd = SET_TC_VERD(skb->tc_verd, 0);
1343 EXPORT_SYMBOL(tc_classify_compat);
1345 int tc_classify(struct sk_buff *skb, struct tcf_proto *tp,
1346 struct tcf_result *res)
1350 #ifdef CONFIG_NET_CLS_ACT
1351 struct tcf_proto *otp = tp;
1354 protocol = skb->protocol;
1356 err = tc_classify_compat(skb, tp, res);
1357 #ifdef CONFIG_NET_CLS_ACT
1358 if (err == TC_ACT_RECLASSIFY) {
1359 u32 verd = G_TC_VERD(skb->tc_verd);
1362 if (verd++ >= MAX_REC_LOOP) {
1363 printk("rule prio %u protocol %02x reclassify loop, "
1365 tp->prio&0xffff, ntohs(tp->protocol));
1368 skb->tc_verd = SET_TC_VERD(skb->tc_verd, verd);
1374 EXPORT_SYMBOL(tc_classify);
1376 void tcf_destroy(struct tcf_proto *tp)
1378 tp->ops->destroy(tp);
1379 module_put(tp->ops->owner);
1383 void tcf_destroy_chain(struct tcf_proto **fl)
1385 struct tcf_proto *tp;
1387 while ((tp = *fl) != NULL) {
1392 EXPORT_SYMBOL(tcf_destroy_chain);
1394 #ifdef CONFIG_PROC_FS
1395 static int psched_show(struct seq_file *seq, void *v)
1399 hrtimer_get_res(CLOCK_MONOTONIC, &ts);
1400 seq_printf(seq, "%08x %08x %08x %08x\n",
1401 (u32)NSEC_PER_USEC, (u32)PSCHED_US2NS(1),
1403 (u32)NSEC_PER_SEC/(u32)ktime_to_ns(timespec_to_ktime(ts)));
1408 static int psched_open(struct inode *inode, struct file *file)
1410 return single_open(file, psched_show, PDE(inode)->data);
1413 static const struct file_operations psched_fops = {
1414 .owner = THIS_MODULE,
1415 .open = psched_open,
1417 .llseek = seq_lseek,
1418 .release = single_release,
1422 static int __init pktsched_init(void)
1424 register_qdisc(&pfifo_qdisc_ops);
1425 register_qdisc(&bfifo_qdisc_ops);
1426 proc_net_fops_create(&init_net, "psched", 0, &psched_fops);
1428 rtnl_register(PF_UNSPEC, RTM_NEWQDISC, tc_modify_qdisc, NULL);
1429 rtnl_register(PF_UNSPEC, RTM_DELQDISC, tc_get_qdisc, NULL);
1430 rtnl_register(PF_UNSPEC, RTM_GETQDISC, tc_get_qdisc, tc_dump_qdisc);
1431 rtnl_register(PF_UNSPEC, RTM_NEWTCLASS, tc_ctl_tclass, NULL);
1432 rtnl_register(PF_UNSPEC, RTM_DELTCLASS, tc_ctl_tclass, NULL);
1433 rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass);
1438 subsys_initcall(pktsched_init);