diff --git a/module/compat/pfk_ifq.c b/module/compat/pfk_ifq.c index c8110d7..7b5c6d6 100644 --- a/module/compat/pfk_ifq.c +++ b/module/compat/pfk_ifq.c @@ -360,14 +360,16 @@ ifq_deq_begin(struct ifqueue *ifq) if (!ifq->ifq_ops || !ifq->ifq_ops->ifqop_deq_begin) return NULL; m = ifq->ifq_ops->ifqop_deq_begin(ifq, &cookie); + ifq->ifq_dequeue = cookie; return m; } void ifq_deq_commit(struct ifqueue *ifq, struct mbuf *m) { - void *cookie = NULL; + void *cookie = ifq->ifq_dequeue; + ifq->ifq_dequeue = NULL; if (ifq->ifq_ops && ifq->ifq_ops->ifqop_deq_commit) ifq->ifq_ops->ifqop_deq_commit(ifq, m, cookie); } @@ -389,12 +391,47 @@ ifq_dequeue(struct ifqueue *ifq) return m; } +/* + * Drain the queue as far as the scheduler allows and hand each packet to the + * stack's output path. OpenBSD calls if_start from softint context; here it + * runs either from the pf work queue (enqueue side) or from the scheduler's + * deferred timeout (pacing side), which is what makes hfsc's bandwidth + * limits take effect. + */ void ifq_start(struct ifqueue *ifq) { + struct mbuf *m; + + ifq->ifq_oactive = 1; + while ((m = ifq_dequeue(ifq)) != NULL) { + if (pfk_ifq_transmit(ifq, m)) + break; /* device busy: packet kept */ + } ifq->ifq_oactive = 0; } +/* + * pf data-path entry: hand a packet to the interface's scheduler when one is + * attached (a pf `queue' rule is configured). Returns 1 when the queue took + * ownership of m, 0 when the caller must keep going (priq/no scheduler). + */ +int +pf_kifq_input(struct pfk_ifnet *ifp, struct mbuf *m) +{ + struct ifqueue *ifq; + + if (ifp == NULL || m == NULL) + return 0; + ifq = &ifp->if_snd; + if (ifq->ifq_ops == NULL || ifq->ifq_ops == ifq_priq_ops) + return 0; /* no scheduler: fast path */ + if (ifq_enqueue(ifq, m) != 0) + return 1; /* dropped by the queue: owned */ + ifq_start(ifq); + return 1; +} + unsigned int ifq_purge(struct ifqueue *ifq) { diff --git a/module/compat/pfk_xmit.c b/module/compat/pfk_xmit.c index 249a88f..db64ec3 100644 --- a/module/compat/pfk_xmit.c +++ b/module/compat/pfk_xmit.c @@ -270,3 +270,40 @@ pfk_if_output(struct pfk_ifnet *ifp, struct mbuf *m, void *dst, m_freem(mp); return err; } + +/* + * Transmit one queued packet. Called from ifq_start() - i.e. from the pf + * work queue or from a scheduler's deferred timeout - and always consumes the + * mbuf. Returns 0 (keep draining); a scheduler that wants to defer does so by + * simply not returning a packet from ifqop_deq_begin(). + */ +int +pfk_ifq_transmit(struct ifqueue *ifq, struct mbuf *m) +{ + struct sk_buff *skb = m_to_skb(m); + struct dst_entry *d; + + (void)ifq; + if (skb == NULL) { + m_freem(m); + return 0; + } + d = skb_dst(skb); + if (d == NULL || IS_ERR(d) || d->dev == NULL || !netif_running(d->dev)) { + m_freem(m); + return 0; + } + skb->dev = d->dev; + skb_reset_network_header(skb); + if (skb->protocol == 0) + skb->protocol = htons(ip_hdr(skb)->protocol == IPPROTO_IPV6 ? + ETH_P_IPV6 : ETH_P_IP); + /* the stack owns skb->cb (IPCB) from here; pf already tested it */ + memset(skb->cb, 0, sizeof(skb->cb)); + skb->mark |= PFK_DONE_OUT; + skb_get(skb); /* the output path consumes one reference */ + m->skb = NULL; /* the mbuf's reference moves to the stack */ + m_freem(m); + d->output(&init_net, NULL, skb); + return 0; +} diff --git a/module/include/compat/net/ifq.h b/module/include/compat/net/ifq.h index bc3f089..8f668a9 100644 --- a/module/include/compat/net/ifq.h +++ b/module/include/compat/net/ifq.h @@ -27,6 +27,7 @@ struct ifqueue { struct mbuf_list *ifq_free; /* free list (unused in the port) */ unsigned int ifq_len; unsigned int ifq_oactive; + void *ifq_dequeue; /* cookie between deq_begin/commit */ /* statistics */ u_int64_t ifq_packets; diff --git a/module/include/pfk.h b/module/include/pfk.h index 3618422..b3152f8 100644 --- a/module/include/pfk.h +++ b/module/include/pfk.h @@ -1237,6 +1237,8 @@ struct fqcodel_stats { * queue when it owns the interface queue, NULL otherwise. */ void *ifq_q_enter(struct ifqueue *, const struct ifq_ops *); void ifq_q_leave(struct ifqueue *, void *); +int pfk_ifq_transmit(struct ifqueue *, struct mbuf *); +int pf_kifq_input(struct pfk_ifnet *, struct mbuf *); /* diff --git a/module/pfsrc/pfk_hooks.c b/module/pfsrc/pfk_hooks.c index e5b926f..f4db68b 100644 --- a/module/pfsrc/pfk_hooks.c +++ b/module/pfsrc/pfk_hooks.c @@ -204,6 +204,22 @@ pfk_pkt_work(struct work_struct *w) if (m == m0) { + /* + * Queueing: when a pf `queue' rule attached a scheduler to the + * egress interface, hand the packet to it instead of sending it + * straight out - this is what makes bandwidth limits real. + */ + if (action == PF_PASS && pk->dir == PF_OUT && + pskb_may_pull(pk->skb, sizeof(struct iphdr)) && + skb_dst(pk->skb) && !IS_ERR(skb_dst(pk->skb)) && + pf_kifq_input(skb_dst(pk->skb)->dev ? + pfk_ifp_bydev(skb_dst(pk->skb)->dev) : NULL, m)) { + orig = pk->skb; + pk->skb = NULL; /* the queue owns the packet */ + kfree_skb(orig); + kfree(pk); + return; + } /* same buffer, possibly rewritten in place: our wrapper * held one extra skb reference — release it */ m_freem(m);